aboutsummaryrefslogtreecommitdiffstats
path: root/scripts/lib/devtool/upgrade.py
blob: 8851a6edbed91850c3aa11efd6ecfd2aab79c6af (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
# Development tool - upgrade command plugin
#
# Copyright (C) 2014-2015 Intel Corporation
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
"""Devtool upgrade plugin"""

import os
import sys
import re
import shutil
import tempfile
import logging
import argparse
import scriptutils
import errno
import bb
import oe.recipeutils
from devtool import standard
from devtool import exec_build_env_command, setup_tinfoil, DevtoolError, parse_recipe, use_external_build

logger = logging.getLogger('devtool')

def plugin_init(pluginlist):
    """Plugin initialization"""
    pass

def _run(cmd, cwd=''):
    logger.debug("Running command %s> %s" % (cwd,cmd))
    return bb.process.run('%s' % cmd, cwd=cwd)

def _get_srctree(tmpdir):
    srctree = tmpdir
    dirs = os.listdir(tmpdir)
    if len(dirs) == 1:
        srctree = os.path.join(tmpdir, dirs[0])
    return srctree

def _copy_source_code(orig, dest):
    for path in standard._ls_tree(orig):
        dest_dir = os.path.join(dest, os.path.dirname(path))
        bb.utils.mkdirhier(dest_dir)
        dest_path = os.path.join(dest, path)
        shutil.move(os.path.join(orig, path), dest_path)

def _get_checksums(rf):
    import re
    checksums = {}
    with open(rf) as f:
        for line in f:
            for cs in ['md5sum', 'sha256sum']:
                m = re.match("^SRC_URI\[%s\].*=.*\"(.*)\"" % cs, line)
                if m:
                    checksums[cs] = m.group(1)
    return checksums

def _replace_checksums(rf, md5, sha256):
    if not md5 and not sha256:
        return
    checksums = {'md5sum':md5, 'sha256sum':sha256}
    with open(rf + ".tmp", "w+") as tmprf:
        with open(rf) as f:
            for line in f:
                m = None
                for cs in checksums.keys():
                    m = re.match("^SRC_URI\[%s\].*=.*\"(.*)\"" % cs, line)
                    if m:
                        if checksums[cs]:
                            oldcheck = m.group(1)
                            newcheck = checksums[cs]
                            line = line.replace(oldcheck, newcheck)
                        break
                tmprf.write(line)
    os.rename(rf + ".tmp", rf)


def _remove_patch_dirs(recipefolder):
    for root, dirs, files in os.walk(recipefolder):
        for d in dirs:
            shutil.rmtree(os.path.join(root,d))

def _recipe_contains(rd, var):
    rf = rd.getVar('FILE', True)
    varfiles = oe.recipeutils.get_var_files(rf, [var], rd)
    for var, fn in varfiles.iteritems():
        if fn and fn.startswith(os.path.dirname(rf) + os.sep):
            return True
    return False

def _rename_recipe_dirs(oldpv, newpv, path):
    for root, dirs, files in os.walk(path):
        for olddir in dirs:
            if olddir.find(oldpv) != -1:
                newdir = olddir.replace(oldpv, newpv)
                if olddir != newdir:
                    _run('mv %s %s' % (olddir, newdir))

def _rename_recipe_file(bpn, oldpv, newpv, path):
    oldrecipe = "%s_%s.bb" % (bpn, oldpv)
    newrecipe = "%s_%s.bb" % (bpn, newpv)
    if os.path.isfile(os.path.join(path, oldrecipe)):
        if oldrecipe != newrecipe:
            _run('mv %s %s' % (oldrecipe, newrecipe), cwd=path)
    else:
        recipe = "%s_git.bb" % bpn
        if os.path.isfile(os.path.join(path, recipe)):
            newrecipe = recipe
    return os.path.join(path, newrecipe)

def _rename_recipe_files(bpn, oldpv, newpv, path):
    _rename_recipe_dirs(oldpv, newpv, path)
    return _rename_recipe_file(bpn, oldpv, newpv, path)

def _write_append(rc, srctree, same_dir, no_same_dir, rev, workspace, d):
    """Writes an append file"""
    if not os.path.exists(rc):
        raise DevtoolError("bbappend not created because %s does not exist" % rc)

    appendpath = os.path.join(workspace, 'appends')
    if not os.path.exists(appendpath):
        bb.utils.mkdirhier(appendpath)

    brf = os.path.basename(os.path.splitext(rc)[0]) # rc basename

    srctree = os.path.abspath(srctree)
    pn = d.getVar('PN',True)
    af = os.path.join(appendpath, '%s.bbappend' % brf)
    with open(af, 'w') as f:
        f.write('FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n\n')
        f.write('inherit externalsrc\n')
        f.write(('# NOTE: We use pn- overrides here to avoid affecting'
                 'multiple variants in the case where the recipe uses BBCLASSEXTEND\n'))
        f.write('EXTERNALSRC_pn-%s = "%s"\n' % (pn, srctree))
        b_is_s = use_external_build(same_dir, no_same_dir, d)
        if b_is_s:
            f.write('EXTERNALSRC_BUILD_pn-%s = "%s"\n' % (pn, srctree))
        if rev:
            f.write('\n# initial_rev: %s\n' % rev)
    return af

def _cleanup_on_error(rf, srctree):
    rfp = os.path.split(rf)[0] # recipe folder
    rfpp = os.path.split(rfp)[0] # recipes folder
    if os.path.exists(rfp):
        shutil.rmtree(b)
    if not len(os.listdir(rfpp)):
        os.rmdir(rfpp)
    srctree = os.path.abspath(srctree)
    if os.path.exists(srctree):
        shutil.rmtree(srctree)

def _upgrade_error(e, rf, srctree):
    if rf:
        cleanup_on_error(rf, srctree)
    logger.error(e)
    raise DevtoolError(e)

def _get_uri(rd):
    srcuris = rd.getVar('SRC_URI', True).split()
    if not len(srcuris):
        raise DevtoolError('SRC_URI not found on recipe')
    srcuri = srcuris[0] # it is assumed, URI is at first position
    srcrev = '${AUTOREV}'
    if '://' in srcuri:
        # Fetch a URL
        rev_re = re.compile(';rev=([^;]+)')
        res = rev_re.search(srcuri)
        if res:
            srcrev = res.group(1)
            srcuri = rev_re.sub('', srcuri)
    return srcuri, srcrev

def _extract_new_source(newpv, srctree, no_patch, srcrev, branch, keep_temp, tinfoil, rd):
    """Extract sources of a recipe with a new version"""

    def __run(cmd):
        """Simple wrapper which calls _run with srctree as cwd"""
        return _run(cmd, srctree)

    crd = rd.createCopy()

    pv = crd.getVar('PV', True)
    crd.setVar('PV', newpv)

    tmpsrctree = None
    uri, rev = _get_uri(crd)
    if srcrev:
        rev = srcrev
    if uri.startswith('git://'):
        __run('git checkout %s' % rev)
        __run('git tag -f devtool-base-new')
        md5 = None
        sha256 = None
    else:
        __run('git checkout -b devtool-%s' % newpv)

        tmpdir = tempfile.mkdtemp(prefix='devtool')
        try:
            md5, sha256 = scriptutils.fetch_uri(tinfoil.config_data, uri, tmpdir, rev)
        except bb.fetch2.FetchError as e:
            raise DevtoolError(e)

        tmpsrctree = _get_srctree(tmpdir)

        scrtree = os.path.abspath(srctree)

        _copy_source_code(tmpsrctree, srctree)

        (stdout,_) = __run('git ls-files --modified --others --exclude-standard')
        for f in stdout.splitlines():
            __run('git add "%s"' % f)

        __run('git commit -q -m "Commit of upstream changes at version %s" --allow-empty' % newpv)
        __run('git tag -f devtool-base-%s' % newpv)

    (stdout, _) = __run('git rev-parse HEAD')
    rev = stdout.rstrip()

    if no_patch:
        patches = oe.recipeutils.get_recipe_patches(crd)
        if len(patches):
            logger.warn('By user choice, the following patches will NOT be applied')
            for patch in patches:
                logger.warn("%s" % os.path.basename(patch))
    else:
        try:
            __run('git checkout devtool-patched -b %s' % branch)
            __run('git rebase %s' % rev)
            if uri.startswith('git://'):
                suffix = 'new'
            else:
                suffix = newpv
            __run('git tag -f devtool-patched-%s' % suffix)
        except bb.process.ExecutionError as e:
            logger.warn('Command \'%s\' failed:\n%s' % (e.command, e.stdout))

    if tmpsrctree:
        if keep_temp:
            logger.info('Preserving temporary directory %s' % tmpsrctree)
        else:
            shutil.rmtree(tmpsrctree)

    return (rev, md5, sha256)

def _create_new_recipe(newpv, md5, sha256, srcrev, workspace, tinfoil, rd):
    """Creates the new recipe under workspace"""
    crd = rd.createCopy()

    bpn = crd.getVar('BPN', True)
    path = os.path.join(workspace, 'recipes', bpn)
    bb.utils.mkdirhier(path)
    oe.recipeutils.copy_recipe_files(crd, path)

    oldpv = crd.getVar('PV', True)
    if not newpv:
        newpv = oldpv
    fullpath = _rename_recipe_files(bpn, oldpv, newpv, path)

    newvalues = {}
    if _recipe_contains(rd, 'PV') and newpv != oldpv:
        newvalues['PV'] = newpv

    if srcrev:
        newvalues['SRCREV'] = srcrev

    if newvalues:
        rd = oe.recipeutils.parse_recipe(fullpath, None, tinfoil.config_data)
        oe.recipeutils.patch_recipe(rd, fullpath, newvalues)

    if md5 and sha256:
        # Unfortunately, oe.recipeutils.patch_recipe cannot update flags.
        # once the latter feature is implemented, we should call patch_recipe
        # instead of the following function
        _replace_checksums(fullpath, md5, sha256)

    return fullpath

def upgrade(args, config, basepath, workspace):
    """Entry point for the devtool 'upgrade' subcommand"""

    if args.recipename in workspace:
        raise DevtoolError("recipe %s is already in your workspace" % args.recipename)
    if not args.version and not args.srcrev:
        raise DevtoolError("You must provide a version using the --version/-V option, or for recipes that fetch from an SCM such as git, the --srcrev/-S option")

    reason = oe.recipeutils.validate_pn(args.recipename)
    if reason:
        raise DevtoolError(reason)

    tinfoil = setup_tinfoil(basepath=basepath, tracking=True)

    rd = parse_recipe(config, tinfoil, args.recipename, True)
    if not rd:
        return 1

    pn = rd.getVar('PN', True)
    if pn != args.recipename:
        logger.info('Mapping %s to %s' % (args.recipename, pn))
    if pn in workspace:
        raise DevtoolError("recipe %s is already in your workspace" % pn)

    standard._check_compatible_recipe(pn, rd)
    if rd.getVar('PV', True) == args.version and rd.getVar('SRCREV', True) == args.srcrev:
        raise DevtoolError("Current and upgrade versions are the same version" % version)

    rf = None
    try:
        rev1 = standard._extract_source(args.srctree, False, 'devtool-orig', False, rd)
        rev2, md5, sha256 = _extract_new_source(args.version, args.srctree, args.no_patch,
                                                args.srcrev, args.branch, args.keep_temp,
                                                tinfoil, rd)
        rf = _create_new_recipe(args.version, md5, sha256, args.srcrev, config.workspace_path, tinfoil, rd)
    except bb.process.CmdError as e:
        _upgrade_error(e, rf, args.srctree)
    except DevtoolError as e:
        _upgrade_error(e, rf, args.srctree)
    standard._add_md5(config, pn, os.path.dirname(rf))

    af = _write_append(rf, args.srctree, args.same_dir, args.no_same_dir, rev2,
                       config.workspace_path, rd)
    standard._add_md5(config, pn, af)
    logger.info('Upgraded source extracted to %s' % args.srctree)
    return 0

def register_commands(subparsers, context):
    """Register devtool subcommands from this plugin"""
    parser_upgrade = subparsers.add_parser('upgrade', help='Upgrade an existing recipe',
                                           description='Upgrades an existing recipe to a new upstream version')
    parser_upgrade.add_argument('recipename', help='Name for recipe to extract the source for')
    parser_upgrade.add_argument('srctree', help='Path to where to extract the source tree')
    parser_upgrade.add_argument('--version', '-V', help='Version to upgrade to (PV)')
    parser_upgrade.add_argument('--srcrev', '-S', help='Source revision to upgrade to (if fetching from an SCM such as git)')
    parser_upgrade.add_argument('--branch', '-b', default="devtool", help='Name for new development branch to checkout (default "%(default)s")')
    parser_upgrade.add_argument('--no-patch', action="store_true", help='Do not apply patches from the recipe to the new source code')
    group = parser_upgrade.add_mutually_exclusive_group()
    group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true")
    group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true")
    parser_upgrade.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
    parser_upgrade.set_defaults(func=upgrade)