summaryrefslogtreecommitdiffstats
path: root/scripts/lib/devtool/upgrade.py
blob: fa5b8ef3c77890dc262683a98e7b4fd57cc610ad (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
# Development tool - upgrade command plugin
#
# Copyright (C) 2014-2017 Intel Corporation
#
# SPDX-License-Identifier: GPL-2.0-only
#
"""Devtool upgrade plugin"""

import os
import sys
import re
import shutil
import tempfile
import logging
import argparse
import scriptutils
import errno
import bb

devtool_path = os.path.dirname(os.path.realpath(__file__)) + '/../../../meta/lib'
sys.path = sys.path + [devtool_path]

import oe.recipeutils
from devtool import standard
from devtool import exec_build_env_command, setup_tinfoil, DevtoolError, parse_recipe, use_external_build, update_unlockedsigs, check_prerelease_version

logger = logging.getLogger('devtool')

def _run(cmd, cwd=''):
    logger.debug("Running command %s> %s" % (cwd,cmd))
    return bb.process.run('%s' % cmd, cwd=cwd)

def _get_srctree(tmpdir):
    srctree = tmpdir
    dirs = scriptutils.filter_src_subdirs(tmpdir)
    if len(dirs) == 1:
        srctree = os.path.join(tmpdir, dirs[0])
    else:
        raise DevtoolError("Cannot determine where the source tree is after unpacking in {}: {}".format(tmpdir,dirs))
    return srctree

def _copy_source_code(orig, dest):
    for path in standard._ls_tree(orig):
        dest_dir = os.path.join(dest, os.path.dirname(path))
        bb.utils.mkdirhier(dest_dir)
        dest_path = os.path.join(dest, path)
        shutil.move(os.path.join(orig, path), dest_path)

def _remove_patch_dirs(recipefolder):
    for root, dirs, files in os.walk(recipefolder):
        for d in dirs:
            shutil.rmtree(os.path.join(root,d))

def _recipe_contains(rd, var):
    rf = rd.getVar('FILE')
    varfiles = oe.recipeutils.get_var_files(rf, [var], rd)
    for var, fn in varfiles.items():
        if fn and fn.startswith(os.path.dirname(rf) + os.sep):
            return True
    return False

def _rename_recipe_dirs(oldpv, newpv, path):
    for root, dirs, files in os.walk(path):
        # Rename directories with the version in their name
        for olddir in dirs:
            if olddir.find(oldpv) != -1:
                newdir = olddir.replace(oldpv, newpv)
                if olddir != newdir:
                    shutil.move(os.path.join(path, olddir), os.path.join(path, newdir))
        # Rename any inc files with the version in their name (unusual, but possible)
        for oldfile in files:
            if oldfile.endswith('.inc'):
                if oldfile.find(oldpv) != -1:
                    newfile = oldfile.replace(oldpv, newpv)
                    if oldfile != newfile:
                        bb.utils.rename(os.path.join(path, oldfile),
                              os.path.join(path, newfile))

def _rename_recipe_file(oldrecipe, bpn, oldpv, newpv, path):
    oldrecipe = os.path.basename(oldrecipe)
    if oldrecipe.endswith('_%s.bb' % oldpv):
        newrecipe = '%s_%s.bb' % (bpn, newpv)
        if oldrecipe != newrecipe:
            shutil.move(os.path.join(path, oldrecipe), os.path.join(path, newrecipe))
    else:
        newrecipe = oldrecipe
    return os.path.join(path, newrecipe)

def _rename_recipe_files(oldrecipe, bpn, oldpv, newpv, path):
    _rename_recipe_dirs(oldpv, newpv, path)
    return _rename_recipe_file(oldrecipe, bpn, oldpv, newpv, path)

def _write_append(rc, srctreebase, srctree, same_dir, no_same_dir, revs, copied, workspace, d):
    """Writes an append file"""
    if not os.path.exists(rc):
        raise DevtoolError("bbappend not created because %s does not exist" % rc)

    appendpath = os.path.join(workspace, 'appends')
    if not os.path.exists(appendpath):
        bb.utils.mkdirhier(appendpath)

    brf = os.path.basename(os.path.splitext(rc)[0]) # rc basename

    srctree = os.path.abspath(srctree)
    pn = d.getVar('PN')
    af = os.path.join(appendpath, '%s.bbappend' % brf)
    with open(af, 'w') as f:
        f.write('FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n\n')
        # Local files can be modified/tracked in separate subdir under srctree
        # Mostly useful for packages with S != WORKDIR
        f.write('FILESPATH:prepend := "%s:"\n' %
                os.path.join(srctreebase, 'oe-local-files'))
        f.write('# srctreebase: %s\n' % srctreebase)
        f.write('inherit externalsrc\n')
        f.write(('# NOTE: We use pn- overrides here to avoid affecting'
                 'multiple variants in the case where the recipe uses BBCLASSEXTEND\n'))
        f.write('EXTERNALSRC:pn-%s = "%s"\n' % (pn, srctree))
        b_is_s = use_external_build(same_dir, no_same_dir, d)
        if b_is_s:
            f.write('EXTERNALSRC_BUILD:pn-%s = "%s"\n' % (pn, srctree))
        f.write('\n')
        if revs:
            for name, rev in revs.items():
                f.write('# initial_rev %s: %s\n' % (name, rev))
        if copied:
            f.write('# original_path: %s\n' % os.path.dirname(d.getVar('FILE')))
            f.write('# original_files: %s\n' % ' '.join(copied))
    return af

def _cleanup_on_error(rd, srctree):
    if os.path.exists(rd):
        shutil.rmtree(rd)
    srctree = os.path.abspath(srctree)
    if os.path.exists(srctree):
        shutil.rmtree(srctree)

def _upgrade_error(e, rd, srctree, keep_failure=False, extramsg=None):
    if not keep_failure:
        _cleanup_on_error(rd, srctree)
    logger.error(e)
    if extramsg:
        logger.error(extramsg)
    if keep_failure:
        logger.info('Preserving failed upgrade files (--keep-failure)')
    sys.exit(1)

def _get_uri(rd):
    srcuris = rd.getVar('SRC_URI').split()
    if not len(srcuris):
        raise DevtoolError('SRC_URI not found on recipe')
    # Get first non-local entry in SRC_URI - usually by convention it's
    # the first entry, but not always!
    srcuri = None
    for entry in srcuris:
        if not entry.startswith('file://'):
            srcuri = entry
            break
    if not srcuri:
        raise DevtoolError('Unable to find non-local entry in SRC_URI')
    srcrev = '${AUTOREV}'
    if '://' in srcuri:
        # Fetch a URL
        rev_re = re.compile(';rev=([^;]+)')
        res = rev_re.search(srcuri)
        if res:
            srcrev = res.group(1)
            srcuri = rev_re.sub('', srcuri)
    return srcuri, srcrev

def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, keep_temp, tinfoil, rd):
    """Extract sources of a recipe with a new version"""

    def __run(cmd):
        """Simple wrapper which calls _run with srctree as cwd"""
        return _run(cmd, srctree)

    crd = rd.createCopy()

    pv = crd.getVar('PV')
    crd.setVar('PV', newpv)

    tmpsrctree = None
    uri, rev = _get_uri(crd)
    if srcrev:
        rev = srcrev
    paths = [srctree]
    if uri.startswith('git://') or uri.startswith('gitsm://'):
        __run('git fetch')
        __run('git checkout %s' % rev)
        __run('git tag -f devtool-base-new')
        __run('git submodule update --recursive')
        __run('git submodule foreach \'git tag -f devtool-base-new\'')
        (stdout, _) = __run('git submodule --quiet foreach \'echo $sm_path\'')
        paths += [os.path.join(srctree, p) for p in stdout.splitlines()]
        checksums = {}
        _, _, _, _, _, params = bb.fetch2.decodeurl(uri)
        srcsubdir_rel = params.get('destsuffix', 'git')
        if not srcbranch:
            check_branch, check_branch_err = __run('git branch -r --contains %s' % srcrev)
            get_branch = [x.strip() for x in check_branch.splitlines()]
            # Remove HEAD reference point and drop remote prefix
            get_branch = [x.split('/', 1)[1] for x in get_branch if not x.startswith('origin/HEAD')]
            if len(get_branch) == 1:
                # If srcrev is on only ONE branch, then use that branch
                srcbranch = get_branch[0]
            elif 'main' in get_branch:
                # If srcrev is on multiple branches, then choose 'main' if it is one of them
                srcbranch = 'main'
            elif 'master' in get_branch:
                # Otherwise choose 'master' if it is one of the branches
                srcbranch = 'master'
            else:
                # If get_branch contains more than one objects, then display error and exit.
                mbrch = '\n  ' + '\n  '.join(get_branch)
                raise DevtoolError('Revision %s was found on multiple branches: %s\nPlease provide the correct branch in the devtool command with "--srcbranch" or "-B" option.' % (srcrev, mbrch))
    else:
        __run('git checkout devtool-base -b devtool-%s' % newpv)

        tmpdir = tempfile.mkdtemp(prefix='devtool')
        try:
            checksums, ftmpdir = scriptutils.fetch_url(tinfoil, uri, rev, tmpdir, logger, preserve_tmp=keep_temp)
        except scriptutils.FetchUrlFailure as e:
            raise DevtoolError(e)

        if ftmpdir and keep_temp:
            logger.info('Fetch temp directory is %s' % ftmpdir)

        tmpsrctree = _get_srctree(tmpdir)
        srctree = os.path.abspath(srctree)
        srcsubdir_rel = os.path.relpath(tmpsrctree, tmpdir)

        # Delete all sources so we ensure no stray files are left over
        for item in os.listdir(srctree):
            if item in ['.git', 'oe-local-files']:
                continue
            itempath = os.path.join(srctree, item)
            if os.path.isdir(itempath):
                shutil.rmtree(itempath)
            else:
                os.remove(itempath)

        # Copy in new ones
        _copy_source_code(tmpsrctree, srctree)

        (stdout,_) = __run('git ls-files --modified --others')
        filelist = stdout.splitlines()
        pbar = bb.ui.knotty.BBProgress('Adding changed files', len(filelist))
        pbar.start()
        batchsize = 100
        for i in range(0, len(filelist), batchsize):
            batch = filelist[i:i+batchsize]
            __run('git add -f -A %s' % ' '.join(['"%s"' % item for item in batch]))
            pbar.update(i)
        pbar.finish()

        useroptions = []
        oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=rd)
        __run('git %s commit -q -m "Commit of upstream changes at version %s" --allow-empty' % (' '.join(useroptions), newpv))
        __run('git tag -f devtool-base-%s' % newpv)

    revs = {}
    for path in paths:
        (stdout, _) = _run('git rev-parse HEAD', cwd=path)
        revs[os.path.relpath(path, srctree)] = stdout.rstrip()

    if no_patch:
        patches = oe.recipeutils.get_recipe_patches(crd)
        if patches:
            logger.warning('By user choice, the following patches will NOT be applied to the new source tree:\n  %s' % '\n  '.join([os.path.basename(patch) for patch in patches]))
    else:
        for path in paths:
            _run('git checkout devtool-patched -b %s' % branch, cwd=path)
            (stdout, _) = _run('git branch --list devtool-override-*', cwd=path)
            branches_to_rebase = [branch] + stdout.split()
            target_branch = revs[os.path.relpath(path, srctree)]

            # There is a bug (or feature?) in git rebase where if a commit with
            # a note is fully rebased away by being part of an old commit, the
            # note is still attached to the old commit. Avoid this by making
            # sure all old devtool related commits have a note attached to them
            # (this assumes git config notes.rewriteMode is set to ignore).
            (stdout, _) = __run('git rev-list devtool-base..%s' % target_branch)
            for rev in stdout.splitlines():
                if not oe.patch.GitApplyTree.getNotes(path, rev):
                    oe.patch.GitApplyTree.addNote(path, rev, "dummy")

            for b in branches_to_rebase:
                logger.info("Rebasing {} onto {}".format(b, target_branch))
                _run('git checkout %s' % b, cwd=path)
                try:
                    _run('git rebase %s' % target_branch, cwd=path)
                except bb.process.ExecutionError as e:
                    if 'conflict' in e.stdout:
                        logger.warning('Command \'%s\' failed:\n%s\n\nYou will need to resolve conflicts in order to complete the upgrade.' % (e.command, e.stdout.rstrip()))
                        _run('git rebase --abort', cwd=path)
                    else:
                        logger.warning('Command \'%s\' failed:\n%s' % (e.command, e.stdout))

            # Remove any dummy notes added above.
            (stdout, _) = __run('git rev-list devtool-base..%s' % target_branch)
            for rev in stdout.splitlines():
                oe.patch.GitApplyTree.removeNote(path, rev, "dummy")

            _run('git checkout %s' % branch, cwd=path)

    if tmpsrctree:
        if keep_temp:
            logger.info('Preserving temporary directory %s' % tmpsrctree)
        else:
            shutil.rmtree(tmpsrctree)
            if tmpdir != tmpsrctree:
                shutil.rmtree(tmpdir)

    return (revs, checksums, srcbranch, srcsubdir_rel)

def _add_license_diff_to_recipe(path, diff):
    notice_text = """# FIXME: the LIC_FILES_CHKSUM values have been updated by 'devtool upgrade'.
# The following is the difference between the old and the new license text.
# Please update the LICENSE value if needed, and summarize the changes in
# the commit message via 'License-Update:' tag.
# (example: 'License-Update: copyright years updated.')
#
# The changes:
#
"""
    commented_diff = "\n".join(["# {}".format(l) for l in diff.split('\n')])
    with open(path, 'rb') as f:
        orig_content = f.read()
    with open(path, 'wb') as f:
        f.write(notice_text.encode())
        f.write(commented_diff.encode())
        f.write("\n#\n\n".encode())
        f.write(orig_content)

def _create_new_recipe(newpv, checksums, srcrev, srcbranch, srcsubdir_old, srcsubdir_new, workspace, tinfoil, rd, license_diff, new_licenses, srctree, keep_failure):
    """Creates the new recipe under workspace"""

    bpn = rd.getVar('BPN')
    path = os.path.join(workspace, 'recipes', bpn)
    bb.utils.mkdirhier(path)
    copied, _ = oe.recipeutils.copy_recipe_files(rd, path, all_variants=True)
    if not copied:
        raise DevtoolError('Internal error - no files were copied for recipe %s' % bpn)
    logger.debug('Copied %s to %s' % (copied, path))

    oldpv = rd.getVar('PV')
    if not newpv:
        newpv = oldpv
    origpath = rd.getVar('FILE')
    fullpath = _rename_recipe_files(origpath, bpn, oldpv, newpv, path)
    logger.debug('Upgraded %s => %s' % (origpath, fullpath))

    newvalues = {}
    if _recipe_contains(rd, 'PV') and newpv != oldpv:
        newvalues['PV'] = newpv

    if srcrev:
        newvalues['SRCREV'] = srcrev

    if srcbranch:
        src_uri = oe.recipeutils.split_var_value(rd.getVar('SRC_URI', False) or '')
        changed = False
        replacing = True
        new_src_uri = []
        for entry in src_uri:
            try:
                scheme, network, path, user, passwd, params = bb.fetch2.decodeurl(entry)
            except bb.fetch2.MalformedUrl as e:
                raise DevtoolError("Could not decode SRC_URI: {}".format(e))
            if replacing and scheme in ['git', 'gitsm']:
                branch = params.get('branch', 'master')
                if rd.expand(branch) != srcbranch:
                    # Handle case where branch is set through a variable
                    res = re.match(r'\$\{([^}@]+)\}', branch)
                    if res:
                        newvalues[res.group(1)] = srcbranch
                        # We know we won't change SRC_URI now, so break out
                        break
                    else:
                        params['branch'] = srcbranch
                        entry = bb.fetch2.encodeurl((scheme, network, path, user, passwd, params))
                        changed = True
                replacing = False
            new_src_uri.append(entry)
        if changed:
            newvalues['SRC_URI'] = ' '.join(new_src_uri)

    newvalues['PR'] = None

    # Work out which SRC_URI entries have changed in case the entry uses a name
    crd = rd.createCopy()
    crd.setVar('PV', newpv)
    for var, value in newvalues.items():
        crd.setVar(var, value)
    old_src_uri = (rd.getVar('SRC_URI') or '').split()
    new_src_uri = (crd.getVar('SRC_URI') or '').split()
    newnames = []
    addnames = []
    for newentry in new_src_uri:
        _, _, _, _, _, params = bb.fetch2.decodeurl(newentry)
        if 'name' in params:
            newnames.append(params['name'])
            if newentry not in old_src_uri:
                addnames.append(params['name'])
    # Find what's been set in the original recipe
    oldnames = []
    oldsums = []
    noname = False
    for varflag in rd.getVarFlags('SRC_URI'):
        for checksum in checksums:
            if varflag.endswith('.' + checksum):
                name = varflag.rsplit('.', 1)[0]
                if name not in oldnames:
                    oldnames.append(name)
                oldsums.append(checksum)
            elif varflag == checksum:
                noname = True
                oldsums.append(checksum)
    # Even if SRC_URI has named entries it doesn't have to actually use the name
    if noname and addnames and addnames[0] not in oldnames:
        addnames = []
    # Drop any old names (the name actually might include ${PV})
    for name in oldnames:
        if name not in newnames:
            for checksum in oldsums:
                newvalues['SRC_URI[%s.%s]' % (name, checksum)] = None

    nameprefix = '%s.' % addnames[0] if addnames else ''

    # md5sum is deprecated, remove any traces of it. If it was the only old
    # checksum, then replace it with the default checksums.
    if 'md5sum' in oldsums:
        newvalues['SRC_URI[%smd5sum]' % nameprefix] = None
        oldsums.remove('md5sum')
        if not oldsums:
            oldsums = ["%ssum" % s for s in bb.fetch2.SHOWN_CHECKSUM_LIST]

    for checksum in oldsums:
        newvalues['SRC_URI[%s%s]' % (nameprefix, checksum)] = checksums[checksum]

    if srcsubdir_new != srcsubdir_old:
        s_subdir_old = os.path.relpath(os.path.abspath(rd.getVar('S')), rd.getVar('WORKDIR'))
        s_subdir_new = os.path.relpath(os.path.abspath(crd.getVar('S')), crd.getVar('WORKDIR'))
        if srcsubdir_old == s_subdir_old and srcsubdir_new != s_subdir_new:
            # Subdir for old extracted source matches what S points to (it should!)
            # but subdir for new extracted source doesn't match what S will be
            newvalues['S'] = '${WORKDIR}/%s' % srcsubdir_new.replace(newpv, '${PV}')
            if crd.expand(newvalues['S']) == crd.expand('${WORKDIR}/${BP}'):
                # It's the default, drop it
                # FIXME what if S is being set in a .inc?
                newvalues['S'] = None
                logger.info('Source subdirectory has changed, dropping S value since it now matches the default ("${WORKDIR}/${BP}")')
            else:
                logger.info('Source subdirectory has changed, updating S value')

    if license_diff:
        newlicchksum = " ".join(["file://{}".format(l['path']) +
                                 (";beginline={}".format(l['beginline']) if l['beginline'] else "") +
                                 (";endline={}".format(l['endline']) if l['endline'] else "") +
                                 (";md5={}".format(l['actual_md5'])) for l in new_licenses])
        newvalues["LIC_FILES_CHKSUM"] = newlicchksum
        _add_license_diff_to_recipe(fullpath, license_diff)

    tinfoil.modified_files()
    try:
        rd = tinfoil.parse_recipe_file(fullpath, False)
    except bb.tinfoil.TinfoilCommandFailed as e:
        _upgrade_error(e, os.path.dirname(fullpath), srctree, keep_failure, 'Parsing of upgraded recipe failed')
    oe.recipeutils.patch_recipe(rd, fullpath, newvalues)

    return fullpath, copied


def _check_git_config():
    def getconfig(name):
        try:
            value = bb.process.run('git config %s' % name)[0].strip()
        except bb.process.ExecutionError as e:
            if e.exitcode == 1:
                value = None
            else:
                raise
        return value

    username = getconfig('user.name')
    useremail = getconfig('user.email')
    configerr = []
    if not username:
        configerr.append('Please set your name using:\n  git config --global user.name')
    if not useremail:
        configerr.append('Please set your email using:\n  git config --global user.email')
    if configerr:
        raise DevtoolError('Your git configuration is incomplete which will prevent rebases from working:\n' + '\n'.join(configerr))

def _extract_licenses(srcpath, recipe_licenses):
    licenses = []
    for url in recipe_licenses.split():
        license = {}
        (type, host, path, user, pswd, parm) = bb.fetch.decodeurl(url)
        license['path'] = path
        license['md5'] = parm.get('md5', '')
        license['beginline'], license['endline'] = 0, 0
        if 'beginline' in parm:
            license['beginline'] = int(parm['beginline'])
        if 'endline' in parm:
            license['endline'] = int(parm['endline'])
        license['text'] = []
        with open(os.path.join(srcpath, path), 'rb') as f:
            import hashlib
            actual_md5 = hashlib.md5()
            lineno = 0
            for line in f:
                lineno += 1
                if (lineno >= license['beginline']) and ((lineno <= license['endline']) or not license['endline']):
                    license['text'].append(line.decode(errors='ignore'))
                    actual_md5.update(line)
        license['actual_md5'] = actual_md5.hexdigest()
        licenses.append(license)
    return licenses

def _generate_license_diff(old_licenses, new_licenses):
    need_diff = False
    for l in new_licenses:
        if l['md5'] != l['actual_md5']:
            need_diff = True
            break
    if need_diff == False:
        return None

    import difflib
    diff = ''
    for old, new in zip(old_licenses, new_licenses):
        for line in difflib.unified_diff(old['text'], new['text'], old['path'], new['path']):
            diff = diff + line
    return diff

def upgrade(args, config, basepath, workspace):
    """Entry point for the devtool 'upgrade' subcommand"""

    if args.recipename in workspace:
        raise DevtoolError("recipe %s is already in your workspace" % args.recipename)
    if args.srcbranch and not args.srcrev:
        raise DevtoolError("If you specify --srcbranch/-B then you must use --srcrev/-S to specify the revision" % args.recipename)

    _check_git_config()

    tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
    try:
        rd = parse_recipe(config, tinfoil, args.recipename, True)
        if not rd:
            return 1

        pn = rd.getVar('PN')
        if pn != args.recipename:
            logger.info('Mapping %s to %s' % (args.recipename, pn))
        if pn in workspace:
            raise DevtoolError("recipe %s is already in your workspace" % pn)

        if args.srctree:
            srctree = os.path.abspath(args.srctree)
        else:
            srctree = standard.get_default_srctree(config, pn)

        srctree_s = standard.get_real_srctree(srctree, rd.getVar('S'), rd.getVar('WORKDIR'))

        # try to automatically discover latest version and revision if not provided on command line
        if not args.version and not args.srcrev:
            version_info = oe.recipeutils.get_recipe_upstream_version(rd)
            if version_info['version'] and not version_info['version'].endswith("new-commits-available"):
                args.version = version_info['version']
            if version_info['revision']:
                args.srcrev = version_info['revision']
        if not args.version and not args.srcrev:
            raise DevtoolError("Automatic discovery of latest version/revision failed - you must provide a version using the --version/-V option, or for recipes that fetch from an SCM such as git, the --srcrev/-S option.")

        standard._check_compatible_recipe(pn, rd)
        old_srcrev = rd.getVar('SRCREV')
        if old_srcrev == 'INVALID':
            old_srcrev = None
        if old_srcrev and not args.srcrev:
            raise DevtoolError("Recipe specifies a SRCREV value; you must specify a new one when upgrading")
        old_ver = rd.getVar('PV')
        if old_ver == args.version and old_srcrev == args.srcrev:
            raise DevtoolError("Current and upgrade versions are the same version")
        if args.version:
            if bb.utils.vercmp_string(args.version, old_ver) < 0:
                logger.warning('Upgrade version %s compares as less than the current version %s. If you are using a package feed for on-target upgrades or providing this recipe for general consumption, then you should increment PE in the recipe (or if there is no current PE value set, set it to "1")' % (args.version, old_ver))
            check_prerelease_version(args.version, 'devtool upgrade')

        rf = None
        license_diff = None
        try:
            logger.info('Extracting current version source...')
            rev1, srcsubdir1 = standard._extract_source(srctree, False, 'devtool-orig', False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides)
            old_licenses = _extract_licenses(srctree_s, (rd.getVar('LIC_FILES_CHKSUM') or ""))
            logger.info('Extracting upgraded version source...')
            rev2, checksums, srcbranch, srcsubdir2 = _extract_new_source(args.version, srctree, args.no_patch,
                                                    args.srcrev, args.srcbranch, args.branch, args.keep_temp,
                                                    tinfoil, rd)
            new_licenses = _extract_licenses(srctree_s, (rd.getVar('LIC_FILES_CHKSUM') or ""))
            license_diff = _generate_license_diff(old_licenses, new_licenses)
            rf, copied = _create_new_recipe(args.version, checksums, args.srcrev, srcbranch, srcsubdir1, srcsubdir2, config.workspace_path, tinfoil, rd, license_diff, new_licenses, srctree, args.keep_failure)
        except (bb.process.CmdError, DevtoolError) as e:
            recipedir = os.path.join(config.workspace_path, 'recipes', rd.getVar('BPN'))
            _upgrade_error(e, recipedir, srctree, args.keep_failure)
        standard._add_md5(config, pn, os.path.dirname(rf))

        af = _write_append(rf, srctree, srctree_s, args.same_dir, args.no_same_dir, rev2,
                        copied, config.workspace_path, rd)
        standard._add_md5(config, pn, af)

        update_unlockedsigs(basepath, workspace, args.fixed_setup, [pn])

        logger.info('Upgraded source extracted to %s' % srctree)
        logger.info('New recipe is %s' % rf)
        if license_diff:
            logger.info('License checksums have been updated in the new recipe; please refer to it for the difference between the old and the new license texts.')
        preferred_version = rd.getVar('PREFERRED_VERSION_%s' % rd.getVar('PN'))
        if preferred_version:
            logger.warning('Version is pinned to %s via PREFERRED_VERSION; it may need adjustment to match the new version before any further steps are taken' % preferred_version)
    finally:
        tinfoil.shutdown()
    return 0

def latest_version(args, config, basepath, workspace):
    """Entry point for the devtool 'latest_version' subcommand"""
    tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
    try:
        rd = parse_recipe(config, tinfoil, args.recipename, True)
        if not rd:
            return 1
        version_info = oe.recipeutils.get_recipe_upstream_version(rd)
        # "new-commits-available" is an indication that upstream never issues version tags
        if not version_info['version'].endswith("new-commits-available"):
            logger.info("Current version: {}".format(version_info['current_version']))
            logger.info("Latest version: {}".format(version_info['version']))
            if version_info['revision']:
                logger.info("Latest version's commit: {}".format(version_info['revision']))
        else:
            logger.info("Latest commit: {}".format(version_info['revision']))
    finally:
        tinfoil.shutdown()
    return 0

def check_upgrade_status(args, config, basepath, workspace):
    if not args.recipe:
        logger.info("Checking the upstream status for all recipes may take a few minutes")
    results = oe.recipeutils.get_recipe_upgrade_status(args.recipe)
    for result in results:
        # pn, update_status, current, latest, maintainer, latest_commit, no_update_reason
        if args.all or result[1] != 'MATCH':
            print("{:25} {:15} {:15} {} {} {}".format(   result[0],
                                                               result[2],
                                                               result[1] if result[1] != 'UPDATE' else (result[3] if not result[3].endswith("new-commits-available") else "new commits"),
                                                               result[4],
                                                               result[5] if result[5] != 'N/A' else "",
                                                               "cannot be updated due to: %s" %(result[6]) if result[6] else ""))

def register_commands(subparsers, context):
    """Register devtool subcommands from this plugin"""

    defsrctree = standard.get_default_srctree(context.config)

    parser_upgrade = subparsers.add_parser('upgrade', help='Upgrade an existing recipe',
                                           description='Upgrades an existing recipe to a new upstream version. Puts the upgraded recipe file into the workspace along with any associated files, and extracts the source tree to a specified location (in case patches need rebasing or adding to as a result of the upgrade).',
                                           group='starting')
    parser_upgrade.add_argument('recipename', help='Name of recipe to upgrade (just name - no version, path or extension)')
    parser_upgrade.add_argument('srctree',  nargs='?', help='Path to where to extract the source tree. If not specified, a subdirectory of %s will be used.' % defsrctree)
    parser_upgrade.add_argument('--version', '-V', help='Version to upgrade to (PV). If omitted, latest upstream version will be determined and used, if possible.')
    parser_upgrade.add_argument('--srcrev', '-S', help='Source revision to upgrade to (useful when fetching from an SCM such as git)')
    parser_upgrade.add_argument('--srcbranch', '-B', help='Branch in source repository containing the revision to use (if fetching from an SCM such as git)')
    parser_upgrade.add_argument('--branch', '-b', default="devtool", help='Name for new development branch to checkout (default "%(default)s")')
    parser_upgrade.add_argument('--no-patch', action="store_true", help='Do not apply patches from the recipe to the new source code')
    parser_upgrade.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations')
    group = parser_upgrade.add_mutually_exclusive_group()
    group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true")
    group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true")
    parser_upgrade.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
    parser_upgrade.add_argument('--keep-failure', action="store_true", help='Keep failed upgrade recipe and associated files  (for debugging)')
    parser_upgrade.set_defaults(func=upgrade, fixed_setup=context.fixed_setup)

    parser_latest_version = subparsers.add_parser('latest-version', help='Report the latest version of an existing recipe',
                                                  description='Queries the upstream server for what the latest upstream release is (for git, tags are checked, for tarballs, a list of them is obtained, and one with the highest version number is reported)',
                                                  group='info')
    parser_latest_version.add_argument('recipename', help='Name of recipe to query (just name - no version, path or extension)')
    parser_latest_version.set_defaults(func=latest_version)

    parser_check_upgrade_status = subparsers.add_parser('check-upgrade-status', help="Report upgradability for multiple (or all) recipes",
                                                        description="Prints a table of recipes together with versions currently provided by recipes, and latest upstream versions, when there is a later version available",
                                                        group='info')
    parser_check_upgrade_status.add_argument('recipe', help='Name of the recipe to report (omit to report upgrade info for all recipes)', nargs='*')
    parser_check_upgrade_status.add_argument('--all', '-a', help='Show all recipes, not just recipes needing upgrade', action="store_true")
    parser_check_upgrade_status.set_defaults(func=check_upgrade_status)