diff options
Diffstat (limited to 'scripts/lib/devtool/upgrade.py')
| -rw-r--r-- | scripts/lib/devtool/upgrade.py | 144 |
1 files changed, 87 insertions, 57 deletions
diff --git a/scripts/lib/devtool/upgrade.py b/scripts/lib/devtool/upgrade.py index 39a1910a49..fa5b8ef3c7 100644 --- a/scripts/lib/devtool/upgrade.py +++ b/scripts/lib/devtool/upgrade.py @@ -35,6 +35,8 @@ def _get_srctree(tmpdir): dirs = scriptutils.filter_src_subdirs(tmpdir) if len(dirs) == 1: srctree = os.path.join(tmpdir, dirs[0]) + else: + raise DevtoolError("Cannot determine where the source tree is after unpacking in {}: {}".format(tmpdir,dirs)) return srctree def _copy_source_code(orig, dest): @@ -88,7 +90,7 @@ def _rename_recipe_files(oldrecipe, bpn, oldpv, newpv, path): _rename_recipe_dirs(oldpv, newpv, path) return _rename_recipe_file(oldrecipe, bpn, oldpv, newpv, path) -def _write_append(rc, srctree, same_dir, no_same_dir, rev, copied, workspace, d): +def _write_append(rc, srctreebase, srctree, same_dir, no_same_dir, revs, copied, workspace, d): """Writes an append file""" if not os.path.exists(rc): raise DevtoolError("bbappend not created because %s does not exist" % rc) @@ -104,6 +106,11 @@ def _write_append(rc, srctree, same_dir, no_same_dir, rev, copied, workspace, d) af = os.path.join(appendpath, '%s.bbappend' % brf) with open(af, 'w') as f: f.write('FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n\n') + # Local files can be modified/tracked in separate subdir under srctree + # Mostly useful for packages with S != WORKDIR + f.write('FILESPATH:prepend := "%s:"\n' % + os.path.join(srctreebase, 'oe-local-files')) + f.write('# srctreebase: %s\n' % srctreebase) f.write('inherit externalsrc\n') f.write(('# NOTE: We use pn- overrides here to avoid affecting' 'multiple variants in the case where the recipe uses BBCLASSEXTEND\n')) @@ -112,19 +119,17 @@ def _write_append(rc, srctree, same_dir, no_same_dir, rev, copied, workspace, d) if b_is_s: f.write('EXTERNALSRC_BUILD:pn-%s = "%s"\n' % (pn, srctree)) f.write('\n') - if rev: - f.write('# initial_rev: %s\n' % rev) + if revs: + for name, rev in revs.items(): + f.write('# initial_rev %s: %s\n' % (name, rev)) if copied: f.write('# original_path: %s\n' % os.path.dirname(d.getVar('FILE'))) f.write('# original_files: %s\n' % ' '.join(copied)) return af def _cleanup_on_error(rd, srctree): - rdp = os.path.split(rd)[0] # recipes folder if os.path.exists(rd): shutil.rmtree(rd) - if not len(os.listdir(rdp)): - os.rmdir(rdp) srctree = os.path.abspath(srctree) if os.path.exists(srctree): shutil.rmtree(srctree) @@ -178,12 +183,16 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, kee uri, rev = _get_uri(crd) if srcrev: rev = srcrev + paths = [srctree] if uri.startswith('git://') or uri.startswith('gitsm://'): __run('git fetch') __run('git checkout %s' % rev) __run('git tag -f devtool-base-new') - md5 = None - sha256 = None + __run('git submodule update --recursive') + __run('git submodule foreach \'git tag -f devtool-base-new\'') + (stdout, _) = __run('git submodule --quiet foreach \'echo $sm_path\'') + paths += [os.path.join(srctree, p) for p in stdout.splitlines()] + checksums = {} _, _, _, _, _, params = bb.fetch2.decodeurl(uri) srcsubdir_rel = params.get('destsuffix', 'git') if not srcbranch: @@ -216,9 +225,6 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, kee if ftmpdir and keep_temp: logger.info('Fetch temp directory is %s' % ftmpdir) - md5 = checksums['md5sum'] - sha256 = checksums['sha256sum'] - tmpsrctree = _get_srctree(tmpdir) srctree = os.path.abspath(srctree) srcsubdir_rel = os.path.relpath(tmpsrctree, tmpdir) @@ -252,29 +258,50 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, kee __run('git %s commit -q -m "Commit of upstream changes at version %s" --allow-empty' % (' '.join(useroptions), newpv)) __run('git tag -f devtool-base-%s' % newpv) - (stdout, _) = __run('git rev-parse HEAD') - rev = stdout.rstrip() + revs = {} + for path in paths: + (stdout, _) = _run('git rev-parse HEAD', cwd=path) + revs[os.path.relpath(path, srctree)] = stdout.rstrip() if no_patch: patches = oe.recipeutils.get_recipe_patches(crd) if patches: logger.warning('By user choice, the following patches will NOT be applied to the new source tree:\n %s' % '\n '.join([os.path.basename(patch) for patch in patches])) else: - __run('git checkout devtool-patched -b %s' % branch) - (stdout, _) = __run('git branch --list devtool-override-*') - branches_to_rebase = [branch] + stdout.split() - for b in branches_to_rebase: - logger.info("Rebasing {} onto {}".format(b, rev)) - __run('git checkout %s' % b) - try: - __run('git rebase %s' % rev) - except bb.process.ExecutionError as e: - if 'conflict' in e.stdout: - logger.warning('Command \'%s\' failed:\n%s\n\nYou will need to resolve conflicts in order to complete the upgrade.' % (e.command, e.stdout.rstrip())) - __run('git rebase --abort') - else: - logger.warning('Command \'%s\' failed:\n%s' % (e.command, e.stdout)) - __run('git checkout %s' % branch) + for path in paths: + _run('git checkout devtool-patched -b %s' % branch, cwd=path) + (stdout, _) = _run('git branch --list devtool-override-*', cwd=path) + branches_to_rebase = [branch] + stdout.split() + target_branch = revs[os.path.relpath(path, srctree)] + + # There is a bug (or feature?) in git rebase where if a commit with + # a note is fully rebased away by being part of an old commit, the + # note is still attached to the old commit. Avoid this by making + # sure all old devtool related commits have a note attached to them + # (this assumes git config notes.rewriteMode is set to ignore). + (stdout, _) = __run('git rev-list devtool-base..%s' % target_branch) + for rev in stdout.splitlines(): + if not oe.patch.GitApplyTree.getNotes(path, rev): + oe.patch.GitApplyTree.addNote(path, rev, "dummy") + + for b in branches_to_rebase: + logger.info("Rebasing {} onto {}".format(b, target_branch)) + _run('git checkout %s' % b, cwd=path) + try: + _run('git rebase %s' % target_branch, cwd=path) + except bb.process.ExecutionError as e: + if 'conflict' in e.stdout: + logger.warning('Command \'%s\' failed:\n%s\n\nYou will need to resolve conflicts in order to complete the upgrade.' % (e.command, e.stdout.rstrip())) + _run('git rebase --abort', cwd=path) + else: + logger.warning('Command \'%s\' failed:\n%s' % (e.command, e.stdout)) + + # Remove any dummy notes added above. + (stdout, _) = __run('git rev-list devtool-base..%s' % target_branch) + for rev in stdout.splitlines(): + oe.patch.GitApplyTree.removeNote(path, rev, "dummy") + + _run('git checkout %s' % branch, cwd=path) if tmpsrctree: if keep_temp: @@ -284,7 +311,7 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, kee if tmpdir != tmpsrctree: shutil.rmtree(tmpdir) - return (rev, md5, sha256, srcbranch, srcsubdir_rel) + return (revs, checksums, srcbranch, srcsubdir_rel) def _add_license_diff_to_recipe(path, diff): notice_text = """# FIXME: the LIC_FILES_CHKSUM values have been updated by 'devtool upgrade'. @@ -305,7 +332,7 @@ def _add_license_diff_to_recipe(path, diff): f.write("\n#\n\n".encode()) f.write(orig_content) -def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, srcsubdir_old, srcsubdir_new, workspace, tinfoil, rd, license_diff, new_licenses, srctree, keep_failure): +def _create_new_recipe(newpv, checksums, srcrev, srcbranch, srcsubdir_old, srcsubdir_new, workspace, tinfoil, rd, license_diff, new_licenses, srctree, keep_failure): """Creates the new recipe under workspace""" bpn = rd.getVar('BPN') @@ -377,30 +404,39 @@ def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, srcsubdir_old, src addnames.append(params['name']) # Find what's been set in the original recipe oldnames = [] + oldsums = [] noname = False for varflag in rd.getVarFlags('SRC_URI'): - if varflag.endswith(('.md5sum', '.sha256sum')): - name = varflag.rsplit('.', 1)[0] - if name not in oldnames: - oldnames.append(name) - elif varflag in ['md5sum', 'sha256sum']: - noname = True + for checksum in checksums: + if varflag.endswith('.' + checksum): + name = varflag.rsplit('.', 1)[0] + if name not in oldnames: + oldnames.append(name) + oldsums.append(checksum) + elif varflag == checksum: + noname = True + oldsums.append(checksum) # Even if SRC_URI has named entries it doesn't have to actually use the name if noname and addnames and addnames[0] not in oldnames: addnames = [] # Drop any old names (the name actually might include ${PV}) for name in oldnames: if name not in newnames: - newvalues['SRC_URI[%s.md5sum]' % name] = None - newvalues['SRC_URI[%s.sha256sum]' % name] = None + for checksum in oldsums: + newvalues['SRC_URI[%s.%s]' % (name, checksum)] = None - if sha256: - if addnames: - nameprefix = '%s.' % addnames[0] - else: - nameprefix = '' + nameprefix = '%s.' % addnames[0] if addnames else '' + + # md5sum is deprecated, remove any traces of it. If it was the only old + # checksum, then replace it with the default checksums. + if 'md5sum' in oldsums: newvalues['SRC_URI[%smd5sum]' % nameprefix] = None - newvalues['SRC_URI[%ssha256sum]' % nameprefix] = sha256 + oldsums.remove('md5sum') + if not oldsums: + oldsums = ["%ssum" % s for s in bb.fetch2.SHOWN_CHECKSUM_LIST] + + for checksum in oldsums: + newvalues['SRC_URI[%s%s]' % (nameprefix, checksum)] = checksums[checksum] if srcsubdir_new != srcsubdir_old: s_subdir_old = os.path.relpath(os.path.abspath(rd.getVar('S')), rd.getVar('WORKDIR')) @@ -425,6 +461,7 @@ def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, srcsubdir_old, src newvalues["LIC_FILES_CHKSUM"] = newlicchksum _add_license_diff_to_recipe(fullpath, license_diff) + tinfoil.modified_files() try: rd = tinfoil.parse_recipe_file(fullpath, False) except bb.tinfoil.TinfoilCommandFailed as e: @@ -437,7 +474,7 @@ def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, srcsubdir_old, src def _check_git_config(): def getconfig(name): try: - value = bb.process.run('git config --global %s' % name)[0].strip() + value = bb.process.run('git config %s' % name)[0].strip() except bb.process.ExecutionError as e: if e.exitcode == 1: value = None @@ -524,14 +561,7 @@ def upgrade(args, config, basepath, workspace): else: srctree = standard.get_default_srctree(config, pn) - # Check that recipe isn't using a shared workdir - s = os.path.abspath(rd.getVar('S')) - workdir = os.path.abspath(rd.getVar('WORKDIR')) - srctree_s = srctree - if s.startswith(workdir) and s != workdir and os.path.dirname(s) != workdir: - # Handle if S is set to a subdirectory of the source - srcsubdir = os.path.relpath(s, workdir).split(os.sep, 1)[1] - srctree_s = os.path.join(srctree, srcsubdir) + srctree_s = standard.get_real_srctree(srctree, rd.getVar('S'), rd.getVar('WORKDIR')) # try to automatically discover latest version and revision if not provided on command line if not args.version and not args.srcrev: @@ -564,18 +594,18 @@ def upgrade(args, config, basepath, workspace): rev1, srcsubdir1 = standard._extract_source(srctree, False, 'devtool-orig', False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides) old_licenses = _extract_licenses(srctree_s, (rd.getVar('LIC_FILES_CHKSUM') or "")) logger.info('Extracting upgraded version source...') - rev2, md5, sha256, srcbranch, srcsubdir2 = _extract_new_source(args.version, srctree, args.no_patch, + rev2, checksums, srcbranch, srcsubdir2 = _extract_new_source(args.version, srctree, args.no_patch, args.srcrev, args.srcbranch, args.branch, args.keep_temp, tinfoil, rd) new_licenses = _extract_licenses(srctree_s, (rd.getVar('LIC_FILES_CHKSUM') or "")) license_diff = _generate_license_diff(old_licenses, new_licenses) - rf, copied = _create_new_recipe(args.version, md5, sha256, args.srcrev, srcbranch, srcsubdir1, srcsubdir2, config.workspace_path, tinfoil, rd, license_diff, new_licenses, srctree, args.keep_failure) + rf, copied = _create_new_recipe(args.version, checksums, args.srcrev, srcbranch, srcsubdir1, srcsubdir2, config.workspace_path, tinfoil, rd, license_diff, new_licenses, srctree, args.keep_failure) except (bb.process.CmdError, DevtoolError) as e: recipedir = os.path.join(config.workspace_path, 'recipes', rd.getVar('BPN')) _upgrade_error(e, recipedir, srctree, args.keep_failure) standard._add_md5(config, pn, os.path.dirname(rf)) - af = _write_append(rf, srctree_s, args.same_dir, args.no_same_dir, rev2, + af = _write_append(rf, srctree, srctree_s, args.same_dir, args.no_same_dir, rev2, copied, config.workspace_path, rd) standard._add_md5(config, pn, af) @@ -619,7 +649,7 @@ def check_upgrade_status(args, config, basepath, workspace): for result in results: # pn, update_status, current, latest, maintainer, latest_commit, no_update_reason if args.all or result[1] != 'MATCH': - logger.info("{:25} {:15} {:15} {} {} {}".format( result[0], + print("{:25} {:15} {:15} {} {} {}".format( result[0], result[2], result[1] if result[1] != 'UPDATE' else (result[3] if not result[3].endswith("new-commits-available") else "new commits"), result[4], |