aboutsummaryrefslogtreecommitdiff
path: root/scripts
diff options
context:
space:
mode:
authorPaul Eggleton <paul.eggleton@linux.intel.com>2017-10-31 16:48:05 +1300
committerRichard Purdie <richard.purdie@linuxfoundation.org>2017-11-10 14:44:31 +0000
commitaa87603d1ffd695027847f4df75c0406cf4e14d8 (patch)
tree109f789016c878bf1955859c75ddf22fd2a40d7d /scripts
parent05f2d5d2ce00c53825ccea5cd9c2262f9d27a638 (diff)
downloadopenembedded-core-aa87603d1ffd695027847f4df75c0406cf4e14d8.zip
openembedded-core-aa87603d1ffd695027847f4df75c0406cf4e14d8.tar.gz
openembedded-core-aa87603d1ffd695027847f4df75c0406cf4e14d8.tar.bz2
devtool: implement conditional patch handling
If you have a recipe that uses overrides to conditionally extend SRC_URI to add additional patches, then you will often need to update those patches if you're making other changes to the source tree (for example if you're upgrading the underlying source). Make this possible with devtool by creating devtool-override-* branches for each override that conditionally appends/prepends SRC_URI, and have devtool update-recipe / finish check each branch out in turn and update the corresponding patches. A current example of a recipe that does this is the quota recipe - it applies an additional patch if musl is the selected C library (i.e. libc-musl is in OVERRIDES). Note that use of this functionality does require some care - in particular, updates to patches that appear on the main branch (named "devtool" by default) should be made there and not only on one of the specific devtool-override-* branches that are created for each override. The recommended procedure is to make the changes you want to make to the main branch first, then check out and rebase each devtool-override-* branch, testing each one by activating the corresponding configuration, and then finally run devtool finish. Fixes [YOCTO #11516]. Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com> Signed-off-by: Ross Burton <ross.burton@intel.com>
Diffstat (limited to 'scripts')
-rw-r--r--scripts/lib/devtool/standard.py176
-rw-r--r--scripts/lib/devtool/upgrade.py3
2 files changed, 153 insertions, 26 deletions
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index d55d504..26187a0 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -35,6 +35,8 @@ from devtool import parse_recipe
logger = logging.getLogger('devtool')
+override_branch_prefix = 'devtool-override-'
+
def add(args, config, basepath, workspace):
"""Entry point for the devtool 'add' subcommand"""
@@ -425,7 +427,7 @@ def extract(args, config, basepath, workspace):
"""Entry point for the devtool 'extract' subcommand"""
import bb
- tinfoil = setup_tinfoil(basepath=basepath)
+ tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
if not tinfoil:
# Error already shown
return 1
@@ -435,7 +437,7 @@ def extract(args, config, basepath, workspace):
return 1
srctree = os.path.abspath(args.srctree)
- initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil)
+ initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides)
logger.info('Source tree extracted to %s' % srctree)
if initial_rev:
@@ -449,7 +451,7 @@ def sync(args, config, basepath, workspace):
"""Entry point for the devtool 'sync' subcommand"""
import bb
- tinfoil = setup_tinfoil(basepath=basepath)
+ tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
if not tinfoil:
# Error already shown
return 1
@@ -459,7 +461,7 @@ def sync(args, config, basepath, workspace):
return 1
srctree = os.path.abspath(args.srctree)
- initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, True, config, basepath, workspace, args.fixed_setup, rd, tinfoil)
+ initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, True, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=True)
logger.info('Source tree %s synchronized' % srctree)
if initial_rev:
@@ -470,7 +472,7 @@ def sync(args, config, basepath, workspace):
tinfoil.shutdown()
-def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, workspace, fixed_setup, d, tinfoil):
+def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, workspace, fixed_setup, d, tinfoil, no_overrides=False):
"""Extract sources of a recipe"""
import oe.recipeutils
import oe.patch
@@ -500,6 +502,16 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works
bb.utils.mkdirhier(srctree)
os.rmdir(srctree)
+ extra_overrides = []
+ if not no_overrides:
+ history = d.varhistory.variable('SRC_URI')
+ for event in history:
+ if not 'flag' in event:
+ if event['op'].startswith(('_append[', '_prepend[')):
+ extra_overrides.append(event['op'].split('[')[1].split(']')[0])
+ if extra_overrides:
+ logger.info('SRC_URI contains some conditional appends/prepends - will create branches to represent these')
+
initial_rev = None
appendexisted = False
@@ -542,6 +554,8 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works
if not is_kernel_yocto:
f.write('PATCHTOOL = "git"\n')
f.write('PATCH_COMMIT_FUNCTIONS = "1"\n')
+ if extra_overrides:
+ f.write('DEVTOOL_EXTRA_OVERRIDES = "%s"\n' % ':'.join(extra_overrides))
f.write('inherit devtool-source\n')
f.write('###--- _extract_source\n')
@@ -702,12 +716,13 @@ def modify(args, config, basepath, workspace):
"""Entry point for the devtool 'modify' subcommand"""
import bb
import oe.recipeutils
+ import oe.patch
if args.recipename in workspace:
raise DevtoolError("recipe %s is already in your workspace" %
args.recipename)
- tinfoil = setup_tinfoil(basepath=basepath)
+ tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
try:
rd = parse_recipe(config, tinfoil, args.recipename, True)
if not rd:
@@ -742,14 +757,16 @@ def modify(args, config, basepath, workspace):
initial_rev = None
commits = []
+ check_commits = False
if not args.no_extract:
- initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil)
+ initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides)
if not initial_rev:
return 1
logger.info('Source tree extracted to %s' % srctree)
# Get list of commits since this revision
(stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_rev, cwd=srctree)
commits = stdout.split()
+ check_commits = True
else:
if os.path.exists(os.path.join(srctree, '.git')):
# Check if it's a tree previously extracted by us
@@ -757,6 +774,8 @@ def modify(args, config, basepath, workspace):
(stdout, _) = bb.process.run('git branch --contains devtool-base', cwd=srctree)
except bb.process.ExecutionError:
stdout = ''
+ if stdout:
+ check_commits = True
for line in stdout.splitlines():
if line.startswith('*'):
(stdout, _) = bb.process.run('git rev-parse devtool-base', cwd=srctree)
@@ -766,6 +785,30 @@ def modify(args, config, basepath, workspace):
(stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree)
initial_rev = stdout.rstrip()
+ branch_patches = {}
+ if check_commits:
+ # Check if there are override branches
+ (stdout, _) = bb.process.run('git branch', cwd=srctree)
+ branches = []
+ for line in stdout.rstrip().splitlines():
+ branchname = line[2:].rstrip()
+ if branchname.startswith(override_branch_prefix):
+ branches.append(branchname)
+ if branches:
+ logger.warn('SRC_URI is conditionally overridden in this recipe, thus several %s* branches have been created, one for each override that makes changes to SRC_URI. It is recommended that you make changes to the %s branch first, then checkout and rebase each %s* branch and update any unique patches there (duplicates on those branches will be ignored by devtool finish/update-recipe)' % (override_branch_prefix, args.branch, override_branch_prefix))
+ branches.insert(0, args.branch)
+ seen_patches = []
+ for branch in branches:
+ branch_patches[branch] = []
+ (stdout, _) = bb.process.run('git log devtool-base..%s' % branch, cwd=srctree)
+ for line in stdout.splitlines():
+ line = line.strip()
+ if line.startswith(oe.patch.GitApplyTree.patch_line_prefix):
+ origpatch = line[len(oe.patch.GitApplyTree.patch_line_prefix):].split(':', 1)[-1].strip()
+ if not origpatch in seen_patches:
+ seen_patches.append(origpatch)
+ branch_patches[branch].append(origpatch)
+
# Need to grab this here in case the source is within a subdirectory
srctreebase = srctree
@@ -808,6 +851,11 @@ def modify(args, config, basepath, workspace):
f.write('\n# initial_rev: %s\n' % initial_rev)
for commit in commits:
f.write('# commit: %s\n' % commit)
+ if branch_patches:
+ for branch in branch_patches:
+ if branch == args.branch:
+ continue
+ f.write('# patches_%s: %s\n' % (branch, ','.join(branch_patches[branch])))
update_unlockedsigs(basepath, workspace, args.fixed_setup, [pn])
@@ -1019,8 +1067,14 @@ def _get_patchset_revs(srctree, recipe_path, initial_rev=None):
"""
import bb
+ # Get current branch
+ stdout, _ = bb.process.run('git rev-parse --abbrev-ref HEAD',
+ cwd=srctree)
+ branchname = stdout.rstrip()
+
# Parse initial rev from recipe if not specified
commits = []
+ patches = []
with open(recipe_path, 'r') as f:
for line in f:
if line.startswith('# initial_rev:'):
@@ -1028,6 +1082,8 @@ def _get_patchset_revs(srctree, recipe_path, initial_rev=None):
initial_rev = line.split(':')[-1].strip()
elif line.startswith('# commit:'):
commits.append(line.split(':')[-1].strip())
+ elif line.startswith('# patches_%s:' % branchname):
+ patches = line.split(':')[-1].strip().split(',')
update_rev = initial_rev
changed_revs = None
@@ -1054,7 +1110,7 @@ def _get_patchset_revs(srctree, recipe_path, initial_rev=None):
if rev in newcommits:
changed_revs.append(rev)
- return initial_rev, update_rev, changed_revs
+ return initial_rev, update_rev, changed_revs, patches
def _remove_file_entries(srcuri, filelist):
"""Remove file:// entries from SRC_URI"""
@@ -1126,6 +1182,7 @@ def _export_patches(srctree, rd, start_rev, destdir, changed_revs=None):
existing_patches = dict((os.path.basename(path), path) for path in
oe.recipeutils.get_recipe_patches(rd))
+ logger.debug('Existing patches: %s' % existing_patches)
# Generate patches from Git, exclude local files directory
patch_pathspec = _git_exclude_path(srctree, 'oe-local-files')
@@ -1414,7 +1471,7 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
raise DevtoolError('unable to find workspace bbappend for recipe %s' %
recipename)
- initial_rev, update_rev, changed_revs = _get_patchset_revs(srctree, append, initial_rev)
+ initial_rev, update_rev, changed_revs, filter_patches = _get_patchset_revs(srctree, append, initial_rev)
if not initial_rev:
raise DevtoolError('Unable to find initial revision - please specify '
'it with --initial-rev')
@@ -1429,22 +1486,32 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
tempdir = tempfile.mkdtemp(prefix='devtool')
try:
local_files_dir = tempfile.mkdtemp(dir=tempdir)
- srctreebase = workspace[recipename]['srctreebase']
- upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase)
+ if filter_patches:
+ upd_f = {}
+ new_f = {}
+ del_f = {}
+ else:
+ srctreebase = workspace[recipename]['srctreebase']
+ upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase)
remove_files = []
if not no_remove:
# Get all patches from source tree and check if any should be removed
all_patches_dir = tempfile.mkdtemp(dir=tempdir)
- upd_p, new_p, del_p = _export_patches(srctree, rd, initial_rev,
- all_patches_dir)
+ _, _, del_p = _export_patches(srctree, rd, initial_rev,
+ all_patches_dir)
# Remove deleted local files and patches
remove_files = list(del_f.values()) + list(del_p.values())
# Get updated patches from source tree
patches_dir = tempfile.mkdtemp(dir=tempdir)
- upd_p, new_p, del_p = _export_patches(srctree, rd, update_rev,
- patches_dir, changed_revs)
+ upd_p, new_p, _ = _export_patches(srctree, rd, update_rev,
+ patches_dir, changed_revs)
+ logger.debug('Pre-filtering: update: %s, new: %s' % (dict(upd_p), dict(new_p)))
+ if filter_patches:
+ new_p = {}
+ upd_p = {k:v for k,v in upd_p.items() if k in filter_patches}
+ remove_files = [f for f in remove_files if f in filter_patches]
updatefiles = False
updaterecipe = False
destpath = None
@@ -1556,18 +1623,73 @@ def _guess_recipe_update_mode(srctree, rdata):
return 'patch'
-def _update_recipe(recipename, workspace, rd, mode, appendlayerdir, wildcard_version, no_remove, initial_rev, no_report_remove=False, dry_run_outdir=None):
+def _update_recipe(recipename, workspace, rd, mode, appendlayerdir, wildcard_version, no_remove, initial_rev, no_report_remove=False, dry_run_outdir=None, no_overrides=False):
srctree = workspace[recipename]['srctree']
if mode == 'auto':
mode = _guess_recipe_update_mode(srctree, rd)
- if mode == 'srcrev':
- updated, appendfile, removed = _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir)
- elif mode == 'patch':
- updated, appendfile, removed = _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir)
- else:
- raise DevtoolError('update_recipe: invalid mode %s' % mode)
- return updated, appendfile, removed
+ override_branches = []
+ mainbranch = None
+ startbranch = None
+ if not no_overrides:
+ stdout, _ = bb.process.run('git branch', cwd=srctree)
+ other_branches = []
+ for line in stdout.splitlines():
+ branchname = line[2:]
+ if line.startswith('* '):
+ startbranch = branchname
+ if branchname.startswith(override_branch_prefix):
+ override_branches.append(branchname)
+ else:
+ other_branches.append(branchname)
+
+ if override_branches:
+ logger.debug('_update_recipe: override branches: %s' % override_branches)
+ logger.debug('_update_recipe: other branches: %s' % other_branches)
+ if startbranch.startswith(override_branch_prefix):
+ if len(other_branches) == 1:
+ mainbranch = other_branches[1]
+ else:
+ raise DevtoolError('Unable to determine main branch - please check out the main branch in source tree first')
+ else:
+ mainbranch = startbranch
+
+ checkedout = None
+ anyupdated = False
+ appendfile = None
+ allremoved = []
+ if override_branches:
+ logger.info('Handling main branch (%s)...' % mainbranch)
+ if startbranch != mainbranch:
+ bb.process.run('git checkout %s' % mainbranch, cwd=srctree)
+ checkedout = mainbranch
+ try:
+ branchlist = [mainbranch] + override_branches
+ for branch in branchlist:
+ crd = bb.data.createCopy(rd)
+ if branch != mainbranch:
+ logger.info('Handling branch %s...' % branch)
+ override = branch[len(override_branch_prefix):]
+ crd.appendVar('OVERRIDES', ':%s' % override)
+ bb.process.run('git checkout %s' % branch, cwd=srctree)
+ checkedout = branch
+
+ if mode == 'srcrev':
+ updated, appendf, removed = _update_recipe_srcrev(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir)
+ elif mode == 'patch':
+ updated, appendf, removed = _update_recipe_patch(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir)
+ else:
+ raise DevtoolError('update_recipe: invalid mode %s' % mode)
+ if updated:
+ anyupdated = True
+ if appendf:
+ appendfile = appendf
+ allremoved.extend(removed)
+ finally:
+ if startbranch and checkedout != startbranch:
+ bb.process.run('git checkout %s' % startbranch, cwd=srctree)
+
+ return anyupdated, appendfile, allremoved
def update_recipe(args, config, basepath, workspace):
"""Entry point for the devtool 'update-recipe' subcommand"""
@@ -1593,7 +1715,7 @@ def update_recipe(args, config, basepath, workspace):
if args.dry_run:
dry_run_output = tempfile.TemporaryDirectory(prefix='devtool')
dry_run_outdir = dry_run_output.name
- updated, _, _ = _update_recipe(args.recipename, workspace, rd, args.mode, args.append, args.wildcard_version, args.no_remove, args.initial_rev, dry_run_outdir=dry_run_outdir)
+ updated, _, _ = _update_recipe(args.recipename, workspace, rd, args.mode, args.append, args.wildcard_version, args.no_remove, args.initial_rev, dry_run_outdir=dry_run_outdir, no_overrides=args.no_overrides)
if updated:
rf = rd.getVar('FILE')
@@ -1816,7 +1938,7 @@ def finish(args, config, basepath, workspace):
if args.dry_run:
dry_run_output = tempfile.TemporaryDirectory(prefix='devtool')
dry_run_outdir = dry_run_output.name
- updated, appendfile, removed = _update_recipe(args.recipename, workspace, rd, args.mode, appendlayerdir, wildcard_version=True, no_remove=False, no_report_remove=removing_original, initial_rev=args.initial_rev, dry_run_outdir=dry_run_outdir)
+ updated, appendfile, removed = _update_recipe(args.recipename, workspace, rd, args.mode, appendlayerdir, wildcard_version=True, no_remove=False, no_report_remove=removing_original, initial_rev=args.initial_rev, dry_run_outdir=dry_run_outdir, no_overrides=args.no_overrides)
removed = [os.path.relpath(pth, recipedir) for pth in removed]
# Remove any old files in the case of an upgrade
@@ -1959,6 +2081,7 @@ def register_commands(subparsers, context):
group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true")
group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true")
parser_modify.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (when not using -n/--no-extract) (default "%(default)s")')
+ parser_modify.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations')
parser_modify.add_argument('--keep-temp', help='Keep temporary directory (for debugging)', action="store_true")
parser_modify.set_defaults(func=modify, fixed_setup=context.fixed_setup)
@@ -1968,6 +2091,7 @@ def register_commands(subparsers, context):
parser_extract.add_argument('recipename', help='Name of recipe to extract the source for')
parser_extract.add_argument('srctree', help='Path to where to extract the source tree')
parser_extract.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (default "%(default)s")')
+ parser_extract.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations')
parser_extract.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
parser_extract.set_defaults(func=extract, fixed_setup=context.fixed_setup)
@@ -1999,6 +2123,7 @@ def register_commands(subparsers, context):
parser_update_recipe.add_argument('--append', '-a', help='Write changes to a bbappend in the specified layer instead of the recipe', metavar='LAYERDIR')
parser_update_recipe.add_argument('--wildcard-version', '-w', help='In conjunction with -a/--append, use a wildcard to make the bbappend apply to any recipe version', action='store_true')
parser_update_recipe.add_argument('--no-remove', '-n', action="store_true", help='Don\'t remove patches, only add or update')
+ parser_update_recipe.add_argument('--no-overrides', '-O', action="store_true", help='Do not handle other override branches (if they exist)')
parser_update_recipe.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)')
parser_update_recipe.set_defaults(func=update_recipe)
@@ -2023,5 +2148,6 @@ def register_commands(subparsers, context):
parser_finish.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE')
parser_finish.add_argument('--initial-rev', help='Override starting revision for patches')
parser_finish.add_argument('--force', '-f', action="store_true", help='Force continuing even if there are uncommitted changes in the source tree repository')
+ parser_finish.add_argument('--no-overrides', '-O', action="store_true", help='Do not handle other override branches (if they exist)')
parser_finish.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)')
parser_finish.set_defaults(func=finish)
diff --git a/scripts/lib/devtool/upgrade.py b/scripts/lib/devtool/upgrade.py
index 3cb523c..f6141bf 100644
--- a/scripts/lib/devtool/upgrade.py
+++ b/scripts/lib/devtool/upgrade.py
@@ -474,7 +474,7 @@ def upgrade(args, config, basepath, workspace):
rf = None
try:
logger.info('Extracting current version source...')
- rev1, srcsubdir1 = standard._extract_source(srctree, False, 'devtool-orig', False, config, basepath, workspace, args.fixed_setup, rd, tinfoil)
+ rev1, srcsubdir1 = standard._extract_source(srctree, False, 'devtool-orig', False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides)
logger.info('Extracting upgraded version source...')
rev2, md5, sha256, srcbranch, srcsubdir2 = _extract_new_source(args.version, srctree, args.no_patch,
args.srcrev, args.srcbranch, args.branch, args.keep_temp,
@@ -513,6 +513,7 @@ def register_commands(subparsers, context):
parser_upgrade.add_argument('--srcbranch', '-B', help='Branch in source repository containing the revision to use (if fetching from an SCM such as git)')
parser_upgrade.add_argument('--branch', '-b', default="devtool", help='Name for new development branch to checkout (default "%(default)s")')
parser_upgrade.add_argument('--no-patch', action="store_true", help='Do not apply patches from the recipe to the new source code')
+ parser_upgrade.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations')
group = parser_upgrade.add_mutually_exclusive_group()
group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true")
group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true")