From 854119e59af55a565f24d5d069dc29eb48d00dab Mon Sep 17 00:00:00 2001 From: Paul Eggleton Date: Wed, 15 Nov 2017 15:21:49 +1300 Subject: devtool: support extracting multiple source trees If you have multiple source trees being extracted to the work directory within a recipe (e.g. you have two tarballs referred to in SRC_URI) and one isn't being extracted into the other, then devtool failed to extract all the sources because it only took the source tree that S pointed into. To fix this, we need to take a look at the work directory after do_unpack and see if there are any additional subdirectories; if so we need to put the main source tree in a subdirectory and put the additional subdirectories next to it. We also ensure that symlinks from the work directory get created at the end of do_unpack to point to these (so that references in the recipe continue to work). In addition to multiple source trees at the work directory level, this patch also handles multiple nested git trees (where you have multiple git URLs in SRC_URI with one or more inside another). These caused a different problem, where changes in sub-repos are not fully captured at the top level - we need to handle each repo separately. Signed-off-by: Paul Eggleton --- meta/classes/devtool-source.bbclass | 57 ++++- meta/lib/oe/recipeutils.py | 4 +- meta/lib/oe/utils.py | 24 +++ meta/lib/oeqa/selftest/cases/devtool.py | 2 +- scripts/devtool | 61 ++++-- scripts/lib/devtool/__init__.py | 14 ++ scripts/lib/devtool/standard.py | 370 +++++++++++++++++++++++--------- 7 files changed, 396 insertions(+), 136 deletions(-) diff --git a/meta/classes/devtool-source.bbclass b/meta/classes/devtool-source.bbclass index 540a60289a..30c9bdb0f7 100644 --- a/meta/classes/devtool-source.bbclass +++ b/meta/classes/devtool-source.bbclass @@ -69,7 +69,7 @@ python devtool_post_unpack() { import shutil sys.path.insert(0, os.path.join(d.getVar('COREBASE'), 'scripts', 'lib')) import scriptutils - from devtool import setup_git_repo + from devtool import setup_git_repo, find_git_repos tempdir = d.getVar('DEVTOOL_TEMPDIR') workdir = d.getVar('WORKDIR') @@ -95,8 +95,26 @@ python devtool_post_unpack() { oe.recipeutils.get_recipe_patches(d)] local_files = oe.recipeutils.get_recipe_local_files(d) - # Ignore local files with subdir={BP} + excludeitems = recipe_patches + list(local_files.keys()) + pthvars = ['RECIPE_SYSROOT', 'RECIPE_SYSROOT_NATIVE', 'S'] + for pthvar in pthvars: + relp = os.path.relpath(d.getVar(pthvar), d.getVar('WORKDIR')) + if not relp.startswith('..'): + excludeitems.append(relp.split(os.sep)[0]) + extradirs = [] srcabspath = os.path.abspath(srcsubdir) + if srcabspath != os.path.abspath(workdir): + for pth in os.listdir(workdir): + if pth in excludeitems: + continue + wpth = os.path.join(workdir, pth) + if os.path.isdir(wpth) and os.listdir(wpth): + extradirs.append(wpth) + + repos = find_git_repos(srcabspath) + extradirs.extend(repos) + + # Ignore local files with subdir={BP} local_files = [fname for fname in local_files if os.path.exists(os.path.join(workdir, fname)) and (srcabspath == workdir or not @@ -145,11 +163,20 @@ python devtool_post_unpack() { (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srcsubdir) initial_rev = stdout.rstrip() + + initial_revs = {} + for extradir in extradirs: + setup_git_repo(extradir, d.getVar('PV'), devbranch, d=d) + (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=extradir) + initial_revs[extradir] = stdout.rstrip() + with open(os.path.join(tempdir, 'initial_rev'), 'w') as f: f.write(initial_rev) with open(os.path.join(tempdir, 'srcsubdir'), 'w') as f: - f.write(srcsubdir) + f.write('%s\n' % srcsubdir) + for extradir in extradirs: + f.write('%s=%s\n' % (extradir, initial_revs[extradir])) } python devtool_pre_patch() { @@ -160,18 +187,25 @@ python devtool_pre_patch() { python devtool_post_patch() { import shutil tempdir = d.getVar('DEVTOOL_TEMPDIR') + + srcdirs = [] with open(os.path.join(tempdir, 'srcsubdir'), 'r') as f: - srcsubdir = f.read() + for line in f: + line = line.rstrip() + if line: + srcdirs.append(line.split('=')[0]) + srcsubdir = srcdirs[0] + with open(os.path.join(tempdir, 'initial_rev'), 'r') as f: initial_rev = f.read() - def rm_patches(): - patches_dir = os.path.join(srcsubdir, 'patches') + def rm_patches(pth): + patches_dir = os.path.join(pth, 'patches') if os.path.exists(patches_dir): shutil.rmtree(patches_dir) # Restore any "patches" directory that was actually part of the source tree try: - bb.process.run('git checkout -- patches', cwd=srcsubdir) + bb.process.run('git checkout -- patches', cwd=pth) except bb.process.ExecutionError: pass @@ -194,7 +228,7 @@ python devtool_post_patch() { localdata = bb.data.createCopy(d) localdata.setVar('OVERRIDES', ':'.join(no_overrides)) bb.build.exec_func('do_patch', localdata) - rm_patches() + rm_patches(srcsubdir) # Now we need to reconcile the dev branch with the no-overrides one # (otherwise we'd likely be left with identical commits that have different hashes) bb.process.run('git checkout %s' % devbranch, cwd=srcsubdir) @@ -212,12 +246,15 @@ python devtool_post_patch() { # Run do_patch function with the override applied localdata.appendVar('OVERRIDES', ':%s' % override) bb.build.exec_func('do_patch', localdata) - rm_patches() + rm_patches(srcsubdir) # Now we need to reconcile the new branch with the no-overrides one # (otherwise we'd likely be left with identical commits that have different hashes) bb.process.run('git rebase devtool-no-overrides', cwd=srcsubdir) bb.process.run('git checkout %s' % devbranch, cwd=srcsubdir) - bb.process.run('git tag -f devtool-patched', cwd=srcsubdir) + for srcdir in srcdirs: + bb.process.run('git tag -f devtool-patched', cwd=srcdir) + if srcdir != srcsubdir: + rm_patches(srcdir) } python devtool_post_configure() { diff --git a/meta/lib/oe/recipeutils.py b/meta/lib/oe/recipeutils.py index 49287273c9..f4a40c2988 100644 --- a/meta/lib/oe/recipeutils.py +++ b/meta/lib/oe/recipeutils.py @@ -445,10 +445,10 @@ def get_recipe_patches(d): """Get a list of the patches included in SRC_URI within a recipe.""" import oe.patch patches = oe.patch.src_patches(d, expand=False) - patchfiles = [] + patchfiles = OrderedDict() for patch in patches: _, _, local, _, _, parm = bb.fetch.decodeurl(patch) - patchfiles.append(local) + patchfiles[local] = parm return patchfiles diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py index 1897c5faea..9670dd7961 100644 --- a/meta/lib/oe/utils.py +++ b/meta/lib/oe/utils.py @@ -368,3 +368,27 @@ class ImageQAFailed(bb.build.FuncFailed): msg = msg + ' (%s)' % self.description return msg + +def is_path_under(path1, possible_parent): + """ + Return True if a path is underneath another, False otherwise. + Multiple paths to test can be specified (as a list or tuple) in + which case all specified test paths must be under the parent to + return True. + """ + def abs_path_trailing(pth): + pth_abs = os.path.abspath(pth) + if not pth_abs.endswith(os.sep): + pth_abs += os.sep + return pth_abs + + possible_parent_abs = abs_path_trailing(possible_parent) + if isinstance(path1, str): + testpaths = [path1] + else: + testpaths = path1 + for path in testpaths: + path_abs = abs_path_trailing(path) + if not path_abs.startswith(possible_parent_abs): + return False + return True diff --git a/meta/lib/oeqa/selftest/cases/devtool.py b/meta/lib/oeqa/selftest/cases/devtool.py index 43280cdc0e..b3063de698 100644 --- a/meta/lib/oeqa/selftest/cases/devtool.py +++ b/meta/lib/oeqa/selftest/cases/devtool.py @@ -7,7 +7,7 @@ import fnmatch import oeqa.utils.ftools as ftools from oeqa.selftest.case import OESelftestTestCase -from oeqa.utils.commands import runCmd, bitbake, get_bb_var, create_temp_layer +from oeqa.utils.commands import runCmd, bitbake, get_bb_var, create_temp_layer, CommandError from oeqa.utils.commands import get_bb_vars, runqemu, get_test_layer from oeqa.core.decorator.oeid import OETestID diff --git a/scripts/devtool b/scripts/devtool index a651d8f213..bccbb922bc 100755 --- a/scripts/devtool +++ b/scripts/devtool @@ -25,6 +25,8 @@ import re import configparser import subprocess import logging +import json +from collections import OrderedDict basepath = '' workspace = {} @@ -109,31 +111,58 @@ def read_workspace(): if not context.fixed_setup: _enable_workspace_layer(config.workspace_path, config, basepath) + def process_inline_json(strvalue, in_value): + if strvalue.count('{') == strvalue.count('}') and strvalue.count('[') == strvalue.count(']'): + pnvalues[in_value] = json.loads(strvalue, object_pairs_hook=OrderedDict) + return True + return False + logger.debug('Reading workspace in %s' % config.workspace_path) externalsrc_re = re.compile(r'^EXTERNALSRC(_pn-([^ =]+))? *= *"([^"]*)"$') for fn in glob.glob(os.path.join(config.workspace_path, 'appends', '*.bbappend')): with open(fn, 'r') as f: pnvalues = {} + in_value = None + strvalue = '' for line in f: - res = externalsrc_re.match(line.rstrip()) - if res: - pn = res.group(2) or os.path.splitext(os.path.basename(fn))[0].split('_')[0] - # Find the recipe file within the workspace, if any - bbfile = os.path.basename(fn).replace('.bbappend', '.bb').replace('%', '*') - recipefile = glob.glob(os.path.join(config.workspace_path, - 'recipes', - pn, - bbfile)) - if recipefile: - recipefile = recipefile[0] - pnvalues['srctree'] = res.group(3) - pnvalues['bbappend'] = fn - pnvalues['recipefile'] = recipefile - elif line.startswith('# srctreebase: '): - pnvalues['srctreebase'] = line.split(':', 1)[1].strip() + if in_value: + if not line.startswith('#'): + logger.error('Syntax error in %s - invalid in-line json' % fn) + sys.exit(1) + strvalue += line[1:] + if process_inline_json(strvalue, in_value): + in_value = None + else: + res = externalsrc_re.match(line.rstrip()) + if res: + pn = res.group(2) or os.path.splitext(os.path.basename(fn))[0].split('_')[0] + # Find the recipe file within the workspace, if any + bbfile = os.path.basename(fn).replace('.bbappend', '.bb').replace('%', '*') + recipefile = glob.glob(os.path.join(config.workspace_path, + 'recipes', + pn, + bbfile)) + if recipefile: + recipefile = recipefile[0] + pnvalues['srctree'] = res.group(3) + pnvalues['bbappend'] = fn + pnvalues['recipefile'] = recipefile + elif line.startswith('# srctreebase: '): + pnvalues['srctreebase'] = line.split(':', 1)[1].strip() + elif line.startswith('# srctreetop: '): + pnvalues['srctreetop'] = line.split(':', 1)[1].strip() + elif line.startswith('# extradirs: '): + strvalue = line.split(':', 1)[1].strip() + in_value = 'extradirs' + if process_inline_json(strvalue, in_value): + in_value = None if pnvalues: if not pnvalues.get('srctreebase', None): pnvalues['srctreebase'] = pnvalues['srctree'] + if not 'srctreetop' in pnvalues: + pnvalues['srctreetop'] = pnvalues['srctreebase'] + if not 'extradirs' in pnvalues: + pnvalues['extradirs'] = [] logger.debug('Found recipe %s' % pnvalues) workspace[pn] = pnvalues diff --git a/scripts/lib/devtool/__init__.py b/scripts/lib/devtool/__init__.py index 07d774dfb7..5cdf07ad58 100644 --- a/scripts/lib/devtool/__init__.py +++ b/scripts/lib/devtool/__init__.py @@ -223,6 +223,20 @@ def setup_git_repo(repodir, version, devbranch, basetag='devtool-base', d=None): bb.process.run('git checkout -b %s' % devbranch, cwd=repodir) bb.process.run('git tag -f %s' % basetag, cwd=repodir) +def find_git_repos(pth, toplevel=False): + """ + Find git repositories under a path + """ + repos = [] + if toplevel and os.path.isdir(os.path.join(pth, '.git')): + repos.append(pth) + for root, dirs, _ in os.walk(pth): + for dfn in dirs: + dfp = os.path.join(root, dfn) + if os.path.isdir(os.path.join(dfp, '.git')) and dfp not in repos: + repos.append(dfp) + return repos + def recipe_to_append(recipefile, config, wildcard=False): """ Convert a recipe file to a bbappend file path within the workspace. diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py index ae48406bea..cadd038bf6 100644 --- a/scripts/lib/devtool/standard.py +++ b/scripts/lib/devtool/standard.py @@ -29,8 +29,9 @@ import scriptutils import errno import glob import filecmp +import json from collections import OrderedDict, namedtuple -from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, use_external_build, setup_git_repo, recipe_to_append, get_bbclassextend_targets, update_unlockedsigs, check_prerelease_version, check_git_repo_dirty, check_git_repo_op, DevtoolError +from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, use_external_build, setup_git_repo, recipe_to_append, get_bbclassextend_targets, update_unlockedsigs, check_prerelease_version, check_git_repo_dirty, check_git_repo_op, find_git_repos, DevtoolError from devtool import parse_recipe logger = logging.getLogger('devtool') @@ -464,7 +465,7 @@ def sync(args, config, basepath, workspace): return 0 -ExtractSourceResult = namedtuple('ExtractSourceResult', ['initial_rev', 'srcsubdir']) +ExtractSourceResult = namedtuple('ExtractSourceResult', ['initial_rev', 'srcsubdir', 'extradirs']) def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, workspace, fixed_setup, d, tinfoil, no_overrides=False): """Extract sources of a recipe""" @@ -576,15 +577,27 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works if not res: raise DevtoolError('Extracting source for %s failed' % pn) + tempworkdir = os.path.join(tempdir, 'workdir') + extra_revs = OrderedDict() + extradirs = [] try: with open(os.path.join(tempdir, 'initial_rev'), 'r') as f: initial_rev = f.read() with open(os.path.join(tempdir, 'srcsubdir'), 'r') as f: - srcsubdir = f.read() + srcsubdir = f.readline().rstrip() + for line in f: + line = line.rstrip() + if '=' in line: + linesplit = line.rstrip().split('=', 1) + extradir = linesplit[0] + extradirs.append(extradir) + extra_revs[os.path.relpath(extradir, tempworkdir)] = linesplit[1] + except FileNotFoundError as e: raise DevtoolError('Something went wrong with source extraction - the devtool-source class was not active or did not function correctly:\n%s' % str(e)) - srcsubdir_rel = os.path.relpath(srcsubdir, os.path.join(tempdir, 'workdir')) + + srcsubdir_rel = os.path.relpath(srcsubdir, tempworkdir) tempdir_localdir = os.path.join(tempdir, 'oe-local-files') srctree_localdir = os.path.join(srctree, 'oe-local-files') @@ -613,6 +626,19 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works logger.info('Adding local source files to srctree...') shutil.move(tempdir_localdir, srcsubdir) + if extra_revs: + logger.warn('This recipe fetches more than one source tree. Each source tree will be tracked in a separate git repository.') + if not oe.utils.is_path_under(extradirs, srcsubdir): + # There's more than one source directory at the top level, we're going to need to create the parent + os.mkdir(srctree) + for extradir in extradirs: + shutil.move(extradir, srctree) + # Write a file to record which one is the main src in case we need to re-modify the same tree later + with open(os.path.join(srctree, '.devtool'), 'w') as f: + data = OrderedDict() + data['srcsubdir'] = os.path.basename(srcsubdir) + data['extradirs'] = extra_revs + json.dump(data, f, indent=' ') shutil.move(srcsubdir, srctree) if os.path.abspath(d.getVar('S')) == os.path.abspath(d.getVar('WORKDIR')): @@ -640,7 +666,7 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works if is_kernel_yocto: logger.info('Copying kernel config to srctree') - shutil.copy2(os.path.join(tempdir, '.config'), srctree) + shutil.copy2(os.path.join(tempdir, '.config'), os.path.join(srctree, srcsubdir_rel)) finally: if appendbackup: @@ -651,7 +677,7 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works logger.info('Preserving temporary directory %s' % tempdir) else: shutil.rmtree(tempdir) - return ExtractSourceResult(initial_rev, srcsubdir_rel) + return ExtractSourceResult(initial_rev, srcsubdir_rel, extra_revs) def _add_md5(config, recipename, filename): """Record checksum of a file (or recursively for a directory) to the md5-file of the workspace""" @@ -704,6 +730,27 @@ def _check_preserve(config, recipename): tf.write(line) os.rename(newfile, origfile) +def _get_initial_rev(srctree): + """ + Given a source tree, get the initial revision and determine if it was prepared by devtool + """ + initial_rev = None + try: + (stdout, _) = bb.process.run('git branch --contains devtool-base', cwd=srctree) + except bb.process.ExecutionError: + stdout = '' + if stdout: + was_devtool = True + for line in stdout.splitlines(): + if line.startswith('*'): + (stdout, _) = bb.process.run('git rev-parse devtool-base', cwd=srctree) + initial_rev = stdout.rstrip() + if not initial_rev: + # Otherwise, just grab the head revision + (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree) + initial_rev = stdout.rstrip() + return initial_rev, was_devtool + def modify(args, config, basepath, workspace): """Entry point for the devtool 'modify' subcommand""" import bb @@ -751,29 +798,40 @@ def modify(args, config, basepath, workspace): commits = [] check_commits = False torev = 'HEAD' + extradirs = OrderedDict() + srctreetop = None + extra_revs = {} if not args.no_extract: ret = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides) initial_rev = ret.initial_rev + extradirs = ret.extradirs + if extradirs and not oe.utils.is_path_under(extradirs.keys(), ret.srcsubdir): + # Multiple source trees, we need to be inside the main one + srctreetop = srctree + srctree = os.path.join(srctree, ret.srcsubdir) logger.info('Source tree extracted to %s' % srctree) check_commits = True else: + devtoolfile = os.path.join(srctree, '.devtool') + if os.path.exists(devtoolfile): + with open(devtoolfile, 'r') as f: + cfg = json.load(f, object_pairs_hook=OrderedDict) + srcsubdir = cfg.get('srcsubdir', '') + if srcsubdir: + srctreetop = srctree + srctree = os.path.abspath(os.path.join(srctree, srcsubdir)) + extradirs.update(cfg.get('extradirs', {})) + else: + # Find any extra source trees + extradirpaths = find_git_repos(srctree) + for extradirpath in extradirpaths: + extradirs[os.path.relpath(extradirpath, srctree)], _ = _get_initial_rev(extradirpath) + if os.path.exists(os.path.join(srctree, '.git')): # Check if it's a tree previously extracted by us - try: - (stdout, _) = bb.process.run('git branch --contains devtool-base', cwd=srctree) - except bb.process.ExecutionError: - stdout = '' - if stdout: - check_commits = True + initial_rev, check_commits = _get_initial_rev(srctree) + if check_commits: torev = 'devtool-patched' - for line in stdout.splitlines(): - if line.startswith('*'): - (stdout, _) = bb.process.run('git rev-parse devtool-base', cwd=srctree) - initial_rev = stdout.rstrip() - if not initial_rev: - # Otherwise, just grab the head revision - (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree) - initial_rev = stdout.rstrip() if initial_rev: # Get list of commits since this revision @@ -807,6 +865,23 @@ def modify(args, config, basepath, workspace): # Need to grab this here in case the source is within a subdirectory srctreebase = srctree + if extradirs: + extradirentries = [] + for extradir, extrarev in extradirs.items(): + extradirentry = OrderedDict() + if srctreetop: + # Several source trees at the top level + extradirentry['path'] = os.path.join(srctreetop, extradir) + else: + # All of the extra source trees are under the main one, so chop off + # the top level subdirectory + extradirentry['path'] = os.path.join(srctree, os.sep.join(extradir.split(os.sep)[1:])) + extradirentry['initial_rev'] = extrarev + # Get list of commits since this revision + (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % extrarev, cwd=extradirentry['path']) + extradirentry['commits'] = stdout.split() + extradirentries.append(extradirentry) + # Check that recipe isn't using a shared workdir s = os.path.abspath(rd.getVar('S')) workdir = os.path.abspath(rd.getVar('WORKDIR')) @@ -842,6 +917,23 @@ def modify(args, config, basepath, workspace): ' cp ${B}/.config ${S}/.config.baseline\n' ' ln -sfT ${B}/.config ${S}/.config.new\n' '}\n') + + if extradirs: + f.write('\n# srctreetop: %s\n' % srctreetop) + extradir_json = json.dumps(extradirentries, indent=' ') + prefix = '# extradirs:' + for line in extradir_json.splitlines(): + f.write('%s %s\n' % (prefix, line)) + prefix = '#' + f.write('\ndo_unpack[postfuncs] += "devtool_symlink_srctrees"\n') + f.write('devtool_symlink_srctrees () {\n') + # We could have done a loop in the function itself, but given there's + # usually only going to be a small number, there's not much point + for extradiritem in extradirentries: + if not oe.utils.is_path_under(extradiritem['path'], srctree): + f.write(' ln -sf %s ${WORKDIR}/\n' % extradiritem['path']) + f.write('}\n') + if initial_rev: f.write('\n# initial_rev: %s\n' % initial_rev) for commit in commits: @@ -1056,7 +1148,7 @@ def rename(args, config, basepath, workspace): return 0 -def _get_patchset_revs(srctree, recipe_path, initial_rev=None): +def _get_patchset_revs(recipename, workspace, srctree, initial_rev=None): """Get initial and update rev of a recipe. These are the start point of the whole patchset and start point for the patches to be re-generated/updated. """ @@ -1067,10 +1159,15 @@ def _get_patchset_revs(srctree, recipe_path, initial_rev=None): cwd=srctree) branchname = stdout.rstrip() + append = workspace[recipename]['bbappend'] + if not os.path.exists(append): + raise DevtoolError('unable to find workspace bbappend for recipe %s' % + recipename) + # Parse initial rev from recipe if not specified commits = [] patches = [] - with open(recipe_path, 'r') as f: + with open(append, 'r') as f: for line in f: if line.startswith('# initial_rev:'): if not initial_rev: @@ -1080,6 +1177,15 @@ def _get_patchset_revs(srctree, recipe_path, initial_rev=None): elif line.startswith('# patches_%s:' % branchname): patches = line.split(':')[-1].strip().split(',') + if srctree != workspace[recipename]['srctree']: + # Extra dir + for extradirentry in workspace[recipename]['extradirs']: + if srctree == extradirentry['path']: + commits = extradirentry['commits'] + break + else: + raise Exception('Failed to find extradir entry for %s' % srctree) + update_rev = initial_rev changed_revs = None if initial_rev: @@ -1160,7 +1266,7 @@ def _remove_source_files(append, files, destpath, no_report_remove=False, dry_ru raise -def _export_patches(srctree, rd, start_rev, destdir, changed_revs=None): +def _export_patches(srctree, rd, start_rev, destdir, changed_revs=None, origsrcdir=None): """Export patches from srctree to given location. Returns three-tuple of dicts: 1. updated - patches that already exist in SRCURI @@ -1175,9 +1281,21 @@ def _export_patches(srctree, rd, start_rev, destdir, changed_revs=None): added = OrderedDict() seqpatch_re = re.compile('^([0-9]{4}-)?(.+)') + patches = oe.recipeutils.get_recipe_patches(rd) + if origsrcdir: + if not origsrcdir.endswith(os.sep): + origsrcdir = origsrcdir + os.sep + s = rd.getVar('S') + for patch, parms in list(patches.items()): + patchdir = os.path.abspath(os.path.join(s, parms.get('patchdir', '.'))) + if not patchdir.endswith(os.sep): + patchdir = patchdir + os.sep + if not patchdir.startswith(origsrcdir): + del patches[patch] + existing_patches = dict((os.path.basename(path), path) for path in - oe.recipeutils.get_recipe_patches(rd)) - logger.debug('Existing patches: %s' % existing_patches) + patches) + logger.debug('Existing patches (for %s): %s' % (srctree, existing_patches)) # Generate patches from Git, exclude local files directory patch_pathspec = _git_exclude_path(srctree, 'oe-local-files') @@ -1370,6 +1488,11 @@ def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wi recipedir = os.path.basename(recipefile) logger.info('Updating SRCREV in recipe %s%s' % (recipedir, dry_run_suffix)) + # Work out what the extracted path would have been (so we can use it + # to filter patches when there are extra source directories) + s = rd.getVar('S') + origsrcdir = os.path.abspath(os.path.join(s, os.path.relpath(srctree, workspace[recipename]['srctree']))) + # Get HEAD revision try: stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree) @@ -1398,7 +1521,8 @@ def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wi patches_dir = tempfile.mkdtemp(dir=tempdir) old_srcrev = rd.getVar('SRCREV') or '' upd_p, new_p, del_p = _export_patches(srctree, rd, old_srcrev, - patches_dir) + patches_dir, + origsrcdir=origsrcdir) logger.debug('Patches: update %s, new %s, delete %s' % (dict(upd_p), dict(new_p), dict(del_p))) # Remove deleted local files and "overlapping" patches @@ -1444,6 +1568,10 @@ def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wi if update_srcuri: patchfields['SRC_URI'] = ' '.join(srcuri) ret = oe.recipeutils.patch_recipe(rd, recipefile, patchfields, redirect_output=dry_run_outdir) + # Set the values into the datastore for the benefit of the next + # call (if extradirs are set) + for var, value in patchfields.items(): + rd.setVar(var, value) finally: shutil.rmtree(tempdir) if not 'git://' in orig_src_uri: @@ -1461,12 +1589,8 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil recipefile = rd.getVar('FILE') recipedir = os.path.dirname(recipefile) - append = workspace[recipename]['bbappend'] - if not os.path.exists(append): - raise DevtoolError('unable to find workspace bbappend for recipe %s' % - recipename) - initial_rev, update_rev, changed_revs, filter_patches = _get_patchset_revs(srctree, append, initial_rev) + initial_rev, update_rev, changed_revs, filter_patches = _get_patchset_revs(recipename, workspace, srctree, initial_rev) if not initial_rev: raise DevtoolError('Unable to find initial revision - please specify ' 'it with --initial-rev') @@ -1476,6 +1600,17 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil if not dl_dir.endswith('/'): dl_dir += '/' + # Work out what the extracted path would have been (so we can use it + # to filter patches when there are extra source directories) + s = rd.getVar('S') + origsrcdir = os.path.abspath(os.path.join(s, os.path.relpath(srctree, workspace[recipename]['srctree']))) + # Determine patchdir for any new patches + patchdir = os.path.relpath(origsrcdir, s) + if patchdir != '.': + patchsuffix = ';patchdir=%s' % patchdir + else: + patchsuffix = '' + dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' tempdir = tempfile.mkdtemp(prefix='devtool') @@ -1494,14 +1629,16 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil # Get all patches from source tree and check if any should be removed all_patches_dir = tempfile.mkdtemp(dir=tempdir) _, _, del_p = _export_patches(srctree, rd, initial_rev, - all_patches_dir) + all_patches_dir, + origsrcdir=origsrcdir) # Remove deleted local files and patches remove_files = list(del_f.values()) + list(del_p.values()) # Get updated patches from source tree patches_dir = tempfile.mkdtemp(dir=tempdir) upd_p, new_p, _ = _export_patches(srctree, rd, update_rev, - patches_dir, changed_revs) + patches_dir, changed_revs, + origsrcdir=origsrcdir) logger.debug('Pre-filtering: update: %s, new: %s' % (dict(upd_p), dict(new_p))) if filter_patches: new_p = {} @@ -1531,7 +1668,7 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil removevalues=removevalues, redirect_output=dry_run_outdir) else: - logger.info('No patches or local source files needed updating') + return False, None, [] else: # Update existing files files_dir = _determine_files_dir(rd) @@ -1553,7 +1690,7 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil # replace the entry in SRC_URI with our local version logger.info('Replacing remote patch %s with updated local version' % basepath) path = os.path.join(files_dir, basepath) - _replace_srcuri_entry(srcuri, basepath, 'file://%s' % basepath) + _replace_srcuri_entry(srcuri, basepath, 'file://%s%s' % (basepath, patchsuffix)) updaterecipe = True else: logger.info('Updating patch %s%s' % (basepath, dry_run_suffix)) @@ -1575,7 +1712,7 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil os.path.join(files_dir, basepath), dry_run_outdir=dry_run_outdir, base_outdir=recipedir) - srcuri.append('file://%s' % basepath) + srcuri.append('file://%s%s' % (basepath, patchsuffix)) updaterecipe = True # Update recipe, if needed if _remove_file_entries(srcuri, remove_files)[0]: @@ -1586,9 +1723,11 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil ret = oe.recipeutils.patch_recipe(rd, recipefile, {'SRC_URI': ' '.join(srcuri)}, redirect_output=dry_run_outdir) + # Set the value into the datastore for the benefit of the next + # call (if extradirs are set) + rd.setVar('SRC_URI', ' '.join(srcuri)) elif not updatefiles: # Neither patches nor recipe were updated - logger.info('No patches or files need updating') return False, None, [] finally: shutil.rmtree(tempdir) @@ -1619,70 +1758,82 @@ def _guess_recipe_update_mode(srctree, rdata): return 'patch' def _update_recipe(recipename, workspace, rd, mode, appendlayerdir, wildcard_version, no_remove, initial_rev, no_report_remove=False, dry_run_outdir=None, no_overrides=False): - srctree = workspace[recipename]['srctree'] - if mode == 'auto': - mode = _guess_recipe_update_mode(srctree, rd) + main_srctree = workspace[recipename]['srctree'] - override_branches = [] - mainbranch = None - startbranch = None - if not no_overrides: - stdout, _ = bb.process.run('git branch', cwd=srctree) - other_branches = [] - for line in stdout.splitlines(): - branchname = line[2:] - if line.startswith('* '): - startbranch = branchname - if branchname.startswith(override_branch_prefix): - override_branches.append(branchname) - else: - other_branches.append(branchname) + appendfile = workspace[recipename]['bbappend'] + srctrees = OrderedDict([(main_srctree, initial_rev)]) + for extradirentry in workspace[recipename]['extradirs']: + srctrees[extradirentry['path']] = extradirentry['initial_rev'] + + logger.debug('Considering source trees: %s' % srctrees) + if mode == 'auto': + mode = _guess_recipe_update_mode(main_srctree, rd) + + crd = bb.data.createCopy(rd) + for srctree, src_initial_rev in srctrees.items(): + override_branches = [] + mainbranch = None + startbranch = None + if not no_overrides: + stdout, _ = bb.process.run('git branch', cwd=srctree) + other_branches = [] + for line in stdout.splitlines(): + branchname = line[2:] + if line.startswith('* '): + startbranch = branchname + if branchname.startswith(override_branch_prefix): + override_branches.append(branchname) + else: + other_branches.append(branchname) + + if override_branches: + logger.debug('_update_recipe: override branches: %s' % override_branches) + logger.debug('_update_recipe: other branches: %s' % other_branches) + if startbranch.startswith(override_branch_prefix): + if len(other_branches) == 1: + mainbranch = other_branches[1] + else: + raise DevtoolError('Unable to determine main branch - please check out the main branch in source tree first') + else: + mainbranch = startbranch + + checkedout = None + anyupdated = False + appendfile = None + allremoved = [] if override_branches: - logger.debug('_update_recipe: override branches: %s' % override_branches) - logger.debug('_update_recipe: other branches: %s' % other_branches) - if startbranch.startswith(override_branch_prefix): - if len(other_branches) == 1: - mainbranch = other_branches[1] + logger.info('Handling main branch (%s)...' % mainbranch) + if startbranch != mainbranch: + bb.process.run('git checkout %s' % mainbranch, cwd=srctree) + checkedout = mainbranch + try: + branchlist = [mainbranch] + override_branches + for branch in branchlist: + if branch != mainbranch: + logger.info('Handling branch %s...' % branch) + override = branch[len(override_branch_prefix):] + crd.setVar('OVERRIDES', '%s:%s' % rd.getVar('OVERRIDES') + override) + bb.process.run('git checkout %s' % branch, cwd=srctree) + checkedout = branch + + if mode == 'srcrev': + updated, appendf, removed = _update_recipe_srcrev(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir) + elif mode == 'patch': + updated, appendf, removed = _update_recipe_patch(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, src_initial_rev, dry_run_outdir) else: - raise DevtoolError('Unable to determine main branch - please check out the main branch in source tree first') - else: - mainbranch = startbranch + raise DevtoolError('update_recipe: invalid mode %s' % mode) + if updated: + anyupdated = True + if appendf: + appendfile = appendf + allremoved.extend(removed) + finally: + if startbranch and checkedout != startbranch: + bb.process.run('git checkout %s' % startbranch, cwd=srctree) - checkedout = None - anyupdated = False - appendfile = None - allremoved = [] - if override_branches: - logger.info('Handling main branch (%s)...' % mainbranch) - if startbranch != mainbranch: - bb.process.run('git checkout %s' % mainbranch, cwd=srctree) - checkedout = mainbranch - try: - branchlist = [mainbranch] + override_branches - for branch in branchlist: - crd = bb.data.createCopy(rd) - if branch != mainbranch: - logger.info('Handling branch %s...' % branch) - override = branch[len(override_branch_prefix):] - crd.appendVar('OVERRIDES', ':%s' % override) - bb.process.run('git checkout %s' % branch, cwd=srctree) - checkedout = branch - - if mode == 'srcrev': - updated, appendf, removed = _update_recipe_srcrev(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir) - elif mode == 'patch': - updated, appendf, removed = _update_recipe_patch(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir) - else: - raise DevtoolError('update_recipe: invalid mode %s' % mode) - if updated: - anyupdated = True - if appendf: - appendfile = appendf - allremoved.extend(removed) - finally: - if startbranch and checkedout != startbranch: - bb.process.run('git checkout %s' % startbranch, cwd=srctree) + if not anyupdated: + logger.info('No patches or files need updating') return anyupdated, appendfile, allremoved @@ -1808,16 +1959,16 @@ def _reset(recipes, no_clean, config, basepath, workspace): # We don't automatically create this dir next to appends, but the user can preservedir(os.path.join(config.workspace_path, 'appends', pn)) - srctreebase = workspace[pn]['srctreebase'] - if os.path.isdir(srctreebase): - if os.listdir(srctreebase): + srctreetop = workspace[pn]['srctreetop'] + if os.path.isdir(srctreetop): + if os.listdir(srctreetop): # We don't want to risk wiping out any work in progress logger.info('Leaving source tree %s as-is; if you no ' 'longer need it then please delete it manually' - % srctreebase) + % srctreetop) else: # This is unlikely, but if it's empty we can just remove it - os.rmdir(srctreebase) + os.rmdir(srctreetop) clean_preferred_provider(pn, config.workspace_path) @@ -1868,12 +2019,17 @@ def finish(args, config, basepath, workspace): srctree = workspace[args.recipename]['srctree'] check_git_repo_op(srctree, [corebasedir]) - dirty = check_git_repo_dirty(srctree) - if dirty: - if args.force: - logger.warning('Source tree is not clean, continuing as requested by -f/--force') - else: - raise DevtoolError('Source tree is not clean:\n\n%s\nEnsure you have committed your changes or use -f/--force if you are sure there\'s nothing that needs to be committed' % dirty) + srctrees = [srctree] + for extradirentry in workspace[args.recipename]['extradirs']: + srctrees.append(extradirentry['path']) + for pth in srctrees: + dirty = check_git_repo_dirty(pth) + if dirty: + if args.force: + logger.warning('Source tree %s is not clean, continuing as requested by -f/--force' % pth) + else: + raise DevtoolError('Source tree %s is not clean:\n\n%s\nEnsure you have committed your changes or use -f/--force if you are sure there\'s nothing that needs to be committed' % (pth, dirty)) + no_clean = False tinfoil = setup_tinfoil(basepath=basepath, tracking=True) -- cgit 1.2.3-korg