From 3b4fecb217f29b3668dedda487542d3c956340f0 Mon Sep 17 00:00:00 2001 From: Paul Eggleton Date: Mon, 16 Jan 2017 11:09:42 +1300 Subject: update: fix logic for updating layer dependencies at the end * We were passing the incorrect path (to the top of the layer repo) if the layer had a subdirectory, so this doesn't seem to have been able to work for such layers previously. * Doing this update in the main update.py script meant that this could never work across branches requiring a python version change (using PythonEnvironment records) since the code was running within the same environment in which update.py was launched - the entire point of the separation of the two scripts. Move the checking to update_layer.py and call it separately to perform these updates, splitting out some common code in order to do so. Signed-off-by: Paul Eggleton --- layerindex/update.py | 85 ++++++++++++++++++++++------------------------ layerindex/update_layer.py | 34 ++++++++++++++++--- 2 files changed, 71 insertions(+), 48 deletions(-) diff --git a/layerindex/update.py b/layerindex/update.py index ebb0e3489e..5ff7fb4ae7 100755 --- a/layerindex/update.py +++ b/layerindex/update.py @@ -17,7 +17,6 @@ import signal from datetime import datetime, timedelta from distutils.version import LooseVersion import utils -from layerconfparse import LayerConfParse import warnings warnings.filterwarnings("ignore", category=DeprecationWarning) @@ -62,6 +61,30 @@ def run_command_interruptible(cmd): return process.returncode, buf +def prepare_update_layer_command(options, branch, layer, updatedeps=False): + """Prepare the update_layer.py command line""" + if branch.update_environment: + cmdprefix = branch.update_environment.get_command() + else: + cmdprefix = 'python3' + cmd = '%s update_layer.py -l %s -b %s' % (cmdprefix, layer.name, branch.name) + if updatedeps: + cmd += ' --update-dependencies' + if options.reload: + cmd += ' --reload' + if options.fullreload: + cmd += ' --fullreload' + if options.nocheckout: + cmd += ' --nocheckout' + if options.dryrun: + cmd += ' -n' + if options.loglevel == logging.DEBUG: + cmd += ' -d' + elif options.loglevel == logging.ERROR: + cmd += ' -q' + return cmd + + def main(): if LooseVersion(git.__version__) < '0.3.1': logger.error("Version of GitPython is too old, please install GitPython (python-git) 0.3.1 or later in order to use this script") @@ -221,24 +244,7 @@ def main(): urldir = layer.get_fetch_dir() repodir = os.path.join(fetchdir, urldir) - if branchobj.update_environment: - cmdprefix = branchobj.update_environment.get_command() - else: - cmdprefix = 'python3' - cmd = '%s update_layer.py -l %s -b %s' % (cmdprefix, layer.name, branch) - if options.reload: - cmd += ' --reload' - if options.fullreload: - cmd += ' --fullreload' - if options.nocheckout: - cmd += ' --nocheckout' - if options.dryrun: - cmd += ' -n' - if options.loglevel == logging.DEBUG: - cmd += ' -d' - elif options.loglevel == logging.ERROR: - cmd += ' -q' - + cmd = prepare_update_layer_command(options, branchobj, layer) logger.debug('Running layer update command: %s' % cmd) layerupdate.started = datetime.now() ret, output = run_command_interruptible(cmd) @@ -265,32 +271,23 @@ def main(): # dependencies that may have been missed. Note that creating the # dependencies is a best-effort and continues if they are not found. for branch in branches: - layerconfparser = LayerConfParse(logger=logger, bitbakepath=bitbakepath) - try: - for layer in layerquery: - - layerbranch = layer.get_layerbranch(branch) - # Skip layers that did not change. - layer_last_rev = None - if layerbranch: + branchobj = utils.get_branch(branch) + for layer in layerquery: + layerbranch = layer.get_layerbranch(branch) + if layerbranch: + if not (options.reload or options.fullreload): + # Skip layers that did not change. layer_last_rev = last_rev.get(layerbranch, None) - if layer_last_rev is None or layer_last_rev == layerbranch.vcs_last_rev: - continue - - urldir = layer.get_fetch_dir() - repodir = os.path.join(fetchdir, urldir) - - utils.checkout_layer_branch(layerbranch, repodir, logger) - - config_data = layerconfparser.parse_layer(layerbranch, repodir) - if not config_data: - logger.debug("Layer %s does not appear to have branch %s" % (layer.name, branch)) - continue - - utils.add_dependencies(layerbranch, config_data, logger=logger) - utils.add_recommends(layerbranch, config_data, logger=logger) - finally: - layerconfparser.shutdown() + if layer_last_rev is None or layer_last_rev == layerbranch.vcs_last_rev: + continue + + logger.info('Updating layer dependencies for %s on branch %s' % (layer.name, branch)) + cmd = prepare_update_layer_command(options, branchobj, layer, updatedeps=True) + logger.debug('Running update dependencies command: %s' % cmd) + ret, output = run_command_interruptible(cmd) + if ret == 254: + # Interrupted by user, break out of loop + break finally: utils.unlock_file(lockfile) diff --git a/layerindex/update_layer.py b/layerindex/update_layer.py index 3497ac5a90..66337cf069 100644 --- a/layerindex/update_layer.py +++ b/layerindex/update_layer.py @@ -20,6 +20,7 @@ from distutils.version import LooseVersion import itertools import utils import recipeparse +import layerconfparse import warnings warnings.filterwarnings("ignore", category=DeprecationWarning) @@ -170,6 +171,9 @@ def main(): parser.add_option("-n", "--dry-run", help = "Don't write any data back to the database", action="store_true", dest="dryrun") + parser.add_option("", "--update-dependencies", + help = "Update layer dependencies only", + action="store_true", dest="updatedeps") parser.add_option("", "--nocheckout", help = "Don't check out branches", action="store_true", dest="nocheckout") @@ -238,6 +242,30 @@ def main(): branchname = layerbranch.actual_branch branchdesc = "%s (%s)" % (options.branch, branchname) + if options.updatedeps: + # Update layer dependencies only + if not layerbranch: + logger.debug('Skipping dependency update for layer %s on branch %s - no layerbranch record' % (layer, branchdesc)) + sys.exit(0) + if not options.nocheckout: + utils.checkout_layer_branch(layerbranch, repodir, logger=logger) + layerdir = os.path.join(repodir, layerbranch.vcs_subdir) + if not os.path.exists(layerdir): + # If this happens it was already flagged during the main update, so ignore it + logger.debug('Skipping dependency update for layer %s on branch %s - layer directory not found' % (layer, branchdesc)) + sys.exit(0) + + layerconfparser = layerconfparse.LayerConfParse(logger=logger, bitbakepath=bitbakepath, tinfoil=tinfoil) + config_data = layerconfparser.parse_layer(layerbranch, layerdir) + if not config_data: + logger.debug("Layer %s does not appear to be valid for branch %s" % (layer.name, branchdesc)) + sys.exit(0) + + utils.add_dependencies(layerbranch, config_data, logger=logger) + utils.add_recommends(layerbranch, config_data, logger=logger) + + sys.exit(0) + # Collect repo info repo = git.Repo(repodir) assert repo.bare == False @@ -300,8 +328,7 @@ def main(): if layerbranch.vcs_last_rev != topcommit.hexsha or options.reload: # Check out appropriate branch if not options.nocheckout: - out = utils.runcmd("git checkout origin/%s" % branchname, repodir, logger=logger) - out = utils.runcmd("git clean -f -x", repodir, logger=logger) + utils.checkout_layer_branch(layerbranch, repodir, logger=logger) if layerbranch.vcs_subdir and not os.path.exists(layerdir): if newbranch: @@ -316,8 +343,7 @@ def main(): logger.info("Collecting data for layer %s on branch %s" % (layer.name, branchdesc)) - from layerconfparse import LayerConfParse - layerconfparser = LayerConfParse(logger=logger, tinfoil=tinfoil) + layerconfparser = layerconfparse.LayerConfParse(logger=logger, tinfoil=tinfoil) layer_config_data = layerconfparser.parse_layer(layerbranch, layerdir) if not layer_config_data: logger.info("Skipping update of layer %s for branch %s - conf/layer.conf may have parse issues" % (layer.name, branchdesc)) -- cgit 1.2.3-korg