diff options
Diffstat (limited to 'rrs/tools/rrs_upgrade_history.py')
| -rwxr-xr-x | rrs/tools/rrs_upgrade_history.py | 255 |
1 files changed, 71 insertions, 184 deletions
diff --git a/rrs/tools/rrs_upgrade_history.py b/rrs/tools/rrs_upgrade_history.py index ae3493df12..e48eba326b 100755 --- a/rrs/tools/rrs_upgrade_history.py +++ b/rrs/tools/rrs_upgrade_history.py @@ -4,13 +4,13 @@ # # To detect package versions of the recipes the script uses the name of the recipe. # -# Copyright (C) 2015 Intel Corporation -# Author: Anibal Limon <anibal.limon@linux.intel.com> +# Copyright (C) 2015, 2018 Intel Corporation +# Authors: Anibal Limon <anibal.limon@linux.intel.com> +# Paul Eggleton <paul.eggleton@linux.intel.com> # # Licensed under the MIT license, see COPYING.MIT for details -from datetime import datetime -from datetime import timedelta +from datetime import datetime, timedelta import sys import os.path @@ -18,199 +18,73 @@ import optparse import logging sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__)))) -from common import common_setup, get_pv_type, load_recipes, \ - get_logger, DryRunRollbackException +from common import common_setup, get_logger common_setup() -from layerindex import utils, recipeparse -from layerindex.update_layer import split_recipe_fn +from layerindex import utils utils.setup_django() -from django.db import transaction import settings logger = get_logger("HistoryUpgrade", settings) fetchdir = settings.LAYER_FETCH_DIR +bitbakepath = os.path.join(fetchdir, 'bitbake') if not fetchdir: logger.error("Please set LAYER_FETCH_DIR in settings.py") sys.exit(1) -# setup bitbake -bitbakepath = os.path.join(fetchdir, 'bitbake') -sys.path.insert(0, os.path.join(bitbakepath, 'lib')) -from bb.utils import vercmp_string - -""" - Store upgrade into RecipeUpgrade model. -""" -def _save_upgrade(recipe, pv, commit, title, info, logger): - from email.utils import parsedate_tz, mktime_tz - from rrs.models import Maintainer, RecipeUpgrade - - maintainer_name = info.split(';')[0] - maintainer_email = info.split(';')[1] - author_date = info.split(';')[2] - commit_date = info.split(';')[3] - - maintainer = Maintainer.create_or_update(maintainer_name, maintainer_email) - - upgrade = RecipeUpgrade() - upgrade.recipe = recipe - upgrade.maintainer = maintainer - upgrade.author_date = datetime.utcfromtimestamp(mktime_tz( - parsedate_tz(author_date))) - upgrade.commit_date = datetime.utcfromtimestamp(mktime_tz( - parsedate_tz(commit_date))) - upgrade.version = pv - upgrade.sha1 = commit - upgrade.title = title.strip() - upgrade.save() -""" - Create upgrade receives new recipe_data and cmp versions. -""" -def _create_upgrade(recipe_data, layerbranch, ct, title, info, logger, initial=False): - from layerindex.models import Recipe - from rrs.models import RecipeUpgrade - - pn = recipe_data.getVar('PN', True) - pv = recipe_data.getVar('PV', True) - - try: - recipe = Recipe.objects.get(pn=pn, layerbranch=layerbranch) - except Exception as e: - logger.warn("%s: Not found in Layer branch %s." % - (pn, str(layerbranch))) - return - - try: - latest_upgrade = RecipeUpgrade.objects.filter( - recipe = recipe).order_by('-commit_date')[0] - prev_pv = latest_upgrade.version - except: - prev_pv = None - - if prev_pv is None: - logger.debug("%s: Initial upgrade ( -> %s)." % (recipe.pn, pv)) - _save_upgrade(recipe, pv, ct, title, info, logger) +def run_internal(maintplanlayerbranch, commit, commitdate, options, logger, bitbake_map, initial=False): + from layerindex.models import PythonEnvironment + from rrs.models import Release + if commitdate < maintplanlayerbranch.python3_switch_date: + # Python 2 + if maintplanlayerbranch.python2_environment: + cmdprefix = maintplanlayerbranch.python2_environment.get_command() + else: + cmdprefix = 'python' + # Ensure we're using a bitbake version that is python 2 compatible + if commitdate > datetime(2016, 5, 10): + commitdate = datetime(2016, 5, 10) else: - from common import get_recipe_pv_without_srcpv - - (ppv, _, _) = get_recipe_pv_without_srcpv(prev_pv, - get_pv_type(prev_pv)) - (npv, _, _) = get_recipe_pv_without_srcpv(pv, - get_pv_type(pv)) - - try: - if npv == 'git': - logger.debug("%s: Avoiding upgrade to unversioned git." % \ - (recipe.pn)) - elif ppv == 'git' or vercmp_string(ppv, npv) == -1: - if initial is True: - logger.debug("%s: Update initial upgrade ( -> %s)." % \ - (recipe.pn, pv)) - latest_upgrade.version = pv - latest_upgrade.save() - else: - logger.debug("%s: detected upgrade (%s -> %s)" \ - " in ct %s." % (pn, prev_pv, pv, ct)) - _save_upgrade(recipe, pv, ct, title, info, logger) - except: - logger.error("%s: fail to detect upgrade (%s -> %s)" \ - " in ct %s." % (pn, prev_pv, pv, ct)) - - -""" - Returns a list containing the fullpaths to the recipes from a commit. -""" -def _get_recipes_filenames(ct, repodir, layerdir, logger): - ct_files = [] - layerdir_start = os.path.normpath(layerdir) + os.sep - - files = utils.runcmd("git log --name-only --format='%n' -n 1 " + ct, - repodir, logger=logger) - - for f in files.split("\n"): - if f != "": - fullpath = os.path.join(repodir, f) - # Skip deleted files in commit - if not os.path.exists(fullpath): - continue - (typename, _, filename) = recipeparse.detect_file_type(fullpath, - layerdir_start) - if typename == 'recipe': - ct_files.append(fullpath) - - return ct_files - -def do_initial(layerbranch, ct, logger, dry_run): - layer = layerbranch.layer - urldir = str(layer.get_fetch_dir()) - repodir = os.path.join(fetchdir, urldir) - layerdir = os.path.join(repodir, str(layerbranch.vcs_subdir)) - - utils.runcmd("git checkout %s" % ct, - repodir, logger=logger) - utils.runcmd("git clean -dfx", repodir, logger=logger) - - title = "Initial import at 1.6 release start." - info = "No maintainer;;Mon, 11 Nov 2013 00:00:00 +0000;Mon, 11 Nov 2013 00:00:00 +0000" - - (tinfoil, d, recipes) = load_recipes(layerbranch, bitbakepath, - fetchdir, settings, logger, nocheckout=True) - - try: - with transaction.atomic(): - for recipe_data in recipes: - _create_upgrade(recipe_data, layerbranch, '', title, - info, logger, initial=True) - if dry_run: - raise DryRunRollbackException - except DryRunRollbackException: - pass - - tinfoil.shutdown() - -def do_loop(layerbranch, ct, logger, dry_run): - layer = layerbranch.layer - urldir = str(layer.get_fetch_dir()) - repodir = os.path.join(fetchdir, urldir) - layerdir = os.path.join(repodir, str(layerbranch.vcs_subdir)) - - utils.runcmd("git checkout %s" % ct, - repodir, logger=logger) - utils.runcmd("git clean -dfx", repodir, logger=logger) - - fns = _get_recipes_filenames(ct, repodir, layerdir, logger) - if not fns: - return - - (tinfoil, d, recipes) = load_recipes(layerbranch, bitbakepath, - fetchdir, settings, logger, recipe_files=fns, - nocheckout=True) - - title = utils.runcmd("git log --format='%s' -n 1 " + ct, - repodir, logger=logger) - info = utils.runcmd("git log --format='%an;%ae;%ad;%cd' --date=rfc -n 1 " \ - + ct, destdir=repodir, logger=logger) - try: - with transaction.atomic(): - for recipe_data in recipes: - _create_upgrade(recipe_data, layerbranch, ct, title, - info, logger) - if dry_run: - raise DryRunRollbackException - except DryRunRollbackException: - pass - - tinfoil.shutdown() - + # Python 3 + if maintplanlayerbranch.python3_environment: + cmdprefix = maintplanlayerbranch.python3_environment.get_command() + else: + cmdprefix = 'python3' + + bitbake_rev = utils.runcmd('git rev-list -1 --before="%s" origin/master' % str(commitdate), + bitbakepath, logger=logger) + check_rev = bitbake_map.get(bitbake_rev, None) + if check_rev: + logger.debug('Preferring bitbake revision %s over %s' % (check_rev, bitbake_rev)) + bitbake_rev = check_rev + + cmd = '%s upgrade_history_internal.py %s %s' % (cmdprefix, maintplanlayerbranch.layerbranch.id, commit) + if initial: + release = Release.get_by_date(maintplanlayerbranch.plan, commitdate) + if release: + comment = 'Initial import at %s release start.' % release.name + else: + comment = 'Initial import at %s' % commit + cmd += ' --initial="%s"' % comment + if bitbake_rev: + cmd += ' --bitbake-rev %s' % bitbake_rev + if options.dry_run: + cmd += ' --dry-run' + if options.loglevel == logging.DEBUG: + cmd += ' --debug' + logger.debug('Running %s' % cmd) + ret, output = utils.run_command_interruptible(cmd) + if ret == 254: + # Interrupted by user, break out of loop + logger.info('Update interrupted, exiting') + sys.exit(254) """ Upgrade history handler. """ def upgrade_history(options, logger): - from layerindex.models import LayerBranch from rrs.models import MaintenancePlan # start date @@ -221,6 +95,7 @@ def upgrade_history(options, logger): since = "2013-11-11" #RecipeUpgrade.objects.all().delete() else: + # FIXME this is awful - we should be storing the last commit somewhere since = (now - timedelta(days=8)).strftime("%Y-%m-%d") maintplans = MaintenancePlan.objects.filter(updates_enabled=True) @@ -236,21 +111,33 @@ def upgrade_history(options, logger): layerdir = os.path.join(repodir, layerbranch.vcs_subdir) commits = utils.runcmd("git log --since='" + since + - "' --format='%H' --reverse", repodir, + "' --format='%H %ct' --reverse origin/master", repodir, logger=logger) commit_list = commits.split('\n') + bitbake_map = {} + # Filter out some bad commits + bitbake_commits = utils.runcmd("git rev-list fef18b445c0cb6b266cd939b9c78d7cbce38663f^..39780b1ccbd76579db0fc6fb9369c848a3bafa9d^", + bitbakepath, + logger=logger) + bitbake_commit_list = bitbake_commits.splitlines() + for commit in bitbake_commit_list: + bitbake_map[commit] = '39780b1ccbd76579db0fc6fb9369c848a3bafa9d' + if options.initial: logger.debug("Adding initial upgrade history ....") - ct = commit_list.pop(0) - do_initial(layerbranch, ct, logger, options.dry_run) + ct, ctepoch = commit_list.pop(0).split() + ctdate = datetime.fromtimestamp(int(ctepoch)) + run_internal(maintplanbranch, ct, ctdate, options, logger, bitbake_map, initial=True) logger.debug("Adding upgrade history from %s to %s ..." % (since, today)) - for ct in commit_list: - if ct: + for item in commit_list: + if item: + ct, ctepoch = item.split() + ctdate = datetime.fromtimestamp(int(ctepoch)) logger.debug("Analysing commit %s ..." % ct) - do_loop(layerbranch, ct, logger, options.dry_run) + run_internal(maintplanbranch, ct, ctdate, options, logger, bitbake_map) if commit_list: utils.runcmd("git clean -dfx", repodir, logger=logger) |