From ba27f9fee0c3ac0d04ed801d523837a1df03b4b0 Mon Sep 17 00:00:00 2001 From: Paul Eggleton Date: Fri, 23 Mar 2018 01:30:56 +1300 Subject: rrs_upgrade_history.py: enable to work across python2/3 versions If you want to go back and get history for the earlier releases (krogoth and previous) then we need to be able to support both python 2 and 3, which practically means we need the same split for this script as we have for the main layer index update script. The catch here is that since we are going back and following the history of changes forward, we basically need to use the same version of bitbake that was current at that time. This works except for around the transition between python 2 to 3 where the metadata lagged behind a bit, so we need to take that into account. In order to keep things generic we have a date field on the maintenance plan layer branch that specifies the date in the metadata where we should switch over to python 3, and then link to PythonEnvironment records that should be used for python 2 and 3 respectively. Signed-off-by: Paul Eggleton --- rrs/migrations/0007_python23.py | 31 +++++ rrs/models.py | 7 +- rrs/tools/rrs_upgrade_history.py | 255 ++++++++++------------------------ rrs/tools/upgrade_history_internal.py | 251 +++++++++++++++++++++++++++++++++ 4 files changed, 358 insertions(+), 186 deletions(-) create mode 100644 rrs/migrations/0007_python23.py create mode 100644 rrs/tools/upgrade_history_internal.py diff --git a/rrs/migrations/0007_python23.py b/rrs/migrations/0007_python23.py new file mode 100644 index 0000000000..a7a70fba88 --- /dev/null +++ b/rrs/migrations/0007_python23.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models +import datetime + + +class Migration(migrations.Migration): + + dependencies = [ + ('layerindex', '0010_add_dependencies'), + ('rrs', '0006_maintplan_email'), + ] + + operations = [ + migrations.AddField( + model_name='maintenanceplanlayerbranch', + name='python2_environment', + field=models.ForeignKey(blank=True, null=True, help_text='Environment to use for Python 2 commits', related_name='maintplan_layerbranch_python2_set', to='layerindex.PythonEnvironment'), + ), + migrations.AddField( + model_name='maintenanceplanlayerbranch', + name='python3_environment', + field=models.ForeignKey(blank=True, null=True, help_text='Environment to use for Python 3 commits', related_name='maintplan_layerbranch_python3_set', to='layerindex.PythonEnvironment'), + ), + migrations.AddField( + model_name='maintenanceplanlayerbranch', + name='python3_switch_date', + field=models.DateTimeField(verbose_name='Commit date to switch to Python 3', default=datetime.datetime(2016, 6, 2, 0, 0)), + ), + ] diff --git a/rrs/models.py b/rrs/models.py index fcf0876e18..e466da0737 100644 --- a/rrs/models.py +++ b/rrs/models.py @@ -9,11 +9,11 @@ import os import os.path sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '../'))) -from datetime import date +from datetime import date, datetime from django.db import models from django.contrib.auth.models import User -from layerindex.models import Recipe, LayerBranch +from layerindex.models import Recipe, LayerBranch, PythonEnvironment from django.core.exceptions import ObjectDoesNotExist @@ -36,6 +36,9 @@ class MaintenancePlan(models.Model): class MaintenancePlanLayerBranch(models.Model): plan = models.ForeignKey(MaintenancePlan) layerbranch = models.ForeignKey(LayerBranch) + python3_switch_date = models.DateTimeField('Commit date to switch to Python 3', default=datetime(2016, 6, 2)) + python2_environment = models.ForeignKey(PythonEnvironment, related_name='maintplan_layerbranch_python2_set', blank=True, null=True, help_text='Environment to use for Python 2 commits') + python3_environment = models.ForeignKey(PythonEnvironment, related_name='maintplan_layerbranch_python3_set', blank=True, null=True, help_text='Environment to use for Python 3 commits') class Meta: verbose_name_plural = "Maintenance plan layer branches" diff --git a/rrs/tools/rrs_upgrade_history.py b/rrs/tools/rrs_upgrade_history.py index ae3493df12..e48eba326b 100755 --- a/rrs/tools/rrs_upgrade_history.py +++ b/rrs/tools/rrs_upgrade_history.py @@ -4,13 +4,13 @@ # # To detect package versions of the recipes the script uses the name of the recipe. # -# Copyright (C) 2015 Intel Corporation -# Author: Anibal Limon +# Copyright (C) 2015, 2018 Intel Corporation +# Authors: Anibal Limon +# Paul Eggleton # # Licensed under the MIT license, see COPYING.MIT for details -from datetime import datetime -from datetime import timedelta +from datetime import datetime, timedelta import sys import os.path @@ -18,199 +18,73 @@ import optparse import logging sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__)))) -from common import common_setup, get_pv_type, load_recipes, \ - get_logger, DryRunRollbackException +from common import common_setup, get_logger common_setup() -from layerindex import utils, recipeparse -from layerindex.update_layer import split_recipe_fn +from layerindex import utils utils.setup_django() -from django.db import transaction import settings logger = get_logger("HistoryUpgrade", settings) fetchdir = settings.LAYER_FETCH_DIR +bitbakepath = os.path.join(fetchdir, 'bitbake') if not fetchdir: logger.error("Please set LAYER_FETCH_DIR in settings.py") sys.exit(1) -# setup bitbake -bitbakepath = os.path.join(fetchdir, 'bitbake') -sys.path.insert(0, os.path.join(bitbakepath, 'lib')) -from bb.utils import vercmp_string - -""" - Store upgrade into RecipeUpgrade model. -""" -def _save_upgrade(recipe, pv, commit, title, info, logger): - from email.utils import parsedate_tz, mktime_tz - from rrs.models import Maintainer, RecipeUpgrade - - maintainer_name = info.split(';')[0] - maintainer_email = info.split(';')[1] - author_date = info.split(';')[2] - commit_date = info.split(';')[3] - - maintainer = Maintainer.create_or_update(maintainer_name, maintainer_email) - - upgrade = RecipeUpgrade() - upgrade.recipe = recipe - upgrade.maintainer = maintainer - upgrade.author_date = datetime.utcfromtimestamp(mktime_tz( - parsedate_tz(author_date))) - upgrade.commit_date = datetime.utcfromtimestamp(mktime_tz( - parsedate_tz(commit_date))) - upgrade.version = pv - upgrade.sha1 = commit - upgrade.title = title.strip() - upgrade.save() -""" - Create upgrade receives new recipe_data and cmp versions. -""" -def _create_upgrade(recipe_data, layerbranch, ct, title, info, logger, initial=False): - from layerindex.models import Recipe - from rrs.models import RecipeUpgrade - - pn = recipe_data.getVar('PN', True) - pv = recipe_data.getVar('PV', True) - - try: - recipe = Recipe.objects.get(pn=pn, layerbranch=layerbranch) - except Exception as e: - logger.warn("%s: Not found in Layer branch %s." % - (pn, str(layerbranch))) - return - - try: - latest_upgrade = RecipeUpgrade.objects.filter( - recipe = recipe).order_by('-commit_date')[0] - prev_pv = latest_upgrade.version - except: - prev_pv = None - - if prev_pv is None: - logger.debug("%s: Initial upgrade ( -> %s)." % (recipe.pn, pv)) - _save_upgrade(recipe, pv, ct, title, info, logger) +def run_internal(maintplanlayerbranch, commit, commitdate, options, logger, bitbake_map, initial=False): + from layerindex.models import PythonEnvironment + from rrs.models import Release + if commitdate < maintplanlayerbranch.python3_switch_date: + # Python 2 + if maintplanlayerbranch.python2_environment: + cmdprefix = maintplanlayerbranch.python2_environment.get_command() + else: + cmdprefix = 'python' + # Ensure we're using a bitbake version that is python 2 compatible + if commitdate > datetime(2016, 5, 10): + commitdate = datetime(2016, 5, 10) else: - from common import get_recipe_pv_without_srcpv - - (ppv, _, _) = get_recipe_pv_without_srcpv(prev_pv, - get_pv_type(prev_pv)) - (npv, _, _) = get_recipe_pv_without_srcpv(pv, - get_pv_type(pv)) - - try: - if npv == 'git': - logger.debug("%s: Avoiding upgrade to unversioned git." % \ - (recipe.pn)) - elif ppv == 'git' or vercmp_string(ppv, npv) == -1: - if initial is True: - logger.debug("%s: Update initial upgrade ( -> %s)." % \ - (recipe.pn, pv)) - latest_upgrade.version = pv - latest_upgrade.save() - else: - logger.debug("%s: detected upgrade (%s -> %s)" \ - " in ct %s." % (pn, prev_pv, pv, ct)) - _save_upgrade(recipe, pv, ct, title, info, logger) - except: - logger.error("%s: fail to detect upgrade (%s -> %s)" \ - " in ct %s." % (pn, prev_pv, pv, ct)) - - -""" - Returns a list containing the fullpaths to the recipes from a commit. -""" -def _get_recipes_filenames(ct, repodir, layerdir, logger): - ct_files = [] - layerdir_start = os.path.normpath(layerdir) + os.sep - - files = utils.runcmd("git log --name-only --format='%n' -n 1 " + ct, - repodir, logger=logger) - - for f in files.split("\n"): - if f != "": - fullpath = os.path.join(repodir, f) - # Skip deleted files in commit - if not os.path.exists(fullpath): - continue - (typename, _, filename) = recipeparse.detect_file_type(fullpath, - layerdir_start) - if typename == 'recipe': - ct_files.append(fullpath) - - return ct_files - -def do_initial(layerbranch, ct, logger, dry_run): - layer = layerbranch.layer - urldir = str(layer.get_fetch_dir()) - repodir = os.path.join(fetchdir, urldir) - layerdir = os.path.join(repodir, str(layerbranch.vcs_subdir)) - - utils.runcmd("git checkout %s" % ct, - repodir, logger=logger) - utils.runcmd("git clean -dfx", repodir, logger=logger) - - title = "Initial import at 1.6 release start." - info = "No maintainer;;Mon, 11 Nov 2013 00:00:00 +0000;Mon, 11 Nov 2013 00:00:00 +0000" - - (tinfoil, d, recipes) = load_recipes(layerbranch, bitbakepath, - fetchdir, settings, logger, nocheckout=True) - - try: - with transaction.atomic(): - for recipe_data in recipes: - _create_upgrade(recipe_data, layerbranch, '', title, - info, logger, initial=True) - if dry_run: - raise DryRunRollbackException - except DryRunRollbackException: - pass - - tinfoil.shutdown() - -def do_loop(layerbranch, ct, logger, dry_run): - layer = layerbranch.layer - urldir = str(layer.get_fetch_dir()) - repodir = os.path.join(fetchdir, urldir) - layerdir = os.path.join(repodir, str(layerbranch.vcs_subdir)) - - utils.runcmd("git checkout %s" % ct, - repodir, logger=logger) - utils.runcmd("git clean -dfx", repodir, logger=logger) - - fns = _get_recipes_filenames(ct, repodir, layerdir, logger) - if not fns: - return - - (tinfoil, d, recipes) = load_recipes(layerbranch, bitbakepath, - fetchdir, settings, logger, recipe_files=fns, - nocheckout=True) - - title = utils.runcmd("git log --format='%s' -n 1 " + ct, - repodir, logger=logger) - info = utils.runcmd("git log --format='%an;%ae;%ad;%cd' --date=rfc -n 1 " \ - + ct, destdir=repodir, logger=logger) - try: - with transaction.atomic(): - for recipe_data in recipes: - _create_upgrade(recipe_data, layerbranch, ct, title, - info, logger) - if dry_run: - raise DryRunRollbackException - except DryRunRollbackException: - pass - - tinfoil.shutdown() - + # Python 3 + if maintplanlayerbranch.python3_environment: + cmdprefix = maintplanlayerbranch.python3_environment.get_command() + else: + cmdprefix = 'python3' + + bitbake_rev = utils.runcmd('git rev-list -1 --before="%s" origin/master' % str(commitdate), + bitbakepath, logger=logger) + check_rev = bitbake_map.get(bitbake_rev, None) + if check_rev: + logger.debug('Preferring bitbake revision %s over %s' % (check_rev, bitbake_rev)) + bitbake_rev = check_rev + + cmd = '%s upgrade_history_internal.py %s %s' % (cmdprefix, maintplanlayerbranch.layerbranch.id, commit) + if initial: + release = Release.get_by_date(maintplanlayerbranch.plan, commitdate) + if release: + comment = 'Initial import at %s release start.' % release.name + else: + comment = 'Initial import at %s' % commit + cmd += ' --initial="%s"' % comment + if bitbake_rev: + cmd += ' --bitbake-rev %s' % bitbake_rev + if options.dry_run: + cmd += ' --dry-run' + if options.loglevel == logging.DEBUG: + cmd += ' --debug' + logger.debug('Running %s' % cmd) + ret, output = utils.run_command_interruptible(cmd) + if ret == 254: + # Interrupted by user, break out of loop + logger.info('Update interrupted, exiting') + sys.exit(254) """ Upgrade history handler. """ def upgrade_history(options, logger): - from layerindex.models import LayerBranch from rrs.models import MaintenancePlan # start date @@ -221,6 +95,7 @@ def upgrade_history(options, logger): since = "2013-11-11" #RecipeUpgrade.objects.all().delete() else: + # FIXME this is awful - we should be storing the last commit somewhere since = (now - timedelta(days=8)).strftime("%Y-%m-%d") maintplans = MaintenancePlan.objects.filter(updates_enabled=True) @@ -236,21 +111,33 @@ def upgrade_history(options, logger): layerdir = os.path.join(repodir, layerbranch.vcs_subdir) commits = utils.runcmd("git log --since='" + since + - "' --format='%H' --reverse", repodir, + "' --format='%H %ct' --reverse origin/master", repodir, logger=logger) commit_list = commits.split('\n') + bitbake_map = {} + # Filter out some bad commits + bitbake_commits = utils.runcmd("git rev-list fef18b445c0cb6b266cd939b9c78d7cbce38663f^..39780b1ccbd76579db0fc6fb9369c848a3bafa9d^", + bitbakepath, + logger=logger) + bitbake_commit_list = bitbake_commits.splitlines() + for commit in bitbake_commit_list: + bitbake_map[commit] = '39780b1ccbd76579db0fc6fb9369c848a3bafa9d' + if options.initial: logger.debug("Adding initial upgrade history ....") - ct = commit_list.pop(0) - do_initial(layerbranch, ct, logger, options.dry_run) + ct, ctepoch = commit_list.pop(0).split() + ctdate = datetime.fromtimestamp(int(ctepoch)) + run_internal(maintplanbranch, ct, ctdate, options, logger, bitbake_map, initial=True) logger.debug("Adding upgrade history from %s to %s ..." % (since, today)) - for ct in commit_list: - if ct: + for item in commit_list: + if item: + ct, ctepoch = item.split() + ctdate = datetime.fromtimestamp(int(ctepoch)) logger.debug("Analysing commit %s ..." % ct) - do_loop(layerbranch, ct, logger, options.dry_run) + run_internal(maintplanbranch, ct, ctdate, options, logger, bitbake_map) if commit_list: utils.runcmd("git clean -dfx", repodir, logger=logger) diff --git a/rrs/tools/upgrade_history_internal.py b/rrs/tools/upgrade_history_internal.py new file mode 100644 index 0000000000..fb015d86a3 --- /dev/null +++ b/rrs/tools/upgrade_history_internal.py @@ -0,0 +1,251 @@ +# Internal script called by rrs_upgrade_history.py +# +# To detect package versions of the recipes the script uses the name of the recipe. +# +# Copyright (C) 2015, 2018 Intel Corporation +# Authors: Anibal Limon +# Paul Eggleton +# +# Licensed under the MIT license, see COPYING.MIT for details + +from datetime import datetime + +import sys +import os +import optparse +import logging +import re +from distutils.version import LooseVersion + +sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__)))) +from common import common_setup, get_pv_type, load_recipes, \ + get_logger, DryRunRollbackException + +common_setup() +from layerindex import utils, recipeparse +from layerindex.update_layer import split_recipe_fn + +""" + Store upgrade into RecipeUpgrade model. +""" +def _save_upgrade(recipe, pv, commit, title, info, logger): + from email.utils import parsedate_tz, mktime_tz + from rrs.models import Maintainer, RecipeUpgrade + + maintainer_name = info.split(';')[0] + maintainer_email = info.split(';')[1] + author_date = info.split(';')[2] + commit_date = info.split(';')[3] + + maintainer = Maintainer.create_or_update(maintainer_name, maintainer_email) + + upgrade = RecipeUpgrade() + upgrade.recipe = recipe + upgrade.maintainer = maintainer + upgrade.author_date = datetime.utcfromtimestamp(mktime_tz( + parsedate_tz(author_date))) + upgrade.commit_date = datetime.utcfromtimestamp(mktime_tz( + parsedate_tz(commit_date))) + upgrade.version = pv + upgrade.sha1 = commit + upgrade.title = title.strip() + upgrade.save() + +""" + Create upgrade receives new recipe_data and cmp versions. +""" +def _create_upgrade(recipe_data, layerbranch, ct, title, info, logger, initial=False): + from layerindex.models import Recipe + from rrs.models import RecipeUpgrade + from bb.utils import vercmp_string + + pn = recipe_data.getVar('PN', True) + pv = recipe_data.getVar('PV', True) + + try: + recipe = Recipe.objects.get(pn=pn, layerbranch=layerbranch) + except Exception as e: + logger.warn("%s: Not found in Layer branch %s." % + (pn, str(layerbranch))) + return + + try: + latest_upgrade = RecipeUpgrade.objects.filter( + recipe = recipe).order_by('-commit_date')[0] + prev_pv = latest_upgrade.version + except KeyboardInterrupt: + raise + except: + prev_pv = None + + if prev_pv is None: + logger.debug("%s: Initial upgrade ( -> %s)." % (recipe.pn, pv)) + _save_upgrade(recipe, pv, ct, title, info, logger) + else: + from common import get_recipe_pv_without_srcpv + + (ppv, _, _) = get_recipe_pv_without_srcpv(prev_pv, + get_pv_type(prev_pv)) + (npv, _, _) = get_recipe_pv_without_srcpv(pv, + get_pv_type(pv)) + + try: + if npv == 'git': + logger.debug("%s: Avoiding upgrade to unversioned git." % \ + (recipe.pn)) + elif ppv == 'git' or vercmp_string(ppv, npv) == -1: + if initial is True: + logger.debug("%s: Update initial upgrade ( -> %s)." % \ + (recipe.pn, pv)) + latest_upgrade.version = pv + latest_upgrade.save() + else: + logger.debug("%s: detected upgrade (%s -> %s)" \ + " in ct %s." % (pn, prev_pv, pv, ct)) + _save_upgrade(recipe, pv, ct, title, info, logger) + except KeyboardInterrupt: + raise + except: + logger.error("%s: fail to detect upgrade (%s -> %s)" \ + " in ct %s." % (pn, prev_pv, pv, ct)) + + +""" + Returns a list containing the fullpaths to the recipes from a commit. +""" +def _get_recipes_filenames(ct, repodir, layerdir, logger): + ct_files = [] + layerdir_start = os.path.normpath(layerdir) + os.sep + + files = utils.runcmd("git log --name-only --format='%n' -n 1 " + ct, + repodir, logger=logger) + + for f in files.split("\n"): + if f != "": + fullpath = os.path.join(repodir, f) + # Skip deleted files in commit + if not os.path.exists(fullpath): + continue + (typename, _, filename) = recipeparse.detect_file_type(fullpath, + layerdir_start) + if typename == 'recipe': + ct_files.append(fullpath) + + return ct_files + + +def generate_history(options, layerbranch_id, commit, logger): + from layerindex.models import LayerBranch + from rrs.models import Release + layerbranch = LayerBranch.objects.get(id=layerbranch_id) + + fetchdir = settings.LAYER_FETCH_DIR + if not fetchdir: + logger.error("Please set LAYER_FETCH_DIR in settings.py") + sys.exit(1) + + layer = layerbranch.layer + urldir = str(layer.get_fetch_dir()) + repodir = os.path.join(fetchdir, urldir) + layerdir = os.path.join(repodir, str(layerbranch.vcs_subdir)) + + utils.runcmd("git checkout %s" % commit, + repodir, logger=logger) + utils.runcmd("git clean -dfx", repodir, logger=logger) + + if options.initial: + fns = None + else: + fns = _get_recipes_filenames(commit, repodir, layerdir, logger) + if not fns: + return + + # setup bitbake + bitbakepath = os.path.join(fetchdir, 'bitbake') + if options.bitbake_rev: + bitbake_rev = options.bitbake_rev + if not re.match('^[0-9a-f]{40}$', bitbake_rev): + # Branch name, need to check out detached + bitbake_rev = 'origin/%s' % bitbake_rev + else: + commitdate = utils.runcmd("git show -s --format=%ci", repodir, logger=logger) + bitbake_rev = '`git rev-list -1 --before="%s" origin/master`' % commitdate + utils.runcmd('git checkout %s' % bitbake_rev, + bitbakepath, logger=logger) + utils.runcmd("git clean -dfx", bitbakepath, logger=logger) + sys.path.insert(0, os.path.join(bitbakepath, 'lib')) + + (tinfoil, d, recipes) = load_recipes(layerbranch, bitbakepath, + fetchdir, settings, logger, recipe_files=fns, + nocheckout=True) + + if options.initial: + title = options.initial + info = 'No maintainer;;' + utils.runcmd("git log --format='%ad;%cd' --date=rfc -n 1 " \ + + commit, destdir=repodir, logger=logger) + recordcommit = '' + else: + title = utils.runcmd("git log --format='%s' -n 1 " + commit, + repodir, logger=logger) + info = utils.runcmd("git log --format='%an;%ae;%ad;%cd' --date=rfc -n 1 " \ + + commit, destdir=repodir, logger=logger) + recordcommit = commit + + try: + with transaction.atomic(): + for recipe_data in recipes: + _create_upgrade(recipe_data, layerbranch, recordcommit, title, + info, logger, initial=options.initial) + if options.dry_run: + raise DryRunRollbackException + except DryRunRollbackException: + pass + + tinfoil.shutdown() + + +if __name__=="__main__": + logger = None + try: + utils.setup_django() + from django.db import transaction + import settings + + logger = get_logger("HistoryUpgrade", settings) + + parser = optparse.OptionParser(usage = """%prog [options] """) + + parser.add_option("-i", "--initial", + help = "Do initial population of upgrade histories (and specify comment)", + action="store", dest="initial", default='') + + parser.add_option("--bitbake-rev", + help = "Use the specified bitbake revision instead of the most recent one at the metadata commit date", + action="store", dest="bitbake_rev", default='') + + parser.add_option("-d", "--debug", + help = "Enable debug output", + action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO) + + parser.add_option("--dry-run", + help = "Do not write any data back to the database", + action="store_true", dest="dry_run", default=False) + + options, args = parser.parse_args(sys.argv) + + logger.setLevel(options.loglevel) + + if len(args) < 2: + logger.error('Please specify layerbranch ID') + sys.exit(1) + + if len(args) < 3: + logger.error('Please specify commit') + sys.exit(1) + + generate_history(options, int(args[1]), args[2], logger) + except KeyboardInterrupt: + if logger: + logger.info('Update interrupted, exiting') + sys.exit(254) + -- cgit 1.2.3-korg