diff options
| -rw-r--r-- | rrs/migrations/0007_python23.py | 31 | ||||
| -rw-r--r-- | rrs/models.py | 7 | ||||
| -rwxr-xr-x | rrs/tools/rrs_upgrade_history.py | 255 | ||||
| -rw-r--r-- | rrs/tools/upgrade_history_internal.py | 251 |
4 files changed, 358 insertions, 186 deletions
diff --git a/rrs/migrations/0007_python23.py b/rrs/migrations/0007_python23.py new file mode 100644 index 0000000000..a7a70fba88 --- /dev/null +++ b/rrs/migrations/0007_python23.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models +import datetime + + +class Migration(migrations.Migration): + + dependencies = [ + ('layerindex', '0010_add_dependencies'), + ('rrs', '0006_maintplan_email'), + ] + + operations = [ + migrations.AddField( + model_name='maintenanceplanlayerbranch', + name='python2_environment', + field=models.ForeignKey(blank=True, null=True, help_text='Environment to use for Python 2 commits', related_name='maintplan_layerbranch_python2_set', to='layerindex.PythonEnvironment'), + ), + migrations.AddField( + model_name='maintenanceplanlayerbranch', + name='python3_environment', + field=models.ForeignKey(blank=True, null=True, help_text='Environment to use for Python 3 commits', related_name='maintplan_layerbranch_python3_set', to='layerindex.PythonEnvironment'), + ), + migrations.AddField( + model_name='maintenanceplanlayerbranch', + name='python3_switch_date', + field=models.DateTimeField(verbose_name='Commit date to switch to Python 3', default=datetime.datetime(2016, 6, 2, 0, 0)), + ), + ] diff --git a/rrs/models.py b/rrs/models.py index fcf0876e18..e466da0737 100644 --- a/rrs/models.py +++ b/rrs/models.py @@ -9,11 +9,11 @@ import os import os.path sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '../'))) -from datetime import date +from datetime import date, datetime from django.db import models from django.contrib.auth.models import User -from layerindex.models import Recipe, LayerBranch +from layerindex.models import Recipe, LayerBranch, PythonEnvironment from django.core.exceptions import ObjectDoesNotExist @@ -36,6 +36,9 @@ class MaintenancePlan(models.Model): class MaintenancePlanLayerBranch(models.Model): plan = models.ForeignKey(MaintenancePlan) layerbranch = models.ForeignKey(LayerBranch) + python3_switch_date = models.DateTimeField('Commit date to switch to Python 3', default=datetime(2016, 6, 2)) + python2_environment = models.ForeignKey(PythonEnvironment, related_name='maintplan_layerbranch_python2_set', blank=True, null=True, help_text='Environment to use for Python 2 commits') + python3_environment = models.ForeignKey(PythonEnvironment, related_name='maintplan_layerbranch_python3_set', blank=True, null=True, help_text='Environment to use for Python 3 commits') class Meta: verbose_name_plural = "Maintenance plan layer branches" diff --git a/rrs/tools/rrs_upgrade_history.py b/rrs/tools/rrs_upgrade_history.py index ae3493df12..e48eba326b 100755 --- a/rrs/tools/rrs_upgrade_history.py +++ b/rrs/tools/rrs_upgrade_history.py @@ -4,13 +4,13 @@ # # To detect package versions of the recipes the script uses the name of the recipe. # -# Copyright (C) 2015 Intel Corporation -# Author: Anibal Limon <anibal.limon@linux.intel.com> +# Copyright (C) 2015, 2018 Intel Corporation +# Authors: Anibal Limon <anibal.limon@linux.intel.com> +# Paul Eggleton <paul.eggleton@linux.intel.com> # # Licensed under the MIT license, see COPYING.MIT for details -from datetime import datetime -from datetime import timedelta +from datetime import datetime, timedelta import sys import os.path @@ -18,199 +18,73 @@ import optparse import logging sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__)))) -from common import common_setup, get_pv_type, load_recipes, \ - get_logger, DryRunRollbackException +from common import common_setup, get_logger common_setup() -from layerindex import utils, recipeparse -from layerindex.update_layer import split_recipe_fn +from layerindex import utils utils.setup_django() -from django.db import transaction import settings logger = get_logger("HistoryUpgrade", settings) fetchdir = settings.LAYER_FETCH_DIR +bitbakepath = os.path.join(fetchdir, 'bitbake') if not fetchdir: logger.error("Please set LAYER_FETCH_DIR in settings.py") sys.exit(1) -# setup bitbake -bitbakepath = os.path.join(fetchdir, 'bitbake') -sys.path.insert(0, os.path.join(bitbakepath, 'lib')) -from bb.utils import vercmp_string - -""" - Store upgrade into RecipeUpgrade model. -""" -def _save_upgrade(recipe, pv, commit, title, info, logger): - from email.utils import parsedate_tz, mktime_tz - from rrs.models import Maintainer, RecipeUpgrade - - maintainer_name = info.split(';')[0] - maintainer_email = info.split(';')[1] - author_date = info.split(';')[2] - commit_date = info.split(';')[3] - - maintainer = Maintainer.create_or_update(maintainer_name, maintainer_email) - - upgrade = RecipeUpgrade() - upgrade.recipe = recipe - upgrade.maintainer = maintainer - upgrade.author_date = datetime.utcfromtimestamp(mktime_tz( - parsedate_tz(author_date))) - upgrade.commit_date = datetime.utcfromtimestamp(mktime_tz( - parsedate_tz(commit_date))) - upgrade.version = pv - upgrade.sha1 = commit - upgrade.title = title.strip() - upgrade.save() -""" - Create upgrade receives new recipe_data and cmp versions. -""" -def _create_upgrade(recipe_data, layerbranch, ct, title, info, logger, initial=False): - from layerindex.models import Recipe - from rrs.models import RecipeUpgrade - - pn = recipe_data.getVar('PN', True) - pv = recipe_data.getVar('PV', True) - - try: - recipe = Recipe.objects.get(pn=pn, layerbranch=layerbranch) - except Exception as e: - logger.warn("%s: Not found in Layer branch %s." % - (pn, str(layerbranch))) - return - - try: - latest_upgrade = RecipeUpgrade.objects.filter( - recipe = recipe).order_by('-commit_date')[0] - prev_pv = latest_upgrade.version - except: - prev_pv = None - - if prev_pv is None: - logger.debug("%s: Initial upgrade ( -> %s)." % (recipe.pn, pv)) - _save_upgrade(recipe, pv, ct, title, info, logger) +def run_internal(maintplanlayerbranch, commit, commitdate, options, logger, bitbake_map, initial=False): + from layerindex.models import PythonEnvironment + from rrs.models import Release + if commitdate < maintplanlayerbranch.python3_switch_date: + # Python 2 + if maintplanlayerbranch.python2_environment: + cmdprefix = maintplanlayerbranch.python2_environment.get_command() + else: + cmdprefix = 'python' + # Ensure we're using a bitbake version that is python 2 compatible + if commitdate > datetime(2016, 5, 10): + commitdate = datetime(2016, 5, 10) else: - from common import get_recipe_pv_without_srcpv - - (ppv, _, _) = get_recipe_pv_without_srcpv(prev_pv, - get_pv_type(prev_pv)) - (npv, _, _) = get_recipe_pv_without_srcpv(pv, - get_pv_type(pv)) - - try: - if npv == 'git': - logger.debug("%s: Avoiding upgrade to unversioned git." % \ - (recipe.pn)) - elif ppv == 'git' or vercmp_string(ppv, npv) == -1: - if initial is True: - logger.debug("%s: Update initial upgrade ( -> %s)." % \ - (recipe.pn, pv)) - latest_upgrade.version = pv - latest_upgrade.save() - else: - logger.debug("%s: detected upgrade (%s -> %s)" \ - " in ct %s." % (pn, prev_pv, pv, ct)) - _save_upgrade(recipe, pv, ct, title, info, logger) - except: - logger.error("%s: fail to detect upgrade (%s -> %s)" \ - " in ct %s." % (pn, prev_pv, pv, ct)) - - -""" - Returns a list containing the fullpaths to the recipes from a commit. -""" -def _get_recipes_filenames(ct, repodir, layerdir, logger): - ct_files = [] - layerdir_start = os.path.normpath(layerdir) + os.sep - - files = utils.runcmd("git log --name-only --format='%n' -n 1 " + ct, - repodir, logger=logger) - - for f in files.split("\n"): - if f != "": - fullpath = os.path.join(repodir, f) - # Skip deleted files in commit - if not os.path.exists(fullpath): - continue - (typename, _, filename) = recipeparse.detect_file_type(fullpath, - layerdir_start) - if typename == 'recipe': - ct_files.append(fullpath) - - return ct_files - -def do_initial(layerbranch, ct, logger, dry_run): - layer = layerbranch.layer - urldir = str(layer.get_fetch_dir()) - repodir = os.path.join(fetchdir, urldir) - layerdir = os.path.join(repodir, str(layerbranch.vcs_subdir)) - - utils.runcmd("git checkout %s" % ct, - repodir, logger=logger) - utils.runcmd("git clean -dfx", repodir, logger=logger) - - title = "Initial import at 1.6 release start." - info = "No maintainer;;Mon, 11 Nov 2013 00:00:00 +0000;Mon, 11 Nov 2013 00:00:00 +0000" - - (tinfoil, d, recipes) = load_recipes(layerbranch, bitbakepath, - fetchdir, settings, logger, nocheckout=True) - - try: - with transaction.atomic(): - for recipe_data in recipes: - _create_upgrade(recipe_data, layerbranch, '', title, - info, logger, initial=True) - if dry_run: - raise DryRunRollbackException - except DryRunRollbackException: - pass - - tinfoil.shutdown() - -def do_loop(layerbranch, ct, logger, dry_run): - layer = layerbranch.layer - urldir = str(layer.get_fetch_dir()) - repodir = os.path.join(fetchdir, urldir) - layerdir = os.path.join(repodir, str(layerbranch.vcs_subdir)) - - utils.runcmd("git checkout %s" % ct, - repodir, logger=logger) - utils.runcmd("git clean -dfx", repodir, logger=logger) - - fns = _get_recipes_filenames(ct, repodir, layerdir, logger) - if not fns: - return - - (tinfoil, d, recipes) = load_recipes(layerbranch, bitbakepath, - fetchdir, settings, logger, recipe_files=fns, - nocheckout=True) - - title = utils.runcmd("git log --format='%s' -n 1 " + ct, - repodir, logger=logger) - info = utils.runcmd("git log --format='%an;%ae;%ad;%cd' --date=rfc -n 1 " \ - + ct, destdir=repodir, logger=logger) - try: - with transaction.atomic(): - for recipe_data in recipes: - _create_upgrade(recipe_data, layerbranch, ct, title, - info, logger) - if dry_run: - raise DryRunRollbackException - except DryRunRollbackException: - pass - - tinfoil.shutdown() - + # Python 3 + if maintplanlayerbranch.python3_environment: + cmdprefix = maintplanlayerbranch.python3_environment.get_command() + else: + cmdprefix = 'python3' + + bitbake_rev = utils.runcmd('git rev-list -1 --before="%s" origin/master' % str(commitdate), + bitbakepath, logger=logger) + check_rev = bitbake_map.get(bitbake_rev, None) + if check_rev: + logger.debug('Preferring bitbake revision %s over %s' % (check_rev, bitbake_rev)) + bitbake_rev = check_rev + + cmd = '%s upgrade_history_internal.py %s %s' % (cmdprefix, maintplanlayerbranch.layerbranch.id, commit) + if initial: + release = Release.get_by_date(maintplanlayerbranch.plan, commitdate) + if release: + comment = 'Initial import at %s release start.' % release.name + else: + comment = 'Initial import at %s' % commit + cmd += ' --initial="%s"' % comment + if bitbake_rev: + cmd += ' --bitbake-rev %s' % bitbake_rev + if options.dry_run: + cmd += ' --dry-run' + if options.loglevel == logging.DEBUG: + cmd += ' --debug' + logger.debug('Running %s' % cmd) + ret, output = utils.run_command_interruptible(cmd) + if ret == 254: + # Interrupted by user, break out of loop + logger.info('Update interrupted, exiting') + sys.exit(254) """ Upgrade history handler. """ def upgrade_history(options, logger): - from layerindex.models import LayerBranch from rrs.models import MaintenancePlan # start date @@ -221,6 +95,7 @@ def upgrade_history(options, logger): since = "2013-11-11" #RecipeUpgrade.objects.all().delete() else: + # FIXME this is awful - we should be storing the last commit somewhere since = (now - timedelta(days=8)).strftime("%Y-%m-%d") maintplans = MaintenancePlan.objects.filter(updates_enabled=True) @@ -236,21 +111,33 @@ def upgrade_history(options, logger): layerdir = os.path.join(repodir, layerbranch.vcs_subdir) commits = utils.runcmd("git log --since='" + since + - "' --format='%H' --reverse", repodir, + "' --format='%H %ct' --reverse origin/master", repodir, logger=logger) commit_list = commits.split('\n') + bitbake_map = {} + # Filter out some bad commits + bitbake_commits = utils.runcmd("git rev-list fef18b445c0cb6b266cd939b9c78d7cbce38663f^..39780b1ccbd76579db0fc6fb9369c848a3bafa9d^", + bitbakepath, + logger=logger) + bitbake_commit_list = bitbake_commits.splitlines() + for commit in bitbake_commit_list: + bitbake_map[commit] = '39780b1ccbd76579db0fc6fb9369c848a3bafa9d' + if options.initial: logger.debug("Adding initial upgrade history ....") - ct = commit_list.pop(0) - do_initial(layerbranch, ct, logger, options.dry_run) + ct, ctepoch = commit_list.pop(0).split() + ctdate = datetime.fromtimestamp(int(ctepoch)) + run_internal(maintplanbranch, ct, ctdate, options, logger, bitbake_map, initial=True) logger.debug("Adding upgrade history from %s to %s ..." % (since, today)) - for ct in commit_list: - if ct: + for item in commit_list: + if item: + ct, ctepoch = item.split() + ctdate = datetime.fromtimestamp(int(ctepoch)) logger.debug("Analysing commit %s ..." % ct) - do_loop(layerbranch, ct, logger, options.dry_run) + run_internal(maintplanbranch, ct, ctdate, options, logger, bitbake_map) if commit_list: utils.runcmd("git clean -dfx", repodir, logger=logger) diff --git a/rrs/tools/upgrade_history_internal.py b/rrs/tools/upgrade_history_internal.py new file mode 100644 index 0000000000..fb015d86a3 --- /dev/null +++ b/rrs/tools/upgrade_history_internal.py @@ -0,0 +1,251 @@ +# Internal script called by rrs_upgrade_history.py +# +# To detect package versions of the recipes the script uses the name of the recipe. +# +# Copyright (C) 2015, 2018 Intel Corporation +# Authors: Anibal Limon <anibal.limon@linux.intel.com> +# Paul Eggleton <paul.eggleton@linux.intel.com> +# +# Licensed under the MIT license, see COPYING.MIT for details + +from datetime import datetime + +import sys +import os +import optparse +import logging +import re +from distutils.version import LooseVersion + +sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__)))) +from common import common_setup, get_pv_type, load_recipes, \ + get_logger, DryRunRollbackException + +common_setup() +from layerindex import utils, recipeparse +from layerindex.update_layer import split_recipe_fn + +""" + Store upgrade into RecipeUpgrade model. +""" +def _save_upgrade(recipe, pv, commit, title, info, logger): + from email.utils import parsedate_tz, mktime_tz + from rrs.models import Maintainer, RecipeUpgrade + + maintainer_name = info.split(';')[0] + maintainer_email = info.split(';')[1] + author_date = info.split(';')[2] + commit_date = info.split(';')[3] + + maintainer = Maintainer.create_or_update(maintainer_name, maintainer_email) + + upgrade = RecipeUpgrade() + upgrade.recipe = recipe + upgrade.maintainer = maintainer + upgrade.author_date = datetime.utcfromtimestamp(mktime_tz( + parsedate_tz(author_date))) + upgrade.commit_date = datetime.utcfromtimestamp(mktime_tz( + parsedate_tz(commit_date))) + upgrade.version = pv + upgrade.sha1 = commit + upgrade.title = title.strip() + upgrade.save() + +""" + Create upgrade receives new recipe_data and cmp versions. +""" +def _create_upgrade(recipe_data, layerbranch, ct, title, info, logger, initial=False): + from layerindex.models import Recipe + from rrs.models import RecipeUpgrade + from bb.utils import vercmp_string + + pn = recipe_data.getVar('PN', True) + pv = recipe_data.getVar('PV', True) + + try: + recipe = Recipe.objects.get(pn=pn, layerbranch=layerbranch) + except Exception as e: + logger.warn("%s: Not found in Layer branch %s." % + (pn, str(layerbranch))) + return + + try: + latest_upgrade = RecipeUpgrade.objects.filter( + recipe = recipe).order_by('-commit_date')[0] + prev_pv = latest_upgrade.version + except KeyboardInterrupt: + raise + except: + prev_pv = None + + if prev_pv is None: + logger.debug("%s: Initial upgrade ( -> %s)." % (recipe.pn, pv)) + _save_upgrade(recipe, pv, ct, title, info, logger) + else: + from common import get_recipe_pv_without_srcpv + + (ppv, _, _) = get_recipe_pv_without_srcpv(prev_pv, + get_pv_type(prev_pv)) + (npv, _, _) = get_recipe_pv_without_srcpv(pv, + get_pv_type(pv)) + + try: + if npv == 'git': + logger.debug("%s: Avoiding upgrade to unversioned git." % \ + (recipe.pn)) + elif ppv == 'git' or vercmp_string(ppv, npv) == -1: + if initial is True: + logger.debug("%s: Update initial upgrade ( -> %s)." % \ + (recipe.pn, pv)) + latest_upgrade.version = pv + latest_upgrade.save() + else: + logger.debug("%s: detected upgrade (%s -> %s)" \ + " in ct %s." % (pn, prev_pv, pv, ct)) + _save_upgrade(recipe, pv, ct, title, info, logger) + except KeyboardInterrupt: + raise + except: + logger.error("%s: fail to detect upgrade (%s -> %s)" \ + " in ct %s." % (pn, prev_pv, pv, ct)) + + +""" + Returns a list containing the fullpaths to the recipes from a commit. +""" +def _get_recipes_filenames(ct, repodir, layerdir, logger): + ct_files = [] + layerdir_start = os.path.normpath(layerdir) + os.sep + + files = utils.runcmd("git log --name-only --format='%n' -n 1 " + ct, + repodir, logger=logger) + + for f in files.split("\n"): + if f != "": + fullpath = os.path.join(repodir, f) + # Skip deleted files in commit + if not os.path.exists(fullpath): + continue + (typename, _, filename) = recipeparse.detect_file_type(fullpath, + layerdir_start) + if typename == 'recipe': + ct_files.append(fullpath) + + return ct_files + + +def generate_history(options, layerbranch_id, commit, logger): + from layerindex.models import LayerBranch + from rrs.models import Release + layerbranch = LayerBranch.objects.get(id=layerbranch_id) + + fetchdir = settings.LAYER_FETCH_DIR + if not fetchdir: + logger.error("Please set LAYER_FETCH_DIR in settings.py") + sys.exit(1) + + layer = layerbranch.layer + urldir = str(layer.get_fetch_dir()) + repodir = os.path.join(fetchdir, urldir) + layerdir = os.path.join(repodir, str(layerbranch.vcs_subdir)) + + utils.runcmd("git checkout %s" % commit, + repodir, logger=logger) + utils.runcmd("git clean -dfx", repodir, logger=logger) + + if options.initial: + fns = None + else: + fns = _get_recipes_filenames(commit, repodir, layerdir, logger) + if not fns: + return + + # setup bitbake + bitbakepath = os.path.join(fetchdir, 'bitbake') + if options.bitbake_rev: + bitbake_rev = options.bitbake_rev + if not re.match('^[0-9a-f]{40}$', bitbake_rev): + # Branch name, need to check out detached + bitbake_rev = 'origin/%s' % bitbake_rev + else: + commitdate = utils.runcmd("git show -s --format=%ci", repodir, logger=logger) + bitbake_rev = '`git rev-list -1 --before="%s" origin/master`' % commitdate + utils.runcmd('git checkout %s' % bitbake_rev, + bitbakepath, logger=logger) + utils.runcmd("git clean -dfx", bitbakepath, logger=logger) + sys.path.insert(0, os.path.join(bitbakepath, 'lib')) + + (tinfoil, d, recipes) = load_recipes(layerbranch, bitbakepath, + fetchdir, settings, logger, recipe_files=fns, + nocheckout=True) + + if options.initial: + title = options.initial + info = 'No maintainer;;' + utils.runcmd("git log --format='%ad;%cd' --date=rfc -n 1 " \ + + commit, destdir=repodir, logger=logger) + recordcommit = '' + else: + title = utils.runcmd("git log --format='%s' -n 1 " + commit, + repodir, logger=logger) + info = utils.runcmd("git log --format='%an;%ae;%ad;%cd' --date=rfc -n 1 " \ + + commit, destdir=repodir, logger=logger) + recordcommit = commit + + try: + with transaction.atomic(): + for recipe_data in recipes: + _create_upgrade(recipe_data, layerbranch, recordcommit, title, + info, logger, initial=options.initial) + if options.dry_run: + raise DryRunRollbackException + except DryRunRollbackException: + pass + + tinfoil.shutdown() + + +if __name__=="__main__": + logger = None + try: + utils.setup_django() + from django.db import transaction + import settings + + logger = get_logger("HistoryUpgrade", settings) + + parser = optparse.OptionParser(usage = """%prog [options] <layerbranchid> <commit>""") + + parser.add_option("-i", "--initial", + help = "Do initial population of upgrade histories (and specify comment)", + action="store", dest="initial", default='') + + parser.add_option("--bitbake-rev", + help = "Use the specified bitbake revision instead of the most recent one at the metadata commit date", + action="store", dest="bitbake_rev", default='') + + parser.add_option("-d", "--debug", + help = "Enable debug output", + action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO) + + parser.add_option("--dry-run", + help = "Do not write any data back to the database", + action="store_true", dest="dry_run", default=False) + + options, args = parser.parse_args(sys.argv) + + logger.setLevel(options.loglevel) + + if len(args) < 2: + logger.error('Please specify layerbranch ID') + sys.exit(1) + + if len(args) < 3: + logger.error('Please specify commit') + sys.exit(1) + + generate_history(options, int(args[1]), args[2], logger) + except KeyboardInterrupt: + if logger: + logger.info('Update interrupted, exiting') + sys.exit(254) + |