From 2da4f5d99beb5d106f9a7f43f75d2486f2928417 Mon Sep 17 00:00:00 2001 From: Paul Eggleton Date: Mon, 18 Dec 2017 22:44:24 +1300 Subject: Implement patch tracking Collect information about patches applied by a recipe, and record each patch along with the upstream status, presenting them in the recipe detail. Implements [YOCTO #7909]. Signed-off-by: Paul Eggleton --- layerindex/admin.py | 1 + layerindex/migrations/0013_patch.py | 28 +++++++++++ layerindex/models.py | 29 ++++++++++++ layerindex/update_layer.py | 85 +++++++++++++++++++++++++++++++--- templates/layerindex/recipedetail.html | 22 +++++++++ 5 files changed, 158 insertions(+), 7 deletions(-) create mode 100644 layerindex/migrations/0013_patch.py diff --git a/layerindex/admin.py b/layerindex/admin.py index e923d0c1fe..3cb5969122 100644 --- a/layerindex/admin.py +++ b/layerindex/admin.py @@ -194,6 +194,7 @@ admin.site.register(Machine, MachineAdmin) admin.site.register(Distro, DistroAdmin) admin.site.register(BBAppend, BBAppendAdmin) admin.site.register(BBClass, BBClassAdmin) +admin.site.register(Patch) admin.site.register(RecipeChangeset, RecipeChangesetAdmin) admin.site.register(ClassicRecipe, ClassicRecipeAdmin) admin.site.register(PythonEnvironment) diff --git a/layerindex/migrations/0013_patch.py b/layerindex/migrations/0013_patch.py new file mode 100644 index 0000000000..9e7180e943 --- /dev/null +++ b/layerindex/migrations/0013_patch.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('layerindex', '0012_layeritem_vcs_commit_url'), + ] + + operations = [ + migrations.CreateModel( + name='Patch', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, verbose_name='ID', serialize=False)), + ('path', models.CharField(max_length=255)), + ('src_path', models.CharField(max_length=255)), + ('status', models.CharField(default='U', choices=[('U', 'Unknown'), ('A', 'Accepted'), ('P', 'Pending'), ('I', 'Inappropriate'), ('B', 'Backport'), ('S', 'Submitted'), ('D', 'Denied')], max_length=1)), + ('status_extra', models.CharField(blank=True, max_length=255)), + ('recipe', models.ForeignKey(to='layerindex.Recipe')), + ], + options={ + 'verbose_name_plural': 'Patches', + }, + ), + ] diff --git a/layerindex/models.py b/layerindex/models.py index af6db374c6..e9751f3b6a 100644 --- a/layerindex/models.py +++ b/layerindex/models.py @@ -407,6 +407,7 @@ class Recipe(models.Model): def __str__(self): return os.path.join(self.filepath, self.filename) + class Source(models.Model): recipe = models.ForeignKey(Recipe) url = models.CharField(max_length=255) @@ -414,6 +415,34 @@ class Source(models.Model): def __str__(self): return '%s - %s' % (self.recipe.pn, self.url) + +class Patch(models.Model): + PATCH_STATUS_CHOICES = [ + ('U', 'Unknown'), + ('A', 'Accepted'), + ('P', 'Pending'), + ('I', 'Inappropriate'), + ('B', 'Backport'), + ('S', 'Submitted'), + ('D', 'Denied'), + ] + recipe = models.ForeignKey(Recipe) + path = models.CharField(max_length=255) + src_path = models.CharField(max_length=255) + status = models.CharField(max_length=1, choices=PATCH_STATUS_CHOICES, default='U') + status_extra = models.CharField(max_length=255, blank=True) + + class Meta: + verbose_name_plural = 'Patches' + + def vcs_web_url(self): + url = self.recipe.layerbranch.file_url(self.path) + return url or '' + + def __str__(self): + return "%s - %s" % (self.recipe, self.src_path) + + class PackageConfig(models.Model): recipe = models.ForeignKey(Recipe) feature = models.CharField(max_length=255) diff --git a/layerindex/update_layer.py b/layerindex/update_layer.py index ecb7cbf9d7..bbfaba9ac3 100644 --- a/layerindex/update_layer.py +++ b/layerindex/update_layer.py @@ -55,11 +55,69 @@ def split_recipe_fn(path): pv = "1.0" return (pn, pv) -def update_recipe_file(tinfoil, data, path, recipe, layerdir_start, repodir): +patch_status_re = re.compile(r"^[\t ]*(Upstream[-_ ]Status:?)[\t ]*(\w+)([\t ]+.*)?", re.IGNORECASE | re.MULTILINE) + +def collect_patch(recipe, patchfn, layerdir_start): + from django.db import DatabaseError + from layerindex.models import Patch + + patchrec = Patch() + patchrec.recipe = recipe + patchrec.path = os.path.relpath(patchfn, layerdir_start) + patchrec.src_path = os.path.relpath(patchrec.path, recipe.filepath) + try: + for encoding in ['utf-8', 'latin-1']: + try: + with open(patchfn, 'r', encoding=encoding) as f: + for line in f: + line = line.rstrip() + if line.startswith('Index: ') or line.startswith('diff -') or line.startswith('+++ '): + break + res = patch_status_re.match(line) + if res: + status = res.group(2).lower() + for key, value in dict(Patch.PATCH_STATUS_CHOICES).items(): + if status == value.lower(): + patchrec.status = key + if res.group(3): + patchrec.status_extra = res.group(3).strip() + break + else: + logger.warn('Invalid upstream status in %s: %s' % (patchfn, line)) + except UnicodeDecodeError: + continue + break + else: + logger.error('Unable to find suitable encoding to read patch %s' % patchfn) + patchrec.save() + except DatabaseError: + raise + except Exception as e: + logger.error("Unable to read patch %s: %s", patchfn, str(e)) + patchrec.save() + +def collect_patches(recipe, envdata, layerdir_start): + from layerindex.models import Patch + + try: + import oe.recipeutils + except ImportError: + logger.warn('Failed to find lib/oe/recipeutils.py in layers - patches will not be imported') + return + + Patch.objects.filter(recipe=recipe).delete() + patches = oe.recipeutils.get_recipe_patches(envdata) + for patch in patches: + if not patch.startswith(layerdir_start): + # Likely a remote patch, skip it + continue + collect_patch(recipe, patch, layerdir_start) + +def update_recipe_file(tinfoil, data, path, recipe, layerdir_start, repodir, skip_patches=False): from django.db import DatabaseError fn = str(os.path.join(path, recipe.filename)) - from layerindex.models import PackageConfig, StaticBuildDep, DynamicBuildDep, Source + from layerindex.models import PackageConfig, StaticBuildDep, DynamicBuildDep, Source, Patch try: logger.debug('Updating recipe %s' % fn) if hasattr(tinfoil, 'parse_recipe_file'): @@ -137,6 +195,10 @@ def update_recipe_file(tinfoil, data, path, recipe, layerdir_start, repodir): dynamic_build_dependency.package_configs.add(package_config) dynamic_build_dependency.recipes.add(recipe) + if not skip_patches: + # Handle patches + collect_patches(recipe, envdata, layerdir_start) + # Get file dependencies within this layer deps = envdata.getVar('__depends', True) filedeps = [] @@ -364,6 +426,15 @@ def main(): # why won't they just fix that?!) tinfoil.config_data.setVar('LICENSE', '') + # Set up for recording patch info + utils.setup_core_layer_sys_path(settings, branch.name) + skip_patches = False + try: + import oe.recipeutils + except ImportError: + logger.warn('Failed to find lib/oe/recipeutils.py in layers - patch information will not be collected') + skip_patches = True + layerconfparser = layerconfparse.LayerConfParse(logger=logger, tinfoil=tinfoil) layer_config_data = layerconfparser.parse_layer(layerdir) if not layer_config_data: @@ -449,7 +520,7 @@ def main(): recipe.filepath = newfilepath recipe.filename = newfilename recipe.save() - update_recipe_file(tinfoil, config_data_copy, os.path.join(layerdir, newfilepath), recipe, layerdir_start, repodir) + update_recipe_file(tinfoil, config_data_copy, os.path.join(layerdir, newfilepath), recipe, layerdir_start, repodir, skip_patches) updatedrecipes.add(os.path.join(oldfilepath, oldfilename)) updatedrecipes.add(os.path.join(newfilepath, newfilename)) else: @@ -581,7 +652,7 @@ def main(): results = layerrecipes.filter(filepath=filepath).filter(filename=filename)[:1] if results: recipe = results[0] - update_recipe_file(tinfoil, config_data_copy, os.path.join(layerdir, filepath), recipe, layerdir_start, repodir) + update_recipe_file(tinfoil, config_data_copy, os.path.join(layerdir, filepath), recipe, layerdir_start, repodir, skip_patches) recipe.save() updatedrecipes.add(recipe.full_path()) elif typename == 'machine': @@ -603,7 +674,7 @@ def main(): for recipe in dirtyrecipes: if not recipe.full_path() in updatedrecipes: - update_recipe_file(tinfoil, config_data_copy, os.path.join(layerdir, recipe.filepath), recipe, layerdir_start, repodir) + update_recipe_file(tinfoil, config_data_copy, os.path.join(layerdir, recipe.filepath), recipe, layerdir_start, repodir, skip_patches) else: # Collect recipe data from scratch @@ -629,7 +700,7 @@ def main(): # Recipe still exists, update it results = layerrecipes.filter(id=v['id'])[:1] recipe = results[0] - update_recipe_file(tinfoil, config_data_copy, root, recipe, layerdir_start, repodir) + update_recipe_file(tinfoil, config_data_copy, root, recipe, layerdir_start, repodir, skip_patches) else: # Recipe no longer exists, mark it for later on layerrecipes_delete.append(v) @@ -698,7 +769,7 @@ def main(): recipe.filename = os.path.basename(added) root = os.path.dirname(added) recipe.filepath = os.path.relpath(root, layerdir) - update_recipe_file(tinfoil, config_data_copy, root, recipe, layerdir_start, repodir) + update_recipe_file(tinfoil, config_data_copy, root, recipe, layerdir_start, repodir, skip_patches) recipe.save() for deleted in layerrecipes_delete: diff --git a/templates/layerindex/recipedetail.html b/templates/layerindex/recipedetail.html index 4164f11513..ef1eebaa41 100644 --- a/templates/layerindex/recipedetail.html +++ b/templates/layerindex/recipedetail.html @@ -148,6 +148,28 @@ +

Patches

+ {% if recipe.patch_set.exists %} + + + + + + + + + {% for patch in recipe.patch_set.all %} + + + + + {% endfor %} + +
PatchStatus
{{ patch.src_path }}{{ patch.get_status_display }} {{ patch.status_extra | urlize }}
+ {% else %} +

None

+ {% endif %} + {% if appends %}

bbappends

This recipe is appended by:

-- cgit 1.2.3-korg