bulkchange.py: use oe.recipeutils code to patch recipes

There were several issues with this code, including that it used
SortedDict which was removed in Django 1.9 and that it seemed not to
have been fully updated to accommodate changes in bitbake's recipe
parsing API. In the end I decided the simplest thing would be to move it
over to using oe.recipeutils.patch_recipe() which is actually a now much
more mature version of the code that originally started life here. With
that we can get the bulk change functionality working again and gain
some of the improvements in behaviour that we've developed in
oe.recipeutils.patch_recipe(), as well as avoiding effectively
duplicated code.

Implements [YOCTO #9730].

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>

2 files changed, 7 insertions, 139 deletions

diff --git a/layerindex/bulkchange.py b/layerindex/bulkchange.py
index 316c7112ae..c06acceda5 100644
--- a/layerindex/bulkchange.py
+++ b/layerindex/bulkchange.py
@@ -2,31 +2,22 @@
 
 # layerindex-web - bulk change implementation
 #
-# Copyright (C) 2013 Intel Corporation
+# Copyright (C) 2013, 2016, 2018 Intel Corporation
 #
 # Licensed under the MIT license, see COPYING.MIT for details
 
 import sys
 import os
-import os.path
 import tempfile
 import tarfile
-import textwrap
-import difflib
 import recipeparse
 import utils
 import shutil
-from django.utils.datastructures import SortedDict
 
 logger = utils.logger_create('LayerIndexImport')
 
-# Help us to find places to insert values
-recipe_progression = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRC_URI', 'do_fetch', 'do_unpack', 'do_patch', 'EXTRA_OECONF', 'do_configure', 'EXTRA_OEMAKE', 'do_compile', 'do_install', 'do_populate_sysroot', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'do_package', 'do_deploy']
-# Variables that sometimes are a bit long but shouldn't be wrapped
-nowrap_vars = ['SUMMARY', 'HOMEPAGE', 'BUGTRACKER', 'LIC_FILES_CHKSUM']
-meta_vars = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION']
-
 def generate_patches(tinfoil, fetchdir, changeset, outputdir):
+    import oe.recipeutils
     tmpoutdir = tempfile.mkdtemp(dir=outputdir)
     last_layer = None
     patchname = ''
@@ -50,13 +41,9 @@ def generate_patches(tinfoil, fetchdir, changeset, outputdir):
                     outfile = open(os.path.join(tmpoutdir, patchname), 'w')
                     last_layer = layer
                 recipefile = str(os.path.join(layerfetchdir, layerbranch.vcs_subdir, change.recipe.filepath, change.recipe.filename))
-                varlist = list(set(list(fields.keys()) + meta_vars))
-                varfiles = recipeparse.get_var_files(recipefile, varlist, config_data_copy)
-                filevars = localise_file_vars(recipefile, varfiles, fields.keys())
-                for f, fvars in filevars.items():
-                    filefields = dict((k, fields[k]) for k in fvars)
-                    patch = patch_recipe(f, layerfetchdir, filefields)
-                    for line in patch:
+                patchdatalist = oe.recipeutils.patch_recipe(config_data_copy, recipefile, fields, patch=True, relpath=layerfetchdir)
+                for patchdata in patchdatalist:
+                    for line in patchdata:
                         outfile.write(line)
     finally:
         if outfile:
@@ -84,113 +71,6 @@ def generate_patches(tinfoil, fetchdir, changeset, outputdir):
     shutil.rmtree(tmpoutdir)
     return ret
 
-
-def patch_recipe(fn, relpath, values):
-    """Update or insert variable values into a recipe file.
-       Note that some manual inspection/intervention may be required
-       since this cannot handle all situations.
-    """
-    remainingnames = {}
-    for k in values.keys():
-        remainingnames[k] = recipe_progression.index(k) if k in recipe_progression else -1
-    remainingnames = SortedDict(sorted(remainingnames.items(), key=lambda x: x[1]))
-
-    with tempfile.NamedTemporaryFile('w', delete=False) as tf:
-        def outputvalue(name):
-            if values[name]:
-                rawtext = '%s = "%s"\n' % (name, values[name])
-                if name in nowrap_vars:
-                    tf.write(rawtext)
-                else:
-                    wrapped = textwrap.wrap(rawtext)
-                    for wrapline in wrapped[:-1]:
-                        tf.write('%s \\\n' % wrapline)
-                    tf.write('%s\n' % wrapped[-1])
-
-        tfn = tf.name
-        with open(fn, 'r') as f:
-            # First runthrough - find existing names (so we know not to insert based on recipe_progression)
-            # Second runthrough - make the changes
-            existingnames = []
-            for runthrough in [1, 2]:
-                currname = None
-                for line in f:
-                    if not currname:
-                        insert = False
-                        for k in remainingnames.keys():
-                            for p in recipe_progression:
-                                if line.startswith(p):
-                                    if remainingnames[k] > -1 and recipe_progression.index(p) > remainingnames[k] and runthrough > 1 and not k in existingnames:
-                                        outputvalue(k)
-                                        del remainingnames[k]
-                                    break
-                        for k in remainingnames.keys():
-                            if line.startswith(k):
-                                currname = k
-                                if runthrough == 1:
-                                    existingnames.append(k)
-                                else:
-                                    del remainingnames[k]
-                                break
-                        if currname and runthrough > 1:
-                            outputvalue(currname)
-
-                    if currname:
-                        sline = line.rstrip()
-                        if not sline.endswith('\\'):
-                            currname = None
-                        continue
-                    if runthrough > 1:
-                        tf.write(line)
-                f.seek(0)
-        if remainingnames:
-            tf.write('\n')
-            for k in remainingnames.keys():
-                outputvalue(k)
-
-    fromlines = open(fn, 'U').readlines()
-    tolines = open(tfn, 'U').readlines()
-    relfn = os.path.relpath(fn, relpath)
-    diff = difflib.unified_diff(fromlines, tolines, 'a/%s' % relfn, 'b/%s' % relfn)
-    os.remove(tfn)
-    return diff
-
-def localise_file_vars(fn, varfiles, varlist):
-    from collections import defaultdict
-
-    fndir = os.path.dirname(fn) + os.sep
-
-    first_meta_file = None
-    for v in meta_vars:
-        f = varfiles.get(v, None)
-        if f:
-            actualdir = os.path.dirname(f) + os.sep
-            if actualdir.startswith(fndir):
-                first_meta_file = f
-                break
-
-    filevars = defaultdict(list)
-    for v in varlist:
-        f = varfiles[v]
-        # Only return files that are in the same directory as the recipe or in some directory below there
-        # (this excludes bbclass files and common inc files that wouldn't be appropriate to set the variable
-        # in if we were going to set a value specific to this recipe)
-        if f:
-            actualfile = f
-        else:
-            # Variable isn't in a file, if it's one of the "meta" vars, use the first file with a meta var in it
-            if first_meta_file:
-                actualfile = first_meta_file
-            else:
-                actualfile = fn
-
-        actualdir = os.path.dirname(actualfile) + os.sep
-        if not actualdir.startswith(fndir):
-            actualfile = fn
-        filevars[actualfile].append(v)
-
-    return filevars
-
 def get_changeset(pk):
     from layerindex.models import RecipeChangeset
     res = list(RecipeChangeset.objects.filter(pk=pk)[:1])
@@ -229,6 +109,8 @@ def main():
             sys.stderr.write("Unable to find changeset with id %s\n" % sys.argv[1])
             sys.exit(1)
 
+        utils.setup_core_layer_sys_path(settings, branch.name)
+
         outp = generate_patches(tinfoil, fetchdir, changeset, sys.argv[2])
     finally:
         tinfoil.shutdown()
diff --git a/layerindex/recipeparse.py b/layerindex/recipeparse.py
index af0c736563..f211cfac4a 100644
--- a/layerindex/recipeparse.py
+++ b/layerindex/recipeparse.py
@@ -110,20 +110,6 @@ def setup_layer(config_data, fetchdir, layerdir, layer, layerbranch):
     config_data_copy.delVar('LAYERDIR')
     return config_data_copy
 
-def get_var_files(fn, varlist, d):
-    import bb.cache
-    varfiles = {}
-    envdata = bb.cache.Cache.loadDataFull(fn, [], d)
-    for v in varlist:
-        history = envdata.varhistory.get_variable_files(v)
-        if history:
-            actualfile = history[-1]
-        else:
-            actualfile = None
-        varfiles[v] = actualfile
-
-    return varfiles
-
 machine_conf_re = re.compile(r'conf/machine/([^/.]*).conf$')
 distro_conf_re = re.compile(r'conf/distro/([^/.]*).conf$')
 bbclass_re = re.compile(r'classes/([^/.]*).bbclass$')