scripts: rename yocto-compat-layer to remove "compatible" nomenclature

"Yocto Project Compatible" [1] is a programme which requires you meet
specific criteria including going through an application process - it is
not sufficient simply to run the script we have created here and have it
produce no warnings/errors. To avoid people being confused by the fact
that this script uses the term "compatible" or variations thereof,
substitute usage of that word with "check" instead. The functionality of
the script is unchanged.

[1] https://www.yoctoproject.org/ecosystem/yocto-project-branding-program

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
Signed-off-by: Ross Burton <ross.burton@intel.com>

1 files changed, 392 insertions, 0 deletions

diff --git a/scripts/lib/checklayer/__init__.py b/scripts/lib/checklayer/__init__.py
new file mode 100644
index 0000000000..6c2b86a79a
--- /dev/null
+++ b/scripts/lib/checklayer/__init__.py
@@ -0,0 +1,392 @@
+# Yocto Project layer check tool
+#
+# Copyright (C) 2017 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+import os
+import re
+import subprocess
+from enum import Enum
+
+import bb.tinfoil
+
+class LayerType(Enum):
+    BSP = 0
+    DISTRO = 1
+    SOFTWARE = 2
+    ERROR_NO_LAYER_CONF = 98
+    ERROR_BSP_DISTRO = 99
+
+def _get_configurations(path):
+    configs = []
+
+    for f in os.listdir(path):
+        file_path = os.path.join(path, f)
+        if os.path.isfile(file_path) and f.endswith('.conf'):
+            configs.append(f[:-5]) # strip .conf
+    return configs
+
+def _get_layer_collections(layer_path, lconf=None, data=None):
+    import bb.parse
+    import bb.data
+
+    if lconf is None:
+        lconf = os.path.join(layer_path, 'conf', 'layer.conf')
+
+    if data is None:
+        ldata = bb.data.init()
+        bb.parse.init_parser(ldata)
+    else:
+        ldata = data.createCopy()
+
+    ldata.setVar('LAYERDIR', layer_path)
+    try:
+        ldata = bb.parse.handle(lconf, ldata, include=True)
+    except BaseException as exc:
+        raise LayerError(exc)
+    ldata.expandVarref('LAYERDIR')
+
+    collections = (ldata.getVar('BBFILE_COLLECTIONS', True) or '').split()
+    if not collections:
+        name = os.path.basename(layer_path)
+        collections = [name]
+
+    collections = {c: {} for c in collections}
+    for name in collections:
+        priority = ldata.getVar('BBFILE_PRIORITY_%s' % name, True)
+        pattern = ldata.getVar('BBFILE_PATTERN_%s' % name, True)
+        depends = ldata.getVar('LAYERDEPENDS_%s' % name, True)
+        collections[name]['priority'] = priority
+        collections[name]['pattern'] = pattern
+        collections[name]['depends'] = depends
+
+    return collections
+
+def _detect_layer(layer_path):
+    """
+        Scans layer directory to detect what type of layer
+        is BSP, Distro or Software.
+
+        Returns a dictionary with layer name, type and path.
+    """
+
+    layer = {}
+    layer_name = os.path.basename(layer_path)
+
+    layer['name'] = layer_name
+    layer['path'] = layer_path
+    layer['conf'] = {}
+
+    if not os.path.isfile(os.path.join(layer_path, 'conf', 'layer.conf')):
+        layer['type'] = LayerType.ERROR_NO_LAYER_CONF
+        return layer
+
+    machine_conf = os.path.join(layer_path, 'conf', 'machine')
+    distro_conf = os.path.join(layer_path, 'conf', 'distro')
+
+    is_bsp = False
+    is_distro = False
+
+    if os.path.isdir(machine_conf):
+        machines = _get_configurations(machine_conf)
+        if machines:
+            is_bsp = True
+
+    if os.path.isdir(distro_conf):
+        distros = _get_configurations(distro_conf)
+        if distros:
+            is_distro = True
+
+    if is_bsp and is_distro:
+        layer['type'] = LayerType.ERROR_BSP_DISTRO
+    elif is_bsp:
+        layer['type'] = LayerType.BSP
+        layer['conf']['machines'] = machines
+    elif is_distro:
+        layer['type'] = LayerType.DISTRO
+        layer['conf']['distros'] = distros
+    else:
+        layer['type'] = LayerType.SOFTWARE
+
+    layer['collections'] = _get_layer_collections(layer['path'])
+
+    return layer
+
+def detect_layers(layer_directories, no_auto):
+    layers = []
+
+    for directory in layer_directories:
+        directory = os.path.realpath(directory)
+        if directory[-1] == '/':
+            directory = directory[0:-1]
+
+        if no_auto:
+            conf_dir = os.path.join(directory, 'conf')
+            if os.path.isdir(conf_dir):
+                layer = _detect_layer(directory)
+                if layer:
+                    layers.append(layer)
+        else:
+            for root, dirs, files in os.walk(directory):
+                dir_name = os.path.basename(root)
+                conf_dir = os.path.join(root, 'conf')
+                if os.path.isdir(conf_dir):
+                    layer = _detect_layer(root)
+                    if layer:
+                        layers.append(layer)
+
+    return layers
+
+def _find_layer_depends(depend, layers):
+    for layer in layers:
+        for collection in layer['collections']:
+            if depend == collection:
+                return layer
+    return None
+
+def add_layer_dependencies(bblayersconf, layer, layers, logger):
+    def recurse_dependencies(depends, layer, layers, logger, ret = []):
+        logger.debug('Processing dependencies %s for layer %s.' % \
+                    (depends, layer['name']))
+
+        for depend in depends.split():
+            # core (oe-core) is suppose to be provided
+            if depend == 'core':
+                continue
+
+            layer_depend = _find_layer_depends(depend, layers)
+            if not layer_depend:
+                logger.error('Layer %s depends on %s and isn\'t found.' % \
+                        (layer['name'], depend))
+                ret = None
+                continue
+
+            # We keep processing, even if ret is None, this allows us to report
+            # multiple errors at once
+            if ret is not None and layer_depend not in ret:
+                ret.append(layer_depend)
+
+            # Recursively process...
+            if 'collections' not in layer_depend:
+                continue
+
+            for collection in layer_depend['collections']:
+                collect_deps = layer_depend['collections'][collection]['depends']
+                if not collect_deps:
+                    continue
+                ret = recurse_dependencies(collect_deps, layer_depend, layers, logger, ret)
+
+        return ret
+
+    layer_depends = []
+    for collection in layer['collections']:
+        depends = layer['collections'][collection]['depends']
+        if not depends:
+            continue
+
+        layer_depends = recurse_dependencies(depends, layer, layers, logger, layer_depends)
+
+    # Note: [] (empty) is allowed, None is not!
+    if layer_depends is None:
+        return False
+    else:
+        # Don't add a layer that is already present.
+        added = set()
+        output = check_command('Getting existing layers failed.', 'bitbake-layers show-layers').decode('utf-8')
+        for layer, path, pri in re.findall(r'^(\S+) +([^\n]*?) +(\d+)$', output, re.MULTILINE):
+            added.add(path)
+
+        for layer_depend in layer_depends:
+            name = layer_depend['name']
+            path = layer_depend['path']
+            if path in added:
+                continue
+            else:
+                added.add(path)
+            logger.info('Adding layer dependency %s' % name)
+            with open(bblayersconf, 'a+') as f:
+                f.write("\nBBLAYERS += \"%s\"\n" % path)
+    return True
+
+def add_layer(bblayersconf, layer, layers, logger):
+    logger.info('Adding layer %s' % layer['name'])
+    with open(bblayersconf, 'a+') as f:
+        f.write("\nBBLAYERS += \"%s\"\n" % layer['path'])
+
+    return True
+
+def check_command(error_msg, cmd):
+    '''
+    Run a command under a shell, capture stdout and stderr in a single stream,
+    throw an error when command returns non-zero exit code. Returns the output.
+    '''
+
+    p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    output, _ = p.communicate()
+    if p.returncode:
+        msg = "%s\nCommand: %s\nOutput:\n%s" % (error_msg, cmd, output.decode('utf-8'))
+        raise RuntimeError(msg)
+    return output
+
+def get_signatures(builddir, failsafe=False, machine=None):
+    import re
+
+    # some recipes needs to be excluded like meta-world-pkgdata
+    # because a layer can add recipes to a world build so signature
+    # will be change
+    exclude_recipes = ('meta-world-pkgdata',)
+
+    sigs = {}
+    tune2tasks = {}
+
+    cmd = ''
+    if machine:
+        cmd += 'MACHINE=%s ' % machine
+    cmd += 'bitbake '
+    if failsafe:
+        cmd += '-k '
+    cmd += '-S none world'
+    sigs_file = os.path.join(builddir, 'locked-sigs.inc')
+    if os.path.exists(sigs_file):
+        os.unlink(sigs_file)
+    try:
+        check_command('Generating signatures failed. This might be due to some parse error and/or general layer incompatibilities.',
+                      cmd)
+    except RuntimeError as ex:
+        if failsafe and os.path.exists(sigs_file):
+            # Ignore the error here. Most likely some recipes active
+            # in a world build lack some dependencies. There is a
+            # separate test_machine_world_build which exposes the
+            # failure.
+            pass
+        else:
+            raise
+
+    sig_regex = re.compile("^(?P<task>.*:.*):(?P<hash>.*) .$")
+    tune_regex = re.compile("(^|\s)SIGGEN_LOCKEDSIGS_t-(?P<tune>\S*)\s*=\s*")
+    current_tune = None
+    with open(sigs_file, 'r') as f:
+        for line in f.readlines():
+            line = line.strip()
+            t = tune_regex.search(line)
+            if t:
+                current_tune = t.group('tune')
+            s = sig_regex.match(line)
+            if s:
+                exclude = False
+                for er in exclude_recipes:
+                    (recipe, task) = s.group('task').split(':')
+                    if er == recipe:
+                        exclude = True
+                        break
+                if exclude:
+                    continue
+
+                sigs[s.group('task')] = s.group('hash')
+                tune2tasks.setdefault(current_tune, []).append(s.group('task'))
+
+    if not sigs:
+        raise RuntimeError('Can\'t load signatures from %s' % sigs_file)
+
+    return (sigs, tune2tasks)
+
+def get_depgraph(targets=['world'], failsafe=False):
+    '''
+    Returns the dependency graph for the given target(s).
+    The dependency graph is taken directly from DepTreeEvent.
+    '''
+    depgraph = None
+    with bb.tinfoil.Tinfoil() as tinfoil:
+        tinfoil.prepare(config_only=False)
+        tinfoil.set_event_mask(['bb.event.NoProvider', 'bb.event.DepTreeGenerated', 'bb.command.CommandCompleted'])
+        if not tinfoil.run_command('generateDepTreeEvent', targets, 'do_build'):
+            raise RuntimeError('starting generateDepTreeEvent failed')
+        while True:
+            event = tinfoil.wait_event(timeout=1000)
+            if event:
+                if isinstance(event, bb.command.CommandFailed):
+                    raise RuntimeError('Generating dependency information failed: %s' % event.error)
+                elif isinstance(event, bb.command.CommandCompleted):
+                    break
+                elif isinstance(event, bb.event.NoProvider):
+                    if failsafe:
+                        # The event is informational, we will get information about the
+                        # remaining dependencies eventually and thus can ignore this
+                        # here like we do in get_signatures(), if desired.
+                        continue
+                    if event._reasons:
+                        raise RuntimeError('Nothing provides %s: %s' % (event._item, event._reasons))
+                    else:
+                        raise RuntimeError('Nothing provides %s.' % (event._item))
+                elif isinstance(event, bb.event.DepTreeGenerated):
+                    depgraph = event._depgraph
+
+    if depgraph is None:
+        raise RuntimeError('Could not retrieve the depgraph.')
+    return depgraph
+
+def compare_signatures(old_sigs, curr_sigs):
+    '''
+    Compares the result of two get_signatures() calls. Returns None if no
+    problems found, otherwise a string that can be used as additional
+    explanation in self.fail().
+    '''
+    # task -> (old signature, new signature)
+    sig_diff = {}
+    for task in old_sigs:
+        if task in curr_sigs and \
+           old_sigs[task] != curr_sigs[task]:
+            sig_diff[task] = (old_sigs[task], curr_sigs[task])
+
+    if not sig_diff:
+        return None
+
+    # Beware, depgraph uses task=<pn>.<taskname> whereas get_signatures()
+    # uses <pn>:<taskname>. Need to convert sometimes. The output follows
+    # the convention from get_signatures() because that seems closer to
+    # normal bitbake output.
+    def sig2graph(task):
+        pn, taskname = task.rsplit(':', 1)
+        return pn + '.' + taskname
+    def graph2sig(task):
+        pn, taskname = task.rsplit('.', 1)
+        return pn + ':' + taskname
+    depgraph = get_depgraph(failsafe=True)
+    depends = depgraph['tdepends']
+
+    # If a task A has a changed signature, but none of its
+    # dependencies, then we need to report it because it is
+    # the one which introduces a change. Any task depending on
+    # A (directly or indirectly) will also have a changed
+    # signature, but we don't need to report it. It might have
+    # its own changes, which will become apparent once the
+    # issues that we do report are fixed and the test gets run
+    # again.
+    sig_diff_filtered = []
+    for task, (old_sig, new_sig) in sig_diff.items():
+        deps_tainted = False
+        for dep in depends.get(sig2graph(task), ()):
+            if graph2sig(dep) in sig_diff:
+                deps_tainted = True
+                break
+        if not deps_tainted:
+            sig_diff_filtered.append((task, old_sig, new_sig))
+
+    msg = []
+    msg.append('%d signatures changed, initial differences (first hash before, second after):' %
+               len(sig_diff))
+    for diff in sorted(sig_diff_filtered):
+        recipe, taskname = diff[0].rsplit(':', 1)
+        cmd = 'bitbake-diffsigs --task %s %s --signature %s %s' % \
+              (recipe, taskname, diff[1], diff[2])
+        msg.append('   %s: %s -> %s' % diff)
+        msg.append('      %s' % cmd)
+        try:
+            output = check_command('Determining signature difference failed.',
+                                   cmd).decode('utf-8')
+        except RuntimeError as error:
+            output = str(error)
+        if output:
+            msg.extend(['      ' + line for line in output.splitlines()])
+            msg.append('')
+    return '\n'.join(msg)