diff options
Diffstat (limited to 'scripts/lib/checklayer')
| -rw-r--r-- | scripts/lib/checklayer/__init__.py | 134 | ||||
| -rw-r--r-- | scripts/lib/checklayer/case.py | 4 | ||||
| -rw-r--r-- | scripts/lib/checklayer/cases/bsp.py | 8 | ||||
| -rw-r--r-- | scripts/lib/checklayer/cases/common.py | 57 | ||||
| -rw-r--r-- | scripts/lib/checklayer/cases/distro.py | 6 | ||||
| -rw-r--r-- | scripts/lib/checklayer/context.py | 4 |
6 files changed, 167 insertions, 46 deletions
diff --git a/scripts/lib/checklayer/__init__.py b/scripts/lib/checklayer/__init__.py index 63952616ba..62ecdfe390 100644 --- a/scripts/lib/checklayer/__init__.py +++ b/scripts/lib/checklayer/__init__.py @@ -1,7 +1,9 @@ # Yocto Project layer check tool # # Copyright (C) 2017 Intel Corporation -# Released under the MIT license (see COPYING.MIT) +# +# SPDX-License-Identifier: MIT +# import os import re @@ -14,6 +16,7 @@ class LayerType(Enum): BSP = 0 DISTRO = 1 SOFTWARE = 2 + CORE = 3 ERROR_NO_LAYER_CONF = 98 ERROR_BSP_DISTRO = 99 @@ -41,9 +44,9 @@ def _get_layer_collections(layer_path, lconf=None, data=None): ldata.setVar('LAYERDIR', layer_path) try: - ldata = bb.parse.handle(lconf, ldata, include=True) - except BaseException as exc: - raise LayerError(exc) + ldata = bb.parse.handle(lconf, ldata, include=True, baseconfig=True) + except: + raise RuntimeError("Parsing of layer.conf from layer: %s failed" % layer_path) ldata.expandVarref('LAYERDIR') collections = (ldata.getVar('BBFILE_COLLECTIONS') or '').split() @@ -56,9 +59,16 @@ def _get_layer_collections(layer_path, lconf=None, data=None): priority = ldata.getVar('BBFILE_PRIORITY_%s' % name) pattern = ldata.getVar('BBFILE_PATTERN_%s' % name) depends = ldata.getVar('LAYERDEPENDS_%s' % name) + compat = ldata.getVar('LAYERSERIES_COMPAT_%s' % name) + try: + depDict = bb.utils.explode_dep_versions2(depends or "") + except bb.utils.VersionStringException as vse: + bb.fatal('Error parsing LAYERDEPENDS_%s: %s' % (name, str(vse))) + collections[name]['priority'] = priority collections[name]['pattern'] = pattern - collections[name]['depends'] = depends + collections[name]['depends'] = ' '.join(depDict.keys()) + collections[name]['compat'] = compat return collections @@ -97,7 +107,13 @@ def _detect_layer(layer_path): if distros: is_distro = True - if is_bsp and is_distro: + layer['collections'] = _get_layer_collections(layer['path']) + + if layer_name == "meta" and "core" in layer['collections']: + layer['type'] = LayerType.CORE + layer['conf']['machines'] = machines + layer['conf']['distros'] = distros + elif is_bsp and is_distro: layer['type'] = LayerType.ERROR_BSP_DISTRO elif is_bsp: layer['type'] = LayerType.BSP @@ -108,8 +124,6 @@ def _detect_layer(layer_path): else: layer['type'] = LayerType.SOFTWARE - layer['collections'] = _get_layer_collections(layer['path']) - return layer def detect_layers(layer_directories, no_auto): @@ -137,14 +151,38 @@ def detect_layers(layer_directories, no_auto): return layers -def _find_layer_depends(depend, layers): +def _find_layer(depend, layers): for layer in layers: + if 'collections' not in layer: + continue + for collection in layer['collections']: if depend == collection: return layer return None -def add_layer_dependencies(bblayersconf, layer, layers, logger): +def sanity_check_layers(layers, logger): + """ + Check that we didn't find duplicate collection names, as the layer that will + be used is non-deterministic. The precise check is duplicate collections + with different patterns, as the same pattern being repeated won't cause + problems. + """ + import collections + + passed = True + seen = collections.defaultdict(set) + for layer in layers: + for name, data in layer.get("collections", {}).items(): + seen[name].add(data["pattern"]) + + for name, patterns in seen.items(): + if len(patterns) > 1: + passed = False + logger.error("Collection %s found multiple times: %s" % (name, ", ".join(patterns))) + return passed + +def get_layer_dependencies(layer, layers, logger): def recurse_dependencies(depends, layer, layers, logger, ret = []): logger.debug('Processing dependencies %s for layer %s.' % \ (depends, layer['name'])) @@ -154,7 +192,7 @@ def add_layer_dependencies(bblayersconf, layer, layers, logger): if depend == 'core': continue - layer_depend = _find_layer_depends(depend, layers) + layer_depend = _find_layer(depend, layers) if not layer_depend: logger.error('Layer %s depends on %s and isn\'t found.' % \ (layer['name'], depend)) @@ -165,6 +203,10 @@ def add_layer_dependencies(bblayersconf, layer, layers, logger): # multiple errors at once if ret is not None and layer_depend not in ret: ret.append(layer_depend) + else: + # we might have processed this dependency already, in which case + # we should not do it again (avoid recursive loop) + continue # Recursively process... if 'collections' not in layer_depend: @@ -187,48 +229,65 @@ def add_layer_dependencies(bblayersconf, layer, layers, logger): layer_depends = recurse_dependencies(depends, layer, layers, logger, layer_depends) # Note: [] (empty) is allowed, None is not! + return layer_depends + +def add_layer_dependencies(bblayersconf, layer, layers, logger): + + layer_depends = get_layer_dependencies(layer, layers, logger) if layer_depends is None: return False else: - # Don't add a layer that is already present. - added = set() - output = check_command('Getting existing layers failed.', 'bitbake-layers show-layers').decode('utf-8') - for layer, path, pri in re.findall(r'^(\S+) +([^\n]*?) +(\d+)$', output, re.MULTILINE): - added.add(path) - - for layer_depend in layer_depends: - name = layer_depend['name'] - path = layer_depend['path'] + add_layers(bblayersconf, layer_depends, logger) + + return True + +def add_layers(bblayersconf, layers, logger): + # Don't add a layer that is already present. + added = set() + output = check_command('Getting existing layers failed.', 'bitbake-layers show-layers').decode('utf-8') + for layer, path, pri in re.findall(r'^(\S+) +([^\n]*?) +(\d+)$', output, re.MULTILINE): + added.add(path) + + with open(bblayersconf, 'a+') as f: + for layer in layers: + logger.info('Adding layer %s' % layer['name']) + name = layer['name'] + path = layer['path'] if path in added: - continue + logger.info('%s is already in %s' % (name, bblayersconf)) else: added.add(path) - logger.info('Adding layer dependency %s' % name) - with open(bblayersconf, 'a+') as f: f.write("\nBBLAYERS += \"%s\"\n" % path) return True -def add_layer(bblayersconf, layer, layers, logger): - logger.info('Adding layer %s' % layer['name']) - with open(bblayersconf, 'a+') as f: - f.write("\nBBLAYERS += \"%s\"\n" % layer['path']) +def check_bblayers(bblayersconf, layer_path, logger): + ''' + If layer_path found in BBLAYERS return True + ''' + import bb.parse + import bb.data - return True + ldata = bb.parse.handle(bblayersconf, bb.data.init(), include=True) + for bblayer in (ldata.getVar('BBLAYERS') or '').split(): + if os.path.normpath(bblayer) == os.path.normpath(layer_path): + return True + + return False -def check_command(error_msg, cmd): +def check_command(error_msg, cmd, cwd=None): ''' Run a command under a shell, capture stdout and stderr in a single stream, throw an error when command returns non-zero exit code. Returns the output. ''' - p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=cwd) output, _ = p.communicate() if p.returncode: msg = "%s\nCommand: %s\nOutput:\n%s" % (error_msg, cmd, output.decode('utf-8')) raise RuntimeError(msg) return output -def get_signatures(builddir, failsafe=False, machine=None): +def get_signatures(builddir, failsafe=False, machine=None, extravars=None): import re # some recipes needs to be excluded like meta-world-pkgdata @@ -239,19 +298,22 @@ def get_signatures(builddir, failsafe=False, machine=None): sigs = {} tune2tasks = {} - cmd = '' + cmd = 'BB_ENV_PASSTHROUGH_ADDITIONS="$BB_ENV_PASSTHROUGH_ADDITIONS BB_SIGNATURE_HANDLER" BB_SIGNATURE_HANDLER="OEBasicHash" ' + if extravars: + cmd += extravars + cmd += ' ' if machine: cmd += 'MACHINE=%s ' % machine cmd += 'bitbake ' if failsafe: cmd += '-k ' - cmd += '-S none world' + cmd += '-S lockedsigs world' sigs_file = os.path.join(builddir, 'locked-sigs.inc') if os.path.exists(sigs_file): os.unlink(sigs_file) try: check_command('Generating signatures failed. This might be due to some parse error and/or general layer incompatibilities.', - cmd) + cmd, builddir) except RuntimeError as ex: if failsafe and os.path.exists(sigs_file): # Ignore the error here. Most likely some recipes active @@ -262,8 +324,8 @@ def get_signatures(builddir, failsafe=False, machine=None): else: raise - sig_regex = re.compile("^(?P<task>.*:.*):(?P<hash>.*) .$") - tune_regex = re.compile("(^|\s)SIGGEN_LOCKEDSIGS_t-(?P<tune>\S*)\s*=\s*") + sig_regex = re.compile(r"^(?P<task>.*:.*):(?P<hash>.*) .$") + tune_regex = re.compile(r"(^|\s)SIGGEN_LOCKEDSIGS_t-(?P<tune>\S*)\s*=\s*") current_tune = None with open(sigs_file, 'r') as f: for line in f.readlines(): diff --git a/scripts/lib/checklayer/case.py b/scripts/lib/checklayer/case.py index 9dd00412e5..fa9dee384e 100644 --- a/scripts/lib/checklayer/case.py +++ b/scripts/lib/checklayer/case.py @@ -1,5 +1,7 @@ # Copyright (C) 2017 Intel Corporation -# Released under the MIT license (see COPYING.MIT) +# +# SPDX-License-Identifier: MIT +# from oeqa.core.case import OETestCase diff --git a/scripts/lib/checklayer/cases/bsp.py b/scripts/lib/checklayer/cases/bsp.py index b6b611be73..b76163fb56 100644 --- a/scripts/lib/checklayer/cases/bsp.py +++ b/scripts/lib/checklayer/cases/bsp.py @@ -1,5 +1,7 @@ # Copyright (C) 2017 Intel Corporation -# Released under the MIT license (see COPYING.MIT) +# +# SPDX-License-Identifier: MIT +# import unittest @@ -9,7 +11,7 @@ from checklayer.case import OECheckLayerTestCase class BSPCheckLayer(OECheckLayerTestCase): @classmethod def setUpClass(self): - if self.tc.layer['type'] != LayerType.BSP: + if self.tc.layer['type'] not in (LayerType.BSP, LayerType.CORE): raise unittest.SkipTest("BSPCheckLayer: Layer %s isn't BSP one." %\ self.tc.layer['name']) @@ -151,7 +153,7 @@ class BSPCheckLayer(OECheckLayerTestCase): # do_build can be ignored: it is know to have # different signatures in some cases, for example in # the allarch ca-certificates due to RDEPENDS=openssl. - # That particular dependency is whitelisted via + # That particular dependency is marked via # SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS, but still shows up # in the sstate signature hash because filtering it # out would be hard and running do_build multiple diff --git a/scripts/lib/checklayer/cases/common.py b/scripts/lib/checklayer/cases/common.py index a13c1088f0..97b16f78c8 100644 --- a/scripts/lib/checklayer/cases/common.py +++ b/scripts/lib/checklayer/cases/common.py @@ -1,18 +1,24 @@ # Copyright (C) 2017 Intel Corporation -# Released under the MIT license (see COPYING.MIT) +# +# SPDX-License-Identifier: MIT +# import glob import os import unittest +import re from checklayer import get_signatures, LayerType, check_command, get_depgraph, compare_signatures from checklayer.case import OECheckLayerTestCase class CommonCheckLayer(OECheckLayerTestCase): def test_readme(self): + if self.tc.layer['type'] == LayerType.CORE: + raise unittest.SkipTest("Core layer's README is top level") + # The top-level README file may have a suffix (like README.rst or README.txt). - readme_files = glob.glob(os.path.join(self.tc.layer['path'], 'README*')) + readme_files = glob.glob(os.path.join(self.tc.layer['path'], '[Rr][Ee][Aa][Dd][Mm][Ee]*')) self.assertTrue(len(readme_files) > 0, - msg="Layer doesn't contains README file.") + msg="Layer doesn't contain a README file.") # There might be more than one file matching the file pattern above # (for example, README.rst and README-COPYING.rst). The one with the shortest @@ -24,6 +30,16 @@ class CommonCheckLayer(OECheckLayerTestCase): self.assertTrue(data, msg="Layer contains a README file but it is empty.") + # If a layer's README references another README, then the checks below are not valid + if re.search('README', data, re.IGNORECASE): + return + + self.assertIn('maintainer', data.lower()) + self.assertIn('patch', data.lower()) + # Check that there is an email address in the README + email_regex = re.compile(r"[^@]+@[^@]+") + self.assertTrue(email_regex.match(data)) + def test_parse(self): check_command('Layer %s failed to parse.' % self.tc.layer['name'], 'bitbake -p') @@ -41,6 +57,36 @@ class CommonCheckLayer(OECheckLayerTestCase): ''' get_signatures(self.td['builddir'], failsafe=False) + def test_world_inherit_class(self): + ''' + This also does "bitbake -S none world" along with inheriting "yocto-check-layer" + class, which can do additional per-recipe test cases. + ''' + msg = [] + try: + get_signatures(self.td['builddir'], failsafe=False, machine=None, extravars='BB_ENV_PASSTHROUGH_ADDITIONS="$BB_ENV_PASSTHROUGH_ADDITIONS INHERIT" INHERIT="yocto-check-layer"') + except RuntimeError as ex: + msg.append(str(ex)) + if msg: + msg.insert(0, 'Layer %s failed additional checks from yocto-check-layer.bbclass\nSee below log for specific recipe parsing errors:\n' % \ + self.tc.layer['name']) + self.fail('\n'.join(msg)) + + @unittest.expectedFailure + def test_patches_upstream_status(self): + import sys + sys.path.append(os.path.join(sys.path[0], '../../../../meta/lib/')) + import oe.qa + patches = [] + for dirpath, dirs, files in os.walk(self.tc.layer['path']): + for filename in files: + if filename.endswith(".patch"): + ppath = os.path.join(dirpath, filename) + if oe.qa.check_upstream_status(ppath): + patches.append(ppath) + self.assertEqual(len(patches), 0 , \ + msg="Found following patches with malformed or missing upstream status:\n%s" % '\n'.join([str(patch) for patch in patches])) + def test_signatures(self): if self.tc.layer['type'] == LayerType.SOFTWARE and \ not self.tc.test_software_layer_signatures: @@ -51,3 +97,8 @@ class CommonCheckLayer(OECheckLayerTestCase): msg = compare_signatures(self.td['sigs'], curr_sigs) if msg is not None: self.fail('Adding layer %s changed signatures.\n%s' % (self.tc.layer['name'], msg)) + + def test_layerseries_compat(self): + for collection_name, collection_data in self.tc.layer['collections'].items(): + self.assertTrue(collection_data['compat'], "Collection %s from layer %s does not set compatible oe-core versions via LAYERSERIES_COMPAT_collection." \ + % (collection_name, self.tc.layer['name'])) diff --git a/scripts/lib/checklayer/cases/distro.py b/scripts/lib/checklayer/cases/distro.py index df1b3035eb..a35332451c 100644 --- a/scripts/lib/checklayer/cases/distro.py +++ b/scripts/lib/checklayer/cases/distro.py @@ -1,5 +1,7 @@ # Copyright (C) 2017 Intel Corporation -# Released under the MIT license (see COPYING.MIT) +# +# SPDX-License-Identifier: MIT +# import unittest @@ -9,7 +11,7 @@ from checklayer.case import OECheckLayerTestCase class DistroCheckLayer(OECheckLayerTestCase): @classmethod def setUpClass(self): - if self.tc.layer['type'] != LayerType.DISTRO: + if self.tc.layer['type'] not in (LayerType.DISTRO, LayerType.CORE): raise unittest.SkipTest("DistroCheckLayer: Layer %s isn't Distro one." %\ self.tc.layer['name']) diff --git a/scripts/lib/checklayer/context.py b/scripts/lib/checklayer/context.py index 1bec2c4103..4de8f668fd 100644 --- a/scripts/lib/checklayer/context.py +++ b/scripts/lib/checklayer/context.py @@ -1,5 +1,7 @@ # Copyright (C) 2017 Intel Corporation -# Released under the MIT license (see COPYING.MIT) +# +# SPDX-License-Identifier: MIT +# import os import sys |