#!/usr/bin/env python3 # Yocto Project layer checking tool # # Copyright (C) 2017 Intel Corporation # # SPDX-License-Identifier: MIT # import os import sys import argparse import logging import time import signal import shutil import collections scripts_path = os.path.dirname(os.path.realpath(__file__)) lib_path = scripts_path + '/lib' sys.path = sys.path + [lib_path] import scriptutils import scriptpath scriptpath.add_oe_lib_path() scriptpath.add_bitbake_lib_path() from checklayer import LayerType, detect_layers, add_layers, add_layer_dependencies, get_layer_dependencies, get_signatures, check_bblayers, sanity_check_layers from oeqa.utils.commands import get_bb_vars PROGNAME = 'yocto-check-layer' CASES_PATHS = [os.path.join(os.path.abspath(os.path.dirname(__file__)), 'lib', 'checklayer', 'cases')] logger = scriptutils.logger_create(PROGNAME, stream=sys.stdout) def test_layer(td, layer, test_software_layer_signatures): from checklayer.context import CheckLayerTestContext logger.info("Starting to analyze: %s" % layer['name']) logger.info("----------------------------------------------------------------------") tc = CheckLayerTestContext(td=td, logger=logger, layer=layer, test_software_layer_signatures=test_software_layer_signatures) tc.loadTests(CASES_PATHS) return tc.runTests() def dump_layer_debug(layer): logger.debug("Found layer %s (%s)" % (layer["name"], layer["path"])) collections = layer.get("collections", {}) if collections: logger.debug("%s collections: %s" % (layer["name"], ", ".join(collections))) def main(): parser = argparse.ArgumentParser( description="Yocto Project layer checking tool", add_help=False) parser.add_argument('layers', metavar='LAYER_DIR', nargs='+', help='Layer to check') parser.add_argument('-o', '--output-log', help='File to output log (optional)', action='store') parser.add_argument('--dependency', nargs="+", help='Layers to process for dependencies', action='store') parser.add_argument('--no-auto-dependency', help='Disable automatic testing of dependencies', action='store_true') parser.add_argument('--machines', nargs="+", help='List of MACHINEs to be used during testing', action='store') parser.add_argument('--additional-layers', nargs="+", help='List of additional layers to add during testing', action='store') group = parser.add_mutually_exclusive_group() group.add_argument('--with-software-layer-signature-check', action='store_true', dest='test_software_layer_signatures', default=True, help='check that software layers do not change signatures (on by default)') group.add_argument('--without-software-layer-signature-check', action='store_false', dest='test_software_layer_signatures', help='disable signature checking for software layers') parser.add_argument('-n', '--no-auto', help='Disable auto layer discovery', action='store_true') parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true') parser.add_argument('-q', '--quiet', help='Print only errors', action='store_true') parser.add_argument('-h', '--help', action='help', default=argparse.SUPPRESS, help='show this help message and exit') args = parser.parse_args() if args.output_log: fh = logging.FileHandler(args.output_log) fh.setFormatter(logging.Formatter("%(levelname)s: %(message)s")) logger.addHandler(fh) if args.debug: logger.setLevel(logging.DEBUG) elif args.quiet: logger.setLevel(logging.ERROR) if not 'BUILDDIR' in os.environ: logger.error("You must source the environment before running this script.") logger.error("$ source oe-init-build-env") return 1 builddir = os.environ['BUILDDIR'] bblayersconf = os.path.join(builddir, 'conf', 'bblayers.conf') layers = detect_layers(args.layers, args.no_auto) if not layers: logger.error("Failed to detect layers") return 1 if args.additional_layers: additional_layers = detect_layers(args.additional_layers, args.no_auto) else: additional_layers = [] if args.dependency: dep_layers = detect_layers(args.dependency, args.no_auto) dep_layers = dep_layers + layers else: dep_layers = layers logger.debug("Found additional layers:") for l in additional_layers: dump_layer_debug(l) logger.debug("Found dependency layers:") for l in dep_layers: dump_layer_debug(l) if not sanity_check_layers(additional_layers + dep_layers, logger): logger.error("Failed layer validation") return 1 logger.info("Detected layers:") for layer in layers: if layer['type'] == LayerType.ERROR_BSP_DISTRO: logger.error("%s: Can't be DISTRO and BSP type at the same time."\ " Both conf/distro and conf/machine folders were found."\ % layer['name']) layers.remove(layer) elif layer['type'] == LayerType.ERROR_NO_LAYER_CONF: logger.info("%s: Doesn't have conf/layer.conf file, so ignoring"\ % layer['name']) layers.remove(layer) else: logger.info("%s: %s, %s" % (layer['name'], layer['type'], layer['path'])) if not layers: return 1 # Find all dependencies, and get them checked too if not args.no_auto_dependency: depends = [] for layer in layers: layer_depends = get_layer_dependencies(layer, dep_layers, logger) if layer_depends: for d in layer_depends: if d not in depends: depends.append(d) for d in depends: if d not in layers: logger.info("Adding %s to the list of layers to test, as a dependency", d['name']) layers.append(d) shutil.copyfile(bblayersconf, bblayersconf + '.backup') def cleanup_bblayers(signum, frame): shutil.copyfile(bblayersconf + '.backup', bblayersconf) os.unlink(bblayersconf + '.backup') signal.signal(signal.SIGTERM, cleanup_bblayers) signal.signal(signal.SIGINT, cleanup_bblayers) td = {} results = collections.OrderedDict() results_status = collections.OrderedDict() layers_tested = 0 for layer in layers: if layer['type'] == LayerType.ERROR_NO_LAYER_CONF or \ layer['type'] == LayerType.ERROR_BSP_DISTRO: continue # Reset to a clean backup copy for each run shutil.copyfile(bblayersconf + '.backup', bblayersconf) if check_bblayers(bblayersconf, layer['path'], logger): logger.info("%s already in %s. To capture initial signatures, layer under test should not present " "in BBLAYERS. Please remove %s from BBLAYERS." % (layer['name'], bblayersconf, layer['name'])) results[layer['name']] = None results_status[layer['name']] = 'SKIPPED (Layer under test should not present in BBLAYERS)' continue logger.info('') logger.info("Setting up for %s(%s), %s" % (layer['name'], layer['type'], layer['path'])) missing_dependencies = not add_layer_dependencies(bblayersconf, layer, dep_layers, logger) if not missing_dependencies: for additional_layer in additional_layers: if not add_layer_dependencies(bblayersconf, additional_layer, dep_layers, logger): missing_dependencies = True break if missing_dependencies: logger.info('Skipping %s due to missing dependencies.' % layer['name']) results[layer['name']] = None results_status[layer['name']] = 'SKIPPED (Missing dependencies)' layers_tested = layers_tested + 1 continue if any(map(lambda additional_layer: not add_layers(bblayersconf, [additional_layer], logger), additional_layers)): logger.info('Skipping %s due to missing additional layers.' % layer['name']) results[layer['name']] = None results_status[layer['name']] = 'SKIPPED (Missing additional layers)' layers_tested = layers_tested + 1 continue logger.info('Getting initial bitbake variables ...') td['bbvars'] = get_bb_vars() logger.info('Getting initial signatures ...') td['builddir'] = builddir try: td['sigs'], td['tunetasks'] = get_signatures(td['builddir']) except RuntimeError as e: logger.info(str(e)) results[layer['name']] = None results_status[layer['name']] = 'FAIL (Generating world signatures)' layers_tested = layers_tested + 1 continue td['machines'] = args.machines if not add_layers(bblayersconf, [layer], logger): logger.info('Skipping %s ???.' % layer['name']) results[layer['name']] = None results_status[layer['name']] = 'SKIPPED (Unknown)' layers_tested = layers_tested + 1 continue result = test_layer(td, layer, args.test_software_layer_signatures) results[layer['name']] = result results_status[layer['name']] = 'PASS' if results[layer['name']].wasSuccessful() else 'FAIL' layers_tested = layers_tested + 1 ret = 0 if layers_tested: logger.info('') logger.info('Summary of results:') logger.info('') for layer_name in results_status: logger.info('%s ... %s' % (layer_name, results_status[layer_name])) if not results[layer_name] or not results[layer_name].wasSuccessful(): ret = 2 # ret = 1 used for initialization errors cleanup_bblayers(None, None) return ret if __name__ == '__main__': try: ret = main() except Exception: ret = 1 import traceback traceback.print_exc() sys.exit(ret)