#!/usr/bin/env python3 # OpenEmbedded Development tool # # Copyright (C) 2014-2015 Intel Corporation # # SPDX-License-Identifier: GPL-2.0-only # import sys import os import argparse import glob import re import configparser import subprocess import logging basepath = '' workspace = {} config = None context = None scripts_path = os.path.dirname(os.path.realpath(__file__)) lib_path = scripts_path + '/lib' sys.path = sys.path + [lib_path] from devtool import DevtoolError, setup_tinfoil import scriptutils import argparse_oe logger = scriptutils.logger_create('devtool') plugins = [] class ConfigHandler(object): config_file = '' config_obj = None init_path = '' workspace_path = '' def __init__(self, filename): self.config_file = filename self.config_obj = configparser.ConfigParser() def get(self, section, option, default=None): try: ret = self.config_obj.get(section, option) except (configparser.NoOptionError, configparser.NoSectionError): if default != None: ret = default else: raise return ret def read(self): if os.path.exists(self.config_file): self.config_obj.read(self.config_file) if self.config_obj.has_option('General', 'init_path'): pth = self.get('General', 'init_path') self.init_path = os.path.join(basepath, pth) if not os.path.exists(self.init_path): logger.error('init_path %s specified in config file cannot be found' % pth) return False else: self.config_obj.add_section('General') self.workspace_path = self.get('General', 'workspace_path', os.path.join(basepath, 'workspace')) return True def write(self): logger.debug('writing to config file %s' % self.config_file) self.config_obj.set('General', 'workspace_path', self.workspace_path) with open(self.config_file, 'w') as f: self.config_obj.write(f) def set(self, section, option, value): if not self.config_obj.has_section(section): self.config_obj.add_section(section) self.config_obj.set(section, option, value) class Context: def __init__(self, **kwargs): self.__dict__.update(kwargs) def read_workspace(): global workspace workspace = {} if not os.path.exists(os.path.join(config.workspace_path, 'conf', 'layer.conf')): if context.fixed_setup: logger.error("workspace layer not set up") sys.exit(1) else: logger.info('Creating workspace layer in %s' % config.workspace_path) _create_workspace(config.workspace_path, config, basepath) if not context.fixed_setup: _enable_workspace_layer(config.workspace_path, config, basepath) logger.debug('Reading workspace in %s' % config.workspace_path) externalsrc_re = re.compile(r'^EXTERNALSRC(:pn-([^ =]+))? *= *"([^"]*)"$') for fn in glob.glob(os.path.join(config.workspace_path, 'appends', '*.bbappend')): with open(fn, 'r') as f: pnvalues = {} for line in f: res = externalsrc_re.match(line.rstrip()) if res: recipepn = os.path.splitext(os.path.basename(fn))[0].split('_')[0] pn = res.group(2) or recipepn # Find the recipe file within the workspace, if any bbfile = os.path.basename(fn).replace('.bbappend', '.bb').replace('%', '*') recipefile = glob.glob(os.path.join(config.workspace_path, 'recipes', recipepn, bbfile)) if recipefile: recipefile = recipefile[0] pnvalues['srctree'] = res.group(3) pnvalues['bbappend'] = fn pnvalues['recipefile'] = recipefile elif line.startswith('# srctreebase: '): pnvalues['srctreebase'] = line.split(':', 1)[1].strip() if pnvalues: if not pnvalues.get('srctreebase', None): pnvalues['srctreebase'] = pnvalues['srctree'] logger.debug('Found recipe %s' % pnvalues) workspace[pn] = pnvalues def create_workspace(args, config, basepath, workspace): if args.layerpath: workspacedir = os.path.abspath(args.layerpath) else: workspacedir = os.path.abspath(os.path.join(basepath, 'workspace')) _create_workspace(workspacedir, config, basepath) if not args.create_only: _enable_workspace_layer(workspacedir, config, basepath) def _create_workspace(workspacedir, config, basepath): import bb confdir = os.path.join(workspacedir, 'conf') if os.path.exists(os.path.join(confdir, 'layer.conf')): logger.info('Specified workspace already set up, leaving as-is') else: # Add a config file bb.utils.mkdirhier(confdir) with open(os.path.join(confdir, 'layer.conf'), 'w') as f: f.write('# ### workspace layer auto-generated by devtool ###\n') f.write('BBPATH =. "$' + '{LAYERDIR}:"\n') f.write('BBFILES += "$' + '{LAYERDIR}/recipes/*/*.bb \\\n') f.write(' $' + '{LAYERDIR}/appends/*.bbappend"\n') f.write('BBFILE_COLLECTIONS += "workspacelayer"\n') f.write('BBFILE_PATTERN_workspacelayer = "^$' + '{LAYERDIR}/"\n') f.write('BBFILE_PATTERN_IGNORE_EMPTY_workspacelayer = "1"\n') f.write('BBFILE_PRIORITY_workspacelayer = "99"\n') f.write('LAYERSERIES_COMPAT_workspacelayer = "${LAYERSERIES_COMPAT_core}"\n') # Add a README file with open(os.path.join(workspacedir, 'README'), 'w') as f: f.write('This layer was created by the OpenEmbedded devtool utility in order to\n') f.write('contain recipes and bbappends that are currently being worked on. The idea\n') f.write('is that the contents is temporary - once you have finished working on a\n') f.write('recipe you use the appropriate method to move the files you have been\n') f.write('working on to a proper layer. In most instances you should use the\n') f.write('devtool utility to manage files within it rather than modifying files\n') f.write('directly (although recipes added with "devtool add" will often need\n') f.write('direct modification.)\n') f.write('\nIf you no longer need to use devtool or the workspace layer\'s contents\n') f.write('you can remove the path to this workspace layer from your conf/bblayers.conf\n') f.write('file (and then delete the layer, if you wish).\n') f.write('\nNote that by default, if devtool fetches and unpacks source code, it\n') f.write('will place it in a subdirectory of a "sources" subdirectory of the\n') f.write('layer. If you prefer it to be elsewhere you can specify the source\n') f.write('tree path on the command line.\n') def _enable_workspace_layer(workspacedir, config, basepath): """Ensure the workspace layer is in bblayers.conf""" import bb bblayers_conf = os.path.join(basepath, 'conf', 'bblayers.conf') if not os.path.exists(bblayers_conf): logger.error('Unable to find bblayers.conf') return if os.path.abspath(workspacedir) != os.path.abspath(config.workspace_path): removedir = config.workspace_path else: removedir = None _, added = bb.utils.edit_bblayers_conf(bblayers_conf, workspacedir, removedir) if added: logger.info('Enabling workspace layer in bblayers.conf') if config.workspace_path != workspacedir: # Update our config to point to the new location config.workspace_path = workspacedir config.write() def main(): global basepath global config global context if sys.getfilesystemencoding() != "utf-8": sys.exit("Please use a locale setting which supports utf-8.\nPython can't change the filesystem locale after loading so we need a utf-8 when python starts or things won't work.") context = Context(fixed_setup=False) # Default basepath basepath = os.path.dirname(os.path.abspath(__file__)) parser = argparse_oe.ArgumentParser(description="OpenEmbedded development tool", add_help=False, epilog="Use %(prog)s <subcommand> --help to get help on a specific command") parser.add_argument('--basepath', help='Base directory of SDK / build directory') parser.add_argument('--bbpath', help='Explicitly specify the BBPATH, rather than getting it from the metadata') parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true') parser.add_argument('-q', '--quiet', help='Print only errors', action='store_true') parser.add_argument('--color', choices=['auto', 'always', 'never'], default='auto', help='Colorize output (where %(metavar)s is %(choices)s)', metavar='COLOR') global_args, unparsed_args = parser.parse_known_args() # Help is added here rather than via add_help=True, as we don't want it to # be handled by parse_known_args() parser.add_argument('-h', '--help', action='help', default=argparse.SUPPRESS, help='show this help message and exit') if global_args.debug: logger.setLevel(logging.DEBUG) elif global_args.quiet: logger.setLevel(logging.ERROR) if global_args.basepath: # Override basepath = global_args.basepath if os.path.exists(os.path.join(basepath, '.devtoolbase')): context.fixed_setup = True else: pth = basepath while pth != '' and pth != os.sep: if os.path.exists(os.path.join(pth, '.devtoolbase')): context.fixed_setup = True basepath = pth break pth = os.path.dirname(pth) if not context.fixed_setup: basepath = os.environ.get('BUILDDIR') if not basepath: logger.error("This script can only be run after initialising the build environment (e.g. by using oe-init-build-env)") sys.exit(1) logger.debug('Using basepath %s' % basepath) config = ConfigHandler(os.path.join(basepath, 'conf', 'devtool.conf')) if not config.read(): return -1 context.config = config bitbake_subdir = config.get('General', 'bitbake_subdir', '') if bitbake_subdir: # Normally set for use within the SDK logger.debug('Using bitbake subdir %s' % bitbake_subdir) sys.path.insert(0, os.path.join(basepath, bitbake_subdir, 'lib')) core_meta_subdir = config.get('General', 'core_meta_subdir') sys.path.insert(0, os.path.join(basepath, core_meta_subdir, 'lib')) else: # Standard location import scriptpath bitbakepath = scriptpath.add_bitbake_lib_path() if not bitbakepath: logger.error("Unable to find bitbake by searching parent directory of this script or PATH") sys.exit(1) logger.debug('Using standard bitbake path %s' % bitbakepath) scriptpath.add_oe_lib_path() scriptutils.logger_setup_color(logger, global_args.color) if global_args.bbpath is None: try: tinfoil = setup_tinfoil(config_only=True, basepath=basepath) try:<# # Sanity check the users setup for common misconfigurations # SANITY_REQUIRED_UTILITIES ?= "patch diffstat makeinfo git bzip2 tar gzip gawk chrpath wget cpio" def bblayers_conf_file(d): return os.path.join(d.getVar('TOPDIR', True), 'conf/bblayers.conf') def sanity_conf_read(fn): with open(fn, 'r') as f: lines = f.readlines() return lines def sanity_conf_find_line(pattern, lines): import re return next(((index, line) for index, line in enumerate(lines) if re.search(pattern, line)), (None, None)) def sanity_conf_update(fn, lines, version_var_name, new_version): index, line = sanity_conf_find_line(version_var_name, lines) lines[index] = '%s = "%d"\n' % (version_var_name, new_version) with open(fn, "w") as f: f.write(''.join(lines)) EXPORT_FUNCTIONS bblayers_conf_file sanity_conf_read sanity_conf_find_line sanity_conf_update # Functions added to this variable MUST throw an exception (or sys.exit()) unless they # successfully changed LCONF_VERSION in bblayers.conf BBLAYERS_CONF_UPDATE_FUNCS += "oecore_update_bblayers" python oecore_update_bblayers() { # bblayers.conf is out of date, so see if we can resolve that current_lconf = int(d.getVar('LCONF_VERSION', True)) if not current_lconf: sys.exit() lconf_version = int(d.getVar('LAYER_CONF_VERSION', True)) lines = [] if current_lconf < 4: sys.exit() bblayers_fn = bblayers_conf_file(d) lines = sanity_conf_read(bblayers_fn) if current_lconf == 4 and lconf_version > 4: topdir_var = '$' + '{TOPDIR}' index, bbpath_line = sanity_conf_find_line('BBPATH', lines) if bbpath_line: start = bbpath_line.find('"') if start != -1 and (len(bbpath_line) != (start + 1)): if bbpath_line[start + 1] == '"': lines[index] = (bbpath_line[:start + 1] + topdir_var + bbpath_line[start + 1:]) else: if not topdir_var in bbpath_line: lines[index] = (bbpath_line[:start + 1] + topdir_var + ':' + bbpath_line[start + 1:]) else: sys.exit() else: index, bbfiles_line = sanity_conf_find_line('BBFILES', lines) if bbfiles_line: lines.insert(index, 'BBPATH = "' + topdir_var + '"\n') else: sys.exit() current_lconf += 1 sanity_conf_update(bblayers_fn, lines, 'LCONF_VERSION', current_lconf) return sys.exit() } def raise_sanity_error(msg, d, network_error=False): if d.getVar("SANITY_USE_EVENTS", True) == "1": try: bb.event.fire(bb.event.SanityCheckFailed(msg, network_error), d) except TypeError: bb.event.fire(bb.event.SanityCheckFailed(msg), d) return bb.fatal(""" OE-core's config sanity checker detected a potential misconfiguration. Either fix the cause of this error or at your own risk disable the checker (see sanity.conf). Following is the list of potential problems / advisories: %s""" % msg) # Check a single tune for validity. def check_toolchain_tune(data, tune, multilib): tune_errors = [] if not tune: return "No tuning found for %s multilib." % multilib bb.debug(2, "Sanity-checking tuning '%s' (%s) features:" % (tune, multilib)) features = (data.getVar("TUNE_FEATURES_tune-%s" % tune, True) or "").split() if not features: return "Tuning '%s' has no defined features, and cannot be used." % tune valid_tunes = data.getVarFlags('TUNEVALID') or {} conflicts = data.getVarFlags('TUNECONFLICTS') or {} # [doc] is the documentation for the variable, not a real feature if 'doc' in valid_tunes: del valid_tunes['doc'] if 'doc' in conflicts: del conflicts['doc'] for feature in features: if feature in conflicts: for conflict in conflicts[feature].split(): if conflict in features: tune_errors.append("Feature '%s' conflicts with '%s'." % (feature, conflict)) if feature in valid_tunes: bb.debug(2, " %s: %s" % (feature, valid_tunes[feature])) else: tune_errors.append("Feature '%s' is not defined." % feature) whitelist = data.getVar("TUNEABI_WHITELIST", True) or '' override = data.getVar("TUNEABI_OVERRIDE", True) or '' if whitelist: tuneabi = data.getVar("TUNEABI_tune-%s" % tune, True) or '' if not tuneabi: tuneabi = tune if True not in [x in whitelist.split() for x in tuneabi.split()]: tune_errors.append("Tuning '%s' (%s) cannot be used with any supported tuning/ABI." % (tune, tuneabi)) if tune_errors: return "Tuning '%s' has the following errors:\n" % tune + '\n'.join(tune_errors) def check_toolchain(data): tune_error_set = [] deftune = data.getVar("DEFAULTTUNE", True) tune_errors = check_toolchain_tune(data, deftune, 'default') if tune_errors: tune_error_set.append(tune_errors) multilibs = (data.getVar("MULTILIB_VARIANTS", True) or "").split() global_multilibs = (data.getVar("MULTILIB_GLOBAL_VARIANTS", True) or "").split() if multilibs: seen_libs = [] seen_tunes = [] for lib in multilibs: if lib in seen_libs: tune_error_set.append("The multilib '%s' appears more than once." % lib) else: seen_libs.append(lib) if not lib in global_multilibs: tune_error_set.append("Multilib %s is not present in MULTILIB_GLOBAL_VARIANTS" % lib) tune = data.getVar("DEFAULTTUNE_virtclass-multilib-%s" % lib, True) if tune in seen_tunes: tune_error_set.append("The tuning '%s' appears in more than one multilib." % tune) else: seen_libs.append(tune) if tune == deftune: tune_error_set.append("Multilib '%s' (%s) is also the default tuning." % (lib, deftune)) else: tune_errors = check_toolchain_tune(data, tune, lib) if tune_errors: tune_error_set.append(tune_errors) if tune_error_set: return "Toolchain tunings invalid:\n" + '\n'.join(tune_error_set) + "\n" return "" def check_conf_exists(fn, data): bbpath = [] fn = data.expand(fn) vbbpath = data.getVar("BBPATH") if vbbpath: bbpath += vbbpath.split(":") for p in bbpath: currname = os.path.join(data.expand(p), fn) if os.access(currname, os.R_OK): return True return False def check_create_long_filename(filepath, pathname): testfile = os.path.join(filepath, ''.join([`num`[-1] for num in xrange(1,200)])) try: if not os.path.exists(filepath): bb.utils.mkdirhier(filepath) f = open(testfile, "w") f.close() os.remove(testfile) except IOError as e: errno, strerror = e.args if errno == 36: # ENAMETOOLONG return "Failed to create a file with a long name in %s. Please use a filesystem that does not unreasonably limit filename length.\n" % pathname else: return "Failed to create a file in %s: %s.\n" % (pathname, strerror) except OSError as e: errno, strerror = e.args return "Failed to create %s directory in which to run long name sanity check: %s.\n" % (pathname, strerror) return "" def check_path_length(filepath, pathname, limit): if len(filepath) > limit: return "The length of %s is longer than 410, this would cause unexpected errors, please use a shorter path.\n" % pathname return "" def check_connectivity(d): # URI's to check can be set in the CONNECTIVITY_CHECK_URIS variable # using the same syntax as for SRC_URI. If the variable is not set # the check is skipped test_uris = (d.getVar('CONNECTIVITY_CHECK_URIS', True) or "").split() retval = "" # Only check connectivity if network enabled and the # CONNECTIVITY_CHECK_URIS are set network_enabled = not d.getVar('BB_NO_NETWORK', True) check_enabled = len(test_uris) # Take a copy of the data store and unset MIRRORS and PREMIRROS data = bb.data.createCopy(d) data.delVar('PREMIRRORS') data.delVar('MIRRORS') if check_enabled and network_enabled: try: fetcher = bb.fetch2.Fetch(test_uris, data) fetcher.checkstatus() except Exception: # Allow the message to be configured so that users can be # pointed to a support mechanism. msg = data.getVar('CONNECTIVITY_CHECK_MSG', True) or "" if len(msg) == 0: msg = "Failed to fetch test data from the network. Please ensure your network is configured correctly.\n" retval = msg return retval def check_supported_distro(sanity_data): tested_distros = sanity_data.getVar('SANITY_TESTED_DISTROS', True) if not tested_distros: return try: distro = oe.lsb.distro_identifier() except Exception: distro = None if distro: if distro not in [x.strip() for x in tested_distros.split('\\n')]: bb.warn('Host distribution "%s" has not been validated with this version of the build system; you may possibly experience unexpected failures. It is recommended that you use a tested distribution.' % distro) else: bb.warn('Host distribution could not be determined; you may possibly experience unexpected failures. It is recommended that you use a tested distribution.') # Checks we should only make if MACHINE is set correctly def check_sanity_validmachine(sanity_data): messages = "" # Check TUNE_ARCH is set if sanity_data.getVar('TUNE_ARCH', True) == 'INVALID': messages = messages + 'TUNE_ARCH is unset. Please ensure your MACHINE configuration includes a valid tune configuration file which will set this correctly.\n' # Check TARGET_OS is set if sanity_data.getVar('TARGET_OS', True) == 'INVALID': messages = messages + 'Please set TARGET_OS directly, or choose a MACHINE or DISTRO that does so.\n' # Check that we don't have duplicate entries in PACKAGE_ARCHS & that TUNE_PKGARCH is in PACKAGE_ARCHS pkgarchs = sanity_data.getVar('PACKAGE_ARCHS', True) tunepkg = sanity_data.getVar('TUNE_PKGARCH', True) tunefound = False seen = {} dups = [] for pa in pkgarchs.split(): if seen.get(pa, 0) == 1: dups.append(pa) else: seen[pa] = 1 if pa == tunepkg: tunefound = True if len(dups): messages = messages + "Error, the PACKAGE_ARCHS variable contains duplicates. The following archs are listed more than once: %s" % " ".join(dups) if tunefound == False: messages = messages + "Error, the PACKAGE_ARCHS variable does not contain TUNE_PKGARCH (%s)." % tunepkg return messages # Checks if necessary to add option march to host gcc def check_gcc_march(sanity_data): result = True message = "" # Check if -march not in BUILD_CFLAGS if sanity_data.getVar("BUILD_CFLAGS",True).find("-march") < 0: result = False # Construct a test file f = open("gcc_test.c", "w") f.write("int main (){ volatile int atomic = 2; __sync_bool_compare_and_swap (&atomic, 2, 3); return 0; }\n") f.close() # Check if GCC could work without march if not result: status,res = oe.utils.getstatusoutput("${BUILD_PREFIX}gcc gcc_test.c -o gcc_test") if status == 0: result = True; if not result: status,res = oe.utils.getstatusoutput("${BUILD_PREFIX}gcc -march=native gcc_test.c -o gcc_test") if status == 0: message = "BUILD_CFLAGS_append = \" -march=native\"" result = True; if not result: build_arch = sanity_data.getVar('BUILD_ARCH', True) status,res = oe.utils.getstatusoutput("${BUILD_PREFIX}gcc -march=%s gcc_test.c -o gcc_test" % build_arch) if status == 0: message = "BUILD_CFLAGS_append = \" -march=%s\"" % build_arch result = True; os.remove("gcc_test.c") if os.path.exists("gcc_test"): os.remove("gcc_test") return (result, message) # Unpatched versions of make 3.82 are known to be broken. See GNU Savannah Bug 30612. # Use a modified reproducer from http://savannah.gnu.org/bugs/?30612 to validate. def check_make_version(sanity_data): from distutils.version import LooseVersion status, result = oe.utils.getstatusoutput("make --version") if status != 0: return "Unable to execute make --version, exit code %s\n" % status version = result.split()[2] if LooseVersion(version) == LooseVersion("3.82"): # Construct a test file f = open("makefile_test", "w") f.write("makefile_test.a: makefile_test_a.c makefile_test_b.c makefile_test.a( makefile_test_a.c makefile_test_b.c)\n") f.write("\n") f.write("makefile_test_a.c:\n") f.write(" touch $@\n") f.write("\n") f.write("makefile_test_b.c:\n") f.write(" touch $@\n") f.close() # Check if make 3.82 has been patched status,result = oe.utils.getstatusoutput("make -f makefile_test") os.remove("makefile_test") if os.path.exists("makefile_test_a.c"): os.remove("makefile_test_a.c") if os.path.exists("makefile_test_b.c"): os.remove("makefile_test_b.c") if os.path.exists("makefile_test.a"): os.remove("makefile_test.a") if status != 0: return "Your version of make 3.82 is broken. Please revert to 3.81 or install a patched version.\n" return None # Tar version 1.24 and onwards handle overwriting symlinks correctly # but earlier versions do not; this needs to work properly for sstate def check_tar_version(sanity_data): from distutils.version import LooseVersion status, result = oe.utils.getstatusoutput("tar --version") if status != 0: return "Unable to execute tar --version, exit code %s\n" % status version = result.split()[3] if LooseVersion(version) < LooseVersion("1.24"): return "Your version of tar is older than 1.24 and has bugs which will break builds. Please install a newer version of tar.\n" return None # We use git parameters and functionality only found in 1.7.5 or later def check_git_version(sanity_data): from distutils.version import LooseVersion status, result = oe.utils.getstatusoutput("git --version 2> /dev/null") if status != 0: return "Unable to execute git --version, exit code %s\n" % status version = result.split()[2] if LooseVersion(version) < LooseVersion("1.7.5"): return "Your version of git is older than 1.7.5 and has bugs which will break builds. Please install a newer version of git.\n" return None def sanity_check_conffiles(status, d): # Check we are using a valid local.conf current_conf = d.getVar('CONF_VERSION', True) conf_version = d.getVar('LOCALCONF_VERSION', True) if current_conf != conf_version: status.addresult("Your version of local.conf was generated from an older/newer version of local.conf.sample and there have been updates made to this file. Please compare the two files and merge any changes before continuing.\nMatching the version numbers will remove this message.\n\"meld conf/local.conf ${COREBASE}/meta*/conf/local.conf.sample\" is a good way to visualise the changes.\n") # Check bblayers.conf is valid current_lconf = d.getVar('LCONF_VERSION', True) lconf_version = d.getVar('LAYER_CONF_VERSION', True) if current_lconf != lconf_version: funcs = d.getVar('BBLAYERS_CONF_UPDATE_FUNCS', True).split() for func in funcs: success = True try: bb.build.exec_func(func, d) except Exception: success = False if success: bb.note("Your conf/bblayers.conf has been automatically updated.") status.reparse = True break if not status.reparse: status.addresult("Your version of bblayers.conf has the wrong LCONF_VERSION (has %s, expecting %s).\nPlease compare the your file against bblayers.conf.sample and merge any changes before continuing.\n\"meld conf/bblayers.conf ${COREBASE}/meta*/conf/bblayers.conf.sample\" is a good way to visualise the changes.\n" % (current_lconf, lconf_version)) # If we have a site.conf, check it's valid if check_conf_exists("conf/site.conf", d): current_sconf = d.getVar('SCONF_VERSION', True) sconf_version = d.getVar('SITE_CONF_VERSION', True) if current_sconf != sconf_version: status.addresult("Your version of site.conf was generated from an older version of site.conf.sample and there have been updates made to this file. Please compare the two files and merge any changes before continuing.\nMatching the version numbers will remove this message.\n\"meld conf/site.conf ${COREBASE}/meta*/conf/site.conf.sample\" is a good way to visualise the changes.\n") def sanity_handle_abichanges(status, d): # # Check the 'ABI' of TMPDIR # current_abi = d.getVar('OELAYOUT_ABI', True) abifile = d.getVar('SANITY_ABIFILE', True) if os.path.exists(abifile): with open(abifile, "r") as f: abi = f.read().strip() if not abi.isdigit(): with open(abifile, "w") as f: f.write(current_abi) elif abi == "2" and current_abi == "3": bb.note("Converting staging from layout version 2 to layout version 3") subprocess.call(d.expand("mv ${TMPDIR}/staging ${TMPDIR}/sysroots"), shell=True) subprocess.call(d.expand("ln -s sysroots ${TMPDIR}/staging"), shell=True) subprocess.call(d.expand("cd ${TMPDIR}/stamps; for i in */*do_populate_staging; do new=`echo $i | sed -e 's/do_populate_staging/do_populate_sysroot/'`; mv $i $new; done"), shell=True) with open(abifile, "w") as f: f.write(current_abi) elif abi == "3" and current_abi == "4": bb.note("Converting staging layout from version 3 to layout version 4") if os.path.exists(d.expand("${STAGING_DIR_NATIVE}${bindir_native}/${MULTIMACH_HOST_SYS}")): subprocess.call(d.expand("mv ${STAGING_DIR_NATIVE}${bindir_native}/${MULTIMACH_HOST_SYS} ${STAGING_BINDIR_CROSS}"), shell=True) subprocess.call(d.expand("ln -s ${STAGING_BINDIR_CROSS} ${STAGING_DIR_NATIVE}${bindir_native}/${MULTIMACH_HOST_SYS}"), shell=True) with open(abifile, "w") as f: f.write(current_abi) elif abi == "4": status.addresult("Staging layout has changed. The cross directory has been deprecated and cross packages are now built under the native sysroot.\nThis requires a rebuild.\n") elif abi == "5" and current_abi == "6": bb.note("Converting staging layout from version 5 to layout version 6") subprocess.call(d.expand("mv ${TMPDIR}/pstagelogs ${SSTATE_MANIFESTS}"), shell=True) with open(abifile, "w") as f: f.write(current_abi) elif abi == "7" and current_abi == "8": status.addresult("Your configuration is using stamp files including the sstate hash but your build directory was built with stamp files that do not include this.\nTo continue, either rebuild or switch back to the OEBasic signature handler with BB_SIGNATURE_HANDLER = 'OEBasic'.\n") elif (abi != current_abi and current_abi == "9"): status.addresult("The layout of the TMPDIR STAMPS directory has changed. Please clean out TMPDIR and rebuild (sstate will be still be valid and reused)\n") elif (abi != current_abi): # Code to convert from one ABI to another could go here if possible. status.addresult("Error, TMPDIR has changed its layout version number (%s to %s) and you need to either rebuild, revert or adjust it at your own risk.\n" % (abi, current_abi)) else: with open(abifile, "w") as f: f.write(current_abi) def check_sanity_sstate_dir_change(sstate_dir, data): # Sanity checks to be done when the value of SSTATE_DIR changes # Check that SSTATE_DIR isn't on a filesystem with limited filename length (eg. eCryptFS) testmsg = "" if sstate_dir != "": testmsg = check_create_long_filename(sstate_dir, "SSTATE_DIR") # If we don't have permissions to SSTATE_DIR, suggest the user set it as an SSTATE_MIRRORS try: err = testmsg.split(': ')[1].strip() if err == "Permission denied.": testmsg = testmsg + "You could try using %s in SSTATE_MIRRORS rather than as an SSTATE_CACHE.\n" % (sstate_dir) except IndexError: pass return testmsg def check_sanity_version_change(status, d): # Sanity checks to be done when SANITY_VERSION changes # In other words, these tests run once in a given build directory and then # never again until the sanity version changes. # Check the python install is complete. glib-2.0-natives requries # xml.parsers.expat try: import xml.parsers.expat except ImportError: status.addresult('Your python is not a full install. Please install the module xml.parsers.expat (python-xml on openSUSE and SUSE Linux).\n') status.addresult(check_make_version(d)) status.addresult(check_tar_version(d)) status.addresult(check_git_version(d)) missing = "" if not check_app_exists("${MAKE}", d): missing = missing + "GNU make," if not check_app_exists('${BUILD_PREFIX}gcc', d): missing = missing + "C Compiler (%sgcc)," % d.getVar("BUILD_PREFIX", True) if not check_app_exists('${BUILD_PREFIX}g++', d): missing = missing + "C++ Compiler (%sg++)," % d.getVar("BUILD_PREFIX", True) required_utilities = d.getVar('SANITY_REQUIRED_UTILITIES', True) for util in required_utilities.split(): if not check_app_exists(util, d): missing = missing + "%s," % util if missing: missing = missing.rstrip(',') status.addresult("Please install the following missing utilities: %s\n" % missing) assume_provided = d.getVar('ASSUME_PROVIDED', True).split() # Check user doesn't have ASSUME_PROVIDED = instead of += in local.conf if "diffstat-native" not in assume_provided: status.addresult('Please use ASSUME_PROVIDED +=, not ASSUME_PROVIDED = in your local.conf\n') if "qemu-native" in assume_provided: if not check_app_exists("qemu-arm", d): status.addresult("qemu-native was in ASSUME_PROVIDED but the QEMU binaries (qemu-arm) can't be found in PATH") (result, message) = check_gcc_march(d) if result and message: status.addresult("Your gcc version is older than 4.5, please add the following param to local.conf\n \ %s\n" % message) if not result: status.addresult("Your gcc version is older than 4.5 or is not working properly. Please verify you can build") status.addresult(" and link something that uses atomic operations, such as: \n") status.addresult(" __sync_bool_compare_and_swap (&atomic, 2, 3);\n") # Check that TMPDIR isn't on a filesystem with limited filename length (eg. eCryptFS) tmpdir = d.getVar('TMPDIR', True) status.addresult(check_create_long_filename(tmpdir, "TMPDIR")) # Some third-party software apparently relies on chmod etc. being suid root (!!) import stat suid_check_bins = "chown chmod mknod".split() for bin_cmd in suid_check_bins: bin_path = bb.utils.which(os.environ["PATH"], bin_cmd) if bin_path: bin_stat = os.stat(bin_path) if bin_stat.st_uid == 0 and bin_stat.st_mode & stat.S_ISUID: status.addresult('%s has the setuid bit set. This interferes with pseudo and may cause other issues that break the build process.\n' % bin_path) # Check that we can fetch from various network transports netcheck = check_connectivity(d) status.addresult(netcheck) if netcheck: status.network_error = True nolibs = d.getVar('NO32LIBS', True) if not nolibs: lib32path = '/lib' if os.path.exists('/lib64') and ( os.path.islink('/lib64') or os.path.islink('/lib') ): lib32path = '/lib32' if os.path.exists('%s/libc.so.6' % lib32path) and not os.path.exists('/usr/include/gnu/stubs-32.h'): status.addresult("You have a 32-bit libc, but no 32-bit headers. You must install the 32-bit libc headers.\n") bbpaths = d.getVar('BBPATH', True).split(":") if ("." in bbpaths or "" in bbpaths) and not status.reparse: status.addresult("BBPATH references the current directory, either through " \ "an empty entry, or a '.'.\n\t This is unsafe and means your "\ "layer configuration is adding empty elements to BBPATH.\n\t "\ "Please check your layer.conf files and other BBPATH " \ "settings to remove the current working directory " \ "references.\n" \ "Parsed BBPATH is" + str(bbpaths)); oes_bb_conf = d.getVar( 'OES_BITBAKE_CONF', True) if not oes_bb_conf: status.addresult('You are not using the OpenEmbedded version of conf/bitbake.conf. This means your environment is misconfigured, in particular check BBPATH.\n') # The length of tmpdir can't be longer than 410 status.addresult(check_path_length(tmpdir, "TMPDIR", 410)) def check_sanity_everybuild(status, d): # Sanity tests which test the users environment so need to run at each build (or are so cheap # it makes sense to always run them. if 0 == os.getuid(): raise_sanity_error("Do not use Bitbake as root.", d) # Check the Python version, we now have a minimum of Python 2.7.3 import sys if sys.hexversion < 0x020703F0: status.addresult('The system requires at least Python 2.7.3 to run. Please update your Python interpreter.\n') # Check the bitbake version meets minimum requirements from distutils.version import LooseVersion minversion = d.getVar('BB_MIN_VERSION', True) if (LooseVersion(bb.__version__) < LooseVersion(minversion)): status.addresult('Bitbake version %s is required and version %s was found\n' % (minversion, bb.__version__)) sanity_check_conffiles(status, d) paths = d.getVar('PATH', True).split(":") if "." in paths or "" in paths: status.addresult("PATH contains '.' or '' (empty element), which will break the build, please remove this.\nParsed PATH is " + str(paths) + "\n") # Check that the DISTRO is valid, if set # need to take into account DISTRO renaming DISTRO distro = d.getVar('DISTRO', True) if distro and distro != "nodistro": if not ( check_conf_exists("conf/distro/${DISTRO}.conf", d) or check_conf_exists("conf/distro/include/${DISTRO}.inc", d) ): status.addresult("DISTRO '%s' not found. Please set a valid DISTRO in your local.conf\n" % d.getVar("DISTRO", True)) # Check that DL_DIR is set, exists and is writable. In theory, we should never even hit the check if DL_DIR isn't # set, since so much relies on it being set. dldir = d.getVar('DL_DIR', True) if not dldir: status.addresult("DL_DIR is not set. Your environment is misconfigured, check that DL_DIR is set, and if the directory exists, that it is writable. \n") if os.path.exists(dldir) and not os.access(dldir, os.W_OK): status.addresult("DL_DIR: %s exists but you do not appear to have write access to it. \n" % dldir) # Check that the MACHINE is valid, if it is set machinevalid = True if d.getVar('MACHINE', True): if not check_conf_exists("conf/machine/${MACHINE}.conf", d): status.addresult('Please set a valid MACHINE in your local.conf or environment\n') machinevalid = False else: status.addresult(check_sanity_validmachine(d)) else: status.addresult('Please set a MACHINE in your local.conf or environment\n') machinevalid = False if machinevalid: status.addresult(check_toolchain(d)) # Check that the SDKMACHINE is valid, if it is set if d.getVar('SDKMACHINE', True): if not check_conf_exists("conf/machine-sdk/${SDKMACHINE}.conf", d): status.addresult('Specified SDKMACHINE value is not valid\n') check_supported_distro(d) # Check if DISPLAY is set if TEST_IMAGE is set if d.getVar('TEST_IMAGE', True) == '1' or d.getVar('DEFAULT_TEST_SUITES', True): display = d.getVar("BB_ORIGENV", False).getVar("DISPLAY", True) if not display: status.addresult('testimage needs an X desktop to start qemu, please set DISPLAY correctly (e.g. DISPLAY=:1.0)\n') omask = os.umask(022) if omask & 0755: status.addresult("Please use a umask which allows a+rx and u+rwx\n") os.umask(omask) if d.getVar('TARGET_ARCH', True) == "arm": # This path is no longer user-readable in modern (very recent) Linux try: if os.path.exists("/proc/sys/vm/mmap_min_addr"): f = open("/proc/sys/vm/mmap_min_addr", "r") try: if (int(f.read().strip()) > 65536): status.addresult("/proc/sys/vm/mmap_min_addr is not <= 65536. This will cause problems with qemu so please fix the value (as root).\n\nTo fix this in later reboots, set vm.mmap_min_addr = 65536 in /etc/sysctl.conf.\n") finally: f.close() except: pass oeroot = d.getVar('COREBASE', True) if oeroot.find('+') != -1: status.addresult("Error, you have an invalid character (+) in your COREBASE directory path. Please move the installation to a directory which doesn't include any + characters.") if oeroot.find('@') != -1: status.addresult("Error, you have an invalid character (@) in your COREBASE directory path. Please move the installation to a directory which doesn't include any @ characters.") if oeroot.find(' ') != -1: status.addresult("Error, you have a space in your COREBASE directory path. Please move the installation to a directory which doesn't include a space since autotools doesn't support this.") # Check that TMPDIR hasn't changed location since the last time we were run tmpdir = d.getVar('TMPDIR', True) checkfile = os.path.join(tmpdir, "saved_tmpdir") if os.path.exists(checkfile): with open(checkfile, "r") as f: saved_tmpdir = f.read().strip() if (saved_tmpdir != tmpdir): status.addresult("Error, TMPDIR has changed location. You need to either move it back to %s or rebuild\n" % saved_tmpdir) else: bb.utils.mkdirhier(tmpdir) with open(checkfile, "w") as f: f.write(tmpdir) def check_sanity(sanity_data): import subprocess class SanityStatus(object): def __init__(self): self.messages = "" self.network_error = False self.reparse = False def addresult(self, message): if message: self.messages = self.messages + message status = SanityStatus() tmpdir = sanity_data.getVar('TMPDIR', True) sstate_dir = sanity_data.getVar('SSTATE_DIR', True) # Check saved sanity info last_sanity_version = 0 last_tmpdir = "" last_sstate_dir = "" sanityverfile = sanity_data.expand("${TOPDIR}/conf/sanity_info") if os.path.exists(sanityverfile): with open(sanityverfile, 'r') as f: for line in f: if line.startswith('SANITY_VERSION'): last_sanity_version = int(line.split()[1]) if line.startswith('TMPDIR'): last_tmpdir = line.split()[1] if line.startswith('SSTATE_DIR'): last_sstate_dir = line.split()[1] check_sanity_everybuild(status, sanity_data) sanity_version = int(sanity_data.getVar('SANITY_VERSION', True) or 1) network_error = False if last_sanity_version < sanity_version: check_sanity_version_change(status, sanity_data) status.addresult(check_sanity_sstate_dir_change(sstate_dir, sanity_data)) else: if last_sstate_dir != sstate_dir: status.addresult(check_sanity_sstate_dir_change(sstate_dir, sanity_data)) if os.path.exists(os.path.dirname(sanityverfile)) and not status.messages: with open(sanityverfile, 'w') as f: f.write("SANITY_VERSION %s\n" % sanity_version) f.write("TMPDIR %s\n" % tmpdir) f.write("SSTATE_DIR %s\n" % sstate_dir) sanity_handle_abichanges(status, sanity_data) if status.messages != "": raise_sanity_error(sanity_data.expand(status.messages), sanity_data, status.network_error) return status.reparse # Create a copy of the datastore and finalise it to ensure appends and # overrides are set - the datastore has yet to be finalised at ConfigParsed def copy_data(e): sanity_data = bb.data.createCopy(e.data) sanity_data.finalize() return sanity_data addhandler check_sanity_eventhandler check_sanity_eventhandler[eventmask] = "bb.event.ConfigParsed bb.event.SanityCheck bb.event.NetworkTest" python check_sanity_eventhandler() { if bb.event.getName(e) == "ConfigParsed" and e.data.getVar("BB_WORKERCONTEXT", True) != "1" and e.data.getVar("DISABLE_SANITY_CHECKS", True) != "1": sanity_data = copy_data(e) reparse = check_sanity(sanity_data) e.data.setVar("BB_INVALIDCONF", reparse) elif bb.event.getName(e) == "SanityCheck": sanity_data = copy_data(e) sanity_data.setVar("SANITY_USE_EVENTS", "1") reparse = check_sanity(sanity_data) e.data.setVar("BB_INVALIDCONF", reparse) bb.event.fire(bb.event.SanityCheckPassed(), e.data) elif bb.event.getName(e) == "NetworkTest": sanity_data = copy_data(e) bb.event.fire(bb.event.NetworkTestFailed() if check_connectivity(sanity_data) else bb.event.NetworkTestPassed(), e.data) return }
# # Sanity check the users setup for common misconfigurations # SANITY_REQUIRED_UTILITIES ?= "patch diffstat makeinfo git bzip2 tar gzip gawk chrpath wget cpio" def bblayers_conf_file(d): return os.path.join(d.getVar('TOPDIR', True), 'conf/bblayers.conf') def sanity_conf_read(fn): with open(fn, 'r') as f: lines = f.readlines() return lines def sanity_conf_find_line(pattern, lines): import re return next(((index, line) for index, line in enumerate(lines) if re.search(pattern, line)), (None, None)) def sanity_conf_update(fn, lines, version_var_name, new_version): index, line = sanity_conf_find_line(version_var_name, lines) lines[index] = '%s = "%d"\n' % (version_var_name, new_version) with open(fn, "w") as f: f.write(''.join(lines)) EXPORT_FUNCTIONS bblayers_conf_file sanity_conf_read sanity_conf_find_line sanity_conf_update # Functions added to this variable MUST throw an exception (or sys.exit()) unless they # successfully changed LCONF_VERSION in bblayers.conf BBLAYERS_CONF_UPDATE_FUNCS += "oecore_update_bblayers" python oecore_update_bblayers() { # bblayers.conf is out of date, so see if we can resolve that current_lconf = int(d.getVar('LCONF_VERSION', True)) if not current_lconf: sys.exit() lconf_version = int(d.getVar('LAYER_CONF_VERSION', True)) lines = [] if current_lconf < 4: sys.exit() bblayers_fn = bblayers_conf_file(d) lines = sanity_conf_read(bblayers_fn) if current_lconf == 4 and lconf_version > 4: topdir_var = '$' + '{TOPDIR}' index, bbpath_line = sanity_conf_find_line('BBPATH', lines) if bbpath_line: start = bbpath_line.find('"') if start != -1 and (len(bbpath_line) != (start + 1)): if bbpath_line[start + 1] == '"': lines[index] = (bbpath_line[:start + 1] + topdir_var + bbpath_line[start + 1:]) else: if not topdir_var in bbpath_line: lines[index] = (bbpath_line[:start + 1] + topdir_var + ':' + bbpath_line[start + 1:]) else: sys.exit() else: index, bbfiles_line = sanity_conf_find_line('BBFILES', lines) if bbfiles_line: lines.insert(index, 'BBPATH = "' + topdir_var + '"\n') else: sys.exit() current_lconf += 1 sanity_conf_update(bblayers_fn, lines, 'LCONF_VERSION', current_lconf) return sys.exit() } def raise_sanity_error(msg, d, network_error=False): if d.getVar("SANITY_USE_EVENTS", True) == "1": try: bb.event.fire(bb.event.SanityCheckFailed(msg, network_error), d) except TypeError: bb.event.fire(bb.event.SanityCheckFailed(msg), d) return bb.fatal(""" OE-core's config sanity checker detected a potential misconfiguration. Either fix the cause of this error or at your own risk disable the checker (see sanity.conf). Following is the list of potential problems / advisories: %s""" % msg) # Check a single tune for validity. def check_toolchain_tune(data, tune, multilib): tune_errors = [] if not tune: return "No tuning found for %s multilib." % multilib bb.debug(2, "Sanity-checking tuning '%s' (%s) features:" % (tune, multilib)) features = (data.getVar("TUNE_FEATURES_tune-%s" % tune, True) or "").split() if not features: return "Tuning '%s' has no defined features, and cannot be used." % tune valid_tunes = data.getVarFlags('TUNEVALID') or {} conflicts = data.getVarFlags('TUNECONFLICTS') or {} # [doc] is the documentation for the variable, not a real feature if 'doc' in valid_tunes: del valid_tunes['doc'] if 'doc' in conflicts: del conflicts['doc'] for feature in features: if feature in conflicts: for conflict in conflicts[feature].split(): if conflict in features: tune_errors.append("Feature '%s' conflicts with '%s'." % (feature, conflict)) if feature in valid_tunes: bb.debug(2, " %s: %s" % (feature, valid_tunes[feature])) else: tune_errors.append("Feature '%s' is not defined." % feature) whitelist = data.getVar("TUNEABI_WHITELIST", True) or '' override = data.getVar("TUNEABI_OVERRIDE", True) or '' if whitelist: tuneabi = data.getVar("TUNEABI_tune-%s" % tune, True) or '' if not tuneabi: tuneabi = tune if True not in [x in whitelist.split() for x in tuneabi.split()]: tune_errors.append("Tuning '%s' (%s) cannot be used with any supported tuning/ABI." % (tune, tuneabi)) if tune_errors: return "Tuning '%s' has the following errors:\n" % tune + '\n'.join(tune_errors) def check_toolchain(data): tune_error_set = [] deftune = data.getVar("DEFAULTTUNE", True) tune_errors = check_toolchain_tune(data, deftune, 'default') if tune_errors: tune_error_set.append(tune_errors) multilibs = (data.getVar("MULTILIB_VARIANTS", True) or "").split() global_multilibs = (data.getVar("MULTILIB_GLOBAL_VARIANTS", True) or "").split() if multilibs: seen_libs = [] seen_tunes = [] for lib in multilibs: if lib in seen_libs: tune_error_set.append("The multilib '%s' appears more than once." % lib) else: seen_libs.append(lib) if not lib in global_multilibs: tune_error_set.append("Multilib %s is not present in MULTILIB_GLOBAL_VARIANTS" % lib) tune = data.getVar("DEFAULTTUNE_virtclass-multilib-%s" % lib, True) if tune in seen_tunes: tune_error_set.append("The tuning '%s' appears in more than one multilib." % tune) else: seen_libs.append(tune) if tune == deftune: tune_error_set.append("Multilib '%s' (%s) is also the default tuning." % (lib, deftune)) else: tune_errors = check_toolchain_tune(data, tune, lib) if tune_errors: tune_error_set.append(tune_errors) if tune_error_set: return "Toolchain tunings invalid:\n" + '\n'.join(tune_error_set) + "\n" return "" def check_conf_exists(fn, data): bbpath = [] fn = data.expand(fn) vbbpath = data.getVar("BBPATH") if vbbpath: bbpath += vbbpath.split(":") for p in bbpath: currname = os.path.join(data.expand(p), fn) if os.access(currname, os.R_OK): return True return False def check_create_long_filename(filepath, pathname): testfile = os.path.join(filepath, ''.join([`num`[-1] for num in xrange(1,200)])) try: if not os.path.exists(filepath): bb.utils.mkdirhier(filepath) f = open(testfile, "w") f.close() os.remove(testfile) except IOError as e: errno, strerror = e.args if errno == 36: # ENAMETOOLONG return "Failed to create a file with a long name in %s. Please use a filesystem that does not unreasonably limit filename length.\n" % pathname else: return "Failed to create a file in %s: %s.\n" % (pathname, strerror) except OSError as e: errno, strerror = e.args return "Failed to create %s directory in which to run long name sanity check: %s.\n" % (pathname, strerror) return "" def check_path_length(filepath, pathname, limit): if len(filepath) > limit: return "The length of %s is longer than 410, this would cause unexpected errors, please use a shorter path.\n" % pathname return "" def check_connectivity(d): # URI's to check can be set in the CONNECTIVITY_CHECK_URIS variable # using the same syntax as for SRC_URI. If the variable is not set # the check is skipped test_uris = (d.getVar('CONNECTIVITY_CHECK_URIS', True) or "").split() retval = "" # Only check connectivity if network enabled and the # CONNECTIVITY_CHECK_URIS are set network_enabled = not d.getVar('BB_NO_NETWORK', True) check_enabled = len(test_uris) # Take a copy of the data store and unset MIRRORS and PREMIRROS data = bb.data.createCopy(d) data.delVar('PREMIRRORS') data.delVar('MIRRORS') if check_enabled and network_enabled: try: fetcher = bb.fetch2.Fetch(test_uris, data) fetcher.checkstatus() except Exception: # Allow the message to be configured so that users can be # pointed to a support mechanism. msg = data.getVar('CONNECTIVITY_CHECK_MSG', True) or "" if len(msg) == 0: msg = "Failed to fetch test data from the network. Please ensure your network is configured correctly.\n" retval = msg return retval def check_supported_distro(sanity_data): tested_distros = sanity_data.getVar('SANITY_TESTED_DISTROS', True) if not tested_distros: return try: distro = oe.lsb.distro_identifier() except Exception: distro = None if distro: if distro not in [x.strip() for x in tested_distros.split('\\n')]: bb.warn('Host distribution "%s" has not been validated with this version of the build system; you may possibly experience unexpected failures. It is recommended that you use a tested distribution.' % distro) else: bb.warn('Host distribution could not be determined; you may possibly experience unexpected failures. It is recommended that you use a tested distribution.') # Checks we should only make if MACHINE is set correctly def check_sanity_validmachine(sanity_data): messages = "" # Check TUNE_ARCH is set if sanity_data.getVar('TUNE_ARCH', True) == 'INVALID': messages = messages + 'TUNE_ARCH is unset. Please ensure your MACHINE configuration includes a valid tune configuration file which will set this correctly.\n' # Check TARGET_OS is set if sanity_data.getVar('TARGET_OS', True) == 'INVALID': messages = messages + 'Please set TARGET_OS directly, or choose a MACHINE or DISTRO that does so.\n' # Check that we don't have duplicate entries in PACKAGE_ARCHS & that TUNE_PKGARCH is in PACKAGE_ARCHS pkgarchs = sanity_data.getVar('PACKAGE_ARCHS', True) tunepkg = sanity_data.getVar('TUNE_PKGARCH', True) tunefound = False seen = {} dups = [] for pa in pkgarchs.split(): if seen.get(pa, 0) == 1: dups.append(pa) else: seen[pa] = 1 if pa == tunepkg: tunefound = True if len(dups): messages = messages + "Error, the PACKAGE_ARCHS variable contains duplicates. The following archs are listed more than once: %s" % " ".join(dups) if tunefound == False: messages = messages + "Error, the PACKAGE_ARCHS variable does not contain TUNE_PKGARCH (%s)." % tunepkg return messages # Checks if necessary to add option march to host gcc def check_gcc_march(sanity_data): result = True message = "" # Check if -march not in BUILD_CFLAGS if sanity_data.getVar("BUILD_CFLAGS",True).find("-march") < 0: result = False # Construct a test file f = open("gcc_test.c", "w") f.write("int main (){ volatile int atomic = 2; __sync_bool_compare_and_swap (&atomic, 2, 3); return 0; }\n") f.close() # Check if GCC could work without march if not result: status,res = oe.utils.getstatusoutput("${BUILD_PREFIX}gcc gcc_test.c -o gcc_test") if status == 0: result = True; if not result: status,res = oe.utils.getstatusoutput("${BUILD_PREFIX}gcc -march=native gcc_test.c -o gcc_test") if status == 0: message = "BUILD_CFLAGS_append = \" -march=native\"" result = True; if not result: build_arch = sanity_data.getVar('BUILD_ARCH', True) status,res = oe.utils.getstatusoutput("${BUILD_PREFIX}gcc -march=%s gcc_test.c -o gcc_test" % build_arch) if status == 0: message = "BUILD_CFLAGS_append = \" -march=%s\"" % build_arch result = True; os.remove("gcc_test.c") if os.path.exists("gcc_test"): os.remove("gcc_test") return (result, message) # Unpatched versions of make 3.82 are known to be broken. See GNU Savannah Bug 30612. # Use a modified reproducer from http://savannah.gnu.org/bugs/?30612 to validate. def check_make_version(sanity_data): from distutils.version import LooseVersion status, result = oe.utils.getstatusoutput("make --version") if status != 0: return "Unable to execute make --version, exit code %s\n" % status version = result.split()[2] if LooseVersion(version) == LooseVersion("3.82"): # Construct a test file f = open("makefile_test", "w") f.write("makefile_test.a: makefile_test_a.c makefile_test_b.c makefile_test.a( makefile_test_a.c makefile_test_b.c)\n") f.write("\n") f.write("makefile_test_a.c:\n") f.write(" touch $@\n") f.write("\n") f.write("makefile_test_b.c:\n") f.write(" touch $@\n") f.close() # Check if make 3.82 has been patched status,result = oe.utils.getstatusoutput("make -f makefile_test") os.remove("makefile_test") if os.path.exists("makefile_test_a.c"): os.remove("makefile_test_a.c") if os.path.exists("makefile_test_b.c"): os.remove("makefile_test_b.c") if os.path.exists("makefile_test.a"): os.remove("makefile_test.a") if status != 0: return "Your version of make 3.82 is broken. Please revert to 3.81 or install a patched version.\n" return None # Tar version 1.24 and onwards handle overwriting symlinks correctly # but earlier versions do not; this needs to work properly for sstate def check_tar_version(sanity_data): from distutils.version import LooseVersion status, result = oe.utils.getstatusoutput("tar --version") if status != 0: return "Unable to execute tar --version, exit code %s\n" % status version = result.split()[3] if LooseVersion(version) < LooseVersion("1.24"): return "Your version of tar is older than 1.24 and has bugs which will break builds. Please install a newer version of tar.\n" return None # We use git parameters and functionality only found in 1.7.5 or later def check_git_version(sanity_data): from distutils.version import LooseVersion status, result = oe.utils.getstatusoutput("git --version 2> /dev/null") if status != 0: return "Unable to execute git --version, exit code %s\n" % status version = result.split()[2] if LooseVersion(version) < LooseVersion("1.7.5"): return "Your version of git is older than 1.7.5 and has bugs which will break builds. Please install a newer version of git.\n" return None def sanity_check_conffiles(status, d): # Check we are using a valid local.conf current_conf = d.getVar('CONF_VERSION', True) conf_version = d.getVar('LOCALCONF_VERSION', True) if current_conf != conf_version: status.addresult("Your version of local.conf was generated from an older/newer version of local.conf.sample and there have been updates made to this file. Please compare the two files and merge any changes before continuing.\nMatching the version numbers will remove this message.\n\"meld conf/local.conf ${COREBASE}/meta*/conf/local.conf.sample\" is a good way to visualise the changes.\n") # Check bblayers.conf is valid current_lconf = d.getVar('LCONF_VERSION', True) lconf_version = d.getVar('LAYER_CONF_VERSION', True) if current_lconf != lconf_version: funcs = d.getVar('BBLAYERS_CONF_UPDATE_FUNCS', True).split() for func in funcs: success = True try: bb.build.exec_func(func, d) except Exception: success = False if success: bb.note("Your conf/bblayers.conf has been automatically updated.") status.reparse = True break if not status.reparse: status.addresult("Your version of bblayers.conf has the wrong LCONF_VERSION (has %s, expecting %s).\nPlease compare the your file against bblayers.conf.sample and merge any changes before continuing.\n\"meld conf/bblayers.conf ${COREBASE}/meta*/conf/bblayers.conf.sample\" is a good way to visualise the changes.\n" % (current_lconf, lconf_version)) # If we have a site.conf, check it's valid if check_conf_exists("conf/site.conf", d): current_sconf = d.getVar('SCONF_VERSION', True) sconf_version = d.getVar('SITE_CONF_VERSION', True) if current_sconf != sconf_version: status.addresult("Your version of site.conf was generated from an older version of site.conf.sample and there have been updates made to this file. Please compare the two files and merge any changes before continuing.\nMatching the version numbers will remove this message.\n\"meld conf/site.conf ${COREBASE}/meta*/conf/site.conf.sample\" is a good way to visualise the changes.\n") def sanity_handle_abichanges(status, d): # # Check the 'ABI' of TMPDIR # current_abi = d.getVar('OELAYOUT_ABI', True) abifile = d.getVar('SANITY_ABIFILE', True) if os.path.exists(abifile): with open(abifile, "r") as f: abi = f.read().strip() if not abi.isdigit(): with open(abifile, "w") as f: f.write(current_abi) elif abi == "2" and current_abi == "3": bb.note("Converting staging from layout version 2 to layout version 3") subprocess.call(d.expand("mv ${TMPDIR}/staging ${TMPDIR}/sysroots"), shell=True) subprocess.call(d.expand("ln -s sysroots ${TMPDIR}/staging"), shell=True) subprocess.call(d.expand("cd ${TMPDIR}/stamps; for i in */*do_populate_staging; do new=`echo $i | sed -e 's/do_populate_staging/do_populate_sysroot/'`; mv $i $new; done"), shell=True) with open(abifile, "w") as f: f.write(current_abi) elif abi == "3" and current_abi == "4": bb.note("Converting staging layout from version 3 to layout version 4") if os.path.exists(d.expand("${STAGING_DIR_NATIVE}${bindir_native}/${MULTIMACH_HOST_SYS}")): subprocess.call(d.expand("mv ${STAGING_DIR_NATIVE}${bindir_native}/${MULTIMACH_HOST_SYS} ${STAGING_BINDIR_CROSS}"), shell=True) subprocess.call(d.expand("ln -s ${STAGING_BINDIR_CROSS} ${STAGING_DIR_NATIVE}${bindir_native}/${MULTIMACH_HOST_SYS}"), shell=True) with open(abifile, "w") as f: f.write(current_abi) elif abi == "4": status.addresult("Staging layout has changed. The cross directory has been deprecated and cross packages are now built under the native sysroot.\nThis requires a rebuild.\n") elif abi == "5" and current_abi == "6": bb.note("Converting staging layout from version 5 to layout version 6") subprocess.call(d.expand("mv ${TMPDIR}/pstagelogs ${SSTATE_MANIFESTS}"), shell=True) with open(abifile, "w") as f: f.write(current_abi) elif abi == "7" and current_abi == "8": status.addresult("Your configuration is using stamp files including the sstate hash but your build directory was built with stamp files that do not include this.\nTo continue, either rebuild or switch back to the OEBasic signature handler with BB_SIGNATURE_HANDLER = 'OEBasic'.\n") elif (abi != current_abi and current_abi == "9"): status.addresult("The layout of the TMPDIR STAMPS directory has changed. Please clean out TMPDIR and rebuild (sstate will be still be valid and reused)\n") elif (abi != current_abi): # Code to convert from one ABI to another could go here if possible. status.addresult("Error, TMPDIR has changed its layout version number (%s to %s) and you need to either rebuild, revert or adjust it at your own risk.\n" % (abi, current_abi)) else: with open(abifile, "w") as f: f.write(current_abi) def check_sanity_sstate_dir_change(sstate_dir, data): # Sanity checks to be done when the value of SSTATE_DIR changes # Check that SSTATE_DIR isn't on a filesystem with limited filename length (eg. eCryptFS) testmsg = "" if sstate_dir != "": testmsg = check_create_long_filename(sstate_dir, "SSTATE_DIR") # If we don't have permissions to SSTATE_DIR, suggest the user set it as an SSTATE_MIRRORS try: err = testmsg.split(': ')[1].strip() if err == "Permission denied.": testmsg = testmsg + "You could try using %s in SSTATE_MIRRORS rather than as an SSTATE_CACHE.\n" % (sstate_dir) except IndexError: pass return testmsg def check_sanity_version_change(status, d): # Sanity checks to be done when SANITY_VERSION changes # In other words, these tests run once in a given build directory and then # never again until the sanity version changes. # Check the python install is complete. glib-2.0-natives requries # xml.parsers.expat try: import xml.parsers.expat except ImportError: status.addresult('Your python is not a full install. Please install the module xml.parsers.expat (python-xml on openSUSE and SUSE Linux).\n') status.addresult(check_make_version(d)) status.addresult(check_tar_version(d)) status.addresult(check_git_version(d)) missing = "" if not check_app_exists("${MAKE}", d): missing = missing + "GNU make," if not check_app_exists('${BUILD_PREFIX}gcc', d): missing = missing + "C Compiler (%sgcc)," % d.getVar("BUILD_PREFIX", True) if not check_app_exists('${BUILD_PREFIX}g++', d): missing = missing + "C++ Compiler (%sg++)," % d.getVar("BUILD_PREFIX", True) required_utilities = d.getVar('SANITY_REQUIRED_UTILITIES', True) for util in required_utilities.split(): if not check_app_exists(util, d): missing = missing + "%s," % util if missing: missing = missing.rstrip(',') status.addresult("Please install the following missing utilities: %s\n" % missing) assume_provided = d.getVar('ASSUME_PROVIDED', True).split() # Check user doesn't have ASSUME_PROVIDED = instead of += in local.conf if "diffstat-native" not in assume_provided: status.addresult('Please use ASSUME_PROVIDED +=, not ASSUME_PROVIDED = in your local.conf\n') if "qemu-native" in assume_provided: if not check_app_exists("qemu-arm", d): status.addresult("qemu-native was in ASSUME_PROVIDED but the QEMU binaries (qemu-arm) can't be found in PATH") (result, message) = check_gcc_march(d) if result and message: status.addresult("Your gcc version is older than 4.5, please add the following param to local.conf\n \ %s\n" % message) if not result: status.addresult("Your gcc version is older than 4.5 or is not working properly. Please verify you can build") status.addresult(" and link something that uses atomic operations, such as: \n") status.addresult(" __sync_bool_compare_and_swap (&atomic, 2, 3);\n") # Check that TMPDIR isn't on a filesystem with limited filename length (eg. eCryptFS) tmpdir = d.getVar('TMPDIR', True) status.addresult(check_create_long_filename(tmpdir, "TMPDIR")) # Some third-party software apparently relies on chmod etc. being suid root (!!) import stat suid_check_bins = "chown chmod mknod".split() for bin_cmd in suid_check_bins: bin_path = bb.utils.which(os.environ["PATH"], bin_cmd) if bin_path: bin_stat = os.stat(bin_path) if bin_stat.st_uid == 0 and bin_stat.st_mode & stat.S_ISUID: status.addresult('%s has the setuid bit set. This interferes with pseudo and may cause other issues that break the build process.\n' % bin_path) # Check that we can fetch from various network transports netcheck = check_connectivity(d) status.addresult(netcheck) if netcheck: status.network_error = True nolibs = d.getVar('NO32LIBS', True) if not nolibs: lib32path = '/lib' if os.path.exists('/lib64') and ( os.path.islink('/lib64') or os.path.islink('/lib') ): lib32path = '/lib32' if os.path.exists('%s/libc.so.6' % lib32path) and not os.path.exists('/usr/include/gnu/stubs-32.h'): status.addresult("You have a 32-bit libc, but no 32-bit headers. You must install the 32-bit libc headers.\n") bbpaths = d.getVar('BBPATH', True).split(":") if ("." in bbpaths or "" in bbpaths) and not status.reparse: status.addresult("BBPATH references the current directory, either through " \ "an empty entry, or a '.'.\n\t This is unsafe and means your "\ "layer configuration is adding empty elements to BBPATH.\n\t "\ "Please check your layer.conf files and other BBPATH " \ "settings to remove the current working directory " \ "references.\n" \ "Parsed BBPATH is" + str(bbpaths)); oes_bb_conf = d.getVar( 'OES_BITBAKE_CONF', True) if not oes_bb_conf: status.addresult('You are not using the OpenEmbedded version of conf/bitbake.conf. This means your environment is misconfigured, in particular check BBPATH.\n') # The length of tmpdir can't be longer than 410 status.addresult(check_path_length(tmpdir, "TMPDIR", 410)) def check_sanity_everybuild(status, d): # Sanity tests which test the users environment so need to run at each build (or are so cheap # it makes sense to always run them. if 0 == os.getuid(): raise_sanity_error("Do not use Bitbake as root.", d) # Check the Python version, we now have a minimum of Python 2.7.3 import sys if sys.hexversion < 0x020703F0: status.addresult('The system requires at least Python 2.7.3 to run. Please update your Python interpreter.\n') # Check the bitbake version meets minimum requirements from distutils.version import LooseVersion minversion = d.getVar('BB_MIN_VERSION', True) if (LooseVersion(bb.__version__) < LooseVersion(minversion)): status.addresult('Bitbake version %s is required and version %s was found\n' % (minversion, bb.__version__)) sanity_check_conffiles(status, d) paths = d.getVar('PATH', True).split(":") if "." in paths or "" in paths: status.addresult("PATH contains '.' or '' (empty element), which will break the build, please remove this.\nParsed PATH is " + str(paths) + "\n") # Check that the DISTRO is valid, if set # need to take into account DISTRO renaming DISTRO distro = d.getVar('DISTRO', True) if distro and distro != "nodistro": if not ( check_conf_exists("conf/distro/${DISTRO}.conf", d) or check_conf_exists("conf/distro/include/${DISTRO}.inc", d) ): status.addresult("DISTRO '%s' not found. Please set a valid DISTRO in your local.conf\n" % d.getVar("DISTRO", True)) # Check that DL_DIR is set, exists and is writable. In theory, we should never even hit the check if DL_DIR isn't # set, since so much relies on it being set. dldir = d.getVar('DL_DIR', True) if not dldir: status.addresult("DL_DIR is not set. Your environment is misconfigured, check that DL_DIR is set, and if the directory exists, that it is writable. \n") if os.path.exists(dldir) and not os.access(dldir, os.W_OK): status.addresult("DL_DIR: %s exists but you do not appear to have write access to it. \n" % dldir) # Check that the MACHINE is valid, if it is set machinevalid = True if d.getVar('MACHINE', True): if not check_conf_exists("conf/machine/${MACHINE}.conf", d): status.addresult('Please set a valid MACHINE in your local.conf or environment\n') machinevalid = False else: status.addresult(check_sanity_validmachine(d)) else: status.addresult('Please set a MACHINE in your local.conf or environment\n') machinevalid = False if machinevalid: status.addresult(check_toolchain(d)) # Check that the SDKMACHINE is valid, if it is set if d.getVar('SDKMACHINE', True): if not check_conf_exists("conf/machine-sdk/${SDKMACHINE}.conf", d): status.addresult('Specified SDKMACHINE value is not valid\n') check_supported_distro(d) # Check if DISPLAY is set if TEST_IMAGE is set if d.getVar('TEST_IMAGE', True) == '1' or d.getVar('DEFAULT_TEST_SUITES', True): display = d.getVar("BB_ORIGENV", False).getVar("DISPLAY", True) if not display: status.addresult('testimage needs an X desktop to start qemu, please set DISPLAY correctly (e.g. DISPLAY=:1.0)\n') omask = os.umask(022) if omask & 0755: status.addresult("Please use a umask which allows a+rx and u+rwx\n") os.umask(omask) if d.getVar('TARGET_ARCH', True) == "arm": # This path is no longer user-readable in modern (very recent) Linux try: if os.path.exists("/proc/sys/vm/mmap_min_addr"): f = open("/proc/sys/vm/mmap_min_addr", "r") try: if (int(f.read().strip()) > 65536): status.addresult("/proc/sys/vm/mmap_min_addr is not <= 65536. This will cause problems with qemu so please fix the value (as root).\n\nTo fix this in later reboots, set vm.mmap_min_addr = 65536 in /etc/sysctl.conf.\n") finally: f.close() except: pass oeroot = d.getVar('COREBASE', True) if oeroot.find('+') != -1: status.addresult("Error, you have an invalid character (+) in your COREBASE directory path. Please move the installation to a directory which doesn't include any + characters.") if oeroot.find('@') != -1: status.addresult("Error, you have an invalid character (@) in your COREBASE directory path. Please move the installation to a directory which doesn't include any @ characters.") if oeroot.find(' ') != -1: status.addresult("Error, you have a space in your COREBASE directory path. Please move the installation to a directory which doesn't include a space since autotools doesn't support this.") # Check that TMPDIR hasn't changed location since the last time we were run tmpdir = d.getVar('TMPDIR', True) checkfile = os.path.join(tmpdir, "saved_tmpdir") if os.path.exists(checkfile): with open(checkfile, "r") as f: saved_tmpdir = f.read().strip() if (saved_tmpdir != tmpdir): status.addresult("Error, TMPDIR has changed location. You need to either move it back to %s or rebuild\n" % saved_tmpdir) else: bb.utils.mkdirhier(tmpdir) with open(checkfile, "w") as f: f.write(tmpdir) def check_sanity(sanity_data): import subprocess class SanityStatus(object): def __init__(self): self.messages = "" self.network_error = False self.reparse = False def addresult(self, message): if message: self.messages = self.messages + message status = SanityStatus() tmpdir = sanity_data.getVar('TMPDIR', True) sstate_dir = sanity_data.getVar('SSTATE_DIR', True) # Check saved sanity info last_sanity_version = 0 last_tmpdir = "" last_sstate_dir = "" sanityverfile = sanity_data.expand("${TOPDIR}/conf/sanity_info") if os.path.exists(sanityverfile): with open(sanityverfile, 'r') as f: for line in f: if line.startswith('SANITY_VERSION'): last_sanity_version = int(line.split()[1]) if line.startswith('TMPDIR'): last_tmpdir = line.split()[1] if line.startswith('SSTATE_DIR'): last_sstate_dir = line.split()[1] check_sanity_everybuild(status, sanity_data) sanity_version = int(sanity_data.getVar('SANITY_VERSION', True) or 1) network_error = False if last_sanity_version < sanity_version: check_sanity_version_change(status, sanity_data) status.addresult(check_sanity_sstate_dir_change(sstate_dir, sanity_data)) else: if last_sstate_dir != sstate_dir: status.addresult(check_sanity_sstate_dir_change(sstate_dir, sanity_data)) if os.path.exists(os.path.dirname(sanityverfile)) and not status.messages: with open(sanityverfile, 'w') as f: f.write("SANITY_VERSION %s\n" % sanity_version) f.write("TMPDIR %s\n" % tmpdir) f.write("SSTATE_DIR %s\n" % sstate_dir) sanity_handle_abichanges(status, sanity_data) if status.messages != "": raise_sanity_error(sanity_data.expand(status.messages), sanity_data, status.network_error) return status.reparse # Create a copy of the datastore and finalise it to ensure appends and # overrides are set - the datastore has yet to be finalised at ConfigParsed def copy_data(e): sanity_data = bb.data.createCopy(e.data) sanity_data.finalize() return sanity_data addhandler check_sanity_eventhandler check_sanity_eventhandler[eventmask] = "bb.event.ConfigParsed bb.event.SanityCheck bb.event.NetworkTest" python check_sanity_eventhandler() { if bb.event.getName(e) == "ConfigParsed" and e.data.getVar("BB_WORKERCONTEXT", True) != "1" and e.data.getVar("DISABLE_SANITY_CHECKS", True) != "1": sanity_data = copy_data(e) reparse = check_sanity(sanity_data) e.data.setVar("BB_INVALIDCONF", reparse) elif bb.event.getName(e) == "SanityCheck": sanity_data = copy_data(e) sanity_data.setVar("SANITY_USE_EVENTS", "1") reparse = check_sanity(sanity_data) e.data.setVar("BB_INVALIDCONF", reparse) bb.event.fire(bb.event.SanityCheckPassed(), e.data) elif bb.event.getName(e) == "NetworkTest": sanity_data = copy_data(e) bb.event.fire(bb.event.NetworkTestFailed() if check_connectivity(sanity_data) else bb.event.NetworkTestPassed(), e.data) return }