diff options
Diffstat (limited to 'meta/classes/icecc.bbclass')
-rw-r--r-- | meta/classes/icecc.bbclass | 296 |
1 files changed, 211 insertions, 85 deletions
diff --git a/meta/classes/icecc.bbclass b/meta/classes/icecc.bbclass index e1c06c49cb..bc3d6f4cc8 100644 --- a/meta/classes/icecc.bbclass +++ b/meta/classes/icecc.bbclass @@ -28,70 +28,106 @@ #Error checking is kept to minimum so double check any parameters you pass to the class ########################################################################################### -BB_HASHBASE_WHITELIST += "ICECC_PARALLEL_MAKE ICECC_DISABLED ICECC_USER_PACKAGE_BL ICECC_USER_CLASS_BL ICECC_USER_PACKAGE_WL ICECC_PATH ICECC_ENV_EXEC" +BB_HASHBASE_WHITELIST += "ICECC_PARALLEL_MAKE ICECC_DISABLED ICECC_USER_PACKAGE_BL \ + ICECC_USER_CLASS_BL ICECC_USER_PACKAGE_WL ICECC_PATH ICECC_ENV_EXEC \ + ICECC_CARET_WORKAROUND ICECC_CFLAGS ICECC_ENV_VERSION \ + ICECC_DEBUG ICECC_LOGFILE ICECC_REPEAT_RATE ICECC_PREFERRED_HOST \ + ICECC_CLANG_REMOTE_CPP ICECC_IGNORE_UNVERIFIED ICECC_TEST_SOCKET \ + ICECC_ENV_DEBUG ICECC_SYSTEM_PACKAGE_BL ICECC_SYSTEM_CLASS_BL \ + ICECC_REMOTE_CPP \ + " ICECC_ENV_EXEC ?= "${STAGING_BINDIR_NATIVE}/icecc-create-env" +HOSTTOOLS_NONFATAL += "icecc patchelf" + +# This version can be incremented when changes are made to the environment that +# invalidate the version on the compile nodes. Changing it will cause a new +# environment to be created. +# +# A useful thing to do for testing Icecream changes locally is to add a +# subversion in local.conf: +# ICECC_ENV_VERSION_append = "-my-ver-1" +ICECC_ENV_VERSION = "2" + +# Default to disabling the caret workaround, If set to "1" in local.conf, icecc +# will locally recompile any files that have warnings, which can adversely +# affect performance. +# +# See: https://github.com/icecc/icecream/issues/190 +export ICECC_CARET_WORKAROUND ??= "0" + +export ICECC_REMOTE_CPP ??= "0" + +ICECC_CFLAGS = "" +CFLAGS += "${ICECC_CFLAGS}" +CXXFLAGS += "${ICECC_CFLAGS}" + +# Debug flags when generating environments +ICECC_ENV_DEBUG ??= "" + +# "system" recipe blacklist contains a list of packages that can not distribute +# compile tasks for one reason or the other. When adding new entry, please +# document why (how it failed) so that we can re-evaluate it later e.g. when +# there is new version +# +# libgcc-initial - fails with CPP sanity check error if host sysroot contains +# cross gcc built for another target tune/variant +# pixman - prng_state: TLS reference mismatches non-TLS reference, possibly due to +# pragma omp threadprivate(prng_state) +# systemtap - _HelperSDT.c undefs macros and uses the identifiers in macros emitting +# inline assembly +# target-sdk-provides-dummy - ${HOST_PREFIX} is empty which triggers the "NULL +# prefix" error. +ICECC_SYSTEM_PACKAGE_BL += "\ + libgcc-initial \ + pixman \ + systemtap \ + target-sdk-provides-dummy \ + " + +# "system" classes that should be blacklisted. When adding new entry, please +# document why (how it failed) so that we can re-evaluate it later +# +# image - Image aren't compiling, but the testing framework for images captures +# PARALLEL_MAKE as part of the test environment. Many tests won't use +# icecream, but leaving the high level of parallelism can cause them to +# consume an unnecessary amount of resources. +ICECC_SYSTEM_CLASS_BL += "\ + image \ + " + def icecc_dep_prepend(d): # INHIBIT_DEFAULT_DEPS doesn't apply to the patch command. Whether or not # we need that built is the responsibility of the patch function / class, not # the application. - if not d.getVar('INHIBIT_DEFAULT_DEPS', False): + if not d.getVar('INHIBIT_DEFAULT_DEPS'): return "icecc-create-env-native" return "" DEPENDS_prepend += "${@icecc_dep_prepend(d)} " +get_cross_kernel_cc[vardepsexclude] += "KERNEL_CC" def get_cross_kernel_cc(bb,d): - kernel_cc = d.getVar('KERNEL_CC', False) + if not icecc_is_kernel(bb, d): + return None # evaluate the expression by the shell if necessary + kernel_cc = d.getVar('KERNEL_CC') if '`' in kernel_cc or '$(' in kernel_cc: - kernel_cc = os.popen("echo %s" % kernel_cc).read()[:-1] + import subprocess + kernel_cc = subprocess.check_output("echo %s" % kernel_cc, shell=True).decode("utf-8")[:-1] - kernel_cc = d.expand(kernel_cc) kernel_cc = kernel_cc.replace('ccache', '').strip() kernel_cc = kernel_cc.split(' ')[0] kernel_cc = kernel_cc.strip() return kernel_cc def get_icecc(d): - return d.getVar('ICECC_PATH', False) or bb.utils.which(os.getenv("PATH"), "icecc") - -def create_path(compilers, bb, d): - """ - Create Symlinks for the icecc in the staging directory - """ - staging = os.path.join(d.expand('${STAGING_BINDIR}'), "ice") - if icecc_is_kernel(bb, d): - staging += "-kernel" - - #check if the icecc path is set by the user - icecc = get_icecc(d) - - # Create the dir if necessary - try: - os.stat(staging) - except: - try: - os.makedirs(staging) - except: - pass - - for compiler in compilers: - gcc_path = os.path.join(staging, compiler) - try: - os.stat(gcc_path) - except: - try: - os.symlink(icecc, gcc_path) - except: - pass - - return staging + return d.getVar('ICECC_PATH') or bb.utils.which(os.getenv("PATH"), "icecc") def use_icecc(bb,d): - if d.getVar('ICECC_DISABLED', False) == "1": + if d.getVar('ICECC_DISABLED') == "1": # don't even try it, when explicitly disabled return "no" @@ -99,10 +135,24 @@ def use_icecc(bb,d): if icecc_is_allarch(bb, d): return "no" - pn = d.getVar('PN', True) + if icecc_is_cross_canadian(bb, d): + return "no" - system_class_blacklist = [] - user_class_blacklist = (d.getVar('ICECC_USER_CLASS_BL', False) or "none").split() + if d.getVar('INHIBIT_DEFAULT_DEPS', False): + # We don't have a compiler, so no icecc + return "no" + + pn = d.getVar('PN') + bpn = d.getVar('BPN') + + # Blacklist/whitelist checks are made against BPN, because there is a good + # chance that if icecc should be skipped for a recipe, it should be skipped + # for all the variants of that recipe. PN is still checked in case a user + # specified a more specific recipe. + check_pn = set([pn, bpn]) + + system_class_blacklist = (d.getVar('ICECC_SYSTEM_CLASS_BL') or "").split() + user_class_blacklist = (d.getVar('ICECC_USER_CLASS_BL') or "none").split() package_class_blacklist = system_class_blacklist + user_class_blacklist for black in package_class_blacklist: @@ -110,35 +160,27 @@ def use_icecc(bb,d): bb.debug(1, "%s: class %s found in blacklist, disable icecc" % (pn, black)) return "no" - # "system" recipe blacklist contains a list of packages that can not distribute compile tasks - # for one reason or the other - # this is the old list (which doesn't seem to be valid anymore, because I was able to build - # all these with icecc enabled) - # system_package_blacklist = [ "uclibc", "glibc", "gcc", "bind", "u-boot", "dhcp-forwarder", "enchant", "connman", "orbit2" ] - # when adding new entry, please document why (how it failed) so that we can re-evaluate it later - # e.g. when there is new version - # building libgcc-initial with icecc fails with CPP sanity check error if host sysroot contains cross gcc built for another target tune/variant - system_package_blacklist = ["libgcc-initial"] - user_package_blacklist = (d.getVar('ICECC_USER_PACKAGE_BL', False) or "").split() - user_package_whitelist = (d.getVar('ICECC_USER_PACKAGE_WL', False) or "").split() + system_package_blacklist = (d.getVar('ICECC_SYSTEM_PACKAGE_BL') or "").split() + user_package_blacklist = (d.getVar('ICECC_USER_PACKAGE_BL') or "").split() + user_package_whitelist = (d.getVar('ICECC_USER_PACKAGE_WL') or "").split() package_blacklist = system_package_blacklist + user_package_blacklist - if pn in package_blacklist: + if check_pn & set(package_blacklist): bb.debug(1, "%s: found in blacklist, disable icecc" % pn) return "no" - if pn in user_package_whitelist: + if check_pn & set(user_package_whitelist): bb.debug(1, "%s: found in whitelist, enable icecc" % pn) return "yes" - if d.getVar('PARALLEL_MAKE', False) == "": + if d.getVar('PARALLEL_MAKE') == "": bb.debug(1, "%s: has empty PARALLEL_MAKE, disable icecc" % pn) return "no" return "yes" def icecc_is_allarch(bb, d): - return d.getVar("PACKAGE_ARCH", True) == "all" or bb.data.inherits_class('allarch', d) + return d.getVar("PACKAGE_ARCH") == "all" def icecc_is_kernel(bb, d): return \ @@ -149,16 +191,27 @@ def icecc_is_native(bb, d): bb.data.inherits_class("cross", d) or \ bb.data.inherits_class("native", d); +def icecc_is_cross_canadian(bb, d): + return bb.data.inherits_class("cross-canadian", d) + +def icecc_dir(bb, d): + return d.expand('${TMPDIR}/work-shared/ice') + # Don't pollute allarch signatures with TARGET_FPU icecc_version[vardepsexclude] += "TARGET_FPU" def icecc_version(bb, d): if use_icecc(bb, d) == "no": return "" - parallel = d.getVar('ICECC_PARALLEL_MAKE', False) or "" - if not d.getVar('PARALLEL_MAKE', False) == "" and parallel: + parallel = d.getVar('ICECC_PARALLEL_MAKE') or "" + if not d.getVar('PARALLEL_MAKE') == "" and parallel: d.setVar("PARALLEL_MAKE", parallel) + # Disable showing the caret in the GCC compiler output if the workaround is + # disabled + if d.getVar('ICECC_CARET_WORKAROUND') == '0': + d.setVar('ICECC_CFLAGS', '-fno-diagnostics-show-caret') + if icecc_is_native(bb, d): archive_name = "local-host-env" elif d.expand('${HOST_PREFIX}') == "": @@ -167,14 +220,18 @@ def icecc_version(bb, d): prefix = d.expand('${HOST_PREFIX}' ) distro = d.expand('${DISTRO}') target_sys = d.expand('${TARGET_SYS}') - float = d.getVar('TARGET_FPU', False) or "hard" + float = d.getVar('TARGET_FPU') or "hard" archive_name = prefix + distro + "-" + target_sys + "-" + float if icecc_is_kernel(bb, d): archive_name += "-kernel" import socket - ice_dir = d.expand('${STAGING_DIR_NATIVE}${prefix_native}') - tar_file = os.path.join(ice_dir, 'ice', archive_name + "-@VERSION@-" + socket.gethostname() + '.tar.gz') + ice_dir = icecc_dir(bb, d) + tar_file = os.path.join(ice_dir, "{archive}-{version}-@VERSION@-{hostname}.tar.gz".format( + archive=archive_name, + version=d.getVar('ICECC_ENV_VERSION'), + hostname=socket.gethostname() + )) return tar_file @@ -183,46 +240,70 @@ def icecc_path(bb,d): # don't create unnecessary directories when icecc is disabled return + staging = os.path.join(d.expand('${STAGING_BINDIR}'), "ice") if icecc_is_kernel(bb, d): - return create_path( [get_cross_kernel_cc(bb,d), ], bb, d) + staging += "-kernel" - else: - prefix = d.expand('${HOST_PREFIX}') - return create_path( [prefix+"gcc", prefix+"g++"], bb, d) + return staging def icecc_get_external_tool(bb, d, tool): external_toolchain_bindir = d.expand('${EXTERNAL_TOOLCHAIN}${bindir_cross}') target_prefix = d.expand('${TARGET_PREFIX}') return os.path.join(external_toolchain_bindir, '%s%s' % (target_prefix, tool)) +def icecc_get_tool_link(tool, d): + import subprocess + try: + return subprocess.check_output("readlink -f %s" % tool, shell=True).decode("utf-8")[:-1] + except subprocess.CalledProcessError as e: + bb.note("icecc: one of the tools probably disappeared during recipe parsing, cmd readlink -f %s returned %d:\n%s" % (tool, e.returncode, e.output.decode("utf-8"))) + return tool + +def icecc_get_path_tool(tool, d): + # This is a little ugly, but we want to make sure we add an actual + # compiler to the toolchain, not ccache. Some distros (e.g. Fedora) + # have ccache enabled by default using symlinks PATH, meaning ccache + # would be found first when looking for the compiler. + paths = os.getenv("PATH").split(':') + while True: + p, hist = bb.utils.which(':'.join(paths), tool, history=True) + if not p or os.path.basename(icecc_get_tool_link(p, d)) != 'ccache': + return p + paths = paths[len(hist):] + + return "" + # Don't pollute native signatures with target TUNE_PKGARCH through STAGING_BINDIR_TOOLCHAIN icecc_get_tool[vardepsexclude] += "STAGING_BINDIR_TOOLCHAIN" def icecc_get_tool(bb, d, tool): if icecc_is_native(bb, d): - return bb.utils.which(os.getenv("PATH"), tool) + return icecc_get_path_tool(tool, d) elif icecc_is_kernel(bb, d): - return bb.utils.which(os.getenv("PATH"), get_cross_kernel_cc(bb, d)) + return icecc_get_path_tool(get_cross_kernel_cc(bb, d), d) else: ice_dir = d.expand('${STAGING_BINDIR_TOOLCHAIN}') target_sys = d.expand('${TARGET_SYS}') - tool_bin = os.path.join(ice_dir, "%s-%s" % (target_sys, tool)) - if os.path.isfile(tool_bin): - return tool_bin - else: - external_tool_bin = icecc_get_external_tool(bb, d, tool) - if os.path.isfile(external_tool_bin): - return external_tool_bin - else: - return "" + for p in ice_dir.split(':'): + tool_bin = os.path.join(p, "%s-%s" % (target_sys, tool)) + if os.path.isfile(tool_bin): + return tool_bin + external_tool_bin = icecc_get_external_tool(bb, d, tool) + if os.path.isfile(external_tool_bin): + return external_tool_bin + return "" def icecc_get_and_check_tool(bb, d, tool): # Check that g++ or gcc is not a symbolic link to icecc binary in # PATH or icecc-create-env script will silently create an invalid # compiler environment package. t = icecc_get_tool(bb, d, tool) - if t and os.popen("readlink -f %s" % t).read()[:-1] == get_icecc(d): - bb.error("%s is a symlink to %s in PATH and this prevents icecc from working" % (t, get_icecc(d))) - return "" + if t: + link_path = icecc_get_tool_link(t, d) + if link_path == get_icecc(d): + bb.error("%s is a symlink to %s in PATH and this prevents icecc from working" % (t, link_path)) + return "" + else: + return t else: return t @@ -245,6 +326,7 @@ def set_icecc_env(): # dummy python version of set_icecc_env return +set_icecc_env[vardepsexclude] += "KERNEL_CC" set_icecc_env() { if [ "${@use_icecc(bb, d)}" = "no" ] then @@ -264,6 +346,16 @@ set_icecc_env() { return fi + ICECC_BIN="${@get_icecc(d)}" + if [ -z "${ICECC_BIN}" ]; then + bbwarn "Cannot use icecc: icecc binary not found" + return + fi + if [ -z "$(which patchelf patchelf-uninative)" ]; then + bbwarn "Cannot use icecc: patchelf not found" + return + fi + ICECC_CC="${@icecc_get_and_check_tool(bb, d, "gcc")}" ICECC_CXX="${@icecc_get_and_check_tool(bb, d, "g++")}" # cannot use icecc_get_and_check_tool here because it assumes as without target_sys prefix @@ -282,6 +374,26 @@ set_icecc_env() { return fi + # Create symlinks to icecc and wrapper-scripts in the recipe-sysroot directory + mkdir -p $ICE_PATH/symlinks + if [ -n "${KERNEL_CC}" ]; then + compilers="${@get_cross_kernel_cc(bb,d)}" + else + compilers="${HOST_PREFIX}gcc ${HOST_PREFIX}g++" + fi + for compiler in $compilers; do + ln -sf $ICECC_BIN $ICE_PATH/symlinks/$compiler + rm -f $ICE_PATH/$compiler + cat <<-__EOF__ > $ICE_PATH/$compiler + #!/bin/sh -e + export ICECC_VERSION=$ICECC_VERSION + export ICECC_CC=$ICECC_CC + export ICECC_CXX=$ICECC_CXX + $ICE_PATH/symlinks/$compiler "\$@" + __EOF__ + chmod 775 $ICE_PATH/$compiler + done + ICECC_AS="`${ICECC_CC} -print-prog-name=as`" # for target recipes should return something like: # /OE/tmp-eglibc/sysroots/x86_64-linux/usr/libexec/arm920tt-oe-linux-gnueabi/gcc/arm-oe-linux-gnueabi/4.8.2/as @@ -298,10 +410,10 @@ set_icecc_env() { # the ICECC_VERSION generation step must be locked by a mutex # in order to prevent race conditions if flock -n "${ICECC_VERSION}.lock" \ - ${ICECC_ENV_EXEC} "${ICECC_CC}" "${ICECC_CXX}" "${ICECC_AS}" "${ICECC_VERSION}" + ${ICECC_ENV_EXEC} ${ICECC_ENV_DEBUG} "${ICECC_CC}" "${ICECC_CXX}" "${ICECC_AS}" "${ICECC_VERSION}" then touch "${ICECC_VERSION}.done" - elif [ ! wait_for_file "${ICECC_VERSION}.done" 30 ] + elif ! wait_for_file "${ICECC_VERSION}.done" 30 then # locking failed so wait for ${ICECC_VERSION}.done to appear bbwarn "Timeout waiting for ${ICECC_VERSION}.done" @@ -309,11 +421,15 @@ set_icecc_env() { fi fi - export ICECC_VERSION ICECC_CC ICECC_CXX - export PATH="$ICE_PATH:$PATH" + # Don't let ccache find the icecream compiler links that have been created, otherwise + # it can end up invoking icecream recursively. export CCACHE_PATH="$PATH" + export CCACHE_DISABLE="1" + + export PATH="$ICE_PATH:$PATH" - bbnote "Using icecc" + bbnote "Using icecc path: $ICE_PATH" + bbnote "Using icecc tarball: $ICECC_VERSION" } do_configure_prepend() { @@ -331,3 +447,13 @@ do_compile_kernelmodules_prepend() { do_install_prepend() { set_icecc_env } + +# IceCream is not (currently) supported in the extensible SDK +ICECC_SDK_HOST_TASK = "nativesdk-icecc-toolchain" +ICECC_SDK_HOST_TASK_task-populate-sdk-ext = "" + +# Don't include IceCream in uninative tarball +ICECC_SDK_HOST_TASK_pn-uninative-tarball = "" + +# Add the toolchain scripts to the SDK +TOOLCHAIN_HOST_TASK_append = " ${ICECC_SDK_HOST_TASK}" |