summaryrefslogtreecommitdiffstats
path: root/meta/classes/icecc.bbclass
diff options
context:
space:
mode:
Diffstat (limited to 'meta/classes/icecc.bbclass')
-rw-r--r--meta/classes/icecc.bbclass296
1 files changed, 211 insertions, 85 deletions
diff --git a/meta/classes/icecc.bbclass b/meta/classes/icecc.bbclass
index e1c06c49cb..bc3d6f4cc8 100644
--- a/meta/classes/icecc.bbclass
+++ b/meta/classes/icecc.bbclass
@@ -28,70 +28,106 @@
#Error checking is kept to minimum so double check any parameters you pass to the class
###########################################################################################
-BB_HASHBASE_WHITELIST += "ICECC_PARALLEL_MAKE ICECC_DISABLED ICECC_USER_PACKAGE_BL ICECC_USER_CLASS_BL ICECC_USER_PACKAGE_WL ICECC_PATH ICECC_ENV_EXEC"
+BB_HASHBASE_WHITELIST += "ICECC_PARALLEL_MAKE ICECC_DISABLED ICECC_USER_PACKAGE_BL \
+ ICECC_USER_CLASS_BL ICECC_USER_PACKAGE_WL ICECC_PATH ICECC_ENV_EXEC \
+ ICECC_CARET_WORKAROUND ICECC_CFLAGS ICECC_ENV_VERSION \
+ ICECC_DEBUG ICECC_LOGFILE ICECC_REPEAT_RATE ICECC_PREFERRED_HOST \
+ ICECC_CLANG_REMOTE_CPP ICECC_IGNORE_UNVERIFIED ICECC_TEST_SOCKET \
+ ICECC_ENV_DEBUG ICECC_SYSTEM_PACKAGE_BL ICECC_SYSTEM_CLASS_BL \
+ ICECC_REMOTE_CPP \
+ "
ICECC_ENV_EXEC ?= "${STAGING_BINDIR_NATIVE}/icecc-create-env"
+HOSTTOOLS_NONFATAL += "icecc patchelf"
+
+# This version can be incremented when changes are made to the environment that
+# invalidate the version on the compile nodes. Changing it will cause a new
+# environment to be created.
+#
+# A useful thing to do for testing Icecream changes locally is to add a
+# subversion in local.conf:
+# ICECC_ENV_VERSION_append = "-my-ver-1"
+ICECC_ENV_VERSION = "2"
+
+# Default to disabling the caret workaround, If set to "1" in local.conf, icecc
+# will locally recompile any files that have warnings, which can adversely
+# affect performance.
+#
+# See: https://github.com/icecc/icecream/issues/190
+export ICECC_CARET_WORKAROUND ??= "0"
+
+export ICECC_REMOTE_CPP ??= "0"
+
+ICECC_CFLAGS = ""
+CFLAGS += "${ICECC_CFLAGS}"
+CXXFLAGS += "${ICECC_CFLAGS}"
+
+# Debug flags when generating environments
+ICECC_ENV_DEBUG ??= ""
+
+# "system" recipe blacklist contains a list of packages that can not distribute
+# compile tasks for one reason or the other. When adding new entry, please
+# document why (how it failed) so that we can re-evaluate it later e.g. when
+# there is new version
+#
+# libgcc-initial - fails with CPP sanity check error if host sysroot contains
+# cross gcc built for another target tune/variant
+# pixman - prng_state: TLS reference mismatches non-TLS reference, possibly due to
+# pragma omp threadprivate(prng_state)
+# systemtap - _HelperSDT.c undefs macros and uses the identifiers in macros emitting
+# inline assembly
+# target-sdk-provides-dummy - ${HOST_PREFIX} is empty which triggers the "NULL
+# prefix" error.
+ICECC_SYSTEM_PACKAGE_BL += "\
+ libgcc-initial \
+ pixman \
+ systemtap \
+ target-sdk-provides-dummy \
+ "
+
+# "system" classes that should be blacklisted. When adding new entry, please
+# document why (how it failed) so that we can re-evaluate it later
+#
+# image - Image aren't compiling, but the testing framework for images captures
+# PARALLEL_MAKE as part of the test environment. Many tests won't use
+# icecream, but leaving the high level of parallelism can cause them to
+# consume an unnecessary amount of resources.
+ICECC_SYSTEM_CLASS_BL += "\
+ image \
+ "
+
def icecc_dep_prepend(d):
# INHIBIT_DEFAULT_DEPS doesn't apply to the patch command. Whether or not
# we need that built is the responsibility of the patch function / class, not
# the application.
- if not d.getVar('INHIBIT_DEFAULT_DEPS', False):
+ if not d.getVar('INHIBIT_DEFAULT_DEPS'):
return "icecc-create-env-native"
return ""
DEPENDS_prepend += "${@icecc_dep_prepend(d)} "
+get_cross_kernel_cc[vardepsexclude] += "KERNEL_CC"
def get_cross_kernel_cc(bb,d):
- kernel_cc = d.getVar('KERNEL_CC', False)
+ if not icecc_is_kernel(bb, d):
+ return None
# evaluate the expression by the shell if necessary
+ kernel_cc = d.getVar('KERNEL_CC')
if '`' in kernel_cc or '$(' in kernel_cc:
- kernel_cc = os.popen("echo %s" % kernel_cc).read()[:-1]
+ import subprocess
+ kernel_cc = subprocess.check_output("echo %s" % kernel_cc, shell=True).decode("utf-8")[:-1]
- kernel_cc = d.expand(kernel_cc)
kernel_cc = kernel_cc.replace('ccache', '').strip()
kernel_cc = kernel_cc.split(' ')[0]
kernel_cc = kernel_cc.strip()
return kernel_cc
def get_icecc(d):
- return d.getVar('ICECC_PATH', False) or bb.utils.which(os.getenv("PATH"), "icecc")
-
-def create_path(compilers, bb, d):
- """
- Create Symlinks for the icecc in the staging directory
- """
- staging = os.path.join(d.expand('${STAGING_BINDIR}'), "ice")
- if icecc_is_kernel(bb, d):
- staging += "-kernel"
-
- #check if the icecc path is set by the user
- icecc = get_icecc(d)
-
- # Create the dir if necessary
- try:
- os.stat(staging)
- except:
- try:
- os.makedirs(staging)
- except:
- pass
-
- for compiler in compilers:
- gcc_path = os.path.join(staging, compiler)
- try:
- os.stat(gcc_path)
- except:
- try:
- os.symlink(icecc, gcc_path)
- except:
- pass
-
- return staging
+ return d.getVar('ICECC_PATH') or bb.utils.which(os.getenv("PATH"), "icecc")
def use_icecc(bb,d):
- if d.getVar('ICECC_DISABLED', False) == "1":
+ if d.getVar('ICECC_DISABLED') == "1":
# don't even try it, when explicitly disabled
return "no"
@@ -99,10 +135,24 @@ def use_icecc(bb,d):
if icecc_is_allarch(bb, d):
return "no"
- pn = d.getVar('PN', True)
+ if icecc_is_cross_canadian(bb, d):
+ return "no"
- system_class_blacklist = []
- user_class_blacklist = (d.getVar('ICECC_USER_CLASS_BL', False) or "none").split()
+ if d.getVar('INHIBIT_DEFAULT_DEPS', False):
+ # We don't have a compiler, so no icecc
+ return "no"
+
+ pn = d.getVar('PN')
+ bpn = d.getVar('BPN')
+
+ # Blacklist/whitelist checks are made against BPN, because there is a good
+ # chance that if icecc should be skipped for a recipe, it should be skipped
+ # for all the variants of that recipe. PN is still checked in case a user
+ # specified a more specific recipe.
+ check_pn = set([pn, bpn])
+
+ system_class_blacklist = (d.getVar('ICECC_SYSTEM_CLASS_BL') or "").split()
+ user_class_blacklist = (d.getVar('ICECC_USER_CLASS_BL') or "none").split()
package_class_blacklist = system_class_blacklist + user_class_blacklist
for black in package_class_blacklist:
@@ -110,35 +160,27 @@ def use_icecc(bb,d):
bb.debug(1, "%s: class %s found in blacklist, disable icecc" % (pn, black))
return "no"
- # "system" recipe blacklist contains a list of packages that can not distribute compile tasks
- # for one reason or the other
- # this is the old list (which doesn't seem to be valid anymore, because I was able to build
- # all these with icecc enabled)
- # system_package_blacklist = [ "uclibc", "glibc", "gcc", "bind", "u-boot", "dhcp-forwarder", "enchant", "connman", "orbit2" ]
- # when adding new entry, please document why (how it failed) so that we can re-evaluate it later
- # e.g. when there is new version
- # building libgcc-initial with icecc fails with CPP sanity check error if host sysroot contains cross gcc built for another target tune/variant
- system_package_blacklist = ["libgcc-initial"]
- user_package_blacklist = (d.getVar('ICECC_USER_PACKAGE_BL', False) or "").split()
- user_package_whitelist = (d.getVar('ICECC_USER_PACKAGE_WL', False) or "").split()
+ system_package_blacklist = (d.getVar('ICECC_SYSTEM_PACKAGE_BL') or "").split()
+ user_package_blacklist = (d.getVar('ICECC_USER_PACKAGE_BL') or "").split()
+ user_package_whitelist = (d.getVar('ICECC_USER_PACKAGE_WL') or "").split()
package_blacklist = system_package_blacklist + user_package_blacklist
- if pn in package_blacklist:
+ if check_pn & set(package_blacklist):
bb.debug(1, "%s: found in blacklist, disable icecc" % pn)
return "no"
- if pn in user_package_whitelist:
+ if check_pn & set(user_package_whitelist):
bb.debug(1, "%s: found in whitelist, enable icecc" % pn)
return "yes"
- if d.getVar('PARALLEL_MAKE', False) == "":
+ if d.getVar('PARALLEL_MAKE') == "":
bb.debug(1, "%s: has empty PARALLEL_MAKE, disable icecc" % pn)
return "no"
return "yes"
def icecc_is_allarch(bb, d):
- return d.getVar("PACKAGE_ARCH", True) == "all" or bb.data.inherits_class('allarch', d)
+ return d.getVar("PACKAGE_ARCH") == "all"
def icecc_is_kernel(bb, d):
return \
@@ -149,16 +191,27 @@ def icecc_is_native(bb, d):
bb.data.inherits_class("cross", d) or \
bb.data.inherits_class("native", d);
+def icecc_is_cross_canadian(bb, d):
+ return bb.data.inherits_class("cross-canadian", d)
+
+def icecc_dir(bb, d):
+ return d.expand('${TMPDIR}/work-shared/ice')
+
# Don't pollute allarch signatures with TARGET_FPU
icecc_version[vardepsexclude] += "TARGET_FPU"
def icecc_version(bb, d):
if use_icecc(bb, d) == "no":
return ""
- parallel = d.getVar('ICECC_PARALLEL_MAKE', False) or ""
- if not d.getVar('PARALLEL_MAKE', False) == "" and parallel:
+ parallel = d.getVar('ICECC_PARALLEL_MAKE') or ""
+ if not d.getVar('PARALLEL_MAKE') == "" and parallel:
d.setVar("PARALLEL_MAKE", parallel)
+ # Disable showing the caret in the GCC compiler output if the workaround is
+ # disabled
+ if d.getVar('ICECC_CARET_WORKAROUND') == '0':
+ d.setVar('ICECC_CFLAGS', '-fno-diagnostics-show-caret')
+
if icecc_is_native(bb, d):
archive_name = "local-host-env"
elif d.expand('${HOST_PREFIX}') == "":
@@ -167,14 +220,18 @@ def icecc_version(bb, d):
prefix = d.expand('${HOST_PREFIX}' )
distro = d.expand('${DISTRO}')
target_sys = d.expand('${TARGET_SYS}')
- float = d.getVar('TARGET_FPU', False) or "hard"
+ float = d.getVar('TARGET_FPU') or "hard"
archive_name = prefix + distro + "-" + target_sys + "-" + float
if icecc_is_kernel(bb, d):
archive_name += "-kernel"
import socket
- ice_dir = d.expand('${STAGING_DIR_NATIVE}${prefix_native}')
- tar_file = os.path.join(ice_dir, 'ice', archive_name + "-@VERSION@-" + socket.gethostname() + '.tar.gz')
+ ice_dir = icecc_dir(bb, d)
+ tar_file = os.path.join(ice_dir, "{archive}-{version}-@VERSION@-{hostname}.tar.gz".format(
+ archive=archive_name,
+ version=d.getVar('ICECC_ENV_VERSION'),
+ hostname=socket.gethostname()
+ ))
return tar_file
@@ -183,46 +240,70 @@ def icecc_path(bb,d):
# don't create unnecessary directories when icecc is disabled
return
+ staging = os.path.join(d.expand('${STAGING_BINDIR}'), "ice")
if icecc_is_kernel(bb, d):
- return create_path( [get_cross_kernel_cc(bb,d), ], bb, d)
+ staging += "-kernel"
- else:
- prefix = d.expand('${HOST_PREFIX}')
- return create_path( [prefix+"gcc", prefix+"g++"], bb, d)
+ return staging
def icecc_get_external_tool(bb, d, tool):
external_toolchain_bindir = d.expand('${EXTERNAL_TOOLCHAIN}${bindir_cross}')
target_prefix = d.expand('${TARGET_PREFIX}')
return os.path.join(external_toolchain_bindir, '%s%s' % (target_prefix, tool))
+def icecc_get_tool_link(tool, d):
+ import subprocess
+ try:
+ return subprocess.check_output("readlink -f %s" % tool, shell=True).decode("utf-8")[:-1]
+ except subprocess.CalledProcessError as e:
+ bb.note("icecc: one of the tools probably disappeared during recipe parsing, cmd readlink -f %s returned %d:\n%s" % (tool, e.returncode, e.output.decode("utf-8")))
+ return tool
+
+def icecc_get_path_tool(tool, d):
+ # This is a little ugly, but we want to make sure we add an actual
+ # compiler to the toolchain, not ccache. Some distros (e.g. Fedora)
+ # have ccache enabled by default using symlinks PATH, meaning ccache
+ # would be found first when looking for the compiler.
+ paths = os.getenv("PATH").split(':')
+ while True:
+ p, hist = bb.utils.which(':'.join(paths), tool, history=True)
+ if not p or os.path.basename(icecc_get_tool_link(p, d)) != 'ccache':
+ return p
+ paths = paths[len(hist):]
+
+ return ""
+
# Don't pollute native signatures with target TUNE_PKGARCH through STAGING_BINDIR_TOOLCHAIN
icecc_get_tool[vardepsexclude] += "STAGING_BINDIR_TOOLCHAIN"
def icecc_get_tool(bb, d, tool):
if icecc_is_native(bb, d):
- return bb.utils.which(os.getenv("PATH"), tool)
+ return icecc_get_path_tool(tool, d)
elif icecc_is_kernel(bb, d):
- return bb.utils.which(os.getenv("PATH"), get_cross_kernel_cc(bb, d))
+ return icecc_get_path_tool(get_cross_kernel_cc(bb, d), d)
else:
ice_dir = d.expand('${STAGING_BINDIR_TOOLCHAIN}')
target_sys = d.expand('${TARGET_SYS}')
- tool_bin = os.path.join(ice_dir, "%s-%s" % (target_sys, tool))
- if os.path.isfile(tool_bin):
- return tool_bin
- else:
- external_tool_bin = icecc_get_external_tool(bb, d, tool)
- if os.path.isfile(external_tool_bin):
- return external_tool_bin
- else:
- return ""
+ for p in ice_dir.split(':'):
+ tool_bin = os.path.join(p, "%s-%s" % (target_sys, tool))
+ if os.path.isfile(tool_bin):
+ return tool_bin
+ external_tool_bin = icecc_get_external_tool(bb, d, tool)
+ if os.path.isfile(external_tool_bin):
+ return external_tool_bin
+ return ""
def icecc_get_and_check_tool(bb, d, tool):
# Check that g++ or gcc is not a symbolic link to icecc binary in
# PATH or icecc-create-env script will silently create an invalid
# compiler environment package.
t = icecc_get_tool(bb, d, tool)
- if t and os.popen("readlink -f %s" % t).read()[:-1] == get_icecc(d):
- bb.error("%s is a symlink to %s in PATH and this prevents icecc from working" % (t, get_icecc(d)))
- return ""
+ if t:
+ link_path = icecc_get_tool_link(t, d)
+ if link_path == get_icecc(d):
+ bb.error("%s is a symlink to %s in PATH and this prevents icecc from working" % (t, link_path))
+ return ""
+ else:
+ return t
else:
return t
@@ -245,6 +326,7 @@ def set_icecc_env():
# dummy python version of set_icecc_env
return
+set_icecc_env[vardepsexclude] += "KERNEL_CC"
set_icecc_env() {
if [ "${@use_icecc(bb, d)}" = "no" ]
then
@@ -264,6 +346,16 @@ set_icecc_env() {
return
fi
+ ICECC_BIN="${@get_icecc(d)}"
+ if [ -z "${ICECC_BIN}" ]; then
+ bbwarn "Cannot use icecc: icecc binary not found"
+ return
+ fi
+ if [ -z "$(which patchelf patchelf-uninative)" ]; then
+ bbwarn "Cannot use icecc: patchelf not found"
+ return
+ fi
+
ICECC_CC="${@icecc_get_and_check_tool(bb, d, "gcc")}"
ICECC_CXX="${@icecc_get_and_check_tool(bb, d, "g++")}"
# cannot use icecc_get_and_check_tool here because it assumes as without target_sys prefix
@@ -282,6 +374,26 @@ set_icecc_env() {
return
fi
+ # Create symlinks to icecc and wrapper-scripts in the recipe-sysroot directory
+ mkdir -p $ICE_PATH/symlinks
+ if [ -n "${KERNEL_CC}" ]; then
+ compilers="${@get_cross_kernel_cc(bb,d)}"
+ else
+ compilers="${HOST_PREFIX}gcc ${HOST_PREFIX}g++"
+ fi
+ for compiler in $compilers; do
+ ln -sf $ICECC_BIN $ICE_PATH/symlinks/$compiler
+ rm -f $ICE_PATH/$compiler
+ cat <<-__EOF__ > $ICE_PATH/$compiler
+ #!/bin/sh -e
+ export ICECC_VERSION=$ICECC_VERSION
+ export ICECC_CC=$ICECC_CC
+ export ICECC_CXX=$ICECC_CXX
+ $ICE_PATH/symlinks/$compiler "\$@"
+ __EOF__
+ chmod 775 $ICE_PATH/$compiler
+ done
+
ICECC_AS="`${ICECC_CC} -print-prog-name=as`"
# for target recipes should return something like:
# /OE/tmp-eglibc/sysroots/x86_64-linux/usr/libexec/arm920tt-oe-linux-gnueabi/gcc/arm-oe-linux-gnueabi/4.8.2/as
@@ -298,10 +410,10 @@ set_icecc_env() {
# the ICECC_VERSION generation step must be locked by a mutex
# in order to prevent race conditions
if flock -n "${ICECC_VERSION}.lock" \
- ${ICECC_ENV_EXEC} "${ICECC_CC}" "${ICECC_CXX}" "${ICECC_AS}" "${ICECC_VERSION}"
+ ${ICECC_ENV_EXEC} ${ICECC_ENV_DEBUG} "${ICECC_CC}" "${ICECC_CXX}" "${ICECC_AS}" "${ICECC_VERSION}"
then
touch "${ICECC_VERSION}.done"
- elif [ ! wait_for_file "${ICECC_VERSION}.done" 30 ]
+ elif ! wait_for_file "${ICECC_VERSION}.done" 30
then
# locking failed so wait for ${ICECC_VERSION}.done to appear
bbwarn "Timeout waiting for ${ICECC_VERSION}.done"
@@ -309,11 +421,15 @@ set_icecc_env() {
fi
fi
- export ICECC_VERSION ICECC_CC ICECC_CXX
- export PATH="$ICE_PATH:$PATH"
+ # Don't let ccache find the icecream compiler links that have been created, otherwise
+ # it can end up invoking icecream recursively.
export CCACHE_PATH="$PATH"
+ export CCACHE_DISABLE="1"
+
+ export PATH="$ICE_PATH:$PATH"
- bbnote "Using icecc"
+ bbnote "Using icecc path: $ICE_PATH"
+ bbnote "Using icecc tarball: $ICECC_VERSION"
}
do_configure_prepend() {
@@ -331,3 +447,13 @@ do_compile_kernelmodules_prepend() {
do_install_prepend() {
set_icecc_env
}
+
+# IceCream is not (currently) supported in the extensible SDK
+ICECC_SDK_HOST_TASK = "nativesdk-icecc-toolchain"
+ICECC_SDK_HOST_TASK_task-populate-sdk-ext = ""
+
+# Don't include IceCream in uninative tarball
+ICECC_SDK_HOST_TASK_pn-uninative-tarball = ""
+
+# Add the toolchain scripts to the SDK
+TOOLCHAIN_HOST_TASK_append = " ${ICECC_SDK_HOST_TASK}"