siteinfo/autotools: Ensure task checksums reflect site files

Currently, if you change the site files, nothing rebuilds since they are
not accounted for in task checksums. They could/should be through the
file-checksums task flag. We need to cache all the files looked for,
whether the exist or not so that if they do exist and didn't,
the checksum also changes.

This gets complicated by the need to clean out hardcoded build
paths from the variable and that other layers can have site files.

This patch adds this functionality. A new variable, SITEINFO_PATHVARS
is added which controls which substitutions to make on the file-checksum
values to remove the hardcoded paths. Layers adding site files will need
to set this to a variable that has the layer path in it and is excluded
from task hashes (COREBASE is the one the core layer uses).

This patch will cause yocto-check-layer to fail for some layers
where site files are added yet the layer isn't a machine specific layer.
This is arguable correct since these additional site files apply to
all recipes and things from a layer like core could be changed by such
changes so it is right they should rebuild. There is a determinism issue
potentially there if not. meta-openembedded does have some such references
but looking at them they should move to core or likely just be removed as
most look obsolete anyway.

[YOCTO #13729]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

5 files changed, 49 insertions, 9 deletions

diff --git a/meta/classes/autotools.bbclass b/meta/classes/autotools.bbclass
index 2c7968e659..bc0c2ea83e 100644
--- a/meta/classes/autotools.bbclass
+++ b/meta/classes/autotools.bbclass
@@ -145,7 +145,13 @@ ACLOCALEXTRAPATH:class-target = " -I ${STAGING_DATADIR_NATIVE}/aclocal/"
 ACLOCALEXTRAPATH:class-nativesdk = " -I ${STAGING_DATADIR_NATIVE}/aclocal/"
 
 python autotools_aclocals () {
-    d.setVar("CONFIG_SITE", siteinfo_get_files(d, sysrootcache=True))
+    sitefiles, searched = siteinfo_get_files(d, sysrootcache=True)
+    d.setVar("CONFIG_SITE", " ".join(sitefiles))
+}
+
+python () {
+    sitefiles, searched = siteinfo_get_files(d, sysrootcache=False)
+    d.appendVarFlag("do_configure", "file-checksums", " " + " ".join(searched))
 }
 
 CONFIGURE_FILES = "${S}/configure.in ${S}/configure.ac ${S}/config.h.in ${S}/acinclude.m4 Makefile.am"
diff --git a/meta/classes/siteinfo.bbclass b/meta/classes/siteinfo.bbclass
index 0bd1f36805..c5f4dfda41 100644
--- a/meta/classes/siteinfo.bbclass
+++ b/meta/classes/siteinfo.bbclass
@@ -176,17 +176,39 @@ python () {
         bb.fatal("Please add your architecture to siteinfo.bbclass")
 }
 
-def siteinfo_get_files(d, sysrootcache = False):
+# Layers with siteconfig need to add a replacement path to this variable so the
+# sstate isn't path specific
+SITEINFO_PATHVARS = "COREBASE"
+
+def siteinfo_get_files(d, sysrootcache=False):
     sitedata = siteinfo_data(d)
-    sitefiles = ""
+    sitefiles = []
+    searched = []
     for path in d.getVar("BBPATH").split(":"):
         for element in sitedata:
             filename = os.path.join(path, "site", element)
             if os.path.exists(filename):
-                sitefiles += filename + " "
+                searched.append(filename + ":True")
+                sitefiles.append(filename)
+            else:
+                searched.append(filename + ":False")
+
+    # Have to parameterise out hardcoded paths such as COREBASE for the main site files
+    for var in d.getVar("SITEINFO_PATHVARS").split():
+        searched2 = []
+        replace = os.path.normpath(d.getVar(var))
+        for s in searched:
+            searched2.append(s.replace(replace, "${" + var + "}"))
+        searched = searched2
+
+    if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross', d) or bb.data.inherits_class('crosssdk', d):
+        # We need sstate sigs for native/cross not to vary upon arch so we can't depend on the site files.
+        # In future we may want to depend upon all site files?
+        # This would show up as breaking sstatetests.SStateTests.test_sstate_32_64_same_hash for example
+        searched = []
 
     if not sysrootcache:
-        return sitefiles
+        return sitefiles, searched
 
     # Now check for siteconfig cache files in sysroots
     path_siteconfig = d.getVar('SITECONFIG_SYSROOTCACHE')
@@ -195,8 +217,8 @@ def siteinfo_get_files(d, sysrootcache = False):
             if not i.endswith("_config"):
                 continue
             filename = os.path.join(path_siteconfig, i)
-            sitefiles += filename + " "
-    return sitefiles
+            sitefiles.append(filename)
+    return sitefiles, searched
 
 #
 # Make some information available via variables
diff --git a/meta/classes/toolchain-scripts.bbclass b/meta/classes/toolchain-scripts.bbclass
index 479f3b706e..fb6261c91d 100644
--- a/meta/classes/toolchain-scripts.bbclass
+++ b/meta/classes/toolchain-scripts.bbclass
@@ -65,6 +65,7 @@ toolchain_create_sdk_env_script () {
 
 # This function creates an environment-setup-script in the TMPDIR which enables
 # a OE-core IDE to integrate with the build tree
+# Caller must ensure CONFIG_SITE is setup
 toolchain_create_tree_env_script () {
 	script=${TMPDIR}/environment-setup-${REAL_MULTIMACH_TARGET_SYS}
 	rm -f $script
@@ -73,7 +74,7 @@ toolchain_create_tree_env_script () {
 	echo 'export PATH=${STAGING_DIR_NATIVE}/usr/bin:${STAGING_BINDIR_TOOLCHAIN}:$PATH' >> $script
 	echo 'export PKG_CONFIG_SYSROOT_DIR=${PKG_CONFIG_SYSROOT_DIR}' >> $script
 	echo 'export PKG_CONFIG_PATH=${PKG_CONFIG_PATH}' >> $script
-	echo 'export CONFIG_SITE="${@siteinfo_get_files(d)}"' >> $script
+	echo 'export CONFIG_SITE="${CONFIG_SITE}"' >> $script
 	echo 'export SDKTARGETSYSROOT=${STAGING_DIR_TARGET}' >> $script
 	echo 'export OECORE_NATIVE_SYSROOT="${STAGING_DIR_NATIVE}"' >> $script
 	echo 'export OECORE_TARGET_SYSROOT="${STAGING_DIR_TARGET}"' >> $script
@@ -161,7 +162,7 @@ EOF
 }
 
 #we get the cached site config in the runtime
-TOOLCHAIN_CONFIGSITE_NOCACHE = "${@siteinfo_get_files(d)}"
+TOOLCHAIN_CONFIGSITE_NOCACHE = "${@' '.join(siteinfo_get_files(d)[0])}"
 TOOLCHAIN_CONFIGSITE_SYSROOTCACHE = "${STAGING_DIR}/${MLPREFIX}${MACHINE}/${target_datadir}/${TARGET_SYS}_config_site.d"
 TOOLCHAIN_NEED_CONFIGSITE_CACHE ??= "virtual/${MLPREFIX}libc ncurses"
 DEPENDS += "${TOOLCHAIN_NEED_CONFIGSITE_CACHE}"
diff --git a/meta/recipes-core/meta/meta-environment.bb b/meta/recipes-core/meta/meta-environment.bb
index 27f0103665..7118fb2aef 100644
--- a/meta/recipes-core/meta/meta-environment.bb
+++ b/meta/recipes-core/meta/meta-environment.bb
@@ -47,6 +47,11 @@ python do_generate_content() {
 }
 addtask generate_content before do_install after do_compile
 
+python () {
+    sitefiles, searched = siteinfo_get_files(d, sysrootcache=False)
+    d.appendVarFlag("do_generate_content", "file-checksums", " " + " ".join(searched))
+}
+
 create_sdk_files() {
 	# Setup site file for external use
 	toolchain_create_sdk_siteconfig ${SDK_OUTPUT}/${SDKPATH}/site-config-${REAL_MULTIMACH_TARGET_SYS}
diff --git a/meta/recipes-core/meta/meta-ide-support.bb b/meta/recipes-core/meta/meta-ide-support.bb
index 768f6f4bb6..5b23f78e51 100644
--- a/meta/recipes-core/meta/meta-ide-support.bb
+++ b/meta/recipes-core/meta/meta-ide-support.bb
@@ -12,4 +12,10 @@ do_populate_ide_support () {
   toolchain_create_tree_env_script
 }
 
+python () {
+    sitefiles, searched = siteinfo_get_files(d, sysrootcache=False)
+    d.setVar("CONFIG_SITE", " ".join(sitefiles))
+    d.appendVarFlag("do_populate_ide_support", "file-checksums", " " + " ".join(searched))
+}
+
 addtask populate_ide_support before do_build after do_install