diff options
Diffstat (limited to 'meta/lib/oeqa/selftest/cases/reproducible.py')
| -rw-r--r-- | meta/lib/oeqa/selftest/cases/reproducible.py | 114 |
1 files changed, 41 insertions, 73 deletions
diff --git a/meta/lib/oeqa/selftest/cases/reproducible.py b/meta/lib/oeqa/selftest/cases/reproducible.py index eee494e5cc..f4dd779842 100644 --- a/meta/lib/oeqa/selftest/cases/reproducible.py +++ b/meta/lib/oeqa/selftest/cases/reproducible.py @@ -9,79 +9,23 @@ import bb.utils import functools import multiprocessing import textwrap -import json -import unittest import tempfile import shutil import stat import os import datetime -# For sample packages, see: -# https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201127-0t7wr_oo/ -# https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201127-4s9ejwyp/ -# https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201127-haiwdlbr/ -# https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201127-hwds3mcl/ -# https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201203-sua0pzvc/ -# (both packages/ and packages-excluded/) exclude_packages = [ - 'acpica-src', - 'babeltrace2-ptest', - 'bootchart2-doc', - 'cups', - 'cwautomacros', - 'dtc', - 'efivar', - 'epiphany', - 'gcr', - 'git', - 'glide', - 'go-dep', - 'go-helloworld', - 'go-runtime', - 'go_', - 'groff', - 'gst-devtools', - 'gstreamer1.0-python', - 'gtk-doc', - 'igt-gpu-tools', - 'kernel-devsrc', - 'libaprutil', - 'libcap-ng', - 'libhandy-1-src', - 'libid3tag', - 'libproxy', - 'libsecret-dev', - 'libsecret-src', - 'lttng-tools-dbg', - 'lttng-tools-ptest', - 'ltp', - 'meson', - 'ovmf-shell-efi', - 'parted-ptest', - 'perf', - 'python3-cython', - 'qemu', - 'quilt-ptest', - 'rsync', - 'ruby', - 'spirv-tools-dev', - 'swig', - 'syslinux-misc', - 'systemd-bootchart', - 'valgrind-ptest', - 'vim', - 'watchdog', - 'xmlto', - 'xorg-minimal-fonts' + 'rust', + 'rust-dbg' ] def is_excluded(package): package_name = os.path.basename(package) for i in exclude_packages: if package_name.startswith(i): - return True - return False + return i + return None MISSING = 'MISSING' DIFFERENT = 'DIFFERENT' @@ -107,14 +51,17 @@ class PackageCompareResults(object): self.different = [] self.different_excluded = [] self.same = [] + self.active_exclusions = set() def add_result(self, r): self.total.append(r) if r.status == MISSING: self.missing.append(r) elif r.status == DIFFERENT: - if is_excluded(r.reference): + exclusion = is_excluded(r.reference) + if exclusion: self.different_excluded.append(r) + self.active_exclusions.add(exclusion) else: self.different.append(r) else: @@ -128,7 +75,10 @@ class PackageCompareResults(object): self.same.sort() def __str__(self): - return 'same=%i different=%i different_excluded=%i missing=%i total=%i' % (len(self.same), len(self.different), len(self.different_excluded), len(self.missing), len(self.total)) + return 'same=%i different=%i different_excluded=%i missing=%i total=%i\nunused_exclusions=%s' % (len(self.same), len(self.different), len(self.different_excluded), len(self.missing), len(self.total), self.unused_exclusions()) + + def unused_exclusions(self): + return sorted(set(exclude_packages) - self.active_exclusions) def compare_file(reference, test, diffutils_sysroot): result = CompareResult() @@ -139,7 +89,7 @@ def compare_file(reference, test, diffutils_sysroot): result.status = MISSING return result - r = runCmd(['cmp', '--quiet', reference, test], native_sysroot=diffutils_sysroot, ignore_status=True) + r = runCmd(['cmp', '--quiet', reference, test], native_sysroot=diffutils_sysroot, ignore_status=True, sync=False) if r.status: result.status = DIFFERENT @@ -148,8 +98,9 @@ def compare_file(reference, test, diffutils_sysroot): result.status = SAME return result -def run_diffoscope(a_dir, b_dir, html_dir, **kwargs): - return runCmd(['diffoscope', '--no-default-limits', '--exclude-directory-metadata', 'yes', '--html-dir', html_dir, a_dir, b_dir], +def run_diffoscope(a_dir, b_dir, html_dir, max_report_size=0, **kwargs): + return runCmd(['diffoscope', '--no-default-limits', '--max-report-size', str(max_report_size), + '--exclude-directory-metadata', 'yes', '--html-dir', html_dir, a_dir, b_dir], **kwargs) class DiffoscopeTests(OESelftestTestCase): @@ -175,8 +126,17 @@ class DiffoscopeTests(OESelftestTestCase): self.assertTrue(os.path.exists(os.path.join(tmpdir, 'index.html')), "HTML index not found!") class ReproducibleTests(OESelftestTestCase): - package_classes = ['deb', 'ipk'] - images = ['core-image-minimal', 'core-image-sato', 'core-image-full-cmdline', 'world'] + # Test the reproducibility of whatever is built between sstate_targets and targets + + package_classes = ['deb', 'ipk', 'rpm'] + + # Maximum report size, in bytes + max_report_size = 250 * 1024 * 1024 + + # targets are the things we want to test the reproducibility of + targets = ['core-image-minimal', 'core-image-sato', 'core-image-full-cmdline', 'core-image-weston', 'world'] + # sstate targets are things to pull from sstate to potentially cut build/debugging time + sstate_targets = [] save_results = False if 'OEQA_DEBUGGING_SAVED_OUTPUT' in os.environ: save_results = os.environ['OEQA_DEBUGGING_SAVED_OUTPUT'] @@ -243,12 +203,11 @@ class ReproducibleTests(OESelftestTestCase): bb.utils.remove(tmpdir, recurse=True) config = textwrap.dedent('''\ - INHERIT += "reproducible_build" PACKAGE_CLASSES = "{package_classes}" INHIBIT_PACKAGE_STRIP = "1" TMPDIR = "{tmpdir}" - LICENSE_FLAGS_WHITELIST = "commercial" - DISTRO_FEATURES_append = ' systemd pam' + LICENSE_FLAGS_ACCEPTED = "commercial" + DISTRO_FEATURES:append = ' systemd pam' USERADDEXTENSION = "useradd-staticids" USERADD_ERROR_DYNAMIC = "skip" USERADD_UID_TABLES += "files/static-passwd" @@ -257,16 +216,23 @@ class ReproducibleTests(OESelftestTestCase): tmpdir=tmpdir) if not use_sstate: + if self.sstate_targets: + self.logger.info("Building prebuild for %s (sstate allowed)..." % (name)) + self.write_config(config) + bitbake(' '.join(self.sstate_targets)) + # This config fragment will disable using shared and the sstate # mirror, forcing a complete build from scratch config += textwrap.dedent('''\ SSTATE_DIR = "${TMPDIR}/sstate" - SSTATE_MIRROR = "" + SSTATE_MIRRORS = "" ''') + self.logger.info("Building %s (sstate%s allowed)..." % (name, '' if use_sstate else ' NOT')) self.write_config(config) d = get_bb_vars(capture_vars) - bitbake(' '.join(self.images)) + # targets used to be called images + bitbake(' '.join(getattr(self, 'images', self.targets))) return d def test_reproducible_builds(self): @@ -290,6 +256,7 @@ class ReproducibleTests(OESelftestTestCase): self.logger.info('Non-reproducible packages will be copied to %s', save_dir) vars_A = self.do_test_build('reproducibleA', self.build_from_sstate) + vars_B = self.do_test_build('reproducibleB', False) # NOTE: The temp directories from the reproducible build are purposely @@ -304,6 +271,7 @@ class ReproducibleTests(OESelftestTestCase): deploy_A = vars_A['DEPLOY_DIR_' + c.upper()] deploy_B = vars_B['DEPLOY_DIR_' + c.upper()] + self.logger.info('Checking %s packages for differences...' % c) result = self.compare_packages(deploy_A, deploy_B, diffutils_sysroot) self.logger.info('Reproducibility summary for %s: %s' % (c, result)) @@ -340,7 +308,7 @@ class ReproducibleTests(OESelftestTestCase): # Copy jquery to improve the diffoscope output usability self.copy_file(os.path.join(jquery_sysroot, 'usr/share/javascript/jquery/jquery.min.js'), os.path.join(package_html_dir, 'jquery.js')) - run_diffoscope('reproducibleA', 'reproducibleB', package_html_dir, + run_diffoscope('reproducibleA', 'reproducibleB', package_html_dir, max_report_size=self.max_report_size, native_sysroot=diffoscope_sysroot, ignore_status=True, cwd=package_dir) if fails: |