1 files changed, 338 insertions, 0 deletions
diff --git a/meta/lib/oeqa/selftest/cases/reproducible.py b/meta/lib/oeqa/selftest/cases/reproducible.py
new file mode 100644
index 0000000000..80e830136f
--- /dev/null
+++ b/meta/lib/oeqa/selftest/cases/reproducible.py
@@ -0,0 +1,338 @@
+#
+# SPDX-License-Identifier: MIT
+#
+# Copyright 2019-2020 by Garmin Ltd. or its subsidiaries
+
+from oeqa.selftest.case import OESelftestTestCase
+from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_bb_vars
+import bb.utils
+import functools
+import multiprocessing
+import textwrap
+import tempfile
+import shutil
+import stat
+import os
+import datetime
+
+exclude_packages = [
+	]
+
+def is_excluded(package):
+    package_name = os.path.basename(package)
+    for i in exclude_packages:
+        if package_name.startswith(i):
+            return i
+    return None
+
+MISSING = 'MISSING'
+DIFFERENT = 'DIFFERENT'
+SAME = 'SAME'
+
+@functools.total_ordering
+class CompareResult(object):
+    def __init__(self):
+        self.reference = None
+        self.test = None
+        self.status = 'UNKNOWN'
+
+    def __eq__(self, other):
+        return (self.status, self.test) == (other.status, other.test)
+
+    def __lt__(self, other):
+        return (self.status, self.test) < (other.status, other.test)
+
+class PackageCompareResults(object):
+    def __init__(self, exclusions):
+        self.total = []
+        self.missing = []
+        self.different = []
+        self.different_excluded = []
+        self.same = []
+        self.active_exclusions = set()
+        exclude_packages.extend((exclusions or "").split())
+
+    def add_result(self, r):
+        self.total.append(r)
+        if r.status == MISSING:
+            self.missing.append(r)
+        elif r.status == DIFFERENT:
+            exclusion = is_excluded(r.reference)
+            if exclusion:
+                self.different_excluded.append(r)
+                self.active_exclusions.add(exclusion)
+            else:
+                self.different.append(r)
+        else:
+            self.same.append(r)
+
+    def sort(self):
+        self.total.sort()
+        self.missing.sort()
+        self.different.sort()
+        self.different_excluded.sort()
+        self.same.sort()
+
+    def __str__(self):
+        return 'same=%i different=%i different_excluded=%i missing=%i total=%i\nunused_exclusions=%s' % (len(self.same), len(self.different), len(self.different_excluded), len(self.missing), len(self.total), self.unused_exclusions())
+
+    def unused_exclusions(self):
+        return sorted(set(exclude_packages) - self.active_exclusions)
+
+def compare_file(reference, test, diffutils_sysroot):
+    result = CompareResult()
+    result.reference = reference
+    result.test = test
+
+    if not os.path.exists(reference):
+        result.status = MISSING
+        return result
+
+    r = runCmd(['cmp', '--quiet', reference, test], native_sysroot=diffutils_sysroot, ignore_status=True, sync=False)
+
+    if r.status:
+        result.status = DIFFERENT
+        return result
+
+    result.status = SAME
+    return result
+
+def run_diffoscope(a_dir, b_dir, html_dir, max_report_size=0, **kwargs):
+    return runCmd(['diffoscope', '--no-default-limits', '--max-report-size', str(max_report_size),
+                   '--exclude-directory-metadata', 'yes', '--html-dir', html_dir, a_dir, b_dir],
+                **kwargs)
+
+class DiffoscopeTests(OESelftestTestCase):
+    diffoscope_test_files = os.path.join(os.path.dirname(os.path.abspath(__file__)), "diffoscope")
+
+    def test_diffoscope(self):
+        bitbake("diffoscope-native -c addto_recipe_sysroot")
+        diffoscope_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "diffoscope-native")
+
+        # Check that diffoscope doesn't return an error when the files compare
+        # the same (a general check that diffoscope is working)
+        with tempfile.TemporaryDirectory() as tmpdir:
+            run_diffoscope('A', 'A', tmpdir,
+                native_sysroot=diffoscope_sysroot, cwd=self.diffoscope_test_files)
+
+        # Check that diffoscope generates an index.html file when the files are
+        # different
+        with tempfile.TemporaryDirectory() as tmpdir:
+            r = run_diffoscope('A', 'B', tmpdir,
+                native_sysroot=diffoscope_sysroot, ignore_status=True, cwd=self.diffoscope_test_files)
+
+            self.assertNotEqual(r.status, 0, msg="diffoscope was successful when an error was expected")
+            self.assertTrue(os.path.exists(os.path.join(tmpdir, 'index.html')), "HTML index not found!")
+
+class ReproducibleTests(OESelftestTestCase):
+    # Test the reproducibility of whatever is built between sstate_targets and targets
+
+    package_classes = ['deb', 'ipk', 'rpm']
+
+    # Maximum report size, in bytes
+    max_report_size = 250 * 1024 * 1024
+
+    # targets are the things we want to test the reproducibility of
+    targets = ['core-image-minimal', 'core-image-sato', 'core-image-full-cmdline', 'core-image-weston', 'world']
+
+    # sstate targets are things to pull from sstate to potentially cut build/debugging time
+    sstate_targets = []
+
+    save_results = False
+    if 'OEQA_DEBUGGING_SAVED_OUTPUT' in os.environ:
+        save_results = os.environ['OEQA_DEBUGGING_SAVED_OUTPUT']
+
+    # This variable controls if one of the test builds is allowed to pull from
+    # an sstate cache/mirror. The other build is always done clean as a point of
+    # comparison.
+    # If you know that your sstate archives are reproducible, enabling this
+    # will test that and also make the test run faster. If your sstate is not
+    # reproducible, disable this in your derived test class
+    build_from_sstate = True
+
+    def setUpLocal(self):
+        super().setUpLocal()
+        needed_vars = [
+            'TOPDIR',
+            'TARGET_PREFIX',
+            'BB_NUMBER_THREADS',
+            'BB_HASHSERVE',
+            'OEQA_REPRODUCIBLE_TEST_PACKAGE',
+            'OEQA_REPRODUCIBLE_TEST_TARGET',
+            'OEQA_REPRODUCIBLE_TEST_SSTATE_TARGETS',
+            'OEQA_REPRODUCIBLE_EXCLUDED_PACKAGES',
+        ]
+        bb_vars = get_bb_vars(needed_vars)
+        for v in needed_vars:
+            setattr(self, v.lower(), bb_vars[v])
+
+        if bb_vars['OEQA_REPRODUCIBLE_TEST_PACKAGE']:
+            self.package_classes = bb_vars['OEQA_REPRODUCIBLE_TEST_PACKAGE'].split()
+
+        if bb_vars['OEQA_REPRODUCIBLE_TEST_TARGET']:
+            self.targets = bb_vars['OEQA_REPRODUCIBLE_TEST_TARGET'].split()
+
+        if bb_vars['OEQA_REPRODUCIBLE_TEST_SSTATE_TARGETS']:
+            self.sstate_targets = bb_vars['OEQA_REPRODUCIBLE_TEST_SSTATE_TARGETS'].split()
+
+        self.extraresults = {}
+        self.extraresults.setdefault('reproducible.rawlogs', {})['log'] = ''
+        self.extraresults.setdefault('reproducible', {}).setdefault('files', {})
+
+    def append_to_log(self, msg):
+        self.extraresults['reproducible.rawlogs']['log'] += msg
+
+    def compare_packages(self, reference_dir, test_dir, diffutils_sysroot):
+        result = PackageCompareResults(self.oeqa_reproducible_excluded_packages)
+
+        old_cwd = os.getcwd()
+        try:
+            file_result = {}
+            os.chdir(test_dir)
+            with multiprocessing.Pool(processes=int(self.bb_number_threads or 0)) as p:
+                for root, dirs, files in os.walk('.'):
+                    async_result = []
+                    for f in files:
+                        reference_path = os.path.join(reference_dir, root, f)
+                        test_path = os.path.join(test_dir, root, f)
+                        async_result.append(p.apply_async(compare_file, (reference_path, test_path, diffutils_sysroot)))
+
+                    for a in async_result:
+                        result.add_result(a.get())
+
+        finally:
+            os.chdir(old_cwd)
+
+        result.sort()
+        return result
+
+    def write_package_list(self, package_class, name, packages):
+        self.extraresults['reproducible']['files'].setdefault(package_class, {})[name] = [
+                {'reference': p.reference, 'test': p.test} for p in packages]
+
+    def copy_file(self, source, dest):
+        bb.utils.mkdirhier(os.path.dirname(dest))
+        shutil.copyfile(source, dest)
+
+    def do_test_build(self, name, use_sstate):
+        capture_vars = ['DEPLOY_DIR_' + c.upper() for c in self.package_classes]
+
+        tmpdir = os.path.join(self.topdir, name, 'tmp')
+        if os.path.exists(tmpdir):
+            bb.utils.remove(tmpdir, recurse=True)
+
+        config = textwrap.dedent('''\
+            PACKAGE_CLASSES = "{package_classes}"
+            TMPDIR = "{tmpdir}"
+            LICENSE_FLAGS_ACCEPTED = "commercial"
+            DISTRO_FEATURES:append = ' pam'
+            USERADDEXTENSION = "useradd-staticids"
+            USERADD_ERROR_DYNAMIC = "skip"
+            USERADD_UID_TABLES += "files/static-passwd"
+            USERADD_GID_TABLES += "files/static-group"
+            ''').format(package_classes=' '.join('package_%s' % c for c in self.package_classes),
+                        tmpdir=tmpdir)
+
+        if not use_sstate:
+            if self.sstate_targets:
+               self.logger.info("Building prebuild for %s (sstate allowed)..." % (name))
+               self.write_config(config)
+               bitbake(' '.join(self.sstate_targets))
+
+            # This config fragment will disable using shared and the sstate
+            # mirror, forcing a complete build from scratch
+            config += textwrap.dedent('''\
+                SSTATE_DIR = "${TMPDIR}/sstate"
+                SSTATE_MIRRORS = "file://.*/.*-native.*  http://sstate.yoctoproject.org/all/PATH;downloadfilename=PATH file://.*/.*-cross.*  http://sstate.yoctoproject.org/all/PATH;downloadfilename=PATH"
+                ''')
+
+        self.logger.info("Building %s (sstate%s allowed)..." % (name, '' if use_sstate else ' NOT'))
+        self.write_config(config)
+        d = get_bb_vars(capture_vars)
+        # targets used to be called images
+        bitbake(' '.join(getattr(self, 'images', self.targets)))
+        return d
+
+    def test_reproducible_builds(self):
+        def strip_topdir(s):
+            if s.startswith(self.topdir):
+                return s[len(self.topdir):]
+            return s
+
+        # Build native utilities
+        self.write_config('')
+        bitbake("diffoscope-native diffutils-native jquery-native -c addto_recipe_sysroot")
+        diffutils_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "diffutils-native")
+        diffoscope_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "diffoscope-native")
+        jquery_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "jquery-native")
+
+        if self.save_results:
+            os.makedirs(self.save_results, exist_ok=True)
+            datestr = datetime.datetime.now().strftime('%Y%m%d')
+            save_dir = tempfile.mkdtemp(prefix='oe-reproducible-%s-' % datestr, dir=self.save_results)
+            os.chmod(save_dir, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
+            self.logger.info('Non-reproducible packages will be copied to %s', save_dir)
+
+        vars_A = self.do_test_build('reproducibleA', self.build_from_sstate)
+
+        vars_B = self.do_test_build('reproducibleB', False)
+
+        # NOTE: The temp directories from the reproducible build are purposely
+        # kept after the build so it can be diffed for debugging.
+
+        fails = []
+
+        for c in self.package_classes:
+            with self.subTest(package_class=c):
+                package_class = 'package_' + c
+
+                deploy_A = vars_A['DEPLOY_DIR_' + c.upper()]
+                deploy_B = vars_B['DEPLOY_DIR_' + c.upper()]
+
+                self.logger.info('Checking %s packages for differences...' % c)
+                result = self.compare_packages(deploy_A, deploy_B, diffutils_sysroot)
+
+                self.logger.info('Reproducibility summary for %s: %s' % (c, result))
+
+                self.append_to_log('\n'.join("%s: %s" % (r.status, r.test) for r in result.total))
+
+                self.write_package_list(package_class, 'missing', result.missing)
+                self.write_package_list(package_class, 'different', result.different)
+                self.write_package_list(package_class, 'different_excluded', result.different_excluded)
+                self.write_package_list(package_class, 'same', result.same)
+
+                if self.save_results:
+                    for d in result.different:
+                        self.copy_file(d.reference, '/'.join([save_dir, 'packages', strip_topdir(d.reference)]))
+                        self.copy_file(d.test, '/'.join([save_dir, 'packages', strip_topdir(d.test)]))
+
+                    for d in result.different_excluded:
+                        self.copy_file(d.reference, '/'.join([save_dir, 'packages-excluded', strip_topdir(d.reference)]))
+                        self.copy_file(d.test, '/'.join([save_dir, 'packages-excluded', strip_topdir(d.test)]))
+
+                if result.different:
+                    fails.append("The following %s packages are different and not in exclusion list:\n%s" %
+                            (c, '\n'.join(r.test for r in (result.different))))
+
+                if result.missing and len(self.sstate_targets) == 0:
+                    fails.append("The following %s packages are missing and not in exclusion list:\n%s" %
+                            (c, '\n'.join(r.test for r in (result.missing))))
+
+        # Clean up empty directories
+        if self.save_results:
+            if not os.listdir(save_dir):
+                os.rmdir(save_dir)
+            else:
+                self.logger.info('Running diffoscope')
+                package_dir = os.path.join(save_dir, 'packages')
+                package_html_dir = os.path.join(package_dir, 'diff-html')
+
+                # Copy jquery to improve the diffoscope output usability
+                self.copy_file(os.path.join(jquery_sysroot, 'usr/share/javascript/jquery/jquery.min.js'), os.path.join(package_html_dir, 'jquery.js'))
+
+                run_diffoscope('reproducibleA', 'reproducibleB', package_html_dir, max_report_size=self.max_report_size,
+                        native_sysroot=diffoscope_sysroot, ignore_status=True, cwd=package_dir)
+
+        if fails:
+            self.fail('\n'.join(fails))
+