1 files changed, 202 insertions, 0 deletions
diff --git a/meta/lib/oeqa/selftest/cases/reproducible.py b/meta/lib/oeqa/selftest/cases/reproducible.py
new file mode 100644
index 0000000000..db538a4f89
--- /dev/null
+++ b/meta/lib/oeqa/selftest/cases/reproducible.py
@@ -0,0 +1,202 @@
+#
+# SPDX-License-Identifier: MIT
+#
+# Copyright 2019 by Garmin Ltd. or its subsidiaries
+
+from oeqa.selftest.case import OESelftestTestCase
+from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_bb_vars
+import bb.utils
+import functools
+import multiprocessing
+import textwrap
+import json
+import unittest
+import tempfile
+import shutil
+import stat
+import os
+
+MISSING = 'MISSING'
+DIFFERENT = 'DIFFERENT'
+SAME = 'SAME'
+
+@functools.total_ordering
+class CompareResult(object):
+    def __init__(self):
+        self.reference = None
+        self.test = None
+        self.status = 'UNKNOWN'
+
+    def __eq__(self, other):
+        return (self.status, self.test) == (other.status, other.test)
+
+    def __lt__(self, other):
+        return (self.status, self.test) < (other.status, other.test)
+
+class PackageCompareResults(object):
+    def __init__(self):
+        self.total = []
+        self.missing = []
+        self.different = []
+        self.same = []
+
+    def add_result(self, r):
+        self.total.append(r)
+        if r.status == MISSING:
+            self.missing.append(r)
+        elif r.status == DIFFERENT:
+            self.different.append(r)
+        else:
+            self.same.append(r)
+
+    def sort(self):
+        self.total.sort()
+        self.missing.sort()
+        self.different.sort()
+        self.same.sort()
+
+    def __str__(self):
+        return 'same=%i different=%i missing=%i total=%i' % (len(self.same), len(self.different), len(self.missing), len(self.total))
+
+def compare_file(reference, test, diffutils_sysroot):
+    result = CompareResult()
+    result.reference = reference
+    result.test = test
+
+    if not os.path.exists(reference):
+        result.status = MISSING
+        return result
+
+    r = runCmd(['cmp', '--quiet', reference, test], native_sysroot=diffutils_sysroot, ignore_status=True)
+
+    if r.status:
+        result.status = DIFFERENT
+        return result
+
+    result.status = SAME
+    return result
+
+class ReproducibleTests(OESelftestTestCase):
+    package_classes = ['deb', 'ipk']
+    images = ['core-image-minimal', 'core-image-sato', 'core-image-full-cmdline']
+    save_results = False
+
+    def setUpLocal(self):
+        super().setUpLocal()
+        needed_vars = ['TOPDIR', 'TARGET_PREFIX', 'BB_NUMBER_THREADS']
+        bb_vars = get_bb_vars(needed_vars)
+        for v in needed_vars:
+            setattr(self, v.lower(), bb_vars[v])
+
+        self.extrasresults = {}
+        self.extrasresults.setdefault('reproducible.rawlogs', {})['log'] = ''
+        self.extrasresults.setdefault('reproducible', {}).setdefault('files', {})
+
+    def append_to_log(self, msg):
+        self.extrasresults['reproducible.rawlogs']['log'] += msg
+
+    def compare_packages(self, reference_dir, test_dir, diffutils_sysroot):
+        result = PackageCompareResults()
+
+        old_cwd = os.getcwd()
+        try:
+            file_result = {}
+            os.chdir(test_dir)
+            with multiprocessing.Pool(processes=int(self.bb_number_threads or 0)) as p:
+                for root, dirs, files in os.walk('.'):
+                    async_result = []
+                    for f in files:
+                        reference_path = os.path.join(reference_dir, root, f)
+                        test_path = os.path.join(test_dir, root, f)
+                        async_result.append(p.apply_async(compare_file, (reference_path, test_path, diffutils_sysroot)))
+
+                    for a in async_result:
+                        result.add_result(a.get())
+
+        finally:
+            os.chdir(old_cwd)
+
+        result.sort()
+        return result
+
+    def write_package_list(self, package_class, name, packages):
+        self.extrasresults['reproducible']['files'].setdefault(package_class, {})[name] = [
+                {'reference': p.reference, 'test': p.test} for p in packages]
+
+    def copy_file(self, source, dest):
+        bb.utils.mkdirhier(os.path.dirname(dest))
+        shutil.copyfile(source, dest)
+
+    def test_reproducible_builds(self):
+        capture_vars = ['DEPLOY_DIR_' + c.upper() for c in self.package_classes]
+
+        if self.save_results:
+            save_dir = tempfile.mkdtemp(prefix='oe-reproducible-')
+            os.chmod(save_dir, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
+            self.logger.info('Non-reproducible packages will be copied to %s', save_dir)
+
+        # Build native utilities
+        self.write_config('')
+        bitbake("diffutils-native -c addto_recipe_sysroot")
+        diffutils_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "diffutils-native")
+
+        # Reproducible builds should not pull from sstate or mirrors, but
+        # sharing DL_DIR is fine
+        common_config = textwrap.dedent('''\
+            INHERIT += "reproducible_build"
+            PACKAGE_CLASSES = "%s"
+            SSTATE_DIR = "${TMPDIR}/sstate"
+            ''') % (' '.join('package_%s' % c for c in self.package_classes))
+
+        # Perform a build.
+        reproducibleA_tmp = os.path.join(self.topdir, 'reproducibleA', 'tmp')
+        if os.path.exists(reproducibleA_tmp):
+            bb.utils.remove(reproducibleA_tmp, recurse=True)
+
+        self.write_config((textwrap.dedent('''\
+            TMPDIR = "%s"
+            ''') % reproducibleA_tmp) + common_config)
+        vars_A = get_bb_vars(capture_vars)
+        bitbake(' '.join(self.images))
+
+        # Perform another build.
+        reproducibleB_tmp = os.path.join(self.topdir, 'reproducibleB', 'tmp')
+        if os.path.exists(reproducibleB_tmp):
+            bb.utils.remove(reproducibleB_tmp, recurse=True)
+
+        self.write_config((textwrap.dedent('''\
+            SSTATE_MIRROR = ""
+            TMPDIR = "%s"
+            ''') % reproducibleB_tmp) + common_config)
+        vars_B = get_bb_vars(capture_vars)
+        bitbake(' '.join(self.images))
+
+        # NOTE: The temp directories from the reproducible build are purposely
+        # kept after the build so it can be diffed for debugging.
+
+        for c in self.package_classes:
+            with self.subTest(package_class=c):
+                package_class = 'package_' + c
+
+                deploy_A = vars_A['DEPLOY_DIR_' + c.upper()]
+                deploy_B = vars_B['DEPLOY_DIR_' + c.upper()]
+
+                result = self.compare_packages(deploy_A, deploy_B, diffutils_sysroot)
+
+                self.logger.info('Reproducibility summary for %s: %s' % (c, result))
+
+                self.append_to_log('\n'.join("%s: %s" % (r.status, r.test) for r in result.total))
+
+                self.write_package_list(package_class, 'missing', result.missing)
+                self.write_package_list(package_class, 'different', result.different)
+                self.write_package_list(package_class, 'same', result.same)
+
+                if self.save_results:
+                    for d in result.different:
+                        self.copy_file(d.reference, '/'.join([save_dir, d.reference]))
+                        self.copy_file(d.test, '/'.join([save_dir, d.test]))
+
+                if result.missing or result.different:
+                    self.fail("The following %s packages are missing or different: %s" %
+                            (c, ' '.join(r.test for r in (result.missing + result.different))))
+