summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAlexander Kanavin <alex.kanavin@gmail.com>2020-12-03 14:37:27 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2020-12-10 13:40:06 +0000
commit99ab08487fed2ad2c23d1a6b9272d24ebc6aed1e (patch)
tree7a228261fdf0b61107b4ebd55edb82689019f19f
parent08d0dbb77e8133d1e2487ee2020311ad88c789aa (diff)
downloadopenembedded-core-contrib-99ab08487fed2ad2c23d1a6b9272d24ebc6aed1e.tar.gz
openembedded-core-contrib-99ab08487fed2ad2c23d1a6b9272d24ebc6aed1e.tar.bz2
openembedded-core-contrib-99ab08487fed2ad2c23d1a6b9272d24ebc6aed1e.zip
selftest/reproducible: add an exclusion list for items that are not yet reproducible
Hopefully over time this list will be reduced to an empty one. Non-reproducible excluded packages are not given to diffoscope and do not cause a failure, but still saved side-by-side with non-reproducible failing ones to make investigation easier. Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--meta/lib/oeqa/selftest/cases/reproducible.py82
1 files changed, 79 insertions, 3 deletions
diff --git a/meta/lib/oeqa/selftest/cases/reproducible.py b/meta/lib/oeqa/selftest/cases/reproducible.py
index 6faeedb544..e3597d5081 100644
--- a/meta/lib/oeqa/selftest/cases/reproducible.py
+++ b/meta/lib/oeqa/selftest/cases/reproducible.py
@@ -17,6 +17,72 @@ import stat
import os
import datetime
+# For sample packages, see:
+# https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201127-0t7wr_oo/
+# https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201127-4s9ejwyp/
+# https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201127-haiwdlbr/
+# https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201127-hwds3mcl/
+# https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201203-sua0pzvc/
+# (both packages/ and packages-excluded/)
+exclude_packages = [
+ 'acpica-src',
+ 'babeltrace2-ptest',
+ 'bootchart2-doc',
+ 'cups',
+ 'cwautomacros',
+ 'dtc',
+ 'efivar',
+ 'epiphany',
+ 'gcr',
+ 'git',
+ 'glide',
+ 'go-dep',
+ 'go-helloworld',
+ 'go-runtime',
+ 'go_',
+ 'groff',
+ 'gst-devtools',
+ 'gstreamer1.0-python',
+ 'gtk-doc',
+ 'igt-gpu-tools',
+ 'kernel-devsrc',
+ 'libaprutil',
+ 'libcap-ng',
+ 'libhandy-1-src',
+ 'libid3tag',
+ 'libproxy',
+ 'libsecret-dev',
+ 'libsecret-src',
+ 'lttng-tools-dbg',
+ 'lttng-tools-ptest',
+ 'ltp',
+ 'meson',
+ 'ovmf-shell-efi',
+ 'parted-ptest',
+ 'perf',
+ 'python3-cython',
+ 'qemu',
+ 'quilt-ptest',
+ 'rsync',
+ 'ruby',
+ 'spirv-tools-dev',
+ 'swig',
+ 'syslinux-misc',
+ 'systemd-bootchart',
+ 'valgrind-ptest',
+ 'vim',
+ 'watchdog',
+ 'xmlto',
+ 'xorg-minimal-fonts'
+ ]
+
+def is_excluded(package):
+ package_name = os.path.basename(package)
+ for i in exclude_packages:
+ if package_name.startswith(i):
+ return True
+ return False
+
MISSING = 'MISSING'
DIFFERENT = 'DIFFERENT'
SAME = 'SAME'
@@ -39,6 +105,7 @@ class PackageCompareResults(object):
self.total = []
self.missing = []
self.different = []
+ self.different_excluded = []
self.same = []
def add_result(self, r):
@@ -46,7 +113,10 @@ class PackageCompareResults(object):
if r.status == MISSING:
self.missing.append(r)
elif r.status == DIFFERENT:
- self.different.append(r)
+ if is_excluded(r.reference):
+ self.different_excluded.append(r)
+ else:
+ self.different.append(r)
else:
self.same.append(r)
@@ -54,10 +124,11 @@ class PackageCompareResults(object):
self.total.sort()
self.missing.sort()
self.different.sort()
+ self.different_excluded.sort()
self.same.sort()
def __str__(self):
- return 'same=%i different=%i missing=%i total=%i' % (len(self.same), len(self.different), len(self.missing), len(self.total))
+ return 'same=%i different=%i different_excluded=%i missing=%i total=%i' % (len(self.same), len(self.different), len(self.different_excluded), len(self.missing), len(self.total))
def compare_file(reference, test, diffutils_sysroot):
result = CompareResult()
@@ -237,6 +308,7 @@ class ReproducibleTests(OESelftestTestCase):
self.write_package_list(package_class, 'missing', result.missing)
self.write_package_list(package_class, 'different', result.different)
+ self.write_package_list(package_class, 'different_excluded', result.different_excluded)
self.write_package_list(package_class, 'same', result.same)
if self.save_results:
@@ -244,8 +316,12 @@ class ReproducibleTests(OESelftestTestCase):
self.copy_file(d.reference, '/'.join([save_dir, 'packages', strip_topdir(d.reference)]))
self.copy_file(d.test, '/'.join([save_dir, 'packages', strip_topdir(d.test)]))
+ for d in result.different_excluded:
+ self.copy_file(d.reference, '/'.join([save_dir, 'packages-excluded', strip_topdir(d.reference)]))
+ self.copy_file(d.test, '/'.join([save_dir, 'packages-excluded', strip_topdir(d.test)]))
+
if result.missing or result.different:
- fails.append("The following %s packages are missing or different: %s" %
+ fails.append("The following %s packages are missing or different and not in exclusion list: %s" %
(c, '\n'.join(r.test for r in (result.missing + result.different))))
# Clean up empty directories