yocto-compat-layer.py: Add script to YP Compatible Layer validation

The yocto-compat-layer script serves as a tool to validate the alignament
of a layer with YP Compatible Layers Programme [1], is based on an RFC
sent to the ML to enable automatic testing of layers [2] that wants to
be YP Compatible.

The tool takes an layer (or set of layers) via command line option -l
and detects what kind of layer is distro, machine or software and then
executes a  set of tests against the layer in order to validate the
compatibility.

The tests currently implemented are:

common.test_readme: Test if a README file exists in the layer and isn't
    empty.
common.test_parse: Test for execute bitbake -p without errors.
common.test_show_environment: Test for execute bitbake -e without errors.
common.test_signatures: Test executed in BSP and DISTRO layers to review
    doesn't comes with recipes that changes the signatures.

bsp.test_bsp_defines_machines: Test if a BSP layers has machines
    configurations.
bsp.test_bsp_no_set_machine: Test the BSP layer to doesn't set
    machine at adding layer.

distro.test_distro_defines_distros: Test if a DISTRO layers has distro
    configurations.
distro.test_distro_no_set_distro: Test the DISTRO layer to doesn't set
    distro at adding layer.

Example of usage:

$ source oe-init-build-env
$ yocto-compat-layer.py LAYER_DIR

[YOCTO #10596]

[1] https://www.yoctoproject.org/webform/yocto-project-compatible-registration
[2] https://lists.yoctoproject.org/pipermail/yocto-ab/2016-October/001801.html

Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
Signed-off-by: Ross Burton <ross.burton@intel.com>

4 files changed, 118 insertions, 0 deletions

diff --git a/scripts/lib/compatlayer/cases/__init__.py b/scripts/lib/compatlayer/cases/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/scripts/lib/compatlayer/cases/__init__.py
diff --git a/scripts/lib/compatlayer/cases/bsp.py b/scripts/lib/compatlayer/cases/bsp.py
new file mode 100644
index 0000000000..5d9bf93e4a
--- /dev/null
+++ b/scripts/lib/compatlayer/cases/bsp.py
@@ -0,0 +1,26 @@
+# Copyright (C) 2017 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+import unittest
+
+from compatlayer import LayerType
+from compatlayer.case import OECompatLayerTestCase
+
+class BSPCompatLayer(OECompatLayerTestCase):
+    @classmethod
+    def setUpClass(self):
+        if self.tc.layer['type'] != LayerType.BSP:
+            raise unittest.SkipTest("BSPCompatLayer: Layer %s isn't BSP one." %\
+                self.tc.layer['name'])
+
+    def test_bsp_defines_machines(self):
+        self.assertTrue(self.tc.layer['conf']['machines'], 
+                "Layer is BSP but doesn't defines machines.")
+
+    def test_bsp_no_set_machine(self):
+        from oeqa.utils.commands import get_bb_var
+
+        machine = get_bb_var('MACHINE')
+        self.assertEqual(self.td['bbvars']['MACHINE'], machine,
+                msg="Layer %s modified machine %s -> %s" % \
+                    (self.tc.layer['name'], self.td['bbvars']['MACHINE'], machine))
diff --git a/scripts/lib/compatlayer/cases/common.py b/scripts/lib/compatlayer/cases/common.py
new file mode 100644
index 0000000000..4d328ec1f1
--- /dev/null
+++ b/scripts/lib/compatlayer/cases/common.py
@@ -0,0 +1,66 @@
+# Copyright (C) 2017 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+import os
+import subprocess
+import unittest
+from compatlayer import get_signatures, LayerType
+from compatlayer.case import OECompatLayerTestCase
+
+class CommonCompatLayer(OECompatLayerTestCase):
+    def test_readme(self):
+        readme_file = os.path.join(self.tc.layer['path'], 'README')
+        self.assertTrue(os.path.isfile(readme_file),
+                msg="Layer doesn't contains README file.")
+
+        data = ''
+        with open(readme_file, 'r') as f:
+            data = f.read()
+        self.assertTrue(data,
+                msg="Layer contains README file but is empty.")
+
+    def test_parse(self):
+        try:
+            output = subprocess.check_output('bitbake -p', shell=True,
+                    stderr=subprocess.PIPE)
+        except subprocess.CalledProcessError as e:
+            import traceback
+            exc = traceback.format_exc()
+            msg = 'Layer %s failed to parse.\n%s\n%s\n' % (self.tc.layer['name'],
+                    exc, e.output.decode('utf-8'))
+            raise RuntimeError(msg)
+
+    def test_show_environment(self):
+        try:
+            output = subprocess.check_output('bitbake -e', shell=True,
+                    stderr=subprocess.PIPE)
+        except subprocess.CalledProcessError as e:
+            import traceback
+            exc = traceback.format_exc()
+            msg = 'Layer %s failed to show environment.\n%s\n%s\n' % \
+                    (self.tc.layer['name'], exc, e.output.decode('utf-8'))
+            raise RuntimeError(msg)
+
+    def test_signatures(self):
+        if self.tc.layer['type'] == LayerType.SOFTWARE:
+            raise unittest.SkipTest("Layer %s isn't BSP or DISTRO one." \
+                     % self.tc.layer['name'])
+
+        sig_diff = {}
+
+        curr_sigs = get_signatures(self.td['builddir'], failsafe=True)
+        for task in self.td['sigs']:
+            if task not in curr_sigs:
+                continue
+
+            if self.td['sigs'][task] != curr_sigs[task]:
+                sig_diff[task] = '%s -> %s' % \
+                        (self.td['sigs'][task], curr_sigs[task])
+
+        detail = ''
+        if sig_diff:
+            for task in sig_diff:
+                detail += "%s changed %s\n" % (task, sig_diff[task])
+        self.assertFalse(bool(sig_diff), "Layer %s changed signatures.\n%s" % \
+                (self.tc.layer['name'], detail))
+
diff --git a/scripts/lib/compatlayer/cases/distro.py b/scripts/lib/compatlayer/cases/distro.py
new file mode 100644
index 0000000000..523acc1e78
--- /dev/null
+++ b/scripts/lib/compatlayer/cases/distro.py
@@ -0,0 +1,26 @@
+# Copyright (C) 2017 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+import unittest
+
+from compatlayer import LayerType
+from compatlayer.case import OECompatLayerTestCase
+
+class DistroCompatLayer(OECompatLayerTestCase):
+    @classmethod
+    def setUpClass(self):
+        if self.tc.layer['type'] != LayerType.DISTRO:
+            raise unittest.SkipTest("DistroCompatLayer: Layer %s isn't Distro one." %\
+                self.tc.layer['name'])
+
+    def test_distro_defines_distros(self):
+        self.assertTrue(self.tc.layer['conf']['distros'], 
+                "Layer is BSP but doesn't defines machines.")
+
+    def test_distro_no_set_distros(self):
+        from oeqa.utils.commands import get_bb_var
+
+        distro = get_bb_var('DISTRO')
+        self.assertEqual(self.td['bbvars']['DISTRO'], distro,
+                msg="Layer %s modified distro %s -> %s" % \
+                    (self.tc.layer['name'], self.td['bbvars']['DISTRO'], distro))