Allow for simultaneous do_rootfs tasks with rpm

Give each rootfs its own RPM channel to use.  This puts the RPM metadata
in a private subdirectory of $WORKDIR, rather than living in DEPLOY_DIR
where other tasks may race with it.

This allows us to reduce the time that the rpm.lock is held to only the
time needed to hardlink the RPMs, allowing the majority of the rootfs
operation to run in parallel.

Also, this fixes the smart tests by generating an index for all packages
at the time of the test, rather than using the one provided by the
rootfs process.

Original credit for the enhancement should go to Steven Walter
stevenrwalter@gmail.com.

Signed-off-by: Stephano Cetola <stephano.cetola@linux.intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

1 files changed, 40 insertions, 1 deletions

diff --git a/meta/lib/oeqa/runtime/smart.py b/meta/lib/oeqa/runtime/smart.py
index c7a5753991..c8ba433cdd 100644
--- a/meta/lib/oeqa/runtime/smart.py
+++ b/meta/lib/oeqa/runtime/smart.py
@@ -1,5 +1,7 @@
 import unittest
 import re
+import oe
+import subprocess
 from oeqa.oetest import oeRuntimeTest, skipModule
 from oeqa.utils.decorators import *
 from oeqa.utils.httpserver import HTTPService
@@ -53,9 +55,46 @@ class SmartBasicTest(SmartTest):
 class SmartRepoTest(SmartTest):
 
     @classmethod
+    def create_index(self, arg):
+        index_cmd = arg
+        try:
+            bb.note("Executing '%s' ..." % index_cmd)
+            result = subprocess.check_output(index_cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8")
+        except subprocess.CalledProcessError as e:
+            return("Index creation command '%s' failed with return code %d:\n%s" %
+                    (e.cmd, e.returncode, e.output.decode("utf-8")))
+        if result:
+            bb.note(result)
+        return None
+
+    @classmethod
     def setUpClass(self):
         self.repolist = []
-        self.repo_server = HTTPService(oeRuntimeTest.tc.d.getVar('DEPLOY_DIR', True), oeRuntimeTest.tc.target.server_ip)
+
+        # Index RPMs
+        rpm_createrepo = bb.utils.which(os.getenv('PATH'), "createrepo")
+        index_cmds = []
+        rpm_dirs_found = False
+        archs = (oeRuntimeTest.tc.d.getVar('ALL_MULTILIB_PACKAGE_ARCHS', True) or "").replace('-', '_').split()
+        for arch in archs:
+            rpm_dir = os.path.join(oeRuntimeTest.tc.d.getVar('DEPLOY_DIR_RPM', True), arch)
+            idx_path = os.path.join(oeRuntimeTest.tc.d.getVar('WORKDIR', True), 'rpm', arch)
+            db_path = os.path.join(oeRuntimeTest.tc.d.getVar('WORKDIR', True), 'rpmdb', arch)
+            if not os.path.isdir(rpm_dir):
+                continue
+            if os.path.exists(db_path):
+                bb.utils.remove(dbpath, True)
+            lockfilename = oeRuntimeTest.tc.d.getVar('DEPLOY_DIR_RPM', True) + "/rpm.lock"
+            lf = bb.utils.lockfile(lockfilename, False)
+            oe.path.copyhardlinktree(rpm_dir, idx_path)
+            bb.utils.unlockfile(lf)
+            index_cmds.append("%s --dbpath %s --update -q %s" % (rpm_createrepo, db_path, idx_path))
+            rpm_dirs_found = True
+         # Create repodata¬
+        result = oe.utils.multiprocess_exec(index_cmds, self.create_index)
+        if result:
+            bb.fatal('%s' % ('\n'.join(result)))
+        self.repo_server = HTTPService(oeRuntimeTest.tc.d.getVar('WORKDIR', True), oeRuntimeTest.tc.target.server_ip)
         self.repo_server.start()
 
     @classmethod