bitbake: Add support for multiconfig dependencies

This patch adds the capability for tasks from different
multiconfigs to depend on one another.

These dependencies can be enabled using the following format:

task[mcdepends] = "multiconfig:FROM-MC:TO-MC:PN:task-to-depend-on"

For the sake of simplicity consider the following example:

Assuming we have set up multiconfig builds, one for qemux86 and one for
qemuarm, named x86 and arm respectively.

Adding the following line to an image recipe (core-image-sato):
do_image[mcdepends] = "multiconfig:x86:arm:core-image-minimal:do_rootfs"

Would state that core-image-sato:do_image from x86 will depend on
core-image-minimal:do_rootfs from arm so it can be executed.

This patch makes modifications to:
- cooker: To glue both multiconfigs in one place and make sure
  the dependencies can be provided.
- taskdata: To parse and add a new kind of dependency (mcdepends) to
  the taskdata object.
- runqueue: To differentiate tasks from different multiconfigs,
  add the specified dependencies to the corresponding tasks, and
  create a working runqueue that contains tasks from both multiconfigs.
- siggen: To avoid looking for tasks from different multiconfigs on
  objects where they dont belong.

The taskdata objects are still not aware of the concept of multiconfig,
so each object doesnt know which multiconfig its building, hence why
the mcdepends are added to all taskdata objects equally (we really
dont expect many of these), but the actual dependencies are added only
to the required tasks by the runqueue.

Signed-off-by: Alejandro Enedino Hernandez Samaniego <alejandr@xilinx.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

5 files changed, 79 insertions, 8 deletions

diff --git a/lib/bb/build.py b/lib/bb/build.py
index 4631abdde..c79354b3f 100644
--- a/lib/bb/build.py
+++ b/lib/bb/build.py
@@ -803,6 +803,7 @@ def add_tasks(tasklist, d):
             if name in flags:
                 deptask = d.expand(flags[name])
                 task_deps[name][task] = deptask
+        getTask('mcdepends')
         getTask('depends')
         getTask('rdepends')
         getTask('deptask')
diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
index 1fda40dd4..946ba9ca0 100644
--- a/lib/bb/cooker.py
+++ b/lib/bb/cooker.py
@@ -609,7 +609,14 @@ class BBCooker:
                 k2 = k.split(":do_")
                 k = k2[0]
                 ktask = k2[1]
-            taskdata[mc].add_provider(localdata[mc], self.recipecaches[mc], k)
+            if mc:
+                # Provider might be from another mc
+                for mcavailable in self.multiconfigs:
+                    # The first element is empty
+                    if mcavailable:
+                        taskdata[mcavailable].add_provider(localdata[mcavailable], self.recipecaches[mcavailable], k)
+            else:
+                taskdata[mc].add_provider(localdata[mc], self.recipecaches[mc], k)
             current += 1
             if not ktask.startswith("do_"):
                 ktask = "do_%s" % ktask
@@ -620,6 +627,27 @@ class BBCooker:
             runlist.append([mc, k, ktask, fn])
             bb.event.fire(bb.event.TreeDataPreparationProgress(current, len(fulltargetlist)), self.data)
 
+        mcdeps = taskdata[mc].get_mcdepends()
+        # No need to do check providers if there are no mcdeps or not an mc build
+        if mcdeps and mc:
+            # Make sure we can provide the multiconfig dependency
+            seen = set()
+            new = True
+            while new:
+                new = False
+                for mc in self.multiconfigs:
+                    for k in mcdeps:
+                        if k in seen:
+                            continue
+                        l = k.split(':')
+                        depmc = l[2]
+                        if depmc not in self.multiconfigs:
+                            bb.fatal("Multiconfig dependency %s depends on nonexistent mc configuration %s" % (k,depmc))
+                        else:
+                            logger.debug(1, "Adding providers for multiconfig dependency %s" % l[3])
+                            taskdata[depmc].add_provider(localdata[depmc], self.recipecaches[depmc], l[3])
+                            seen.add(k)
+                            new = True
         for mc in self.multiconfigs:
             taskdata[mc].add_unresolved(localdata[mc], self.recipecaches[mc])
 
@@ -706,8 +734,8 @@ class BBCooker:
             if not dotname in depend_tree["tdepends"]:
                 depend_tree["tdepends"][dotname] = []
             for dep in rq.rqdata.runtaskentries[tid].depends:
-                (depmc, depfn, deptaskname, deptaskfn) = bb.runqueue.split_tid_mcfn(dep)
-                deppn = self.recipecaches[mc].pkg_fn[deptaskfn]
+                (depmc, depfn, _, deptaskfn) = bb.runqueue.split_tid_mcfn(dep)
+                deppn = self.recipecaches[depmc].pkg_fn[deptaskfn]
                 depend_tree["tdepends"][dotname].append("%s.%s" % (deppn, bb.runqueue.taskname_from_tid(dep)))
             if taskfn not in seen_fns:
                 seen_fns.append(taskfn)
diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py
index ba9bebebc..a43c9983a 100644
--- a/lib/bb/runqueue.py
+++ b/lib/bb/runqueue.py
@@ -602,6 +602,19 @@ class RunQueueData:
                     if t in taskData[mc].taskentries:
                         depends.add(t)
 
+        def add_mc_dependencies(mc, tid):
+            mcdeps = taskData[mc].get_mcdepends()
+            for dep in mcdeps:
+                mcdependency = dep.split(':')
+                pn = mcdependency[3]
+                frommc = mcdependency[1]
+                mcdep = mcdependency[2]
+                deptask = mcdependency[4]
+                if mc == frommc:
+                    fn = taskData[mcdep].build_targets[pn][0]
+                    newdep = '%s:%s' % (fn,deptask)
+                    taskData[mc].taskentries[tid].tdepends.append(newdep)
+
         for mc in taskData:
             for tid in taskData[mc].taskentries:
 
@@ -618,12 +631,16 @@ class RunQueueData:
                 if fn in taskData[mc].failed_fns:
                     continue
 
+                # We add multiconfig dependencies before processing internal task deps (tdepends)
+                if 'mcdepends' in task_deps and taskname in task_deps['mcdepends']:
+                    add_mc_dependencies(mc, tid)
+
                 # Resolve task internal dependencies
                 #
                 # e.g. addtask before X after Y
                 for t in taskData[mc].taskentries[tid].tdepends:
-                    (_, depfn, deptaskname, _) = split_tid_mcfn(t)
-                    depends.add(build_tid(mc, depfn, deptaskname))
+                    (depmc, depfn, deptaskname, _) = split_tid_mcfn(t)
+                    depends.add(build_tid(depmc, depfn, deptaskname))
 
                 # Resolve 'deptask' dependencies
                 #
diff --git a/lib/bb/siggen.py b/lib/bb/siggen.py
index ab228e414..e9bb51d73 100644
--- a/lib/bb/siggen.py
+++ b/lib/bb/siggen.py
@@ -193,15 +193,24 @@ class SignatureGeneratorBasic(SignatureGenerator):
         return taint
 
     def get_taskhash(self, fn, task, deps, dataCache):
+
+        mc = ''
+        if fn.startswith('multiconfig:'):
+            mc = fn.split(':')[1]
         k = fn + "." + task
+
         data = dataCache.basetaskhash[k]
         self.basehash[k] = data
         self.runtaskdeps[k] = []
         self.file_checksum_values[k] = []
         recipename = dataCache.pkg_fn[fn]
-
         for dep in sorted(deps, key=clean_basepath):
-            depname = dataCache.pkg_fn[self.pkgnameextract.search(dep).group('fn')]
+            pkgname = self.pkgnameextract.search(dep).group('fn')
+            if mc:
+                depmc = pkgname.split(':')[1]
+                if mc != depmc:
+                    continue
+            depname = dataCache.pkg_fn[pkgname]
             if not self.rundep_check(fn, recipename, task, dep, depname, dataCache):
                 continue
             if dep not in self.taskhash:
diff --git a/lib/bb/taskdata.py b/lib/bb/taskdata.py
index 0ea6c0bfd..94e822c48 100644
--- a/lib/bb/taskdata.py
+++ b/lib/bb/taskdata.py
@@ -70,6 +70,8 @@ class TaskData:
 
         self.skiplist = skiplist
 
+        self.mcdepends = []
+
     def add_tasks(self, fn, dataCache):
         """
         Add tasks for a given fn to the database
@@ -88,6 +90,13 @@ class TaskData:
 
         self.add_extra_deps(fn, dataCache)
 
+        def add_mcdepends(task):
+            for dep in task_deps['mcdepends'][task].split():
+                if len(dep.split(':')) != 5:
+                    bb.msg.fatal("TaskData", "Error for %s:%s[%s], multiconfig dependency %s does not contain exactly four  ':' characters.\n Task '%s' should be specified in the form 'multiconfig:fromMC:toMC:packagename:task'" % (fn, task, 'mcdepends', dep, 'mcdepends'))
+                if dep not in self.mcdepends:
+                    self.mcdepends.append(dep)
+
         # Common code for dep_name/depends = 'depends'/idepends and 'rdepends'/irdepends
         def handle_deps(task, dep_name, depends, seen):
             if dep_name in task_deps and task in task_deps[dep_name]:
@@ -110,16 +119,20 @@ class TaskData:
             parentids = []
             for dep in task_deps['parents'][task]:
                 if dep not in task_deps['tasks']:
-                    bb.debug(2, "Not adding dependeny of %s on %s since %s does not exist" % (task, dep, dep))
+                    bb.debug(2, "Not adding dependency of %s on %s since %s does not exist" % (task, dep, dep))
                     continue
                 parentid = "%s:%s" % (fn, dep)
                 parentids.append(parentid)
             self.taskentries[tid].tdepends.extend(parentids)
 
+
             # Touch all intertask dependencies
             handle_deps(task, 'depends', self.taskentries[tid].idepends, self.seen_build_target)
             handle_deps(task, 'rdepends', self.taskentries[tid].irdepends, self.seen_run_target)
 
+            if 'mcdepends' in task_deps and task in task_deps['mcdepends']:
+                add_mcdepends(task)
+
         # Work out build dependencies
         if not fn in self.depids:
             dependids = set()
@@ -537,6 +550,9 @@ class TaskData:
                     provmap[name] = provider[0]
         return provmap
 
+    def get_mcdepends(self):
+        return self.mcdepends
+
     def dump_data(self):
         """
         Dump some debug information on the internal data structures