bitbake/codeparser: Improve cache handling

The current codeparser cache handling hurts performance badly even
over a couple of cores and certainly on many core systems, it can
spent huge amounts of time in the codeparser cache save functions.

This patch reworks the cache handling so that each parsing thread
saves out its own "differences" file compared to any existing core
cache and then the main bitbake thread picks these up and merges
things back together.

This was tested on systems with small and large numbers of cores
and was found to perform orders of magnitude better in all cases
despite the more complex code.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

2 files changed, 71 insertions, 12 deletions

diff --git a/lib/bb/codeparser.py b/lib/bb/codeparser.py
index bb2616748..936b74261 100644
--- a/lib/bb/codeparser.py
+++ b/lib/bb/codeparser.py
@@ -70,7 +70,51 @@ def parser_cache_save(d):
     if not cachefile:
         return
 
-    lf = bb.utils.lockfile(cachefile + ".lock")
+    glf = bb.utils.lockfile(cachefile + ".lock", shared=True)
+
+    i = os.getpid()
+    lf = None
+    while not lf:
+        shellcache = {}
+        pythoncache = {}
+
+        lf = bb.utils.lockfile(cachefile + ".lock." + str(i), retry=False)
+        if not lf or os.path.exists(cachefile + "-" + str(i)):
+            if lf:
+               bb.utils.unlockfile(lf) 
+               lf = None
+            i = i + 1
+            continue
+
+        try:
+            p = pickle.Unpickler(file(cachefile, "rb"))
+            data, version = p.load()
+        except (IOError, EOFError):
+            data, version = None, None
+
+        if version != PARSERCACHE_VERSION:
+            shellcache = shellparsecache
+            pythoncache = pythonparsecache
+        else:
+            for h in pythonparsecache:
+                if h not in data[0]:
+                    pythoncache[h] = pythonparsecache[h]
+            for h in shellparsecache:
+                if h not in data[1]:
+                    shellcache[h] = shellparsecache[h]
+
+        p = pickle.Pickler(file(cachefile + "-" + str(i), "wb"), -1)
+        p.dump([[pythoncache, shellcache], PARSERCACHE_VERSION])
+
+    bb.utils.unlockfile(lf)
+    bb.utils.unlockfile(glf)
+
+def parser_cache_savemerge(d):
+    cachefile = parser_cachefile(d)
+    if not cachefile:
+        return
+
+    glf = bb.utils.lockfile(cachefile + ".lock")
 
     try:
         p = pickle.Unpickler(file(cachefile, "rb"))
@@ -78,17 +122,33 @@ def parser_cache_save(d):
     except (IOError, EOFError):
         data, version = None, None
 
-    if version == PARSERCACHE_VERSION:
-        for h in data[0]:
-            if h not in pythonparsecache:
-                pythonparsecache[h] = data[0][h]
-        for h in data[1]:
-            if h not in pythonparsecache:
-                shellparsecache[h] = data[1][h]
+    if version != PARSERCACHE_VERSION:
+        data = [{}, {}]
+
+    for f in [y for y in os.listdir(os.path.dirname(cachefile)) if y.startswith(os.path.basename(cachefile) + '-')]:
+        f = os.path.join(os.path.dirname(cachefile), f)
+        try:
+            p = pickle.Unpickler(file(f, "rb"))
+            extradata, version = p.load()
+        except (IOError, EOFError):
+            extradata, version = [{}, {}], None
+        
+        if version != PARSERCACHE_VERSION:
+            continue
+
+        for h in extradata[0]:
+            if h not in data[0]:
+                data[0][h] = extradata[0][h]
+        for h in extradata[1]:
+            if h not in data[1]:
+                data[1][h] = extradata[1][h]
+        os.unlink(f)
 
     p = pickle.Pickler(file(cachefile, "wb"), -1)
-    p.dump([[pythonparsecache, shellparsecache], PARSERCACHE_VERSION])
-    bb.utils.unlockfile(lf)
+    p.dump([data, PARSERCACHE_VERSION])
+
+    bb.utils.unlockfile(glf)
+
 
 class PythonParser():
     class ValueVisitor():
diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
index 43b36adcd..f00fdb2e9 100644
--- a/lib/bb/cooker.py
+++ b/lib/bb/cooker.py
@@ -1120,11 +1120,10 @@ class CookerParser(object):
             self.pool.terminate()
         self.pool.join()
 
-        bb.codeparser.parser_cache_save(self.cfgdata)
-
         sync = threading.Thread(target=self.bb_cache.sync)
         sync.start()
         multiprocessing.util.Finalize(None, sync.join, exitpriority=-100)
+        bb.codeparser.parser_cache_savemerge(self.cooker.configuration.data)
 
     def load_cached(self):
         for filename, appends in self.fromcache: