From 9f27563d66523f5af1028f173d53ee75e0877d46 Mon Sep 17 00:00:00 2001
From: Richard Purdie <richard.purdie@linuxfoundation.org>
Date: Fri, 27 May 2011 16:14:48 +0100
Subject: bitbake/codeparser: Improve cache handling

The current codeparser cache handling hurts performance badly even
over a couple of cores and certainly on many core systems, it can
spent huge amounts of time in the codeparser cache save functions.

This patch reworks the cache handling so that each parsing thread
saves out its own "differences" file compared to any existing core
cache and then the main bitbake thread picks these up and merges
things back together.

This was tested on systems with small and large numbers of cores
and was found to perform orders of magnitude better in all cases
despite the more complex code.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 lib/bb/codeparser.py | 80 +++++++++++++++++++++++++++++++++++++++++++++-------
 lib/bb/cooker.py     |  3 +-
 2 files changed, 71 insertions(+), 12 deletions(-)

(limited to 'lib')

diff --git a/lib/bb/codeparser.py b/lib/bb/codeparser.py
index bb2616748..936b74261 100644
--- a/lib/bb/codeparser.py
+++ b/lib/bb/codeparser.py
@@ -70,7 +70,51 @@ def parser_cache_save(d):
     if not cachefile:
         return
 
-    lf = bb.utils.lockfile(cachefile + ".lock")
+    glf = bb.utils.lockfile(cachefile + ".lock", shared=True)
+
+    i = os.getpid()
+    lf = None
+    while not lf:
+        shellcache = {}
+        pythoncache = {}
+
+        lf = bb.utils.lockfile(cachefile + ".lock." + str(i), retry=False)
+        if not lf or os.path.exists(cachefile + "-" + str(i)):
+            if lf:
+               bb.utils.unlockfile(lf) 
+               lf = None
+            i = i + 1
+            continue
+
+        try:
+            p = pickle.Unpickler(file(cachefile, "rb"))
+            data, version = p.load()
+        except (IOError, EOFError):
+            data, version = None, None
+
+        if version != PARSERCACHE_VERSION:
+            shellcache = shellparsecache
+            pythoncache = pythonparsecache
+        else:
+            for h in pythonparsecache:
+                if h not in data[0]:
+                    pythoncache[h] = pythonparsecache[h]
+            for h in shellparsecache:
+                if h not in data[1]:
+                    shellcache[h] = shellparsecache[h]
+
+        p = pickle.Pickler(file(cachefile + "-" + str(i), "wb"), -1)
+        p.dump([[pythoncache, shellcache], PARSERCACHE_VERSION])
+
+    bb.utils.unlockfile(lf)
+    bb.utils.unlockfile(glf)
+
+def parser_cache_savemerge(d):
+    cachefile = parser_cachefile(d)
+    if not cachefile:
+        return
+
+    glf = bb.utils.lockfile(cachefile + ".lock")
 
     try:
         p = pickle.Unpickler(file(cachefile, "rb"))
@@ -78,17 +122,33 @@ def parser_cache_save(d):
     except (IOError, EOFError):
         data, version = None, None
 
-    if version == PARSERCACHE_VERSION:
-        for h in data[0]:
-            if h not in pythonparsecache:
-                pythonparsecache[h] = data[0][h]
-        for h in data[1]:
-            if h not in pythonparsecache:
-                shellparsecache[h] = data[1][h]
+    if version != PARSERCACHE_VERSION:
+        data = [{}, {}]
+
+    for f in [y for y in os.listdir(os.path.dirname(cachefile)) if y.startswith(os.path.basename(cachefile) + '-')]:
+        f = os.path.join(os.path.dirname(cachefile), f)
+        try:
+            p = pickle.Unpickler(file(f, "rb"))
+            extradata, version = p.load()
+        except (IOError, EOFError):
+            extradata, version = [{}, {}], None
+        
+        if version != PARSERCACHE_VERSION:
+            continue
+
+        for h in extradata[0]:
+            if h not in data[0]:
+                data[0][h] = extradata[0][h]
+        for h in extradata[1]:
+            if h not in data[1]:
+                data[1][h] = extradata[1][h]
+        os.unlink(f)
 
     p = pickle.Pickler(file(cachefile, "wb"), -1)
-    p.dump([[pythonparsecache, shellparsecache], PARSERCACHE_VERSION])
-    bb.utils.unlockfile(lf)
+    p.dump([data, PARSERCACHE_VERSION])
+
+    bb.utils.unlockfile(glf)
+
 
 class PythonParser():
     class ValueVisitor():
diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
index 43b36adcd..f00fdb2e9 100644
--- a/lib/bb/cooker.py
+++ b/lib/bb/cooker.py
@@ -1120,11 +1120,10 @@ class CookerParser(object):
             self.pool.terminate()
         self.pool.join()
 
-        bb.codeparser.parser_cache_save(self.cfgdata)
-
         sync = threading.Thread(target=self.bb_cache.sync)
         sync.start()
         multiprocessing.util.Finalize(None, sync.join, exitpriority=-100)
+        bb.codeparser.parser_cache_savemerge(self.cooker.configuration.data)
 
     def load_cached(self):
         for filename, appends in self.fromcache:
-- 
cgit 1.2.3-korg