codeparser: Implement persistent cache

For a given input to this code, the output doesn't change to implement a persistent
cache of the data to speed up parsing.

Signed-off-by: Richard Purdie <rpurdie@linux.intel.com>

2 files changed, 68 insertions, 13 deletions

diff --git a/lib/bb/codeparser.py b/lib/bb/codeparser.py
index 0757294fb..06409319e 100644
--- a/lib/bb/codeparser.py
+++ b/lib/bb/codeparser.py
@@ -1,11 +1,20 @@
 import ast
 import codegen
 import logging
-from pysh import pyshyacc, pyshlex
+import os.path
+import bb.utils, bb.data
 from itertools import chain
+from pysh import pyshyacc, pyshlex
 
 
 logger = logging.getLogger('BitBake.CodeParser')
+PARSERCACHE_VERSION = 2
+
+try:
+    import cPickle as pickle
+except ImportError:
+    import pickle
+    logger.info('Importing cPickle failed.  Falling back to a very slow implementation.')
 
 
 def check_indent(codestr):
@@ -24,6 +33,45 @@ def check_indent(codestr):
     return codestr
 
 pythonparsecache = {}
+shellparsecache = {}
+
+def parser_cachefile(d):
+    cachedir = (bb.data.getVar("PERSISTENT_DIR", d, True) or
+                bb.data.getVar("CACHE", d, True))
+    if cachedir in [None, '']:
+        return None
+    bb.utils.mkdirhier(cachedir)
+    cachefile = os.path.join(cachedir, "bb_codeparser.dat")
+    logger.debug(1, "Using cache in '%s' for codeparser cache", cachefile)
+    return cachefile
+
+def parser_cache_init(d):
+    global pythonparsecache
+    global shellparsecache
+
+    cachefile = parser_cachefile(d)
+    if not cachefile:
+        return
+
+    try:
+        p = pickle.Unpickler(file(cachefile, "rb"))
+        data, version = p.load()
+    except:
+        return
+
+    if version != PARSERCACHE_VERSION:
+        return
+
+    pythonparsecache = data[0]
+    shellparsecache = data[1]
+
+def parser_cache_save(d):
+    cachefile = parser_cachefile(d)
+    if not cachefile:
+        return
+
+    p = pickle.Pickler(file(cachefile, "wb"), -1)
+    p.dump([[pythonparsecache, shellparsecache], PARSERCACHE_VERSION])
 
 class PythonParser():
     class ValueVisitor():
@@ -136,9 +184,11 @@ class PythonParser():
 
     def parse_python(self, node):
 
-        if node in pythonparsecache:
-            self.references = pythonparsecache[node].references
-            self.execs = pythonparsecache[node].execs
+        h = hash(str(node))
+
+        if h in pythonparsecache:
+            self.references = pythonparsecache[h]["refs"]
+            self.execs = pythonparsecache[h]["execs"]
             return
 
         code = compile(check_indent(str(node)), "<string>", "exec",
@@ -153,10 +203,9 @@ class PythonParser():
         self.references.update(visitor.var_execs)
         self.execs = visitor.direct_func_calls
 
-        pythonparsecache[node] = self
-
-
-shellparsecache = {}
+        pythonparsecache[h] = {}
+        pythonparsecache[h]["refs"] = self.references
+        pythonparsecache[h]["execs"] = self.execs
 
 class ShellParser():
     def __init__(self):
@@ -169,9 +218,11 @@ class ShellParser():
         commands it executes.
         """
 
-        if value in pythonparsecache:
-            self.execs = shellparsecache[value].execs
-            return
+        h = hash(str(value))
+
+        if h in shellparsecache:
+            self.execs = shellparsecache[h]["execs"]
+            return self.execs
 
         try:
             tokens, _ = pyshyacc.parse(value, eof=True, debug=False)
@@ -182,7 +233,8 @@ class ShellParser():
             self.process_tokens(token)
         self.execs = set(cmd for cmd in self.allexecs if cmd not in self.funcdefs)
 
-        shellparsecache[value] = self
+        shellparsecache[h] = {}
+        shellparsecache[h]["execs"] = self.execs
 
         return self.execs
 
diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
index c8ca09980..1df2bfe8a 100644
--- a/lib/bb/cooker.py
+++ b/lib/bb/cooker.py
@@ -522,6 +522,7 @@ class BBCooker:
             bb.event.register(var, bb.data.getVar(var, self.configuration.data))
 
         bb.fetch.fetcher_init(self.configuration.data)
+        bb.codeparser.parser_cache_init(self.configuration.data)
         bb.event.fire(bb.event.ConfigParsed(), self.configuration.data)
 
     def handleCollections( self, collections ):
@@ -879,7 +880,7 @@ class BBCooker:
 
     def shutdown(self):
         self.state = state.shutdown
- 
+
     def stop(self):
         self.state = state.stop
 
@@ -956,6 +957,8 @@ class CookerParser(object):
             self.pool.terminate()
         self.pool.join()
 
+        bb.codeparser.parser_cache_save(self.cfgdata)
+
         sync = threading.Thread(target=self.bb_cache.sync)
         sync.start()
         multiprocessing.util.Finalize(None, sync.join, exitpriority=-100)