aboutsummaryrefslogtreecommitdiffstats
path: root/bitbake
diff options
context:
space:
mode:
authorRichard Purdie <rpurdie@linux.intel.com>2010-08-31 14:19:22 +0100
committerRichard Purdie <rpurdie@linux.intel.com>2010-08-31 14:49:14 +0100
commit9897b81e56eae97dea11b9489a3b691b8fc86441 (patch)
tree78b9e73fd6c8d13b797b54cb3c3d885d9b4ca7eb /bitbake
parent332c33af188c14dd0051d8a45fe0ad680611db69 (diff)
downloadopenembedded-core-contrib-9897b81e56eae97dea11b9489a3b691b8fc86441.tar.gz
bitbake/codeparser: Implement persistent cache
For a given input to this code, the output doesn't change to implement a persistent cache of the data to speed up parsing. Signed-off-by: Richard Purdie <rpurdie@linux.intel.com>
Diffstat (limited to 'bitbake')
-rw-r--r--bitbake/lib/bb/codeparser.py66
-rw-r--r--bitbake/lib/bb/cooker.py2
2 files changed, 58 insertions, 10 deletions
diff --git a/bitbake/lib/bb/codeparser.py b/bitbake/lib/bb/codeparser.py
index 88a26c82a7e..faffe390d97 100644
--- a/bitbake/lib/bb/codeparser.py
+++ b/bitbake/lib/bb/codeparser.py
@@ -4,6 +4,14 @@ from bb import msg, utils
import ast
import codegen
+PARSERCACHE_VERSION = 1
+
+try:
+ import cPickle as pickle
+except ImportError:
+ import pickle
+ bb.msg.note(1, bb.msg.domain.Cache, "Importing cPickle failed. Falling back to a very slow implementation.")
+
def check_indent(codestr):
"""If the code is indented, add a top level piece of code to 'remove' the indentation"""
@@ -13,6 +21,43 @@ def check_indent(codestr):
return codestr
pythonparsecache = {}
+shellparsecache = {}
+
+def parser_cachefile(d):
+ cachedir = bb.data.getVar("PERSISTENT_DIR", d, True) or bb.data.getVar("CACHE", d, True)
+ if cachedir in [None, '']:
+ return None
+ bb.utils.mkdirhier(cachedir)
+ cachefile = os.path.join(cachedir, "bb_codeparser.dat")
+ bb.msg.debug(1, bb.msg.domain.Cache, "Using cache in '%s' for codeparser cache" % cachefile)
+ return cachefile
+
+def parser_cache_init(d):
+
+ cachefile = parser_cachefile(d)
+ if not cachefile:
+ return
+
+ try:
+ p = pickle.Unpickler(file(cachefile, "rb"))
+ data, version = p.load()
+ except:
+ return
+
+ if version != PARSERCACHE_VERSION:
+ return
+
+ bb.codeparser.pythonparsecache = data[0]
+ bb.codeparser.shellparsecache = data[1]
+
+def parser_cache_save(d):
+
+ cachefile = parser_cachefile(d)
+ if not cachefile:
+ return
+
+ p = pickle.Pickler(file(cachefile, "wb"), -1)
+ p.dump([[bb.codeparser.pythonparsecache, bb.codeparser.shellparsecache], PARSERCACHE_VERSION])
class PythonParser():
class ValueVisitor():
@@ -125,9 +170,11 @@ class PythonParser():
def parse_python(self, node):
- if node in pythonparsecache:
- self.references = pythonparsecache[node].references
- self.execs = pythonparsecache[node].execs
+ h = hash(node)
+
+ if h in pythonparsecache:
+ self.references = pythonparsecache[h].references
+ self.execs = pythonparsecache[h].execs
return
code = compile(check_indent(str(node)), "<string>", "exec",
@@ -142,10 +189,7 @@ class PythonParser():
self.references.update(visitor.var_execs)
self.execs = visitor.direct_func_calls
- pythonparsecache[node] = self
-
-
-shellparsecache = {}
+ pythonparsecache[h] = self
class ShellParser():
def __init__(self):
@@ -158,8 +202,10 @@ class ShellParser():
commands it executes.
"""
- if value in pythonparsecache:
- self.execs = shellparsecache[value].execs
+ h = hash(value)
+
+ if h in shellparsecache:
+ self.execs = shellparsecache[h].execs
return
try:
@@ -171,7 +217,7 @@ class ShellParser():
self.process_tokens(token)
self.execs = set(cmd for cmd in self.allexecs if cmd not in self.funcdefs)
- shellparsecache[value] = self
+ shellparsecache[h] = self
return self.execs
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index dcdcb7d4435..ea33693ddb7 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -546,6 +546,7 @@ class BBCooker:
if bb.data.getVar("BB_WORKERCONTEXT", self.configuration.data) is None:
bb.fetch.fetcher_init(self.configuration.data)
+ bb.codeparser.parser_cache_init(self.configuration.data)
bb.event.fire(bb.event.ConfigParsed(), self.configuration.data)
@@ -1011,6 +1012,7 @@ class CookerParser:
if self.pointer >= self.total:
cooker.bb_cache.sync()
+ bb.codeparser.parser_cache_save(cooker.configuration.data)
if self.error > 0:
raise ParsingErrorsFound
return False