diff options
Diffstat (limited to 'lib/bb')
-rw-r--r-- | lib/bb/codeparser.py | 80 | ||||
-rw-r--r-- | lib/bb/cooker.py | 3 |
2 files changed, 71 insertions, 12 deletions
diff --git a/lib/bb/codeparser.py b/lib/bb/codeparser.py index bb26167481..936b742614 100644 --- a/lib/bb/codeparser.py +++ b/lib/bb/codeparser.py @@ -70,7 +70,51 @@ def parser_cache_save(d): if not cachefile: return - lf = bb.utils.lockfile(cachefile + ".lock") + glf = bb.utils.lockfile(cachefile + ".lock", shared=True) + + i = os.getpid() + lf = None + while not lf: + shellcache = {} + pythoncache = {} + + lf = bb.utils.lockfile(cachefile + ".lock." + str(i), retry=False) + if not lf or os.path.exists(cachefile + "-" + str(i)): + if lf: + bb.utils.unlockfile(lf) + lf = None + i = i + 1 + continue + + try: + p = pickle.Unpickler(file(cachefile, "rb")) + data, version = p.load() + except (IOError, EOFError): + data, version = None, None + + if version != PARSERCACHE_VERSION: + shellcache = shellparsecache + pythoncache = pythonparsecache + else: + for h in pythonparsecache: + if h not in data[0]: + pythoncache[h] = pythonparsecache[h] + for h in shellparsecache: + if h not in data[1]: + shellcache[h] = shellparsecache[h] + + p = pickle.Pickler(file(cachefile + "-" + str(i), "wb"), -1) + p.dump([[pythoncache, shellcache], PARSERCACHE_VERSION]) + + bb.utils.unlockfile(lf) + bb.utils.unlockfile(glf) + +def parser_cache_savemerge(d): + cachefile = parser_cachefile(d) + if not cachefile: + return + + glf = bb.utils.lockfile(cachefile + ".lock") try: p = pickle.Unpickler(file(cachefile, "rb")) @@ -78,17 +122,33 @@ def parser_cache_save(d): except (IOError, EOFError): data, version = None, None - if version == PARSERCACHE_VERSION: - for h in data[0]: - if h not in pythonparsecache: - pythonparsecache[h] = data[0][h] - for h in data[1]: - if h not in pythonparsecache: - shellparsecache[h] = data[1][h] + if version != PARSERCACHE_VERSION: + data = [{}, {}] + + for f in [y for y in os.listdir(os.path.dirname(cachefile)) if y.startswith(os.path.basename(cachefile) + '-')]: + f = os.path.join(os.path.dirname(cachefile), f) + try: + p = pickle.Unpickler(file(f, "rb")) + extradata, version = p.load() + except (IOError, EOFError): + extradata, version = [{}, {}], None + + if version != PARSERCACHE_VERSION: + continue + + for h in extradata[0]: + if h not in data[0]: + data[0][h] = extradata[0][h] + for h in extradata[1]: + if h not in data[1]: + data[1][h] = extradata[1][h] + os.unlink(f) p = pickle.Pickler(file(cachefile, "wb"), -1) - p.dump([[pythonparsecache, shellparsecache], PARSERCACHE_VERSION]) - bb.utils.unlockfile(lf) + p.dump([data, PARSERCACHE_VERSION]) + + bb.utils.unlockfile(glf) + class PythonParser(): class ValueVisitor(): diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py index 43b36adcde..f00fdb2e99 100644 --- a/lib/bb/cooker.py +++ b/lib/bb/cooker.py @@ -1120,11 +1120,10 @@ class CookerParser(object): self.pool.terminate() self.pool.join() - bb.codeparser.parser_cache_save(self.cfgdata) - sync = threading.Thread(target=self.bb_cache.sync) sync.start() multiprocessing.util.Finalize(None, sync.join, exitpriority=-100) + bb.codeparser.parser_cache_savemerge(self.cooker.configuration.data) def load_cached(self): for filename, appends in self.fromcache: |