cache/codeparser: Switch to a new BB_CACHEDIR variable for cache location

Currently the codeparser cache is set from CACHE, which is typically in
bitbake.conf which means we can't read/write any cache until it is found/read.
We may well have python expressions to parse before that happens.
The net result is suboptimal functioning of the codeparser cache since it will
often be invalidated by data that is never written.

This patch changes the codeparser and filechecksum caches to use BB_CACHE as
their setting and defaults it to ${TOPDIR}/cache.

The patch doesn't change where the "persistent" data such as prserver and
hash-equiavalance resides (PERSISTENT_DIR) or where the metadata parsing
cache resists (still currently CACHE). I've left those for a later patch.

The patch does ensure data parsed by the core datastore parsing calls is
written back since this is now much more useful after this change.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

5 files changed, 17 insertions, 11 deletions

diff --git a/lib/bb/cache.py b/lib/bb/cache.py
index 8db4e4767..b309775bb 100644
--- a/lib/bb/cache.py
+++ b/lib/bb/cache.py
@@ -838,11 +838,10 @@ class MultiProcessCache(object):
         self.cachedata = self.create_cachedata()
         self.cachedata_extras = self.create_cachedata()
 
-    def init_cache(self, d, cache_file_name=None):
-        cachedir = (d.getVar("PERSISTENT_DIR") or
-                    d.getVar("CACHE"))
-        if cachedir in [None, '']:
+    def init_cache(self, cachedir, cache_file_name=None):
+        if not cachedir:
             return
+
         bb.utils.mkdirhier(cachedir)
         self.cachefile = os.path.join(cachedir,
                                       cache_file_name or self.__class__.cache_file_name)
diff --git a/lib/bb/codeparser.py b/lib/bb/codeparser.py
index ecae7b080..d6b810258 100644
--- a/lib/bb/codeparser.py
+++ b/lib/bb/codeparser.py
@@ -184,12 +184,12 @@ class CodeParserCache(MultiProcessCache):
         self.shellcachelines[h] = cacheline
         return cacheline
 
-    def init_cache(self, d):
+    def init_cache(self, cachedir):
         # Check if we already have the caches
         if self.pythoncache:
             return
 
-        MultiProcessCache.init_cache(self, d)
+        MultiProcessCache.init_cache(self, cachedir)
 
         # cachedata gets re-assigned in the parent
         self.pythoncache = self.cachedata[0]
@@ -201,8 +201,8 @@ class CodeParserCache(MultiProcessCache):
 
 codeparsercache = CodeParserCache()
 
-def parser_cache_init(d):
-    codeparsercache.init_cache(d)
+def parser_cache_init(cachedir):
+    codeparsercache.init_cache(cachedir)
 
 def parser_cache_save():
     codeparsercache.save_extras()
diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
index f28951cce..c5e9fa294 100644
--- a/lib/bb/cooker.py
+++ b/lib/bb/cooker.py
@@ -2268,7 +2268,7 @@ class CookerParser(object):
             if hasattr(process, "close"):
                 process.close()
 
-
+        bb.codeparser.parser_cache_save()
         bb.codeparser.parser_cache_savemerge()
         bb.cache.SiggenRecipeInfo.reset()
         bb.fetch.fetcher_parse_done()
diff --git a/lib/bb/cookerdata.py b/lib/bb/cookerdata.py
index c6b5658d7..1658bee93 100644
--- a/lib/bb/cookerdata.py
+++ b/lib/bb/cookerdata.py
@@ -271,7 +271,6 @@ class CookerDataBuilder(object):
             if self.data.getVar("BB_WORKERCONTEXT", False) is None and not worker:
                 bb.fetch.fetcher_init(self.data)
             bb.parse.init_parser(self.data)
-            bb.codeparser.parser_cache_init(self.data)
 
             bb.event.fire(bb.event.ConfigParsed(), self.data)
 
@@ -370,6 +369,11 @@ class CookerDataBuilder(object):
             data.setVar("TOPDIR", os.path.dirname(os.path.dirname(layerconf)))
             data = parse_config_file(layerconf, data)
 
+            if not data.getVar("BB_CACHEDIR"):
+                data.setVar("BB_CACHEDIR", "${TOPDIR}/cache")
+
+            bb.codeparser.parser_cache_init(data.getVar("BB_CACHEDIR"))
+
             layers = (data.getVar('BBLAYERS') or "").split()
             broken_layers = []
 
@@ -473,6 +477,9 @@ class CookerDataBuilder(object):
 
         if not data.getVar("TOPDIR"):
             data.setVar("TOPDIR", os.path.abspath(os.getcwd()))
+        if not data.getVar("BB_CACHEDIR"):
+            data.setVar("BB_CACHEDIR", "${TOPDIR}/cache")
+        bb.codeparser.parser_cache_init(data.getVar("BB_CACHEDIR"))
 
         data = parse_config_file(os.path.join("conf", "bitbake.conf"), data)
 
diff --git a/lib/bb/fetch2/__init__.py b/lib/bb/fetch2/__init__.py
index b1cd6b25c..5a7a6024d 100644
--- a/lib/bb/fetch2/__init__.py
+++ b/lib/bb/fetch2/__init__.py
@@ -518,7 +518,7 @@ def fetcher_init(d):
     else:
         raise FetchError("Invalid SRCREV cache policy of: %s" % srcrev_policy)
 
-    _checksum_cache.init_cache(d)
+    _checksum_cache.init_cache(d.getVar("BB_CACHEDIR"))
 
     for m in methods:
         if hasattr(m, "init"):