diff options
Diffstat (limited to 'bitbake/lib/bb/cache.py')
| -rw-r--r-- | bitbake/lib/bb/cache.py | 207 |
1 files changed, 166 insertions, 41 deletions
diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py index 921a9f7589..05c42518a7 100644 --- a/bitbake/lib/bb/cache.py +++ b/bitbake/lib/bb/cache.py @@ -33,15 +33,15 @@ Place, Suite 330, Boston, MA 02111-1307 USA. import os, re import bb.data import bb.utils +from sets import Set try: import cPickle as pickle except ImportError: import pickle - print "NOTE: Importing cPickle failed. Falling back to a very slow implementation." + bb.msg.note(1, bb.msg.domain.Cache, "Importing cPickle failed. Falling back to a very slow implementation.") -# __cache_version__ = "123" -__cache_version__ = "124" # changes the __depends structure +__cache_version__ = "125" class Cache: """ @@ -58,14 +58,12 @@ class Cache: if self.cachedir in [None, '']: self.has_cache = False - if cooker.cb is not None: - print "NOTE: Not using a cache. Set CACHE = <directory> to enable." + bb.msg.note(1, bb.msg.domain.Cache, "Not using a cache. Set CACHE = <directory> to enable.") else: self.has_cache = True self.cachefile = os.path.join(self.cachedir,"bb_cache.dat") - if cooker.cb is not None: - print "NOTE: Using cache in '%s'" % self.cachedir + bb.msg.debug(1, bb.msg.domain.Cache, "Using cache in '%s'" % self.cachedir) try: os.stat( self.cachedir ) except OSError: @@ -80,7 +78,7 @@ class Cache: if version_data['BITBAKE_VER'] != bb.__version__: raise ValueError, 'Bitbake Version Mismatch' except (ValueError, KeyError): - bb.note("Invalid cache found, rebuilding...") + bb.msg.note(1, bb.msg.domain.Cache, "Invalid cache found, rebuilding...") self.depends_cache = {} if self.depends_cache: @@ -108,7 +106,7 @@ class Cache: if fn != self.data_fn: # We're trying to access data in the cache which doesn't exist # yet setData hasn't been called to setup the right access. Very bad. - bb.error("Parsing error data_fn %s and fn %s don't match" % (self.data_fn, fn)) + bb.msg.error(bb.msg.domain.Cache, "Parsing error data_fn %s and fn %s don't match" % (self.data_fn, fn)) result = bb.data.getVar(var, self.data, exp) self.depends_cache[fn][var] = result @@ -127,15 +125,15 @@ class Cache: self.getVar("__depends", fn, True) self.depends_cache[fn]["CACHETIMESTAMP"] = bb.parse.cached_mtime(fn) - def loadDataFull(self, fn, cooker): + def loadDataFull(self, fn, cfgData): """ Return a complete set of data for fn. To do this, we need to parse the file. """ - bb_data, skipped = self.load_bbfile(fn, cooker) + bb_data, skipped = self.load_bbfile(fn, cfgData) return bb_data - def loadData(self, fn, cooker): + def loadData(self, fn, cfgData): """ Load a subset of data for fn. If the cached data is valid we do nothing, @@ -148,7 +146,7 @@ class Cache: return True, True return True, False - bb_data, skipped = self.load_bbfile(fn, cooker) + bb_data, skipped = self.load_bbfile(fn, cfgData) self.setData(fn, bb_data) return False, skipped @@ -175,32 +173,36 @@ class Cache: # Check file still exists if self.mtime(fn) == 0: - bb.debug(2, "Cache: %s not longer exists" % fn) + bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s not longer exists" % fn) self.remove(fn) return False # File isn't in depends_cache if not fn in self.depends_cache: - bb.debug(2, "Cache: %s is not cached" % fn) + bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s is not cached" % fn) self.remove(fn) return False # Check the file's timestamp if bb.parse.cached_mtime(fn) > self.getVar("CACHETIMESTAMP", fn, True): - bb.debug(2, "Cache: %s changed" % fn) + bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s changed" % fn) self.remove(fn) return False # Check dependencies are still valid depends = self.getVar("__depends", fn, True) for f,old_mtime in depends: + # Check if file still exists + if self.mtime(f) == 0: + return False + new_mtime = bb.parse.cached_mtime(f) if (new_mtime > old_mtime): - bb.debug(2, "Cache: %s's dependency %s changed" % (fn, f)) + bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s's dependency %s changed" % (fn, f)) self.remove(fn) return False - bb.debug(2, "Depends Cache: %s is clean" % fn) + bb.msg.debug(2, bb.msg.domain.Cache, "Depends Cache: %s is clean" % fn) if not fn in self.clean: self.clean[fn] = "" @@ -220,7 +222,7 @@ class Cache: Remove a fn from the cache Called from the parser in error cases """ - bb.debug(1, "Removing %s from cache" % fn) + bb.msg.debug(1, bb.msg.domain.Cache, "Removing %s from cache" % fn) if fn in self.depends_cache: del self.depends_cache[fn] if fn in self.clean: @@ -229,7 +231,7 @@ class Cache: def sync(self): """ Save the cache - Called from the parser when complete (or exitting) + Called from the parser when complete (or exiting) """ if not self.has_cache: @@ -243,12 +245,103 @@ class Cache: p.dump([self.depends_cache, version_data]) def mtime(self, cachefile): - try: - return os.stat(cachefile)[8] - except OSError: - return 0 + return bb.parse.cached_mtime_noerror(cachefile) - def load_bbfile( self, bbfile , cooker): + def handle_data(self, file_name, cacheData): + """ + Save data we need into the cache + """ + + pn = self.getVar('PN', file_name, True) + pv = self.getVar('PV', file_name, True) + pr = self.getVar('PR', file_name, True) + dp = int(self.getVar('DEFAULT_PREFERENCE', file_name, True) or "0") + provides = Set([pn] + (self.getVar("PROVIDES", file_name, True) or "").split()) + depends = bb.utils.explode_deps(self.getVar("DEPENDS", file_name, True) or "") + packages = (self.getVar('PACKAGES', file_name, True) or "").split() + packages_dynamic = (self.getVar('PACKAGES_DYNAMIC', file_name, True) or "").split() + rprovides = (self.getVar("RPROVIDES", file_name, True) or "").split() + + cacheData.task_queues[file_name] = self.getVar("_task_graph", file_name, True) + cacheData.task_deps[file_name] = self.getVar("_task_deps", file_name, True) + + # build PackageName to FileName lookup table + if pn not in cacheData.pkg_pn: + cacheData.pkg_pn[pn] = [] + cacheData.pkg_pn[pn].append(file_name) + + cacheData.stamp[file_name] = self.getVar('STAMP', file_name, True) + + # build FileName to PackageName lookup table + cacheData.pkg_fn[file_name] = pn + cacheData.pkg_pvpr[file_name] = (pv,pr) + cacheData.pkg_dp[file_name] = dp + + # Build forward and reverse provider hashes + # Forward: virtual -> [filenames] + # Reverse: PN -> [virtuals] + if pn not in cacheData.pn_provides: + cacheData.pn_provides[pn] = Set() + cacheData.pn_provides[pn] |= provides + + for provide in provides: + if provide not in cacheData.providers: + cacheData.providers[provide] = [] + cacheData.providers[provide].append(file_name) + + cacheData.deps[file_name] = Set() + for dep in depends: + cacheData.all_depends.add(dep) + cacheData.deps[file_name].add(dep) + + # Build reverse hash for PACKAGES, so runtime dependencies + # can be be resolved (RDEPENDS, RRECOMMENDS etc.) + for package in packages: + if not package in cacheData.packages: + cacheData.packages[package] = [] + cacheData.packages[package].append(file_name) + rprovides += (self.getVar("RPROVIDES_%s" % package, file_name, 1) or "").split() + + for package in packages_dynamic: + if not package in cacheData.packages_dynamic: + cacheData.packages_dynamic[package] = [] + cacheData.packages_dynamic[package].append(file_name) + + for rprovide in rprovides: + if not rprovide in cacheData.rproviders: + cacheData.rproviders[rprovide] = [] + cacheData.rproviders[rprovide].append(file_name) + + # Build hash of runtime depends and rececommends + + def add_dep(deplist, deps): + for dep in deps: + if not dep in deplist: + deplist[dep] = "" + + if not file_name in cacheData.rundeps: + cacheData.rundeps[file_name] = {} + if not file_name in cacheData.runrecs: + cacheData.runrecs[file_name] = {} + + for package in packages + [pn]: + if not package in cacheData.rundeps[file_name]: + cacheData.rundeps[file_name][package] = {} + if not package in cacheData.runrecs[file_name]: + cacheData.runrecs[file_name][package] = {} + + add_dep(cacheData.rundeps[file_name][package], bb.utils.explode_deps(self.getVar('RDEPENDS', file_name, True) or "")) + add_dep(cacheData.runrecs[file_name][package], bb.utils.explode_deps(self.getVar('RRECOMMENDS', file_name, True) or "")) + add_dep(cacheData.rundeps[file_name][package], bb.utils.explode_deps(self.getVar("RDEPENDS_%s" % package, file_name, True) or "")) + add_dep(cacheData.runrecs[file_name][package], bb.utils.explode_deps(self.getVar("RRECOMMENDS_%s" % package, file_name, True) or "")) + + # Collect files we may need for possible world-dep + # calculations + if not self.getVar('BROKEN', file_name, True) and not self.getVar('EXCLUDE_FROM_WORLD', file_name, True): + cacheData.possible_world.append(file_name) + + + def load_bbfile( self, bbfile , config): """ Load and parse one .bb build file Return the data and whether parsing resulted in the file being skipped @@ -257,25 +350,15 @@ class Cache: import bb from bb import utils, data, parse, debug, event, fatal - topdir = data.getVar('TOPDIR', cooker.configuration.data) - if not topdir: - topdir = os.path.abspath(os.getcwd()) - # set topdir to here - data.setVar('TOPDIR', topdir, cooker.configuration) - bbfile = os.path.abspath(bbfile) - bbfile_loc = os.path.abspath(os.path.dirname(bbfile)) # expand tmpdir to include this topdir - data.setVar('TMPDIR', data.getVar('TMPDIR', cooker.configuration.data, 1) or "", cooker.configuration.data) - # set topdir to location of .bb file - topdir = bbfile_loc - #data.setVar('TOPDIR', topdir, cfg) - # go there + data.setVar('TMPDIR', data.getVar('TMPDIR', config, 1) or "", config) + bbfile_loc = os.path.abspath(os.path.dirname(bbfile)) oldpath = os.path.abspath(os.getcwd()) - if self.mtime(topdir): - os.chdir(topdir) - bb_data = data.init_db(cooker.configuration.data) + if self.mtime(bbfile_loc): + os.chdir(bbfile_loc) + bb_data = data.init_db(config) try: - parse.handle(bbfile, bb_data) # read .bb data + bb_data = parse.handle(bbfile, bb_data) # read .bb data os.chdir(oldpath) return bb_data, False except bb.parse.SkipPackage: @@ -304,3 +387,45 @@ def init(cooker): """ return Cache(cooker) + + +#============================================================================# +# CacheData +#============================================================================# +class CacheData: + """ + The data structures we compile from the cached data + """ + + def __init__(self): + """ + Direct cache variables + (from Cache.handle_data) + """ + self.providers = {} + self.rproviders = {} + self.packages = {} + self.packages_dynamic = {} + self.possible_world = [] + self.pkg_pn = {} + self.pkg_fn = {} + self.pkg_pvpr = {} + self.pkg_dp = {} + self.pn_provides = {} + self.all_depends = Set() + self.deps = {} + self.rundeps = {} + self.runrecs = {} + self.task_queues = {} + self.task_deps = {} + self.stamp = {} + self.preferred = {} + + """ + Indirect Cache variables + (set elsewhere) + """ + self.ignored_dependencies = [] + self.world_target = Set() + self.bbfile_priority = {} + self.bbfile_config_priorities = [] |