diff options
author | Richard Purdie <rpurdie@linux.intel.com> | 2008-03-13 22:46:17 +0000 |
---|---|---|
committer | Richard Purdie <rpurdie@linux.intel.com> | 2008-03-13 22:46:17 +0000 |
commit | c93014671c0cfe7670335e204f3e14f0d42d0eb6 (patch) | |
tree | 914d626e4466033f02be54165c66a58c1573aaaf /lib/bb | |
parent | 4ad6a9e946f7128112f147610aedf69b0432ec0b (diff) | |
download | bitbake-c93014671c0cfe7670335e204f3e14f0d42d0eb6.tar.gz |
cache.py: Improve the way various data structures are handled to speed up the hot cache data path
Diffstat (limited to 'lib/bb')
-rw-r--r-- | lib/bb/cache.py | 76 | ||||
-rw-r--r-- | lib/bb/taskdata.py | 4 |
2 files changed, 41 insertions, 39 deletions
diff --git a/lib/bb/cache.py b/lib/bb/cache.py index dad82a9b3..8ca3aba71 100644 --- a/lib/bb/cache.py +++ b/lib/bb/cache.py @@ -80,7 +80,7 @@ class Cache: if old_mtime > newest_mtime: newest_mtime = old_mtime - if self.mtime(self.cachefile) >= newest_mtime: + if bb.parse.cached_mtime_noerror(self.cachefile) >= newest_mtime: try: p = pickle.Unpickler(file(self.cachefile, "rb")) self.depends_cache, version_data = p.load() @@ -199,31 +199,34 @@ class Cache: self.remove(fn) return False + mtime = bb.parse.cached_mtime_noerror(fn) + # Check file still exists - if self.mtime(fn) == 0: + if mtime == 0: bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s not longer exists" % fn) self.remove(fn) return False # Check the file's timestamp - if bb.parse.cached_mtime(fn) > self.getVar("CACHETIMESTAMP", fn, True): + if mtime > self.getVar("CACHETIMESTAMP", fn, True): bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s changed" % fn) self.remove(fn) return False # Check dependencies are still valid depends = self.getVar("__depends", fn, True) - for f,old_mtime in depends: - # Check if file still exists - if self.mtime(f) == 0: - self.remove(fn) - return False - - new_mtime = bb.parse.cached_mtime(f) - if (new_mtime > old_mtime): - bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s's dependency %s changed" % (fn, f)) - self.remove(fn) - return False + if depends: + for f,old_mtime in depends: + fmtime = bb.parse.cached_mtime_noerror(f) + # Check if file still exists + if fmtime == 0: + self.remove(fn) + return False + + if (fmtime > old_mtime): + bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s's dependency %s changed" % (fn, f)) + self.remove(fn) + return False #bb.msg.debug(2, bb.msg.domain.Cache, "Depends Cache: %s is clean" % fn) if not fn in self.clean: @@ -284,7 +287,6 @@ class Cache: pv = self.getVar('PV', file_name, True) pr = self.getVar('PR', file_name, True) dp = int(self.getVar('DEFAULT_PREFERENCE', file_name, True) or "0") - provides = Set([pn] + (self.getVar("PROVIDES", file_name, True) or "").split()) depends = bb.utils.explode_deps(self.getVar("DEPENDS", file_name, True) or "") packages = (self.getVar('PACKAGES', file_name, True) or "").split() packages_dynamic = (self.getVar('PACKAGES_DYNAMIC', file_name, True) or "").split() @@ -304,24 +306,31 @@ class Cache: cacheData.pkg_pepvpr[file_name] = (pe,pv,pr) cacheData.pkg_dp[file_name] = dp + provides = [pn] + for provide in (self.getVar("PROVIDES", file_name, True) or "").split(): + if provide not in provides: + provides.append(provide) + # Build forward and reverse provider hashes # Forward: virtual -> [filenames] # Reverse: PN -> [virtuals] if pn not in cacheData.pn_provides: - cacheData.pn_provides[pn] = Set() - cacheData.pn_provides[pn] |= provides + cacheData.pn_provides[pn] = [] - cacheData.fn_provides[file_name] = Set() + cacheData.fn_provides[file_name] = provides for provide in provides: if provide not in cacheData.providers: cacheData.providers[provide] = [] cacheData.providers[provide].append(file_name) - cacheData.fn_provides[file_name].add(provide) + if not provide in cacheData.pn_provides[pn]: + cacheData.pn_provides[pn].append(provide) - cacheData.deps[file_name] = Set() + cacheData.deps[file_name] = [] for dep in depends: - cacheData.all_depends.add(dep) - cacheData.deps[file_name].add(dep) + if not dep in cacheData.deps[file_name]: + cacheData.deps[file_name].append(dep) + if not dep in cacheData.all_depends: + cacheData.all_depends.append(dep) # Build reverse hash for PACKAGES, so runtime dependencies # can be be resolved (RDEPENDS, RRECOMMENDS etc.) @@ -343,28 +352,21 @@ class Cache: # Build hash of runtime depends and rececommends - def add_dep(deplist, deps): - for dep in deps: - if not dep in deplist: - deplist[dep] = "" - if not file_name in cacheData.rundeps: cacheData.rundeps[file_name] = {} if not file_name in cacheData.runrecs: cacheData.runrecs[file_name] = {} - rdepends = bb.utils.explode_deps(self.getVar('RDEPENDS', file_name, True) or "") - rrecommends = bb.utils.explode_deps(self.getVar('RRECOMMENDS', file_name, True) or "") + rdepends = self.getVar('RDEPENDS', file_name, True) or "" + rrecommends = self.getVar('RRECOMMENDS', file_name, True) or "" for package in packages + [pn]: if not package in cacheData.rundeps[file_name]: - cacheData.rundeps[file_name][package] = {} + cacheData.rundeps[file_name][package] = [] if not package in cacheData.runrecs[file_name]: - cacheData.runrecs[file_name][package] = {} + cacheData.runrecs[file_name][package] = [] - add_dep(cacheData.rundeps[file_name][package], rdepends) - add_dep(cacheData.runrecs[file_name][package], rrecommends) - add_dep(cacheData.rundeps[file_name][package], bb.utils.explode_deps(self.getVar("RDEPENDS_%s" % package, file_name, True) or "")) - add_dep(cacheData.runrecs[file_name][package], bb.utils.explode_deps(self.getVar("RRECOMMENDS_%s" % package, file_name, True) or "")) + cacheData.rundeps[file_name][package] = rdepends + " " + (self.getVar("RDEPENDS_%s" % package, file_name, True) or "") + cacheData.runrecs[file_name][package] = rrecommends + " " + (self.getVar("RRECOMMENDS_%s" % package, file_name, True) or "") # Collect files we may need for possible world-dep # calculations @@ -385,7 +387,7 @@ class Cache: data.setVar('TMPDIR', data.getVar('TMPDIR', config, 1) or "", config) bbfile_loc = os.path.abspath(os.path.dirname(bbfile)) oldpath = os.path.abspath(os.getcwd()) - if self.mtime(bbfile_loc): + if bb.parse.cached_mtime_noerror(bbfile_loc): os.chdir(bbfile_loc) bb_data = data.init_db(config) try: @@ -444,7 +446,7 @@ class CacheData: self.pkg_dp = {} self.pn_provides = {} self.fn_provides = {} - self.all_depends = Set() + self.all_depends = [] self.deps = {} self.rundeps = {} self.runrecs = {} diff --git a/lib/bb/taskdata.py b/lib/bb/taskdata.py index a50188b41..0fb34ad74 100644 --- a/lib/bb/taskdata.py +++ b/lib/bb/taskdata.py @@ -167,11 +167,11 @@ class TaskData: rdepends = dataCache.rundeps[fn] rrecs = dataCache.runrecs[fn] for package in rdepends: - for rdepend in rdepends[package]: + for rdepend in bb.utils.explode_deps(rdepends[package]): bb.msg.debug(2, bb.msg.domain.TaskData, "Added runtime dependency %s for %s" % (rdepend, fn)) rdependids[self.getrun_id(rdepend)] = None for package in rrecs: - for rdepend in rrecs[package]: + for rdepend in bb.utils.explode_deps(rrecs[package]): bb.msg.debug(2, bb.msg.domain.TaskData, "Added runtime recommendation %s for %s" % (rdepend, fn)) rdependids[self.getrun_id(rdepend)] = None self.rdepids[fnid] = rdependids.keys() |