diff options
Diffstat (limited to 'lib/bb/data_smart.py')
| -rw-r--r-- | lib/bb/data_smart.py | 256 |
1 files changed, 219 insertions, 37 deletions
diff --git a/lib/bb/data_smart.py b/lib/bb/data_smart.py index 5fdfeee2c..ddf98e6a2 100644 --- a/lib/bb/data_smart.py +++ b/lib/bb/data_smart.py @@ -28,7 +28,7 @@ BitBake build tools. # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # Based on functions from the base bb module, Copyright 2003 Holger Schurig -import copy, re +import copy, re, sys, traceback from collections import MutableMapping import logging import hashlib @@ -43,6 +43,42 @@ __setvar_regexp__ = re.compile('(?P<base>.*?)(?P<keyword>_append|_prepend)(_(?P< __expand_var_regexp__ = re.compile(r"\${[^{}]+}") __expand_python_regexp__ = re.compile(r"\${@.+?}") +def infer_caller_details(loginfo, parent = False, varval = True): + """Save the caller the trouble of specifying everything.""" + # Save effort. + if 'ignore' in loginfo and loginfo['ignore']: + return + # If nothing was provided, mark this as possibly unneeded. + if not loginfo: + loginfo['ignore'] = True + return + # Infer caller's likely values for variable (var) and value (value), + # to reduce clutter in the rest of the code. + if varval and ('variable' not in loginfo or 'detail' not in loginfo): + try: + raise Exception + except Exception: + tb = sys.exc_info()[2] + if parent: + above = tb.tb_frame.f_back.f_back + else: + above = tb.tb_frame.f_back + lcls = above.f_locals.items() + for k, v in lcls: + if k == 'value' and 'detail' not in loginfo: + loginfo['detail'] = v + if k == 'var' and 'variable' not in loginfo: + loginfo['variable'] = v + # Infer file/line/function from traceback + if 'file' not in loginfo: + depth = 3 + if parent: + depth = 4 + file, line, func, text = traceback.extract_stack(limit = depth)[0] + loginfo['file'] = file + loginfo['line'] = line + if func not in loginfo: + loginfo['func'] = func class VariableParse: def __init__(self, varname, d, val = None): @@ -157,11 +193,80 @@ class IncludeHistory(object): o.write("\n") child.emit(o, level) +class VariableHistory(object): + def __init__(self, dataroot): + self.dataroot = dataroot + self.variables = COWDictBase.copy() + + def copy(self): + new = VariableHistory(self.dataroot) + new.variables = self.variables.copy() + return new + + def record(self, *kwonly, **loginfo): + if not self.dataroot._tracking: + return + if len(kwonly) > 0: + raise TypeError + infer_caller_details(loginfo, parent = True) + if 'ignore' in loginfo and loginfo['ignore']: + return + if 'op' not in loginfo or not loginfo['op']: + loginfo['op'] = 'set' + if 'detail' in loginfo: + loginfo['detail'] = str(loginfo['detail']) + if 'variable' not in loginfo or 'file' not in loginfo: + raise ValueError("record() missing variable or file.") + var = loginfo['variable'] + + if var not in self.variables: + self.variables[var] = [] + self.variables[var].append(loginfo.copy()) + + def variable(self, var): + if var in self.variables: + return self.variables[var] + else: + return [] + + def emit(self, var, oval, val, o): + history = self.variable(var) + commentVal = re.sub('\n', '\n#', str(oval)) + if history: + if len(history) == 1: + o.write("#\n# $%s\n" % var) + else: + o.write("#\n# $%s [%d operations]\n" % (var, len(history))) + for event in history: + # o.write("# %s\n" % str(event)) + if 'func' in event: + # If we have a function listed, this is internal + # code, not an operation in a config file, and the + # full path is distracting. + event['file'] = re.sub('.*/', '', event['file']) + display_func = ' [%s]' % event['func'] + else: + display_func = '' + if 'flag' in event: + flag = '[%s] ' % (event['flag']) + else: + flag = '' + o.write("# %s %s:%s%s\n# %s\"%s\"\n" % (event['op'], event['file'], event['line'], display_func, flag, re.sub('\n', '\n# ', event['detail']))) + if len(history) > 1: + o.write("# computed:\n") + o.write('# "%s"\n' % (commentVal)) + else: + o.write("#\n# $%s\n# [no history recorded]\n#\n" % var) + o.write('# "%s"\n' % (commentVal)) + + class DataSmart(MutableMapping): def __init__(self, special = COWDictBase.copy(), seen = COWDictBase.copy() ): self.dict = {} self.inchistory = IncludeHistory() + self.varhistory = VariableHistory(self) + self._tracking = False # cookie monster tribute self._special_values = special @@ -169,6 +274,12 @@ class DataSmart(MutableMapping): self.expand_cache = {} + def enableTracking(self): + self._tracking = True + + def disableTracking(self): + self._tracking = False + def expandWithRefs(self, s, varname): if not isinstance(s, basestring): # sanity check @@ -204,10 +315,14 @@ class DataSmart(MutableMapping): return self.expandWithRefs(s, varname).value - def finalize(self): + def finalize(self, parent = False): """Performs final steps upon the datastore, including application of overrides""" overrides = (self.getVar("OVERRIDES", True) or "").split(":") or [] + finalize_caller = { + 'op': 'finalize', + } + infer_caller_details(finalize_caller, parent = parent, varval = False) # # Well let us see what breaks here. We used to iterate @@ -224,6 +339,9 @@ class DataSmart(MutableMapping): # Then we will handle _append and _prepend # + # We only want to report finalization once per variable overridden. + finalizes_reported = {} + for o in overrides: # calculate '_'+override l = len(o) + 1 @@ -236,7 +354,19 @@ class DataSmart(MutableMapping): for var in vars: name = var[:-l] try: - self.setVar(name, self.getVar(var, False)) + # Report only once, even if multiple changes. + if name not in finalizes_reported: + finalizes_reported[name] = True + finalize_caller['variable'] = name + finalize_caller['detail'] = 'was: ' + str(self.getVar(name, False)) + self.varhistory.record(**finalize_caller) + # Copy history of the override over. + for event in self.varhistory.variable(var): + loginfo = event.copy() + loginfo['variable'] = name + loginfo['op'] = 'override[%s]:%s' % (o, loginfo['op']) + self.varhistory.record(**loginfo) + self.setVar(name, self.getVar(var, False), op = 'finalize', file = 'override[%s]' % o, line = '') self.delVar(var) except Exception: logger.info("Untracked delVar") @@ -267,9 +397,9 @@ class DataSmart(MutableMapping): # We save overrides that may be applied at some later stage if keep: - self.setVarFlag(append, op, keep) + self.setVarFlag(append, op, keep, ignore=True) else: - self.delVarFlag(append, op) + self.delVarFlag(append, op, ignore=True) def initVar(self, var): self.expand_cache = {} @@ -297,7 +427,10 @@ class DataSmart(MutableMapping): else: self.initVar(var) - def setVar(self, var, value): + + def setVar(self, var, value, **loginfo): + if 'op' not in loginfo: + loginfo['op'] = "set" self.expand_cache = {} match = __setvar_regexp__.match(var) if match and match.group("keyword") in __setvar_keyword__: @@ -306,15 +439,22 @@ class DataSmart(MutableMapping): override = match.group('add') l = self.getVarFlag(base, keyword) or [] l.append([value, override]) - self.setVarFlag(base, keyword, l) - + self.setVarFlag(base, keyword, l, ignore=True) + # And cause that to be recorded: + loginfo['detail'] = value + loginfo['variable'] = base + if override: + loginfo['op'] = '%s[%s]' % (keyword, override) + else: + loginfo['op'] = keyword + self.varhistory.record(**loginfo) # todo make sure keyword is not __doc__ or __module__ # pay the cookie monster try: - self._special_values[keyword].add( base ) + self._special_values[keyword].add(base) except KeyError: self._special_values[keyword] = set() - self._special_values[keyword].add( base ) + self._special_values[keyword].add(base) return @@ -331,6 +471,7 @@ class DataSmart(MutableMapping): # setting var self.dict[var]["_content"] = value + self.varhistory.record(**loginfo) def getVar(self, var, expand=False, noweakdefault=False): value = self.getVarFlag(var, "_content", False, noweakdefault) @@ -340,13 +481,17 @@ class DataSmart(MutableMapping): return self.expand(value, var) return value - def renameVar(self, key, newkey): + def renameVar(self, key, newkey, **loginfo): """ Rename the variable key to newkey """ val = self.getVar(key, 0) if val is not None: - self.setVar(newkey, val) + loginfo['variable'] = newkey + loginfo['op'] = 'rename from %s' % key + loginfo['detail'] = val + self.varhistory.record(**loginfo) + self.setVar(newkey, val, ignore=True) for i in ('_append', '_prepend'): src = self.getVarFlag(key, i) @@ -355,23 +500,34 @@ class DataSmart(MutableMapping): dest = self.getVarFlag(newkey, i) or [] dest.extend(src) - self.setVarFlag(newkey, i, dest) + self.setVarFlag(newkey, i, dest, ignore=True) if i in self._special_values and key in self._special_values[i]: self._special_values[i].remove(key) self._special_values[i].add(newkey) - self.delVar(key) - - def appendVar(self, key, value): - value = (self.getVar(key, False) or "") + value - self.setVar(key, value) - - def prependVar(self, key, value): - value = value + (self.getVar(key, False) or "") - self.setVar(key, value) - - def delVar(self, var): + loginfo['variable'] = key + loginfo['op'] = 'rename (to)' + loginfo['detail'] = newkey + self.varhistory.record(**loginfo) + self.delVar(key, ignore=True) + + def appendVar(self, var, value, **loginfo): + loginfo['op'] = 'append' + self.varhistory.record(**loginfo) + newvalue = (self.getVar(var, False) or "") + value + self.setVar(var, newvalue, ignore=True) + + def prependVar(self, var, value, **loginfo): + loginfo['op'] = 'prepend' + self.varhistory.record(**loginfo) + newvalue = value + (self.getVar(var, False) or "") + self.setVar(var, newvalue, ignore=True) + + def delVar(self, var, **loginfo): + loginfo['detail'] = "" + loginfo['op'] = 'del' + self.varhistory.record(**loginfo) self.expand_cache = {} self.dict[var] = {} if '_' in var: @@ -379,10 +535,14 @@ class DataSmart(MutableMapping): if override and override in self._seen_overrides and var in self._seen_overrides[override]: self._seen_overrides[override].remove(var) - def setVarFlag(self, var, flag, flagvalue): + def setVarFlag(self, var, flag, value, **loginfo): + if 'op' not in loginfo: + loginfo['op'] = "set" + loginfo['flag'] = flag + self.varhistory.record(**loginfo) if not var in self.dict: self._makeShadowCopy(var) - self.dict[var][flag] = flagvalue + self.dict[var][flag] = value def getVarFlag(self, var, flag, expand=False, noweakdefault=False): local_var = self._findVar(var) @@ -396,7 +556,7 @@ class DataSmart(MutableMapping): value = self.expand(value, None) return value - def delVarFlag(self, var, flag): + def delVarFlag(self, var, flag, **loginfo): local_var = self._findVar(var) if not local_var: return @@ -404,23 +564,38 @@ class DataSmart(MutableMapping): self._makeShadowCopy(var) if var in self.dict and flag in self.dict[var]: - del self.dict[var][flag] - - def appendVarFlag(self, key, flag, value): - value = (self.getVarFlag(key, flag, False) or "") + value - self.setVarFlag(key, flag, value) + loginfo['detail'] = "" + loginfo['op'] = 'delFlag' + loginfo['flag'] = flag + self.varhistory.record(**loginfo) - def prependVarFlag(self, key, flag, value): - value = value + (self.getVarFlag(key, flag, False) or "") - self.setVarFlag(key, flag, value) + del self.dict[var][flag] - def setVarFlags(self, var, flags): + def appendVarFlag(self, var, flag, value, **loginfo): + loginfo['op'] = 'append' + loginfo['flag'] = flag + self.varhistory.record(**loginfo) + newvalue = (self.getVarFlag(var, flag, False) or "") + value + self.setVarFlag(var, flag, newvalue, ignore=True) + + def prependVarFlag(self, var, flag, value, **loginfo): + loginfo['op'] = 'prepend' + loginfo['flag'] = flag + self.varhistory.record(**loginfo) + newvalue = value + (self.getVarFlag(var, flag, False) or "") + self.setVarFlag(var, flag, newvalue, ignore=True) + + def setVarFlags(self, var, flags, **loginfo): + infer_caller_details(loginfo) if not var in self.dict: self._makeShadowCopy(var) for i in flags: if i == "_content": continue + loginfo['flag'] = i + loginfo['detail'] = flags[i] + self.varhistory.record(**loginfo) self.dict[var][i] = flags[i] def getVarFlags(self, var): @@ -438,13 +613,16 @@ class DataSmart(MutableMapping): return flags - def delVarFlags(self, var): + def delVarFlags(self, var, **loginfo): if not var in self.dict: self._makeShadowCopy(var) if var in self.dict: content = None + loginfo['op'] = 'delete flags' + self.varhistory.record(**loginfo) + # try to save the content if "_content" in self.dict[var]: content = self.dict[var]["_content"] @@ -461,8 +639,12 @@ class DataSmart(MutableMapping): # we really want this to be a DataSmart... data = DataSmart(seen=self._seen_overrides.copy(), special=self._special_values.copy()) data.dict["_data"] = self.dict + data.varhistory = self.varhistory.copy() + data.varhistory.datasmart = data data.inchistory = self.inchistory.copy() + data._tracking = self._tracking + return data def expandVarref(self, variable, parents=False): |