diff options
Diffstat (limited to 'lib/bb/data_smart.py')
| -rw-r--r-- | lib/bb/data_smart.py | 272 |
1 files changed, 182 insertions, 90 deletions
diff --git a/lib/bb/data_smart.py b/lib/bb/data_smart.py index 7f1b6dcb4..0128a5bb1 100644 --- a/lib/bb/data_smart.py +++ b/lib/bb/data_smart.py @@ -16,8 +16,11 @@ BitBake build tools. # # Based on functions from the base bb module, Copyright 2003 Holger Schurig -import copy, re, sys, traceback -from collections import MutableMapping +import builtins +import copy +import re +import sys +from collections.abc import MutableMapping import logging import hashlib import bb, bb.codeparser @@ -26,13 +29,25 @@ from bb.COW import COWDictBase logger = logging.getLogger("BitBake.Data") -__setvar_keyword__ = ["_append", "_prepend", "_remove"] -__setvar_regexp__ = re.compile(r'(?P<base>.*?)(?P<keyword>_append|_prepend|_remove)(_(?P<add>[^A-Z]*))?$') -__expand_var_regexp__ = re.compile(r"\${[a-zA-Z0-9\-_+./~]+?}") -__expand_python_regexp__ = re.compile(r"\${@.+?}") +__setvar_keyword__ = [":append", ":prepend", ":remove"] +__setvar_regexp__ = re.compile(r'(?P<base>.*?)(?P<keyword>:append|:prepend|:remove)(:(?P<add>[^A-Z]*))?$') +__expand_var_regexp__ = re.compile(r"\${[a-zA-Z0-9\-_+./~:]+?}") +__expand_python_regexp__ = re.compile(r"\${@(?:{.*?}|.)+?}") __whitespace_split__ = re.compile(r'(\s)') __override_regexp__ = re.compile(r'[a-z0-9]+') +bitbake_renamed_vars = { + "BB_ENV_WHITELIST": "BB_ENV_PASSTHROUGH", + "BB_ENV_EXTRAWHITE": "BB_ENV_PASSTHROUGH_ADDITIONS", + "BB_HASHBASE_WHITELIST": "BB_BASEHASH_IGNORE_VARS", + "BB_HASHCONFIG_WHITELIST": "BB_HASHCONFIG_IGNORE_VARS", + "BB_HASHTASK_WHITELIST": "BB_TASKHASH_IGNORE_TASKS", + "BB_SETSCENE_ENFORCE_WHITELIST": "BB_SETSCENE_ENFORCE_IGNORE_TASKS", + "MULTI_PROVIDER_WHITELIST": "BB_MULTI_PROVIDER_ALLOWED", + "BB_STAMP_WHITELIST": "is a deprecated variable and support has been removed", + "BB_STAMP_POLICY": "is a deprecated variable and support has been removed", +} + def infer_caller_details(loginfo, parent = False, varval = True): """Save the caller the trouble of specifying everything.""" # Save effort. @@ -80,10 +95,11 @@ def infer_caller_details(loginfo, parent = False, varval = True): loginfo['func'] = func class VariableParse: - def __init__(self, varname, d, val = None): + def __init__(self, varname, d, unexpanded_value = None, val = None): self.varname = varname self.d = d self.value = val + self.unexpanded_value = unexpanded_value self.references = set() self.execs = set() @@ -107,6 +123,11 @@ class VariableParse: else: code = match.group()[3:-1] + # Do not run code that contains one or more unexpanded variables + # instead return the code with the characters we removed put back + if __expand_var_regexp__.findall(code): + return "${@" + code + "}" + if self.varname: varname = 'Var <%s>' % self.varname else: @@ -132,16 +153,21 @@ class VariableParse: value = utils.better_eval(codeobj, DataContext(self.d), {'d' : self.d}) return str(value) - class DataContext(dict): + excluded = set([i for i in dir(builtins) if not i.startswith('_')] + ['oe']) + def __init__(self, metadata, **kwargs): self.metadata = metadata dict.__init__(self, **kwargs) self['d'] = metadata + self.context = set(bb.utils.get_context()) def __missing__(self, key): + if key in self.excluded or key in self.context: + raise KeyError(key) + value = self.metadata.getVar(key) - if value is None or self.metadata.getVarFlag(key, 'func', False): + if value is None: raise KeyError(key) else: return value @@ -151,6 +177,7 @@ class ExpansionError(Exception): self.expression = expression self.variablename = varname self.exception = exception + self.varlist = [varname or expression or ""] if varname: if expression: self.msg = "Failure expanding variable %s, expression was %s which triggered exception %s: %s" % (varname, expression, type(exception).__name__, exception) @@ -160,8 +187,14 @@ class ExpansionError(Exception): self.msg = "Failure expanding expression %s which triggered exception %s: %s" % (expression, type(exception).__name__, exception) Exception.__init__(self, self.msg) self.args = (varname, expression, exception) + + def addVar(self, varname): + if varname: + self.varlist.append(varname) + def __str__(self): - return self.msg + chain = "\nThe variable dependency chain for the failure is: " + " -> ".join(self.varlist) + return self.msg + chain class IncludeHistory(object): def __init__(self, parent = None, filename = '[TOP LEVEL]'): @@ -189,7 +222,7 @@ class IncludeHistory(object): if self.current.parent: self.current = self.current.parent else: - bb.warn("Include log: Tried to finish '%s' at top level." % filename) + bb.warn("Include log: Tried to finish '%s' at top level." % self.filename) return False def emit(self, o, level = 0): @@ -277,7 +310,7 @@ class VariableHistory(object): for (r, override) in d.overridedata[var]: for event in self.variable(r): loginfo = event.copy() - if 'flag' in loginfo and not loginfo['flag'].startswith("_"): + if 'flag' in loginfo and not loginfo['flag'].startswith(("_", ":")): continue loginfo['variable'] = var loginfo['op'] = 'override[%s]:%s' % (override, loginfo['op']) @@ -329,6 +362,16 @@ class VariableHistory(object): lines.append(line) return lines + def get_variable_refs(self, var): + """Return a dict of file/line references""" + var_history = self.variable(var) + refs = {} + for event in var_history: + if event['file'] not in refs: + refs[event['file']] = [] + refs[event['file']].append(event['line']) + return refs + def get_variable_items_files(self, var): """ Use variable history to map items added to a list variable and @@ -342,7 +385,7 @@ class VariableHistory(object): for event in history: if 'flag' in event: continue - if event['op'] == '_remove': + if event['op'] == ':remove': continue if isset and event['op'] == 'set?': continue @@ -363,6 +406,23 @@ class VariableHistory(object): else: self.variables[var] = [] +def _print_rename_error(var, loginfo, renamedvars, fullvar=None): + info = "" + if "file" in loginfo: + info = " file: %s" % loginfo["file"] + if "line" in loginfo: + info += " line: %s" % loginfo["line"] + if fullvar and fullvar != var: + info += " referenced as: %s" % fullvar + if info: + info = " (%s)" % info.strip() + renameinfo = renamedvars[var] + if " " in renameinfo: + # A space signals a string to display instead of a rename + bb.erroronce('Variable %s %s%s' % (var, renameinfo, info)) + else: + bb.erroronce('Variable %s has been renamed to %s%s' % (var, renameinfo, info)) + class DataSmart(MutableMapping): def __init__(self): self.dict = {} @@ -370,6 +430,8 @@ class DataSmart(MutableMapping): self.inchistory = IncludeHistory() self.varhistory = VariableHistory(self) self._tracking = False + self._var_renames = {} + self._var_renames.update(bitbake_renamed_vars) self.expand_cache = {} @@ -391,9 +453,9 @@ class DataSmart(MutableMapping): def expandWithRefs(self, s, varname): if not isinstance(s, str): # sanity check - return VariableParse(varname, self, s) + return VariableParse(varname, self, s, s) - varparse = VariableParse(varname, self) + varparse = VariableParse(varname, self, s) while s.find('${') != -1: olds = s @@ -403,14 +465,17 @@ class DataSmart(MutableMapping): s = __expand_python_regexp__.sub(varparse.python_sub, s) except SyntaxError as e: # Likely unmatched brackets, just don't expand the expression - if e.msg != "EOL while scanning string literal": + if e.msg != "EOL while scanning string literal" and not e.msg.startswith("unterminated string literal"): raise if s == olds: break - except ExpansionError: + except ExpansionError as e: + e.addVar(varname) raise except bb.parse.SkipRecipe: raise + except bb.BBHandledException: + raise except Exception as exc: tb = sys.exc_info()[2] raise ExpansionError(varname, s, exc).with_traceback(tb) from exc @@ -422,24 +487,19 @@ class DataSmart(MutableMapping): def expand(self, s, varname = None): return self.expandWithRefs(s, varname).value - def finalize(self, parent = False): - return - - def internal_finalize(self, parent = False): - """Performs final steps upon the datastore, including application of overrides""" - self.overrides = None - def need_overrides(self): if self.overrides is not None: return if self.inoverride: return + overrride_stack = [] for count in range(5): self.inoverride = True # Can end up here recursively so setup dummy values self.overrides = [] self.overridesset = set() self.overrides = (self.getVar("OVERRIDES") or "").split(":") or [] + overrride_stack.append(self.overrides) self.overridesset = set(self.overrides) self.inoverride = False self.expand_cache = {} @@ -449,7 +509,7 @@ class DataSmart(MutableMapping): self.overrides = newoverrides self.overridesset = set(self.overrides) else: - bb.fatal("Overrides could not be expanded into a stable state after 5 iterations, overrides must be being referenced by other overridden variables in some recursive fashion. Please provide your configuration to bitbake-devel so we can laugh, er, I mean try and understand how to make it work.") + bb.fatal("Overrides could not be expanded into a stable state after 5 iterations, overrides must be being referenced by other overridden variables in some recursive fashion. Please provide your configuration to bitbake-devel so we can laugh, er, I mean try and understand how to make it work. The list of failing override expansions: %s" % "\n".join(str(s) for s in overrride_stack)) def initVar(self, var): self.expand_cache = {} @@ -460,27 +520,44 @@ class DataSmart(MutableMapping): dest = self.dict while dest: if var in dest: - return dest[var], self.overridedata.get(var, None) + return dest[var] if "_data" not in dest: break dest = dest["_data"] - return None, self.overridedata.get(var, None) + return None def _makeShadowCopy(self, var): if var in self.dict: return - local_var, _ = self._findVar(var) + local_var = self._findVar(var) if local_var: self.dict[var] = copy.copy(local_var) else: self.initVar(var) + def hasOverrides(self, var): + return var in self.overridedata def setVar(self, var, value, **loginfo): #print("var=" + str(var) + " val=" + str(value)) + + if not var.startswith("__anon_") and ("_append" in var or "_prepend" in var or "_remove" in var): + info = "%s" % var + if "file" in loginfo: + info += " file: %s" % loginfo["file"] + if "line" in loginfo: + info += " line: %s" % loginfo["line"] + bb.fatal("Variable %s contains an operation using the old override syntax. Please convert this layer/metadata before attempting to use with a newer bitbake." % info) + + shortvar = var.split(":", 1)[0] + if shortvar in self._var_renames: + _print_rename_error(shortvar, loginfo, self._var_renames, fullvar=var) + # Mark that we have seen a renamed variable + self.setVar("_FAILPARSINGERRORHANDLED", True) + self.expand_cache = {} parsing=False if 'parsing' in loginfo: @@ -509,7 +586,7 @@ class DataSmart(MutableMapping): # pay the cookie monster # more cookies for the cookie monster - if '_' in var: + if ':' in var: self._setvar_update_overrides(base, **loginfo) if base in self.overridevars: @@ -520,27 +597,27 @@ class DataSmart(MutableMapping): self._makeShadowCopy(var) if not parsing: - if "_append" in self.dict[var]: - del self.dict[var]["_append"] - if "_prepend" in self.dict[var]: - del self.dict[var]["_prepend"] - if "_remove" in self.dict[var]: - del self.dict[var]["_remove"] + if ":append" in self.dict[var]: + del self.dict[var][":append"] + if ":prepend" in self.dict[var]: + del self.dict[var][":prepend"] + if ":remove" in self.dict[var]: + del self.dict[var][":remove"] if var in self.overridedata: active = [] self.need_overrides() for (r, o) in self.overridedata[var]: if o in self.overridesset: active.append(r) - elif "_" in o: - if set(o.split("_")).issubset(self.overridesset): + elif ":" in o: + if set(o.split(":")).issubset(self.overridesset): active.append(r) for a in active: self.delVar(a) del self.overridedata[var] # more cookies for the cookie monster - if '_' in var: + if ':' in var: self._setvar_update_overrides(var, **loginfo) # setting var @@ -562,12 +639,12 @@ class DataSmart(MutableMapping): nextnew.update(vardata.references) nextnew.update(vardata.contains.keys()) new = nextnew - self.internal_finalize(True) + self.overrides = None def _setvar_update_overrides(self, var, **loginfo): # aka pay the cookie monster - override = var[var.rfind('_')+1:] - shortvar = var[:var.rfind('_')] + override = var[var.rfind(':')+1:] + shortvar = var[:var.rfind(':')] while override and __override_regexp__.match(override): if shortvar not in self.overridedata: self.overridedata[shortvar] = [] @@ -576,9 +653,9 @@ class DataSmart(MutableMapping): self.overridedata[shortvar] = list(self.overridedata[shortvar]) self.overridedata[shortvar].append([var, override]) override = None - if "_" in shortvar: - override = var[shortvar.rfind('_')+1:] - shortvar = var[:shortvar.rfind('_')] + if ":" in shortvar: + override = var[shortvar.rfind(':')+1:] + shortvar = var[:shortvar.rfind(':')] if len(shortvar) == 0: override = None @@ -602,10 +679,11 @@ class DataSmart(MutableMapping): self.varhistory.record(**loginfo) self.setVar(newkey, val, ignore=True, parsing=True) - for i in (__setvar_keyword__): - src = self.getVarFlag(key, i, False) - if src is None: + srcflags = self.getVarFlags(key, False, True) or {} + for i in srcflags: + if i not in (__setvar_keyword__): continue + src = srcflags[i] dest = self.getVarFlag(newkey, i, False) or [] dest.extend(src) @@ -617,7 +695,7 @@ class DataSmart(MutableMapping): self.overridedata[newkey].append([v.replace(key, newkey), o]) self.renameVar(v, v.replace(key, newkey)) - if '_' in newkey and val is None: + if ':' in newkey and val is None: self._setvar_update_overrides(newkey, **loginfo) loginfo['variable'] = key @@ -629,12 +707,12 @@ class DataSmart(MutableMapping): def appendVar(self, var, value, **loginfo): loginfo['op'] = 'append' self.varhistory.record(**loginfo) - self.setVar(var + "_append", value, ignore=True, parsing=True) + self.setVar(var + ":append", value, ignore=True, parsing=True) def prependVar(self, var, value, **loginfo): loginfo['op'] = 'prepend' self.varhistory.record(**loginfo) - self.setVar(var + "_prepend", value, ignore=True, parsing=True) + self.setVar(var + ":prepend", value, ignore=True, parsing=True) def delVar(self, var, **loginfo): self.expand_cache = {} @@ -645,10 +723,10 @@ class DataSmart(MutableMapping): self.dict[var] = {} if var in self.overridedata: del self.overridedata[var] - if '_' in var: - override = var[var.rfind('_')+1:] - shortvar = var[:var.rfind('_')] - while override and override.islower(): + if ':' in var: + override = var[var.rfind(':')+1:] + shortvar = var[:var.rfind(':')] + while override and __override_regexp__.match(override): try: if shortvar in self.overridedata: # Force CoW by recreating the list first @@ -657,15 +735,23 @@ class DataSmart(MutableMapping): except ValueError as e: pass override = None - if "_" in shortvar: - override = var[shortvar.rfind('_')+1:] - shortvar = var[:shortvar.rfind('_')] + if ":" in shortvar: + override = var[shortvar.rfind(':')+1:] + shortvar = var[:shortvar.rfind(':')] if len(shortvar) == 0: override = None def setVarFlag(self, var, flag, value, **loginfo): self.expand_cache = {} + if var == "BB_RENAMED_VARIABLES": + self._var_renames[flag] = value + + if var in self._var_renames: + _print_rename_error(var, loginfo, self._var_renames) + # Mark that we have seen a renamed variable + self.setVar("_FAILPARSINGERRORHANDLED", True) + if 'op' not in loginfo: loginfo['op'] = "set" loginfo['flag'] = flag @@ -674,7 +760,7 @@ class DataSmart(MutableMapping): self._makeShadowCopy(var) self.dict[var][flag] = value - if flag == "_defaultval" and '_' in var: + if flag == "_defaultval" and ':' in var: self._setvar_update_overrides(var, **loginfo) if flag == "_defaultval" and var in self.overridevars: self._setvar_update_overridevars(var, value) @@ -695,22 +781,27 @@ class DataSmart(MutableMapping): return None cachename = var + "[" + flag + "]" + if not expand and retparser and cachename in self.expand_cache: + return self.expand_cache[cachename].unexpanded_value, self.expand_cache[cachename] + if expand and cachename in self.expand_cache: return self.expand_cache[cachename].value - local_var, overridedata = self._findVar(var) + local_var = self._findVar(var) value = None removes = set() - if flag == "_content" and overridedata is not None and not parsing: + if flag == "_content" and not parsing: + overridedata = self.overridedata.get(var, None) + if flag == "_content" and not parsing and overridedata is not None: match = False active = {} self.need_overrides() for (r, o) in overridedata: - # What about double overrides both with "_" in the name? + # FIXME What about double overrides both with "_" in the name? if o in self.overridesset: active[o] = r - elif "_" in o: - if set(o.split("_")).issubset(self.overridesset): + elif ":" in o: + if set(o.split(":")).issubset(self.overridesset): active[o] = r mod = True @@ -718,10 +809,10 @@ class DataSmart(MutableMapping): mod = False for o in self.overrides: for a in active.copy(): - if a.endswith("_" + o): + if a.endswith(":" + o): t = active[a] del active[a] - active[a.replace("_" + o, "")] = t + active[a.replace(":" + o, "")] = t mod = True elif a == o: match = active[a] @@ -740,31 +831,31 @@ class DataSmart(MutableMapping): value = copy.copy(local_var["_defaultval"]) - if flag == "_content" and local_var is not None and "_append" in local_var and not parsing: - if not value: - value = "" + if flag == "_content" and local_var is not None and ":append" in local_var and not parsing: self.need_overrides() - for (r, o) in local_var["_append"]: + for (r, o) in local_var[":append"]: match = True if o: - for o2 in o.split("_"): + for o2 in o.split(":"): if not o2 in self.overrides: match = False if match: + if value is None: + value = "" value = value + r - if flag == "_content" and local_var is not None and "_prepend" in local_var and not parsing: - if not value: - value = "" + if flag == "_content" and local_var is not None and ":prepend" in local_var and not parsing: self.need_overrides() - for (r, o) in local_var["_prepend"]: + for (r, o) in local_var[":prepend"]: match = True if o: - for o2 in o.split("_"): + for o2 in o.split(":"): if not o2 in self.overrides: match = False if match: + if value is None: + value = "" value = r + value parser = None @@ -773,12 +864,12 @@ class DataSmart(MutableMapping): if expand: value = parser.value - if value and flag == "_content" and local_var is not None and "_remove" in local_var and not parsing: + if value and flag == "_content" and local_var is not None and ":remove" in local_var and not parsing: self.need_overrides() - for (r, o) in local_var["_remove"]: + for (r, o) in local_var[":remove"]: match = True if o: - for o2 in o.split("_"): + for o2 in o.split(":"): if not o2 in self.overrides: match = False if match: @@ -791,7 +882,7 @@ class DataSmart(MutableMapping): expanded_removes[r] = self.expand(r).split() parser.removes = set() - val = "" + val = [] for v in __whitespace_split__.split(parser.value): skip = False for r in removes: @@ -800,8 +891,8 @@ class DataSmart(MutableMapping): skip = True if skip: continue - val = val + v - parser.value = val + val.append(v) + parser.value = "".join(val) if expand: value = parser.value @@ -816,7 +907,7 @@ class DataSmart(MutableMapping): def delVarFlag(self, var, flag, **loginfo): self.expand_cache = {} - local_var, _ = self._findVar(var) + local_var = self._findVar(var) if not local_var: return if not var in self.dict: @@ -859,12 +950,12 @@ class DataSmart(MutableMapping): self.dict[var][i] = flags[i] def getVarFlags(self, var, expand = False, internalflags=False): - local_var, _ = self._findVar(var) + local_var = self._findVar(var) flags = {} if local_var: for i in local_var: - if i.startswith("_") and not internalflags: + if i.startswith(("_", ":")) and not internalflags: continue flags[i] = local_var[i] if expand and i in expand: @@ -905,6 +996,7 @@ class DataSmart(MutableMapping): data.inchistory = self.inchistory.copy() data._tracking = self._tracking + data._var_renames = self._var_renames data.overrides = None data.overridevars = copy.copy(self.overridevars) @@ -927,7 +1019,7 @@ class DataSmart(MutableMapping): value = self.getVar(variable, False) for key in keys: referrervalue = self.getVar(key, False) - if referrervalue and ref in referrervalue: + if referrervalue and isinstance(referrervalue, str) and ref in referrervalue: self.setVar(key, referrervalue.replace(ref, value)) def localkeys(self): @@ -962,8 +1054,8 @@ class DataSmart(MutableMapping): for (r, o) in self.overridedata[var]: if o in self.overridesset: overrides.add(var) - elif "_" in o: - if set(o.split("_")).issubset(self.overridesset): + elif ":" in o: + if set(o.split(":")).issubset(self.overridesset): overrides.add(var) for k in keylist(self.dict): @@ -993,10 +1085,10 @@ class DataSmart(MutableMapping): d = self.createCopy() bb.data.expandKeys(d) - config_whitelist = set((d.getVar("BB_HASHCONFIG_WHITELIST") or "").split()) + config_ignore_vars = set((d.getVar("BB_HASHCONFIG_IGNORE_VARS") or "").split()) keys = set(key for key in iter(d) if not key.startswith("__")) for key in keys: - if key in config_whitelist: + if key in config_ignore_vars: continue value = d.getVar(key, False) or "" @@ -1005,7 +1097,7 @@ class DataSmart(MutableMapping): else: data.update({key:value}) - varflags = d.getVarFlags(key, internalflags = True) + varflags = d.getVarFlags(key, internalflags = True, expand=["vardepvalue"]) if not varflags: continue for f in varflags: |