diff options
Diffstat (limited to 'lib/bb')
-rw-r--r-- | lib/bb/data.py | 190 | ||||
-rw-r--r-- | lib/bb/data_smart.py | 77 |
2 files changed, 138 insertions, 129 deletions
diff --git a/lib/bb/data.py b/lib/bb/data.py index 56ee977f6..6cac3ea19 100644 --- a/lib/bb/data.py +++ b/lib/bb/data.py @@ -7,6 +7,18 @@ BitBake 'Data' implementations Functions for interacting with the data structure used by the BitBake build tools. +The expandData and update_data are the most expensive +operations. At night the cookie monster came by and +suggested 'give me cookies on setting the variables and +things will work out'. Taking this suggestion into account +applying the skills from the not yet passed 'Entwurf und +Analyse von Algorithmen' lecture and the cookie +monster seems to be right. We will track setVar more carefully +to have faster update_data and expandKeys operations. + +This is a treade-off between speed and memory again but +the speed is more critical here. + Copyright (C) 2003, 2004 Chris Larson Copyright (C) 2005 Holger Hans Peter Freyther @@ -273,6 +285,27 @@ def setData(newData, d): """Sets the data object to the supplied value""" d = newData + +## +## Cookie Monsters' query functions +## +def _get_override_vars(d, override): + """ + Internal!!! + + Get the Names of Variables that have a specific + override. This function returns a iterable + Set or an empty list + """ + return [] + +def _get_var_flags_triple(d): + """ + Internal!!! + + """ + return [] + __expand_var_regexp__ = re.compile(r"\${[^{}]+}") __expand_python_regexp__ = re.compile(r"\${@.+?}") @@ -303,43 +336,7 @@ def expand(s, d, varname = None): >>> print expand('${SRC_URI}', d) http://somebug.${TARGET_MOO} """ - def var_sub(match): - key = match.group()[2:-1] - if varname and key: - if varname == key: - raise Exception("variable %s references itself!" % varname) - var = getVar(key, d, 1) - if var is not None: - return var - else: - return match.group() - - def python_sub(match): - import bb - code = match.group()[3:-1] - locals()['d'] = d - s = eval(code) - if type(s) == types.IntType: s = str(s) - return s - - if type(s) is not types.StringType: # sanity check - return s - - while s.find('$') != -1: - olds = s - try: - s = __expand_var_regexp__.sub(var_sub, s) - s = __expand_python_regexp__.sub(python_sub, s) - if s == olds: break - if type(s) is not types.StringType: # sanity check - import bb - bb.error('expansion of %s returned non-string %s' % (olds, s)) - except KeyboardInterrupt: - raise - except: - note("%s:%s while evaluating:\n%s" % (sys.exc_info()[0], sys.exc_info()[1], s)) - raise - return s + return d.expand(s, varname) def expandKeys(alterdata, readdata = None): if readdata == None: @@ -356,7 +353,7 @@ def expandKeys(alterdata, readdata = None): # setVarFlags(ekey, copy.copy(getVarFlags(key, readdata)), alterdata) setVar(ekey, val, alterdata) - for i in ('_append', '_prepend', '_delete'): + for i in ('_append', '_prepend'): dest = getVarFlag(ekey, i, alterdata) or [] src = getVarFlag(key, i, readdata) or [] dest.extend(src) @@ -507,67 +504,76 @@ def update_data(d): >>> print getVar('TEST', d) local """ - debug(2, "update_data()") -# can't do delete env[...] while iterating over the dictionary, so remember them - dodel = [] + # now ask the cookie monster for help + #print "Cookie Monster" + #print "Append/Prepend %s" % d._special_values + #print "Overrides %s" % d._seen_overrides + overrides = (getVar('OVERRIDES', d, 1) or "").split(':') or [] - def applyOverrides(var, d): - if not overrides: - debug(1, "OVERRIDES not defined, nothing to do") - return - val = getVar(var, d) - for o in overrides: - if var.endswith("_" + o): - l = len(o)+1 - name = var[:-l] - d[name] = d[var] + # + # Well let us see what breaks here. We used to iterate + # over each variable and apply the override and then + # do the line expanding. + # If we have bad luck - which we will have - the keys + # where in some order that is so important for this + # method which we don't have anymore. + # Anyway we will fix that and write test cases this + # time. + + # + # First we apply all overrides + # Then we will handle _append and _prepend + # + + for o in overrides: + # calculate '_'+override + l = len(o)+1 + + # see if one should even try + if not o in d._seen_overrides: + continue - for s in keys(d): - applyOverrides(s, d) - sval = getVar(s, d) or "" - -# Handle line appends: - for (a, o) in getVarFlag(s, '_append', d) or []: - # maybe the OVERRIDE was not yet added so keep the append - if (o and o in overrides) or not o: - delVarFlag(s, '_append', d) - if o: - if not o in overrides: + vars = d._seen_overrides[o] + for var in vars: + name = var[:-l] + try: + d[name] = d[var] + except: + note ("Untracked delVar") + + # now on to the appends and prepends + if '_append' in d._special_values: + appends = d._special_values['_append'] or [] + for append in appends: + for (a, o) in getVarFlag(append, '_append', d) or []: + # maybe the OVERRIDE was not yet added so keep the append + if (o and o in overrides) or not o: + delVarFlag(append, '_append', d) + if o and not o in overrides: continue - sval+=a - setVar(s, sval, d) - -# Handle line prepends - for (a, o) in getVarFlag(s, '_prepend', d) or []: - # maybe the OVERRIDE was not yet added so keep the append - if (o and o in overrides) or not o: - delVarFlag(s, '_prepend', d) - if o: - if not o in overrides: + + sval = getVar(append,d) or "" + sval+=a + setVar(append, sval, d) + + + if '_prepend' in d._special_values: + prepends = d._special_values['_prepend'] or [] + + for prepend in prepends: + for (a, o) in getVarFlag(prepend, '_prepend', d) or []: + # maybe the OVERRIDE was not yet added so keep the prepend + if (o and o in overrides) or not o: + delVarFlag(prepend, '_prepend', d) + if o and not o in overrides: continue - sval=a+sval - setVar(s, sval, d) - -# Handle line deletions - name = s + "_delete" - nameval = getVar(name, d) - if nameval: - sval = getVar(s, d) - if sval: - new = '' - pattern = nameval.replace('\n','').strip() - for line in sval.split('\n'): - if line.find(pattern) == -1: - new = new + '\n' + line - setVar(s, new, d) - dodel.append(name) - -# delete all environment vars no longer needed - for s in dodel: - delVar(s, d) + + sval = a + (getVar(prepend,d) or "") + setVar(prepend, sval, d) + def inherits_class(klass, d): val = getVar('__inherit_cache', d) or "" diff --git a/lib/bb/data_smart.py b/lib/bb/data_smart.py index 8b062ee57..c21921ca3 100644 --- a/lib/bb/data_smart.py +++ b/lib/bb/data_smart.py @@ -8,7 +8,7 @@ BitBake build tools. Copyright (C) 2003, 2004 Chris Larson Copyright (C) 2004, 2005 Seb Frankengul -Copyright (C) 2005 Holger Hans Peter Freyther +Copyright (C) 2005, 2006 Holger Hans Peter Freyther Copyright (C) 2005 Uli Luckas Copyright (C) 2005 ROAD GmbH @@ -29,7 +29,8 @@ Based on functions from the base bb module, Copyright 2003 Holger Schurig """ import copy, os, re, sys, time, types -from bb import note, debug, fatal, utils, methodpool +from bb import note, debug, error, fatal, utils, methodpool +from sets import Set try: import cPickle as pickle @@ -37,9 +38,8 @@ except ImportError: import pickle print "NOTE: Importing cPickle failed. Falling back to a very slow implementation." - -__setvar_keyword__ = ["_append","_prepend","_delete"] -__setvar_regexp__ = re.compile('(?P<base>.*?)(?P<keyword>_append|_prepend|_delete)(_(?P<add>.*))?') +__setvar_keyword__ = ["_append","_prepend"] +__setvar_regexp__ = re.compile('(?P<base>.*?)(?P<keyword>_append|_prepend)(_(?P<add>.*))?') __expand_var_regexp__ = re.compile(r"\${[^{}]+}") __expand_python_regexp__ = re.compile(r"\${@.+?}") @@ -48,6 +48,10 @@ class DataSmart: def __init__(self): self.dict = {} + # cookie monster tribute + self._special_values = {} + self._seen_overrides = {} + def expand(self,s, varname): def var_sub(match): key = match.group()[2:-1] @@ -78,8 +82,7 @@ class DataSmart: s = __expand_python_regexp__.sub(python_sub, s) if s == olds: break if type(s) is not types.StringType: # sanity check - import bb - bb.error('expansion of %s returned non-string %s' % (olds, s)) + error('expansion of %s returned non-string %s' % (olds, s)) except KeyboardInterrupt: raise except: @@ -91,18 +94,6 @@ class DataSmart: if not var in self.dict: self.dict[var] = {} - def pickle_prep(self, cfg): - if "_data" in self.dict: - if self.dict["_data"] == cfg: - self.dict["_data"] = "cfg"; - else: # this is an unknown array for the moment - pass - - def unpickle_prep(self, cfg): - if "_data" in self.dict: - if self.dict["_data"] == "cfg": - self.dict["_data"] = cfg; - def _findVar(self,var): _dest = self.dict @@ -116,14 +107,6 @@ class DataSmart: return _dest[var] return None - def _copyVar(self,var,name): - local_var = self._findVar(var) - if local_var: - self.dict[name] = copy.copy(local_var) - else: - debug(1,"Warning, _copyVar %s to %s, %s does not exists" % (var,name,var)) - - def _makeShadowCopy(self, var): if var in self.dict: return @@ -142,11 +125,20 @@ class DataSmart: keyword = match.group("keyword") override = match.group('add') l = self.getVarFlag(base, keyword) or [] - if override == 'delete': - if l.count([value, None]): - del l[l.index([value, None])] l.append([value, override]) - self.setVarFlag(base, match.group("keyword"), l) + self.setVarFlag(base, keyword, l) + + # pay the cookie monster + try: + self._special_values[keyword].add( base ) + except: + self._special_values[keyword] = Set() + self._special_values[keyword].add( base ) + + # SRC_URI_append_simpad is both a flag and a override + #if not override in self._seen_overrides: + # self._seen_overrides[override] = Set() + #self._seen_overrides[override].add( base ) return if not var in self.dict: @@ -155,6 +147,13 @@ class DataSmart: self.delVarFlag(var, 'matchesenv') self.setVarFlag(var, 'export', 1) + # more cookies for the cookie monster + if '_' in var: + override = var[var.rfind('_')+1:] + if not override in self._seen_overrides: + self._seen_overrides[override] = Set() + self._seen_overrides[override].add( var ) + # setting var self.dict[var]["content"] = value @@ -166,6 +165,8 @@ class DataSmart: return value def delVar(self,var): + if not var in self.dict: + self._makeShadowCopy(var) self.dict[var] = {} def setVarFlag(self,var,flag,flagvalue): @@ -284,6 +285,8 @@ class DataSmartPackage(DataSmart): cache_bbfile = self.sanitize_filename(self.bbfile) p = pickle.Unpickler( file("%s/%s"%(self.cache,cache_bbfile),"rb")) self.dict = p.load() + self._seen_overrides = p.load() + self._special_values = p.load() self.unpickle_prep() # compile the functions into global scope @@ -331,6 +334,8 @@ class DataSmartPackage(DataSmart): cache_bbfile = self.sanitize_filename(self.bbfile) p = pickle.Pickler(file("%s/%s" %(self.cache,cache_bbfile), "wb" ), -1 ) p.dump( self.dict ) + p.dump( self._seen_overrides ) + p.dump( self._special_values ) self.unpickle_prep() @@ -347,14 +352,12 @@ class DataSmartPackage(DataSmart): If self.dict contains a _data key and it is a configuration we will remember we had a configuration instance attached """ - if "_data" in self.dict: - if self.dict["_data"] == self.parent: - dest["_data"] = "cfg" + if "_data" in self.dict and self.dict["_data"] == self.parent: + dest["_data"] = "cfg" def unpickle_prep(self): """ If we had a configuration instance attached, we will reattach it """ - if "_data" in self.dict: - if self.dict["_data"] == "cfg": - self.dict["_data"] = self.parent + if "_data" in self.dict and self.dict["_data"] == "cfg": + self.dict["_data"] = self.parent |