bitbake/lib/bb/data.py:

    -Only have one expand method in the dict
     implementation.
    -Apply some magic knowledge by feeding the
     cookie master.
    -Speed up update_data by asking the Cookie
     Monster for help.
     We do not iterate over each key, but the
     keys we know that have the data we care
     about. This gives a huge speed improvement
     to parsing.

bitbake/lib/bb/data_smart.py:
    -Make setVar more expensive by feeding the cookie
     monster

2 files changed, 138 insertions, 129 deletions

diff --git a/lib/bb/data.py b/lib/bb/data.py
index 56ee977f6..6cac3ea19 100644
--- a/lib/bb/data.py
+++ b/lib/bb/data.py
@@ -7,6 +7,18 @@ BitBake 'Data' implementations
 Functions for interacting with the data structure used by the
 BitBake build tools.
 
+The expandData and update_data are the most expensive
+operations. At night the cookie monster came by and
+suggested 'give me cookies on setting the variables and
+things will work out'. Taking this suggestion into account
+applying the skills from the not yet passed 'Entwurf und
+Analyse von Algorithmen' lecture and the cookie 
+monster seems to be right. We will track setVar more carefully
+to have faster update_data and expandKeys operations.
+
+This is a treade-off between speed and memory again but
+the speed is more critical here.
+
 Copyright (C) 2003, 2004  Chris Larson
 Copyright (C) 2005        Holger Hans Peter Freyther
 
@@ -273,6 +285,27 @@ def setData(newData, d):
     """Sets the data object to the supplied value"""
     d = newData
 
+
+##
+## Cookie Monsters' query functions
+##
+def _get_override_vars(d, override):
+    """
+    Internal!!!
+
+    Get the Names of Variables that have a specific
+    override. This function returns a iterable
+    Set or an empty list
+    """
+    return []
+
+def _get_var_flags_triple(d):
+    """
+    Internal!!!
+
+    """
+    return []
+
 __expand_var_regexp__ = re.compile(r"\${[^{}]+}")
 __expand_python_regexp__ = re.compile(r"\${@.+?}")
 
@@ -303,43 +336,7 @@ def expand(s, d, varname = None):
         >>> print expand('${SRC_URI}', d)
         http://somebug.${TARGET_MOO}
     """
-    def var_sub(match):
-        key = match.group()[2:-1]
-        if varname and key:
-            if varname == key:
-                raise Exception("variable %s references itself!" % varname)
-        var = getVar(key, d, 1)
-        if var is not None:
-            return var
-        else:
-            return match.group()
-
-    def python_sub(match):
-        import bb
-        code = match.group()[3:-1]
-        locals()['d'] = d
-        s = eval(code)
-        if type(s) == types.IntType: s = str(s)
-        return s
-
-    if type(s) is not types.StringType: # sanity check
-        return s
-
-    while s.find('$') != -1:
-        olds = s
-        try:
-            s = __expand_var_regexp__.sub(var_sub, s)
-            s = __expand_python_regexp__.sub(python_sub, s)
-            if s == olds: break
-            if type(s) is not types.StringType: # sanity check
-                import bb
-                bb.error('expansion of %s returned non-string %s' % (olds, s))
-        except KeyboardInterrupt:
-            raise
-        except:
-            note("%s:%s while evaluating:\n%s" % (sys.exc_info()[0], sys.exc_info()[1], s))
-            raise
-    return s
+    return d.expand(s, varname)
 
 def expandKeys(alterdata, readdata = None):
     if readdata == None:
@@ -356,7 +353,7 @@ def expandKeys(alterdata, readdata = None):
 #        setVarFlags(ekey, copy.copy(getVarFlags(key, readdata)), alterdata)
         setVar(ekey, val, alterdata)
 
-        for i in ('_append', '_prepend', '_delete'):
+        for i in ('_append', '_prepend'):
             dest = getVarFlag(ekey, i, alterdata) or []
             src = getVarFlag(key, i, readdata) or []
             dest.extend(src)
@@ -507,67 +504,76 @@ def update_data(d):
         >>> print getVar('TEST', d)
         local
     """
-
     debug(2, "update_data()")
 
-#   can't do delete env[...] while iterating over the dictionary, so remember them
-    dodel = []
+    # now ask the cookie monster for help
+    #print "Cookie Monster"
+    #print "Append/Prepend %s" % d._special_values
+    #print "Overrides      %s" % d._seen_overrides
+
     overrides = (getVar('OVERRIDES', d, 1) or "").split(':') or []
 
-    def applyOverrides(var, d):
-        if not overrides:
-            debug(1, "OVERRIDES not defined, nothing to do")
-            return
-        val = getVar(var, d)
-        for o in overrides:
-            if var.endswith("_" + o):
-                l = len(o)+1
-                name = var[:-l]
-                d[name] = d[var]
+    #
+    # Well let us see what breaks here. We used to iterate
+    # over each variable and apply the override and then
+    # do the line expanding.
+    # If we have bad luck - which we will have - the keys
+    # where in some order that is so important for this
+    # method which we don't have anymore.
+    # Anyway we will fix that and write test cases this
+    # time.
+
+    #
+    # First we apply all overrides
+    # Then  we will handle _append and _prepend
+    #
+
+    for o in overrides:
+        # calculate '_'+override
+        l    = len(o)+1
+
+        # see if one should even try
+        if not o in d._seen_overrides:
+            continue
 
-    for s in keys(d):
-        applyOverrides(s, d)
-        sval = getVar(s, d) or ""
-
-#       Handle line appends:
-        for (a, o) in getVarFlag(s, '_append', d) or []:
-            # maybe the OVERRIDE was not yet added so keep the append
-            if (o and o in overrides) or not o:
-                delVarFlag(s, '_append', d)
-            if o:
-                if not o in overrides:
+        vars = d._seen_overrides[o]
+        for var in vars:
+            name = var[:-l]
+            try:
+                d[name] = d[var]
+            except:
+                note ("Untracked delVar")
+
+    # now on to the appends and prepends
+    if '_append' in d._special_values:
+        appends = d._special_values['_append'] or []
+        for append in appends:
+            for (a, o) in getVarFlag(append, '_append', d) or []:
+                # maybe the OVERRIDE was not yet added so keep the append
+                if (o and o in overrides) or not o:
+                    delVarFlag(append, '_append', d)
+                if o and not o in overrides:
                     continue
-            sval+=a
-            setVar(s, sval, d)
-
-#       Handle line prepends
-        for (a, o) in getVarFlag(s, '_prepend', d) or []:
-            # maybe the OVERRIDE was not yet added so keep the append
-            if (o and o in overrides) or not o:
-                delVarFlag(s, '_prepend', d)
-            if o:
-                if not o in overrides:
+
+                sval = getVar(append,d) or ""
+                sval+=a
+                setVar(append, sval, d)
+
+
+    if '_prepend' in d._special_values:
+        prepends = d._special_values['_prepend'] or []
+
+        for prepend in prepends:
+            for (a, o) in getVarFlag(prepend, '_prepend', d) or []:
+                # maybe the OVERRIDE was not yet added so keep the prepend
+                if (o and o in overrides) or not o:
+                    delVarFlag(prepend, '_prepend', d)
+                if o and not o in overrides:
                     continue
-            sval=a+sval
-            setVar(s, sval, d)
-
-#       Handle line deletions
-        name = s + "_delete"
-        nameval = getVar(name, d)
-        if nameval:
-            sval = getVar(s, d)
-            if sval:
-                new = ''
-                pattern = nameval.replace('\n','').strip()
-                for line in sval.split('\n'):
-                    if line.find(pattern) == -1:
-                        new = new + '\n' + line
-                setVar(s, new, d)
-                dodel.append(name)
-
-#   delete all environment vars no longer needed
-    for s in dodel:
-        delVar(s, d)
+
+                sval = a + (getVar(prepend,d) or "")
+                setVar(prepend, sval, d)
+
 
 def inherits_class(klass, d):
     val = getVar('__inherit_cache', d) or ""
diff --git a/lib/bb/data_smart.py b/lib/bb/data_smart.py
index 8b062ee57..c21921ca3 100644
--- a/lib/bb/data_smart.py
+++ b/lib/bb/data_smart.py
@@ -8,7 +8,7 @@ BitBake build tools.
 
 Copyright (C) 2003, 2004  Chris Larson
 Copyright (C) 2004, 2005  Seb Frankengul
-Copyright (C) 2005        Holger Hans Peter Freyther
+Copyright (C) 2005, 2006  Holger Hans Peter Freyther
 Copyright (C) 2005        Uli Luckas
 Copyright (C) 2005        ROAD GmbH
 
@@ -29,7 +29,8 @@ Based on functions from the base bb module, Copyright 2003 Holger Schurig
 """
 
 import copy, os, re, sys, time, types
-from   bb import note, debug, fatal, utils, methodpool
+from bb   import note, debug, error, fatal, utils, methodpool
+from sets import Set
 
 try:
     import cPickle as pickle
@@ -37,9 +38,8 @@ except ImportError:
     import pickle
     print "NOTE: Importing cPickle failed. Falling back to a very slow implementation."
 
-
-__setvar_keyword__ = ["_append","_prepend","_delete"]
-__setvar_regexp__ = re.compile('(?P<base>.*?)(?P<keyword>_append|_prepend|_delete)(_(?P<add>.*))?')
+__setvar_keyword__ = ["_append","_prepend"]
+__setvar_regexp__ = re.compile('(?P<base>.*?)(?P<keyword>_append|_prepend)(_(?P<add>.*))?')
 __expand_var_regexp__ = re.compile(r"\${[^{}]+}")
 __expand_python_regexp__ = re.compile(r"\${@.+?}")
 
@@ -48,6 +48,10 @@ class DataSmart:
     def __init__(self):
         self.dict = {}
 
+        # cookie monster tribute
+        self._special_values = {}
+        self._seen_overrides = {}
+
     def expand(self,s, varname):
         def var_sub(match):
             key = match.group()[2:-1]
@@ -78,8 +82,7 @@ class DataSmart:
                 s = __expand_python_regexp__.sub(python_sub, s)
                 if s == olds: break
                 if type(s) is not types.StringType: # sanity check
-                    import bb
-                    bb.error('expansion of %s returned non-string %s' % (olds, s))
+                    error('expansion of %s returned non-string %s' % (olds, s))
             except KeyboardInterrupt:
                 raise
             except:
@@ -91,18 +94,6 @@ class DataSmart:
         if not var in self.dict:
             self.dict[var] = {}
 
-    def pickle_prep(self, cfg):
-        if "_data" in self.dict:
-            if self.dict["_data"] == cfg:
-                self.dict["_data"] = "cfg";
-            else: # this is an unknown array for the moment
-                pass
-
-    def unpickle_prep(self, cfg):
-        if "_data" in self.dict:
-            if self.dict["_data"] == "cfg":
-                self.dict["_data"] = cfg;
-
     def _findVar(self,var):
         _dest = self.dict
 
@@ -116,14 +107,6 @@ class DataSmart:
             return _dest[var]
         return None
 
-    def _copyVar(self,var,name):
-        local_var = self._findVar(var)
-        if local_var:
-            self.dict[name] = copy.copy(local_var)
-        else:
-            debug(1,"Warning, _copyVar %s to %s, %s does not exists" % (var,name,var))
-
-
     def _makeShadowCopy(self, var):
         if var in self.dict:
             return
@@ -142,11 +125,20 @@ class DataSmart:
             keyword = match.group("keyword")
             override = match.group('add')
             l = self.getVarFlag(base, keyword) or []
-            if override == 'delete':
-                if l.count([value, None]):
-                    del l[l.index([value, None])]
             l.append([value, override])
-            self.setVarFlag(base, match.group("keyword"), l)
+            self.setVarFlag(base, keyword, l)
+
+            # pay the cookie monster
+            try:
+                self._special_values[keyword].add( base )
+            except:
+                self._special_values[keyword] = Set()
+                self._special_values[keyword].add( base )
+
+            # SRC_URI_append_simpad is both a flag and a override
+            #if not override in self._seen_overrides:
+            #    self._seen_overrides[override] = Set()
+            #self._seen_overrides[override].add( base )
             return
 
         if not var in self.dict:
@@ -155,6 +147,13 @@ class DataSmart:
             self.delVarFlag(var, 'matchesenv')
             self.setVarFlag(var, 'export', 1)
 
+        # more cookies for the cookie monster
+        if '_' in var:
+            override = var[var.rfind('_')+1:]
+            if not override in self._seen_overrides:
+                self._seen_overrides[override] = Set()
+            self._seen_overrides[override].add( var )
+
         # setting var
         self.dict[var]["content"] = value
 
@@ -166,6 +165,8 @@ class DataSmart:
         return value
 
     def delVar(self,var):
+        if not var in self.dict:
+            self._makeShadowCopy(var)
         self.dict[var] = {}
 
     def setVarFlag(self,var,flag,flagvalue):
@@ -284,6 +285,8 @@ class DataSmartPackage(DataSmart):
         cache_bbfile = self.sanitize_filename(self.bbfile)
         p = pickle.Unpickler( file("%s/%s"%(self.cache,cache_bbfile),"rb"))
         self.dict = p.load()
+        self._seen_overrides = p.load()
+        self._special_values = p.load()
         self.unpickle_prep()
 
         # compile the functions into global scope
@@ -331,6 +334,8 @@ class DataSmartPackage(DataSmart):
         cache_bbfile = self.sanitize_filename(self.bbfile)
         p = pickle.Pickler(file("%s/%s" %(self.cache,cache_bbfile), "wb" ), -1 )
         p.dump( self.dict )
+        p.dump( self._seen_overrides )
+        p.dump( self._special_values )
 
         self.unpickle_prep()
 
@@ -347,14 +352,12 @@ class DataSmartPackage(DataSmart):
         If self.dict contains a _data key and it is a configuration
         we will remember we had a configuration instance attached
         """
-        if "_data" in self.dict:
-            if self.dict["_data"] == self.parent:
-                dest["_data"] = "cfg"
+        if "_data" in self.dict and  self.dict["_data"] == self.parent:
+            dest["_data"] = "cfg"
 
     def unpickle_prep(self):
         """
         If we had a configuration instance attached, we will reattach it
         """
-        if "_data" in self.dict:
-            if self.dict["_data"] == "cfg":
-                self.dict["_data"] = self.parent
+        if "_data" in self.dict and  self.dict["_data"] == "cfg":
+            self.dict["_data"] = self.parent