bitbake: [parser] Cary a Statement Node through the parsing

When parsing we will collect a number of statements
that can be evaluated...The plan is to be evaluate
things twice (old+new) and then compare the result,
it should be the same.

Signed-off-by: Richard Purdie <rpurdie@linux.intel.com>

4 files changed, 92 insertions, 79 deletions

diff --git a/bitbake/lib/bb/parse/__init__.py b/bitbake/lib/bb/parse/__init__.py
index c6a925c7a8..f97233fc2b 100644
--- a/bitbake/lib/bb/parse/__init__.py
+++ b/bitbake/lib/bb/parse/__init__.py
@@ -68,11 +68,15 @@ def supports(fn, data):
             return 1
     return 0
 
-def handle(fn, data, include = 0):
+def handle(fn, data, include = 0, statements = None):
     """Call the handler that is appropriate for this file"""
+    if not statements:
+        import ast
+        statements = ast.StatementGroup()
+
     for h in handlers:
         if h['supports'](fn, data):
-            return h['handle'](fn, data, include)
+            return h['handle'](fn, data, include, statements)
     raise ParseError("%s is not a BitBake file" % fn)
 
 def init(fn, data):
diff --git a/bitbake/lib/bb/parse/ast.py b/bitbake/lib/bb/parse/ast.py
index 032b4961ba..8e7ba7fde8 100644
--- a/bitbake/lib/bb/parse/ast.py
+++ b/bitbake/lib/bb/parse/ast.py
@@ -21,23 +21,33 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-import bb, re
+import bb, re, string
 
 __word__ = re.compile(r"\S+")
-__parsed_methods__ = methodpool.get_parsed_dict()
+__parsed_methods__ = bb.methodpool.get_parsed_dict()
 
-def handleInclude(m, fn, lineno, data, force):
+class StatementGroup:
+    def __init__(self):
+        self.statements = []
+
+    def eval(self, data):
+        """
+        Apply each statement on the data... in order
+        """
+        map(lambda x: x.eval(data), self.statements)
+
+def handleInclude(statements, m, fn, lineno, data, force):
     s = bb.data.expand(m.group(1), data)
     bb.msg.debug(3, bb.msg.domain.Parsing, "CONF %s:%d: including %s" % (fn, lineno, s))
     if force:
-        include(fn, s, data, "include required")
+        bb.parse.ConfHandler.include(statements, fn, s, data, "include required")
     else:
-        include(fn, s, data, False)
+        bb.parse.ConfHandler.include(statements, fn, s, data, False)
 
-def handleExport(m, data):
+def handleExport(statements, m, data):
     bb.data.setVarFlag(m.group(1), "export", 1, data)
 
-def handleData(groupd, data):
+def handleData(statements, groupd, data):
     key = groupd["var"]
     if "exp" in groupd and groupd["exp"] != None:
         bb.data.setVarFlag(key, "export", 1, data)
@@ -71,45 +81,43 @@ def getFunc(groupd, key, data):
     else:
         return bb.data.getVar(key, data)
 
-def handleMethod(func_name, lineno, fn, body, d):
+def handleMethod(statements, func_name, lineno, fn, body, d):
     if func_name == "__anonymous":
         funcname = ("__anon_%s_%s" % (lineno, fn.translate(string.maketrans('/.+-', '____'))))
-        if not funcname in methodpool._parsed_fns:
+        if not funcname in bb.methodpool._parsed_fns:
             text = "def %s(d):\n" % (funcname) + '\n'.join(body)
-            methodpool.insert_method(funcname, text, fn)
-        anonfuncs = data.getVar('__BBANONFUNCS', d) or []
+            bb.methodpool.insert_method(funcname, text, fn)
+        anonfuncs = bb.data.getVar('__BBANONFUNCS', d) or []
         anonfuncs.append(funcname)
-        data.setVar('__BBANONFUNCS', anonfuncs, d)
+        bb.data.setVar('__BBANONFUNCS', anonfuncs, d)
     else:
-        data.setVarFlag(func_name, "func", 1, d)
-        data.setVar(func_name, '\n'.join(body), d)
-
-
+        bb.data.setVarFlag(func_name, "func", 1, d)
+        bb.data.setVar(func_name, '\n'.join(body), d)
 
-def handlePythonMethod(root, body, fn):
+def handlePythonMethod(statements, root, body, fn):
     # Note we will add root to parsedmethods after having parse
     # 'this' file. This means we will not parse methods from
     # bb classes twice
     if not root in __parsed_methods__:
         text = '\n'.join(body)
-        methodpool.insert_method(root, text, fn)
+        bb.methodpool.insert_method(root, text, fn)
 
-def handleMethodFlags(key, m, d):
-    if data.getVar(key, d):
+def handleMethodFlags(statements, key, m, d):
+    if bb.data.getVar(key, d):
         # Clean up old version of this piece of metadata, as its
         # flags could cause problems
-        data.setVarFlag(key, 'python', None, d)
-        data.setVarFlag(key, 'fakeroot', None, d)
+        bb.data.setVarFlag(key, 'python', None, d)
+        bb.data.setVarFlag(key, 'fakeroot', None, d)
     if m.group("py") is not None:
-        data.setVarFlag(key, "python", "1", d)
+        bb.data.setVarFlag(key, "python", "1", d)
     else:
-        data.delVarFlag(key, "python", d)
+        bb.data.delVarFlag(key, "python", d)
     if m.group("fr") is not None:
-        data.setVarFlag(key, "fakeroot", "1", d)
+        bb.data.setVarFlag(key, "fakeroot", "1", d)
     else:
-        data.delVarFlag(key, "fakeroot", d)
+        bb.data.delVarFlag(key, "fakeroot", d)
 
-def handleExportFuncs(m, d):
+def handleExportFuncs(statements, m, classes, d):
     fns = m.group(1)
     n = __word__.findall(fns)
     for f in n:
@@ -125,27 +133,27 @@ def handleExportFuncs(m, d):
             vars.append([allvars[0], allvars[2]])
 
         for (var, calledvar) in vars:
-            if data.getVar(var, d) and not data.getVarFlag(var, 'export_func', d):
+            if bb.data.getVar(var, d) and not bb.data.getVarFlag(var, 'export_func', d):
                 continue
 
-            if data.getVar(var, d):
-                data.setVarFlag(var, 'python', None, d)
-                data.setVarFlag(var, 'func', None, d)
+            if bb.data.getVar(var, d):
+                bb.data.setVarFlag(var, 'python', None, d)
+                bb.data.setVarFlag(var, 'func', None, d)
 
             for flag in [ "func", "python" ]:
-                if data.getVarFlag(calledvar, flag, d):
-                    data.setVarFlag(var, flag, data.getVarFlag(calledvar, flag, d), d)
+                if bb.data.getVarFlag(calledvar, flag, d):
+                    bb.data.setVarFlag(var, flag, bb.data.getVarFlag(calledvar, flag, d), d)
             for flag in [ "dirs" ]:
-                if data.getVarFlag(var, flag, d):
-                    data.setVarFlag(calledvar, flag, data.getVarFlag(var, flag, d), d)
+                if bb.data.getVarFlag(var, flag, d):
+                    bb.data.setVarFlag(calledvar, flag, bb.data.getVarFlag(var, flag, d), d)
 
-            if data.getVarFlag(calledvar, "python", d):
-                data.setVar(var, "\tbb.build.exec_func('" + calledvar + "', d)\n", d)
+            if bb.data.getVarFlag(calledvar, "python", d):
+                bb.data.setVar(var, "\tbb.build.exec_func('" + calledvar + "', d)\n", d)
             else:
-                data.setVar(var, "\t" + calledvar + "\n", d)
-            data.setVarFlag(var, 'export_func', '1', d)
+                bb.data.setVar(var, "\t" + calledvar + "\n", d)
+            bb.data.setVarFlag(var, 'export_func', '1', d)
 
-def handleAddTask(m, d):
+def handleAddTask(statements, m, d):
     func = m.group("func")
     before = m.group("before")
     after = m.group("after")
@@ -154,38 +162,38 @@ def handleAddTask(m, d):
     if func[:3] != "do_":
         var = "do_" + func
 
-    data.setVarFlag(var, "task", 1, d)
+    bb.data.setVarFlag(var, "task", 1, d)
 
-    bbtasks = data.getVar('__BBTASKS', d) or []
+    bbtasks = bb.data.getVar('__BBTASKS', d) or []
     if not var in bbtasks:
         bbtasks.append(var)
-    data.setVar('__BBTASKS', bbtasks, d)
+    bb.data.setVar('__BBTASKS', bbtasks, d)
 
-    existing = data.getVarFlag(var, "deps", d) or []
+    existing = bb.data.getVarFlag(var, "deps", d) or []
     if after is not None:
         # set up deps for function
         for entry in after.split():
             if entry not in existing:
                 existing.append(entry)
-    data.setVarFlag(var, "deps", existing, d)
+    bb.data.setVarFlag(var, "deps", existing, d)
     if before is not None:
         # set up things that depend on this func
         for entry in before.split():
-            existing = data.getVarFlag(entry, "deps", d) or []
+            existing = bb.data.getVarFlag(entry, "deps", d) or []
             if var not in existing:
-                data.setVarFlag(entry, "deps", [var] + existing, d)
+                bb.data.setVarFlag(entry, "deps", [var] + existing, d)
 
-def handleBBHandlers(m, d):
+def handleBBHandlers(statements, m, d):
     fns = m.group(1)
     hs = __word__.findall(fns)
-    bbhands = data.getVar('__BBHANDLERS', d) or []
+    bbhands = bb.data.getVar('__BBHANDLERS', d) or []
     for h in hs:
         bbhands.append(h)
-        data.setVarFlag(h, "handler", 1, d)
-    data.setVar('__BBHANDLERS', bbhands, d)
+        bb.data.setVarFlag(h, "handler", 1, d)
+    bb.data.setVar('__BBHANDLERS', bbhands, d)
 
-def handleInherit(m, d):
+def handleInherit(statements, m, d):
     files = m.group(1)
     n = __word__.findall(files)
-    inherit(n, d)
+    bb.parse.BBHandler.inherit(statements, n, d)
 
diff --git a/bitbake/lib/bb/parse/parse_py/BBHandler.py b/bitbake/lib/bb/parse/parse_py/BBHandler.py
index 1343ec114f..46413c5dc3 100644
--- a/bitbake/lib/bb/parse/parse_py/BBHandler.py
+++ b/bitbake/lib/bb/parse/parse_py/BBHandler.py
@@ -59,7 +59,7 @@ IN_PYTHON_EOF = -9999999999999
 def supports(fn, d):
     return fn[-3:] == ".bb" or fn[-8:] == ".bbclass" or fn[-4:] == ".inc"
 
-def inherit(files, d):
+def inherit(statements, files, d):
     __inherit_cache = data.getVar('__inherit_cache', d) or []
     fn = ""
     lineno = 0
@@ -72,7 +72,7 @@ def inherit(files, d):
             bb.msg.debug(2, bb.msg.domain.Parsing, "BB %s:%d: inheriting %s" % (fn, lineno, file))
             __inherit_cache.append( file )
             data.setVar('__inherit_cache', __inherit_cache, d)
-            include(fn, file, d, "inherit")
+            include(statements, fn, file, d, "inherit")
             __inherit_cache = data.getVar('__inherit_cache', d) or []
 
 
@@ -116,13 +116,14 @@ def finalise(fn, d):
     bb.event.fire(bb.event.RecipeParsed(fn), d)
 
 
-def handle(fn, d, include = 0):
+def handle(fn, d, include, statements):
     global __func_start_regexp__, __inherit_regexp__, __export_func_regexp__, __addtask_regexp__, __addhandler_regexp__, __infunc__, __body__, __residue__
     __body__ = []
     __infunc__ = ""
     __classname__ = ""
     __residue__ = []
 
+
     if include == 0:
         bb.msg.debug(2, bb.msg.domain.Parsing, "BB " + fn + ": handle(data)")
     else:
@@ -159,10 +160,10 @@ def handle(fn, d, include = 0):
         s = f.readline()
         if not s: break
         s = s.rstrip()
-        feeder(lineno, s, fn, base_name, d)
+        feeder(lineno, s, fn, base_name, d, statements)
     if __inpython__:
         # add a blank line to close out any python definition
-        feeder(IN_PYTHON_EOF, "", fn, base_name, d)
+        feeder(IN_PYTHON_EOF, "", fn, base_name, d, statements)
     if ext == ".bbclass":
         classes.remove(__classname__)
     else:
@@ -182,7 +183,7 @@ def handle(fn, d, include = 0):
                 pn = data.getVar('PN', d, True)
                 based = bb.data.createCopy(d)
                 data.setVar('PN', pn + '-' + cls, based)
-                inherit([cls], based)
+                inherit(statements, [cls], based)
                 try:
                     finalise(fn, based)
                 except bb.parse.SkipPackage:
@@ -199,12 +200,12 @@ def handle(fn, d, include = 0):
 
     return d
 
-def feeder(lineno, s, fn, root, d):
+def feeder(lineno, s, fn, root, d, statements):
     global __func_start_regexp__, __inherit_regexp__, __export_func_regexp__, __addtask_regexp__, __addhandler_regexp__, __def_regexp__, __python_func_regexp__, __inpython__,__infunc__, __body__, classes, bb, __residue__
     if __infunc__:
         if s == '}':
             __body__.append('')
-            ast.handleMethod(__infunc__, __body__, d)
+            ast.handleMethod(statements, __infunc__, lineno, fn, __body__, d)
             __infunc__ = ""
             __body__ = []
         else:
@@ -217,7 +218,7 @@ def feeder(lineno, s, fn, root, d):
             __body__.append(s)
             return
         else:
-            ast.handlePythonMethod(root, __body__, fn)
+            ast.handlePythonMethod(statements, root, __body__, fn)
             __body__ = []
             __inpython__ = False
 
@@ -238,7 +239,7 @@ def feeder(lineno, s, fn, root, d):
     m = __func_start_regexp__.match(s)
     if m:
         __infunc__ = m.group("func") or "__anonymous"
-        ast.handleMethodFlags(__infunc__, m, d)
+        ast.handleMethodFlags(statements, __infunc__, m, d)
         return
 
     m = __def_regexp__.match(s)
@@ -249,26 +250,26 @@ def feeder(lineno, s, fn, root, d):
 
     m = __export_func_regexp__.match(s)
     if m:
-        ast.handleExportFuncs(m, classes, d)
+        ast.handleExportFuncs(statements, m, classes, d)
         return
 
     m = __addtask_regexp__.match(s)
     if m:
-        ast.handleAddTask(m, d  )
+        ast.handleAddTask(statements, m, d)
         return
 
     m = __addhandler_regexp__.match(s)
     if m:
-        ast.handleBBHandlers(m, d)
+        ast.handleBBHandlers(statements, m, d)
         return
 
     m = __inherit_regexp__.match(s)
     if m:
-        ast.handleInherit(m, d)
+        ast.handleInherit(statements, m, d)
         return
 
     from bb.parse import ConfHandler
-    return ConfHandler.feeder(lineno, s, fn, d)
+    return ConfHandler.feeder(lineno, s, fn, d, statements)
 
 __pkgsplit_cache__={}
 def vars_from_file(mypkg, d):
diff --git a/bitbake/lib/bb/parse/parse_py/ConfHandler.py b/bitbake/lib/bb/parse/parse_py/ConfHandler.py
index 43a3a69bbb..69f2eea1b3 100644
--- a/bitbake/lib/bb/parse/parse_py/ConfHandler.py
+++ b/bitbake/lib/bb/parse/parse_py/ConfHandler.py
@@ -49,7 +49,7 @@ def init(data):
 def supports(fn, d):
     return fn[-5:] == ".conf"
 
-def include(oldfn, fn, data, error_out):
+def include(statements, oldfn, fn, data, error_out):
     """
 
     error_out If True a ParseError will be reaised if the to be included
@@ -70,13 +70,13 @@ def include(oldfn, fn, data, error_out):
 
     from bb.parse import handle
     try:
-        ret = handle(fn, data, True)
+        ret = handle(fn, data, True, statements)
     except IOError:
         if error_out:
             raise ParseError("Could not %(error_out)s file %(fn)s" % vars() )
         bb.msg.debug(2, bb.msg.domain.Parsing, "CONF file '%s' not found" % fn)
 
-def handle(fn, data, include = 0):
+def handle(fn, data, include, statements):
     init(data)
 
     if include == 0:
@@ -103,32 +103,32 @@ def handle(fn, data, include = 0):
             s2 = f.readline()[:-1].strip()
             lineno = lineno + 1
             s = s[:-1] + s2
-        feeder(lineno, s, fn, data)
+        feeder(lineno, s, fn, data, statements)
 
     if oldfile:
         bb.data.setVar('FILE', oldfile, data)
     return data
 
-def feeder(lineno, s, fn, data):
+def feeder(lineno, s, fn, data, statements):
     m = __config_regexp__.match(s)
     if m:
         groupd = m.groupdict()
-        ast.handleData(groupd, data)
+        ast.handleData(statements, groupd, data)
         return
 
     m = __include_regexp__.match(s)
     if m:
-        ast.handleInclude(m, fn, lineno, data, False)
+        ast.handleInclude(statements, m, fn, lineno, data, False)
         return
 
     m = __require_regexp__.match(s)
     if m:
-        ast.handleInclude(m, fn, lineno, data, True)
+        ast.handleInclude(statements, m, fn, lineno, data, True)
         return
 
     m = __export_regexp__.match(s)
     if m:
-        ast.handleExport(m, data)
+        ast.handleExport(statements, m, data)
         return
 
     raise ParseError("%s:%d: unparsed line: '%s'" % (fn, lineno, s));