More things done on the Bitbake C parser:

    -Implement token_t and switch the grammar actions
     to the new class
    -Start with the lex_t as well
    -Start with implementing the high level python
     side.

5 files changed, 340 insertions, 118 deletions

diff --git a/lib/bb/parse/parse_c/bitbakeparser.l b/lib/bb/parse/parse_c/bitbakeparser.l
index be749ea81..73d8c4001 100644
--- a/lib/bb/parse/parse_c/bitbakeparser.l
+++ b/lib/bb/parse/parse_c/bitbakeparser.l
@@ -69,11 +69,8 @@
 
 %{
 
-#include "standard.h"
 #include "token.h"
-#include "filestack.h"
 #include "lexer.h"
-#include "arguments.h"
 #include <ctype.h>
 
 extern void *bbparseAlloc(void *(*mallocProc)(size_t));
@@ -205,9 +202,9 @@ PROC            \({C_SP}*\)
 
 <INITIAL>{K_FAKEROOT}           { yyextra->accept (T_FAKEROOT); }
 <INITIAL>{K_PYTHON}             { yyextra->accept (T_PYTHON); }
-{PROC}{C_SP}*{BOPEN}{C_SP}*\n*  { BEGIN S_PROC;
+{PROC}{C_SP}*{B_OPEN}{C_SP}*\n*  { BEGIN S_PROC;
                                   yyextra->accept (T_PROC_OPEN); }
-<S_PROC>{BCLOSE}{C_SP}*\n*      { BEGIN INITIAL;
+<S_PROC>{B_CLOSE}{C_SP}*\n*      { BEGIN INITIAL;
                                   yyextra->accept (T_PROC_CLOSE); }
 <S_PROC>([^}][^\n]*)?\n*        { yyextra->accept (T_PROC_BODY, yytext); }
 
@@ -246,68 +243,49 @@ PROC            \({C_SP}*\)
 
 void lex_t::accept (int token, const char* sz) 
 {
-  token_t t;
-  memset (&t, 0, sizeof (t));
-  if (sz) {
-    t.sz = new char [strlen (sz) + 1];
-    strcpy (t.sz, sz);
-  }
-  struct yyguts_t * yyg = (struct yyguts_t*)scanner;
-  extern char* token_name[];
-  extern char* state_name[];
-  if (arguments.show_tokens)
-    printf (" <%s> %s %-*.*s\n",
-            state_name[(yyg->yy_start - 1)/2], token_name[token],
-            60, 60, sz ? sz : "");
-  parse (parser, token, t, this);
+    token_t t;
+    memset (&t, 0, sizeof (t));
+    t.copyString(sz);
+
+    /* tell lemon to parse the token */
+    parse (parser, token, t, this);
 }
 
-int lex_t::line (void)
+int lex_t::line ()const
 {
-  return yyget_lineno (scanner);
+    return yyget_lineno (scanner);
 }
 
-const char* lex_t::filename (void)
+const char* lex_t::filename ()const
 {
-  return mf->m_szPath;
+    return m_fileName;
 }
 
 void parse (MappedFile* mf)
 {
-  void* parser = bbparseAlloc (malloc);
-  yyscan_t scanner;
-  lex_t lex;
-
-  yylex_init (&scanner);
-//  printf ("scanner create %p\n", scanner);
-
-  lex.parser = parser;
-  lex.scanner = scanner;
-  lex.mf = mf;
-  lex.rgbInput = mf->m_rgb;
-  lex.cbInput = mf->m_cb;
-  lex.parse = bbparse;
-  yyset_extra (&lex, scanner);
-
-#if defined (USE_TRACE_LOG)
-  FILE* fp = fopen ("trace.log", "a+");
-  bbparseTrace (fp, "");
-#endif
-
-  int result = yylex (scanner);
-
-  lex.accept (0);
-  bbparseTrace (NULL, NULL);;
-
-  if (result != T_EOF)
-    WARNING ("premature end of file\n");
-//  printf ("scanner release %p\n", scanner);
-  yylex_destroy (scanner);
-
-//  printf ("yylex return %d\n", result);
-
-#if defined (USE_TRACE_LOG)
-  fclose (fp);
-#endif
-  bbparseFree (parser, free);
+    void* parser = bbparseAlloc (malloc);
+    yyscan_t scanner;
+    lex_t lex;
+
+    yylex_init (&scanner);
+
+    lex.parser = parser;
+    lex.scanner = scanner;
+    lex.mf = mf;
+    lex.rgbInput = mf->m_rgb;
+    lex.cbInput = mf->m_cb;
+    lex.parse = bbparse;
+    yyset_extra (&lex, scanner);
+
+
+    int result = yylex (scanner);
+
+    lex.accept (0);
+    bbparseTrace (NULL, NULL);
+
+    if (result != T_EOF)
+        WARNING ("premature end of file\n");
+
+    yylex_destroy (scanner);
+    bbparseFree (parser, free);
 }
diff --git a/lib/bb/parse/parse_c/bitbakeparser.py b/lib/bb/parse/parse_c/bitbakeparser.py
new file mode 100644
index 000000000..cd60cac27
--- /dev/null
+++ b/lib/bb/parse/parse_c/bitbakeparser.py
@@ -0,0 +1,128 @@
+"""
+
+BitBake C Parser Python Code
+
+Copyright (C) 2005 Holger Hans Peter Freyther
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
+THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+"""
+
+__version__ = "0xdeadbeef"
+
+class CParser:
+    """
+    The C-based Parser for Bitbake
+    """
+    def __init__(self, data):
+        """
+        Constructor
+        """
+        self._data = data
+
+    def _syntax_error(self, file, line):
+        """
+        lemon/flex reports an syntax error to us and we will
+        raise an exception
+        """
+        pass
+
+    def _export(self, data):
+        """
+        EXPORT VAR = "MOO"
+        we will now export VAR
+        """
+        pass
+
+    def _assign(self, key, value):
+        """
+        VAR = "MOO"
+        we will assign moo to VAR
+        """
+        pass
+
+    def _append(self, key, value):
+        """
+        VAR += "MOO"
+        we will append " MOO" to var
+        """
+        pass
+
+    def _prepend(self, key, value):
+        """
+        VAR =+ "MOO"
+        we will prepend "MOO " to var
+        """
+        pass
+
+    def _immediate(self, key, value):
+        """
+        VAR := "MOO ${CVSDATE}"
+        we will assign immediately and expand vars
+        """
+        pass
+
+    def _conditional(self, key, value):
+        """
+        """
+        pass
+
+    def _add_task(self, task, before = None, after = None):
+        """
+        """
+        pass
+
+    def _include(self, file):
+        """
+        """
+        pass
+
+    def _inherit(self, file):
+        """
+        """
+        pass
+
+    def _shell_procedure(self, name, body):
+        """
+        """
+        pass
+
+    def _python_procedure(self, name, body):
+        """
+        """
+        pass
+
+    def _fakeroot_procedure(self, name, body):
+        """
+        """
+        pass
+
+    def _def_procedure(self, a, b, c):
+        """
+        """
+        pass
+
+    def _export_func(self, name):
+        """
+        """
+        pass
+
+    def _add_handler(self, handler):
+        """
+        """
+        pass
diff --git a/lib/bb/parse/parse_c/bitbakeparser.y b/lib/bb/parse/parse_c/bitbakeparser.y
index b4cd73fb6..c6ae71125 100644
--- a/lib/bb/parse/parse_c/bitbakeparser.y
+++ b/lib/bb/parse/parse_c/bitbakeparser.y
@@ -37,11 +37,10 @@
 
 %token_type {token_t}
 %name bbparse
-%token_prefix	T_
+%token_prefix   T_
 %extra_argument  {lex_t* lex}
 
 %include {
-#include <iostream>
 #include "standard.h"
 }
 
@@ -57,112 +56,106 @@ statements ::= statements statement.
 statements ::= .
 
 variable(r) ::= SYMBOL(s).
-        { r.sz = s.sz; s.sz = NULL;
-          s.release_this (); }
+        { r.assignString( s.string() );
+          s.assignString( 0 );
+          s.release_this(); }
 variable(r) ::= VARIABLE(v).
-        { char* sz = e_interpolate (v.sz);
-          if (sz) { r.sz = sz; delete v.sz; }
-          else    { r.sz = v.sz; }
-          v.sz = NULL;
-          v.release_this (); }
+        {
+          r.assignString( v.string() );
+          v.assignString( 0 );
+          v.release_this(); }
 
 statement ::= EXPORT variable(s) OP_ASSIGN STRING(v).
-        { e_assign (s.sz, v.sz); e_export (s.sz);
-          s.release_this (); v.release_this (); }
+        { e_assign( s.string(), v.string() );
+          e_export( s.string() );
+          s.release_this(); v.release_this(); }
 statement ::= EXPORT variable(s) OP_IMMEDIATE STRING(v).
-        { e_immediate (s.sz, v.sz); e_export (s.sz);
-          s.release_this (); v.release_this (); }
-
+        { e_immediate (s.string(), v.string() );
+          e_export( s.string() );
+          s.release_this(); v.release_this(); }
 statement ::= EXPORT variable(s) OP_COND STRING(v).
-        { e_cond (s.sz, v.sz); e_export (s.sz);
-          s.release_this (); v.release_this (); }
+        { e_cond( s.string(), v.string() );
+          s.release_this(); v.release_this(); }
 
 statement ::= variable(s) OP_ASSIGN STRING(v).
-        { e_assign (s.sz, v.sz);
-          s.release_this (); v.release_this (); }
+        { e_assign( s.string(), v.string() );
+          s.release_this(); v.release_this(); }
 statement ::= variable(s) OP_PREPEND STRING(v).
-        { e_prepend (s.sz, v.sz);
-          s.release_this (); v.release_this (); }
+        { e_prepend( s.string(), v.string() );
+          s.release_this(); v.release_this(); }
 statement ::= variable(s) OP_APPEND STRING(v).
-        { e_append (s.sz, v.sz);
-          s.release_this (); v.release_this (); }
+        { e_append( s.string() , v.string() );
+          s.release_this(); v.release_this(); }
 statement ::= variable(s) OP_IMMEDIATE STRING(v).
-        { e_immediate (s.sz, v.sz);
-          s.release_this (); v.release_this (); }
+        { e_immediate( s.string(), v.string() );
+          s.release_this(); v.release_this(); }
 statement ::= variable(s) OP_COND STRING(v).
-        { e_cond (s.sz, v.sz);
-          s.release_this (); v.release_this (); }
+        { e_cond( s.string(), v.string() );
+          s.release_this(); v.release_this(); }
 
 task ::= TSYMBOL(t) BEFORE TSYMBOL(b) AFTER  TSYMBOL(a).
-        { e_addtask (t.sz, b.sz, a.sz);
-          t.release_this (); b.release_this (); a.release_this (); }
+        { e_addtask( t.string(), b.string(), a.string() );
+          t.release_this(); b.release_this(); a.release_this(); }
 task ::= TSYMBOL(t) AFTER  TSYMBOL(a) BEFORE TSYMBOL(b).
-        { e_addtask (t.sz, b.sz, a.sz);
-          t.release_this (); a.release_this (); b.release_this (); }
+        { e_addtask( t.string(), b.string(), a.string());
+          t.release_this(); a.release_this(); b.release_this(); }
 task ::= TSYMBOL(t).
-        { e_addtask (t.sz, NULL, NULL); 
-          t.release_this ();}
+        { e_addtask( t.string(), NULL, NULL);
+          t.release_this();}
 task ::= TSYMBOL(t) BEFORE TSYMBOL(b).
-        { e_addtask (t.sz, b.sz, NULL);
-          t.release_this (); b.release_this (); }
+        { e_addtask( t.string(), b.string(), NULL);
+          t.release_this(); b.release_this(); }
 task ::= TSYMBOL(t) AFTER  TSYMBOL(a).
-        { e_addtask (t.sz, NULL, a.sz); 
-          t.release_this (); a.release_this (); }
+        { e_addtask( t.string(), NULL, a.string());
+          t.release_this(); a.release_this(); }
 tasks ::= tasks task.
 tasks ::= task.
 statement ::= ADDTASK tasks.
 
 statement ::= ADDHANDLER SYMBOL(s).
-        { e_addhandler (s.sz); s.release_this (); }
+        { e_addhandler( s.string()); s.release_this (); }
 
-func ::= FSYMBOL(f). { e_export_func (f.sz); f.release_this (); }
+func ::= FSYMBOL(f). { e_export_func(f.string()); f.release_this(); }
 funcs ::= funcs func.
 funcs ::= func.
 statement ::= EXPORT_FUNC funcs.
 
-inherit ::= ISYMBOL(i). { e_inherit (i.sz); i.release_this (); }
+inherit ::= ISYMBOL(i). { e_inherit(i.string() ); i.release_this (); }
 inherits ::= inherits inherit.
 inherits ::= inherit.
 statement ::= INHERIT inherits.
 
 statement ::= INCLUDE ISYMBOL(i).
-        { e_include (i.sz); i.release_this (); }
+        { e_include(i.string() ); i.release_this(); }
 
 proc_body(r) ::= proc_body(l) PROC_BODY(b). 
         { /* concatenate body lines */
-          size_t cb = (l.sz ? strlen (l.sz) : 0) + strlen (b.sz) + 1;
-          r.sz = new char[cb];
-          *r.sz = 0;
-          if (l.sz) strcat (r.sz, l.sz);
-          strcat (r.sz, b.sz);
+          r.assignString( token_t::concatString(l.string(), b.string()) );
           l.release_this ();
           b.release_this ();
         }
-proc_body(b) ::= . { b.sz = 0; }
+proc_body(b) ::= . { b.assignString(0); }
 statement ::= variable(p) PROC_OPEN proc_body(b) PROC_CLOSE.
-        { e_proc (p.sz, b.sz); 
-          p.release_this (); b.release_this (); }
+        { e_proc( p.string(), b.string() );
+          p.release_this(); b.release_this(); }
 statement ::= PYTHON SYMBOL(p) PROC_OPEN proc_body(b) PROC_CLOSE.
-        { e_proc_python (p.sz, b.sz); 
-          p.release_this (); b.release_this (); }
+        { e_proc_python (p.string(), b.string() );
+          p.release_this(); b.release_this(); }
 statement ::= PYTHON PROC_OPEN proc_body(b) PROC_CLOSE.
-        { e_proc_python (NULL, b.sz);
+        { e_proc_python( NULL, b.string());
           b.release_this (); }
 
 statement ::= FAKEROOT SYMBOL(p) PROC_OPEN proc_body(b) PROC_CLOSE.
-        { e_proc_fakeroot (p.sz, b.sz);
+        { e_proc_fakeroot(p.string(), b.string() );
           p.release_this (); b.release_this (); }
 
 def_body(r) ::= def_body(l) DEF_BODY(b).
         { /* concatenate body lines */
-          size_t cb = (l.sz ? strlen (l.sz) : 0) + strlen (b.sz);
-          r.sz = new char[cb + 1];
-          *r.sz = 0;
-          if (l.sz) strcat (r.sz, l.sz);
-          strcat (r.sz, b.sz); 
+          r.assignString( token_t::concatString(l.string(), b.string());
           l.release_this (); b.release_this ();
         }
 def_body(b) ::= . { b.sz = 0; }
 statement ::= SYMBOL(p) DEF_ARGS(a) def_body(b).
-        { e_def (p.sz, a.sz, b.sz);
-          p.release_this(); a.release_this (); b.release_this (); }
+        { e_def( p.string(), a.string(), b.string());
+          p.release_this(); a.release_this(); b.release_this(); }
+
diff --git a/lib/bb/parse/parse_c/lexer.h b/lib/bb/parse/parse_c/lexer.h
new file mode 100644
index 000000000..74f6295f3
--- /dev/null
+++ b/lib/bb/parse/parse_c/lexer.h
@@ -0,0 +1,40 @@
+/*
+Copyright (C) 2005 Holger Hans Peter Freyther
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
+THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+*/
+
+#ifndef LEXER_H
+#define LEXER_H
+
+struct lex_t {
+    void *parser;
+    void *scanner;
+    void* (*parse)(void*, int, token_t, lex_t*);
+
+    void accept(int token, const char* string = 0);
+    int  line()const;
+    const char* filename()const;
+private:
+    const char* m_fileName;
+};
+
+
+#endif
diff --git a/lib/bb/parse/parse_c/token.h b/lib/bb/parse/parse_c/token.h
new file mode 100644
index 000000000..2351fda6b
--- /dev/null
+++ b/lib/bb/parse/parse_c/token.h
@@ -0,0 +1,83 @@
+/*
+Copyright (C) 2005 Holger Hans Peter Freyther
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
+THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+*/
+
+#ifndef TOKEN_H
+#define TOKEN_H
+
+#define PURE_METHOD
+
+struct token_t {
+    const char* string()const PURE_METHOD;
+
+    static char* concatString(const char* l, const char* r);
+    void assignString(const char* str);
+    void copyString(const char* str);
+
+    void release_this();
+
+private:
+    char  *m_string;
+    size_t m_stringLen;
+};
+
+inline const char* token_t::string()const
+{
+    return m_string;
+}
+
+/*
+ * append str to the current string
+ */
+inline char* token_t::concatString(const char* l, const char* r)
+{
+    size_t cb = (l ? strlen (l) : 0) + strlen (r) + 1;
+    r_sz = new char[cb];
+    *r_sz = 0;
+    if (l) strcat (r_sz, l);
+    strcat (r_sz, r);
+
+    return r_sz;
+}
+
+inline void token_t::assignString(const char* str)
+{
+    m_string = str;
+    m_stringLen = str ? strlen(str) : 0;
+}
+
+inline void token_t::copyString(const char* str)
+{
+    if( str ) {
+        m_stringLen = strlen(str);
+        m_string = new char[m_stringLen+1];
+        strcpy(m_string, str)
+    }
+}
+
+inline void token_t::release_this()
+{
+    delete m_string;
+    m_string = 0;
+}
+
+#endif