data_smart: implement remote datastore functionality

This allows you to maintain a local reference to a remote datastore. The
actual implementation of the remote connection is delegated to a
connector object that the caller must define and supply. There is
support for getting variable values and expanding python references
(i.e. ${@...} remotely, however setting variables remotely is not
supported - any variable setting is done locally as if the datastore
were a copy (which it kind of is).

Loosely based on an earlier prototype implementation by Qing He.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

1 files changed, 33 insertions, 6 deletions

diff --git a/lib/bb/data_smart.py b/lib/bb/data_smart.py
index 17768c8f4..5d0ed12d6 100644
--- a/lib/bb/data_smart.py
+++ b/lib/bb/data_smart.py
@@ -116,7 +116,15 @@ class VariableParse:
                 return match.group()
 
     def python_sub(self, match):
-            code = match.group()[3:-1]
+            if isinstance(match, str):
+                code = match
+            else:
+                code = match.group()[3:-1]
+
+            if "_remote_data" in self.d:
+                connector = self.d["_remote_data"]
+                return connector.expandPythonRef(self.varname, code)
+
             codeobj = compile(code.strip(), self.varname or "<expansion>", "eval")
 
             parser = bb.codeparser.PythonParser(self.varname, logger)
@@ -247,10 +255,15 @@ class VariableHistory(object):
         self.variables[var].append(loginfo.copy())
 
     def variable(self, var):
-        if var in self.variables:
-            return self.variables[var]
+        remote_connector = self.dataroot.getVar('_remote_data', False)
+        if remote_connector:
+            varhistory = remote_connector.getVarHistory(var)
         else:
-            return []
+            varhistory = []
+
+        if var in self.variables:
+            varhistory.extend(self.variables[var])
+        return varhistory
 
     def emit(self, var, oval, val, o, d):
         history = self.variable(var)
@@ -449,6 +462,10 @@ class DataSmart(MutableMapping):
             if var in dest:
                 return dest[var]
 
+            if "_remote_data" in dest:
+                connector = dest["_remote_data"]["_content"]
+                return connector.getVar(var)
+
             if "_data" not in dest:
                 break
             dest = dest["_data"]
@@ -471,6 +488,12 @@ class DataSmart(MutableMapping):
         if 'parsing' in loginfo:
             parsing=True
 
+        if '_remote_data' in self.dict:
+            connector = self.dict["_remote_data"]["_content"]
+            res = connector.setVar(var, value)
+            if not res:
+                return
+
         if 'op' not in loginfo:
             loginfo['op'] = "set"
         self.expand_cache = {}
@@ -875,7 +898,7 @@ class DataSmart(MutableMapping):
 
     def localkeys(self):
         for key in self.dict:
-            if key != '_data':
+            if key not in ['_data', '_remote_data']:
                 yield key
 
     def __iter__(self):
@@ -884,7 +907,7 @@ class DataSmart(MutableMapping):
         def keylist(d):        
             klist = set()
             for key in d:
-                if key == "_data":
+                if key in ["_data", "_remote_data"]:
                     continue
                 if key in deleted:
                     continue
@@ -898,6 +921,10 @@ class DataSmart(MutableMapping):
             if "_data" in d:
                 klist |= keylist(d["_data"])
 
+            if "_remote_data" in d:
+                connector = d["_remote_data"]["_content"]
+                klist |= connector.getKeys()
+
             return klist
 
         self.need_overrides()