fetch2: Add support for upstream source tracing

This is a temporary commit, just to show the new
idea of adding hooks to the fetcher

Signed-off-by: Alberto Pianon <alberto@pianon.eu>

7 files changed, 84 insertions, 0 deletions

diff --git a/lib/bb/fetch2/__init__.py b/lib/bb/fetch2/__init__.py
index 765aedd51..4bfd70dc7 100644
--- a/lib/bb/fetch2/__init__.py
+++ b/lib/bb/fetch2/__init__.py
@@ -27,6 +27,7 @@ import bb.persist_data, bb.utils
 import bb.checksum
 import bb.process
 import bb.event
+from .trace import Trace
 
 __version__ = "2"
 _checksum_cache = bb.checksum.FileChecksumCache()
@@ -1298,6 +1299,7 @@ class FetchData(object):
         if not self.pswd and "pswd" in self.parm:
             self.pswd = self.parm["pswd"]
         self.setup = False
+        self.destdir = None
 
         def configure_checksum(checksum_id):
             if "name" in self.parm:
@@ -1576,6 +1578,8 @@ class FetchMethod(object):
             bb.utils.mkdirhier(unpackdir)
         else:
             unpackdir = rootdir
+        urldata.destdir = unpackdir
+        urldata.is_unpacked_archive = unpack and cmd
 
         if not unpack or not cmd:
             # If file == dest, then avoid any copies, as we already put the file into dest!
@@ -1591,6 +1595,7 @@ class FetchMethod(object):
                     if urlpath.find("/") != -1:
                         destdir = urlpath.rsplit("/", 1)[0] + '/'
                         bb.utils.mkdirhier("%s/%s" % (unpackdir, destdir))
+                        urldata.destdir = "%s/%s" % (unpackdir, destdir)
                 cmd = 'cp -fpPRH "%s" "%s"' % (file, destdir)
 
         if not cmd:
@@ -1882,6 +1887,11 @@ class Fetch(object):
         if not urls:
             urls = self.urls
 
+        if d.getVar("SRCTRACE_ENABLE") == "1" or os.environ.get("SRCTRACE_ENABLE") == "1":
+            trace = Trace(root, self.d, self.ud)
+        else:
+            trace = None
+
         for u in urls:
             ud = self.ud[u]
             ud.setup_localpath(self.d)
@@ -1890,10 +1900,16 @@ class Fetch(object):
                 lf = bb.utils.lockfile(ud.lockfile)
 
             ud.method.unpack(ud, root, self.d)
+            if trace is not None:
+                trace.commit(u, ud)
 
             if ud.lockfile:
                 bb.utils.unlockfile(lf)
 
+        if trace is not None:
+            trace.write_data()
+
+
     def clean(self, urls=None):
         """
         Clean files that the fetcher gets or places
diff --git a/lib/bb/fetch2/crate.py b/lib/bb/fetch2/crate.py
index 3310ed005..68250974c 100644
--- a/lib/bb/fetch2/crate.py
+++ b/lib/bb/fetch2/crate.py
@@ -101,8 +101,10 @@ class Crate(Wget):
         bp = d.getVar('BP')
         if bp == ud.parm.get('name'):
             cmd = "tar -xz --no-same-owner -f %s" % thefile
+            ud.destdir = rootdir
         else:
             cargo_bitbake = self._cargo_bitbake_path(rootdir)
+            ud.destdir = cargo_bitbake
 
             cmd = "tar -xz --no-same-owner -f %s -C %s" % (thefile, cargo_bitbake)
 
diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py
index a87361ccf..f6aee3d20 100644
--- a/lib/bb/fetch2/gitsm.py
+++ b/lib/bb/fetch2/gitsm.py
@@ -34,6 +34,12 @@ class GitSM(Git):
         """
         return ud.type in ['gitsm']
 
+    def urldata_init(self, ud, d):
+        super(GitSM, self).urldata_init(ud, d)
+        ud.module_data = []
+        ud.checkout_destdir = None
+        ud.is_module = False
+
     def process_submodules(self, ud, workdir, function, d):
         """
         Iterate over all of the submodules in this repository and execute
@@ -145,6 +151,15 @@ class GitSM(Git):
 
             function(ud, url, module, paths[module], workdir, ld)
 
+            if function.__name__ == "unpack_submodules":
+                destdir = os.path.join(ud.checkout_destdir, paths[module])
+                ud.module_data.append({
+                    "url": url,
+                    "destdir": destdir.rstrip("/"),
+                    "parent_destdir": ud.checkout_destdir.rstrip("/"),
+                    "revision": subrevision[module]
+                })
+
         return submodules != []
 
     def need_update(self, ud, d):
@@ -216,9 +231,15 @@ class GitSM(Git):
             else:
                 repo_conf = os.path.join(ud.destdir, '.git')
 
+            checkout_destdir = os.path.join(ud.checkout_destdir, modpath)
+
             try:
                 newfetch = Fetch([url], d, cache=False)
+                newud = newfetch.ud[url]
+                newud.checkout_destdir = checkout_destdir
+                newud.is_module = True
                 newfetch.unpack(root=os.path.dirname(os.path.join(repo_conf, 'modules', module)))
+                ud.module_data += newud.module_data
             except Exception as e:
                 logger.error('gitsm: submodule unpack failed: %s %s' % (type(e).__name__, str(e)))
                 raise
@@ -240,6 +261,10 @@ class GitSM(Git):
 
         Git.unpack(self, ud, destdir, d)
 
+        if not ud.checkout_destdir:
+            # for main git repo, checkout destdir corresponds with unpack destdir
+            ud.checkout_destdir = ud.destdir
+
         ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d)
 
         if not ud.bareclone and ret:
diff --git a/lib/bb/fetch2/hg.py b/lib/bb/fetch2/hg.py
index 063e13008..0fd69db76 100644
--- a/lib/bb/fetch2/hg.py
+++ b/lib/bb/fetch2/hg.py
@@ -242,6 +242,7 @@ class Hg(FetchMethod):
         revflag = "-r %s" % ud.revision
         subdir = ud.parm.get("destsuffix", ud.module)
         codir = "%s/%s" % (destdir, subdir)
+        ud.destdir = codir
 
         scmdata = ud.parm.get("scmdata", "")
         if scmdata != "nokeep":
diff --git a/lib/bb/fetch2/npm.py b/lib/bb/fetch2/npm.py
index f83485ad8..4ddb53e71 100644
--- a/lib/bb/fetch2/npm.py
+++ b/lib/bb/fetch2/npm.py
@@ -298,6 +298,7 @@ class Npm(FetchMethod):
         destsuffix = ud.parm.get("destsuffix", "npm")
         destdir = os.path.join(rootdir, destsuffix)
         npm_unpack(ud.localpath, destdir, d)
+        ud.destdir = destdir
 
     def clean(self, ud, d):
         """Clean any existing full or partial download"""
diff --git a/lib/bb/fetch2/npmsw.py b/lib/bb/fetch2/npmsw.py
index 4ff2c8ffc..633d00c6f 100644
--- a/lib/bb/fetch2/npmsw.py
+++ b/lib/bb/fetch2/npmsw.py
@@ -79,6 +79,8 @@ class NpmShrinkWrap(FetchMethod):
 
     def urldata_init(self, ud, d):
         """Init npmsw specific variables within url data"""
+        ud.module_data = []
+        ud.is_module = False
 
         # Get the 'shrinkwrap' parameter
         ud.shrinkwrap_file = re.sub(r"^npmsw://", "", ud.url.split(";")[0])
@@ -192,6 +194,7 @@ class NpmShrinkWrap(FetchMethod):
                 raise ParameterError("Unsupported dependency: %s" % name, ud.url)
 
             ud.deps.append({
+                "name": name,
                 "url": url,
                 "localpath": localpath,
                 "extrapaths": extrapaths,
@@ -270,15 +273,28 @@ class NpmShrinkWrap(FetchMethod):
         destsuffix = ud.parm.get("destsuffix")
         if destsuffix:
             destdir = os.path.join(rootdir, destsuffix)
+        ud.destdir = destdir
 
         bb.utils.mkdirhier(destdir)
         bb.utils.copyfile(ud.shrinkwrap_file,
                           os.path.join(destdir, "npm-shrinkwrap.json"))
 
+        for dep in ud.deps:
+            dep_destdir = os.path.join(destdir, dep["destsuffix"])
+            dep_parent_destdir = re.sub("/node_modules/"+dep["name"]+"$", "", dep_destdir)
+            ud.module_data.append({
+                "url": dep["url"] or dep["localpath"],
+                "destdir": dep_destdir.rstrip("/"),
+                "parent_destdir": dep_parent_destdir.rstrip("/"),
+                "revision": None
+            })
+
         auto = [dep["url"] for dep in ud.deps if not dep["localpath"]]
         manual = [dep for dep in ud.deps if dep["localpath"]]
 
         if auto:
+            for url in ud.proxy.ud:
+                ud.proxy.ud[url].is_module = True
             ud.proxy.unpack(destdir, auto)
 
         for dep in manual:
diff --git a/lib/bb/fetch2/trace.py b/lib/bb/fetch2/trace.py
new file mode 100644
index 000000000..10a30369d
--- /dev/null
+++ b/lib/bb/fetch2/trace.py
@@ -0,0 +1,23 @@
+class Trace:
+
+    def __init__(self, root, d, ud_dict):
+        self.root = root
+        for url, ud in ud_dict.items():
+            if hasattr(ud, "is_module") and ud.is_module:
+               self.is_module = True
+               return
+        self.is_module = False
+        self.d = d
+        self.td = {}
+        # TODO: do stuff to take a snapshot of the initial state of root dir
+
+    def commit(self, url, ud):
+        if self.is_module:
+            return
+        # TODO: do stuff to take a snapshot of the sources unpacked from url
+        # and commit data into self.td
+
+    def write_data(self):
+        if self.is_module:
+            return
+        # TODO: write self.td to some file in <root>/temp