From 2bc8aac6c023f200d84dd3354d6bf29ccf09c1d8 Mon Sep 17 00:00:00 2001 From: Mark Hatle Date: Thu, 20 Sep 2018 11:08:06 -0400 Subject: WIP - gitsm fetcher rework Signed-off-by: Mark Hatle --- lib/bb/fetch2/gitsm.py | 195 +++++++++++++++++++++++++------------------------ 1 file changed, 98 insertions(+), 97 deletions(-) diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py index 86773094d..8d6171127 100644 --- a/lib/bb/fetch2/gitsm.py +++ b/lib/bb/fetch2/gitsm.py @@ -34,6 +34,8 @@ import bb from bb.fetch2.git import Git from bb.fetch2 import runfetchcmd from bb.fetch2 import logger +from bb.fetch2 import Fetch +from bb.fetch2 import BBFetchException class GitSM(Git): def supports(self, ud, d): @@ -51,87 +53,62 @@ class GitSM(Git): pass return False - def _set_relative_paths(self, repopath): - """ - Fix submodule paths to be relative instead of absolute, - so that when we move the repo it doesn't break - (In Git 1.7.10+ this is done automatically) - """ + def update_submodules(self, ud, d, allow_network): submodules = [] - with open(os.path.join(repopath, '.gitmodules'), 'r') as f: - for line in f.readlines(): + paths = {} + uris = {} + local_paths = {} + + for name in ud.names: + gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=ud.clonedir) + + module = "" + for line in gitmodules.splitlines(): if line.startswith('[submodule'): - submodules.append(line.split('"')[1]) + module = line.split('"')[1] + submodules.append(module) + elif module and line.strip().startswith('path'): + path = line.split('=')[1].strip() + paths[module] = path + elif module and line.strip().startswith('url'): + url = line.split('=')[1].strip() + uris[module] = url for module in submodules: - repo_conf = os.path.join(repopath, module, '.git') - if os.path.exists(repo_conf): - with open(repo_conf, 'r') as f: - lines = f.readlines() - newpath = '' - for i, line in enumerate(lines): - if line.startswith('gitdir:'): - oldpath = line.split(': ')[-1].rstrip() - if oldpath.startswith('/'): - newpath = '../' * (module.count('/') + 1) + '.git/modules/' + module - lines[i] = 'gitdir: %s\n' % newpath - break - if newpath: - with open(repo_conf, 'w') as f: - for line in lines: - f.write(line) - - repo_conf2 = os.path.join(repopath, '.git', 'modules', module, 'config') - if os.path.exists(repo_conf2): - with open(repo_conf2, 'r') as f: - lines = f.readlines() - newpath = '' - for i, line in enumerate(lines): - if line.lstrip().startswith('worktree = '): - oldpath = line.split(' = ')[-1].rstrip() - if oldpath.startswith('/'): - newpath = '../' * (module.count('/') + 3) + module - lines[i] = '\tworktree = %s\n' % newpath - break - if newpath: - with open(repo_conf2, 'w') as f: - for line in lines: - f.write(line) + module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], paths[module]), d, quiet=True, workdir=ud.clonedir) + module_hash = module_hash.split()[2] - def update_submodules(self, ud, d, allow_network): - # We have to convert bare -> full repo, do the submodule bit, then convert back - tmpclonedir = ud.clonedir + ".tmp" - gitdir = tmpclonedir + os.sep + ".git" - bb.utils.remove(tmpclonedir, True) - os.mkdir(tmpclonedir) - os.rename(ud.clonedir, gitdir) - runfetchcmd("sed " + gitdir + "/config -i -e 's/bare.*=.*true/bare = false/'", d) - runfetchcmd(ud.basecmd + " reset --hard", d, workdir=tmpclonedir) - runfetchcmd(ud.basecmd + " checkout -f " + ud.revisions[ud.names[0]], d, workdir=tmpclonedir) - - try: - if allow_network: - fetch_flags = "" - else: - fetch_flags = "--no-fetch" - - # The 'git submodule sync' sandwiched between two successive 'git submodule update' commands is - # intentional. See the notes on the similar construction in download() for an explanation. - runfetchcmd("%(basecmd)s submodule update --init --recursive %(fetch_flags)s || (%(basecmd)s submodule sync --recursive && %(basecmd)s submodule update --init --recursive %(fetch_flags)s)" % {'basecmd': ud.basecmd, 'fetch_flags' : fetch_flags}, d, workdir=tmpclonedir) - except bb.fetch.FetchError: - if allow_network: - raise - else: - # This method was called as a probe to see whether the submodule history - # is complete enough to allow the current working copy to have its - # modules filled in. It's not, so swallow up the exception and report - # the negative result. - return False - finally: - self._set_relative_paths(tmpclonedir) - runfetchcmd("sed " + gitdir + "/config -i -e 's/bare.*=.*false/bare = true/'", d, workdir=tmpclonedir) - os.rename(gitdir, ud.clonedir,) - bb.utils.remove(tmpclonedir, True) + try: + url = uris[module] + if url.startswith('http:'): + url = url.replace('http:', 'git:', 1) + ';protocol=http' + elif url.startswith('https:'): + url = url.replace('https:', 'git:', 1) + ';protocol=https' + elif url.startswith('ssh:'): + url = url.replace('ssh:', 'git:', 1) + ';protocol=ssh' + elif url.startswith('rsync:'): + url = url.replace('rsync:', 'git:', 1) + ';protocol=https' + url += ";bareclone=1;nocheckout=1;name=%s" % (module) + ld = d.createCopy() + ld.setVar('SRCREV', module_hash) + ld.setVar('SRCPV', d.getVar('SRCPV')) + ld.setVar('SRCREV_FORMAT', module) + ld.setVar('SRC_URI', url) + newfetch = Fetch([url], ld) + newfetch.download() + local_paths[module] = newfetch.localpath(url) + + # Correct the submodule references to the local download version... + bb.warn("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_paths[module]}) + runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_paths[module]}, d, workdir=ud.clonedir) + try: + os.mkdir(os.path.join(ud.clonedir, 'modules')) + except OSError: + pass + os.symlink(local_paths[module], os.path.join(ud.clonedir, 'modules', paths[module])) + + except BBFetchException as e: + bb.error(str(e)) return True @@ -155,12 +132,12 @@ class GitSM(Git): if not ud.shallow or ud.localpath != ud.fullshallow: submodules = self.uses_submodules(ud, d, ud.clonedir) if submodules: - self.update_submodules(ud, d, allow_network=True) + self.update_submodules(ud, d, allow_network=False) def clone_shallow_local(self, ud, dest, d): super(GitSM, self).clone_shallow_local(ud, dest, d) - runfetchcmd('cp -fpPRH "%s/modules" "%s/"' % (ud.clonedir, os.path.join(dest, '.git')), d) + runfetchcmd('cp -fpLR "%s/modules" "%s/"' % (ud.clonedir, os.path.join(dest, '.git')), d) def unpack(self, ud, destdir, d): Git.unpack(self, ud, destdir, d) @@ -174,29 +151,53 @@ class GitSM(Git): else: repo_conf = os.path.join(ud.destdir, '.git') - if os.path.exists(ud.clonedir): + if os.path.exists(os.path.join(ud.clonedir, 'modules')): # This is not a copy unpacked from a shallow mirror clone. So # the manual intervention to populate the .git/modules done # in clone_shallow_local() won't have been done yet. - runfetchcmd("cp -fpPRH %s %s" % (os.path.join(ud.clonedir, 'modules'), repo_conf), d) - fetch_flags = "--no-fetch" + bb.warn('cp -fpLR') + runfetchcmd("cp -fpLR %s %s" % (os.path.join(ud.clonedir, 'modules'), repo_conf), d) elif os.path.exists(os.path.join(repo_conf, 'modules')): # Unpacked from a shallow mirror clone. Manual population of # .git/modules is already done. - fetch_flags = "--no-fetch" + bb.warn('shallow mirror') + pass else: - # This isn't fatal; git-submodule will just fetch it - # during do_unpack(). - fetch_flags = "" - bb.error("submodule history not retrieved during do_fetch()") - - # Careful not to hit the network during unpacking; all history should already - # be fetched. - # - # The repeated attempts to do the submodule initialization sandwiched around a sync to - # install the correct remote URLs into the submodules' .git/config metadata are deliberate. - # Bad remote URLs are leftover in the modules' .git/config files from the unpack of bare - # clone tarballs and an initial 'git submodule update' is necessary to prod them back to - # enough life so that the 'git submodule sync' realizes the existing module .git/config - # files exist to be updated. - runfetchcmd("%(basecmd)s submodule update --init --recursive %(fetch_flags)s || (%(basecmd)s submodule sync --recursive && %(basecmd)s submodule update --init --recursive %(fetch_flags)s)" % {'basecmd': ud.basecmd, 'fetch_flags': fetch_flags}, d, workdir=ud.destdir) + # This is fatal; git-submodule would fetch it, but that is not allowed + raise bb.fetch2.FetchError("submodule contents not retrieved during do_fetch()") + + submodules = [] + paths = {} + uris = {} + local_paths = {} + for name in ud.names: + gitmodules = runfetchcmd("%s show HEAD:.gitmodules" % (ud.basecmd), d, quiet=True, workdir=ud.destdir) + bb.warn('gitmodules: %s' % gitmodules) + + module = "" + for line in gitmodules.splitlines(): + if line.startswith('[submodule'): + module = line.split('"')[1] + submodules.append(module) + elif module and line.strip().startswith('path'): + path = line.split('=')[1].strip() + paths[module] = path + elif module and line.strip().startswith('url'): + url = line.split('=')[1].strip() + uris[module] = url + + for module in submodules: + modpath = os.path.join(repo_conf, 'modules', module) + + # Determine (from the submodule) the correct url to reference + bb.warn('%s: %s' % (modpath, "%(basecmd)s config remote.origin.url" % {'basecmd': ud.basecmd})) + local_paths[module] = runfetchcmd("%(basecmd)s config remote.origin.url" % {'basecmd': ud.basecmd}, d, workdir=modpath) + + # Setup the local URL properly (like git submodule init or sync would do...) + runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_paths[module]}, d, workdir=ud.destdir) + + # Ensure the submodule repository is NOT set to bare, since we're checking it out... + runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=modpath) + + # Run submodule update, this sets up the directories -- without touching the config + runfetchcmd("%s submodule update --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir) -- cgit 1.2.3-korg