aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMark Hatle <mark.hatle@windriver.com>2018-09-20 11:08:06 -0400
committerMark Hatle <mark.hatle@windriver.com>2018-09-20 11:10:17 -0400
commit2bc8aac6c023f200d84dd3354d6bf29ccf09c1d8 (patch)
treeb019e2d2fda3c9728513b0793700a3a13b51cce5
parentb6ea045a972928fc99a72596823b4f8a7ef07000 (diff)
downloadbitbake-contrib-mgh/wip.tar.gz
WIP - gitsm fetcher reworkmgh/wip
Signed-off-by: Mark Hatle <mark.hatle@windriver.com>
-rw-r--r--lib/bb/fetch2/gitsm.py195
1 files changed, 98 insertions, 97 deletions
diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py
index 86773094d..8d6171127 100644
--- a/lib/bb/fetch2/gitsm.py
+++ b/lib/bb/fetch2/gitsm.py
@@ -34,6 +34,8 @@ import bb
from bb.fetch2.git import Git
from bb.fetch2 import runfetchcmd
from bb.fetch2 import logger
+from bb.fetch2 import Fetch
+from bb.fetch2 import BBFetchException
class GitSM(Git):
def supports(self, ud, d):
@@ -51,87 +53,62 @@ class GitSM(Git):
pass
return False
- def _set_relative_paths(self, repopath):
- """
- Fix submodule paths to be relative instead of absolute,
- so that when we move the repo it doesn't break
- (In Git 1.7.10+ this is done automatically)
- """
+ def update_submodules(self, ud, d, allow_network):
submodules = []
- with open(os.path.join(repopath, '.gitmodules'), 'r') as f:
- for line in f.readlines():
+ paths = {}
+ uris = {}
+ local_paths = {}
+
+ for name in ud.names:
+ gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=ud.clonedir)
+
+ module = ""
+ for line in gitmodules.splitlines():
if line.startswith('[submodule'):
- submodules.append(line.split('"')[1])
+ module = line.split('"')[1]
+ submodules.append(module)
+ elif module and line.strip().startswith('path'):
+ path = line.split('=')[1].strip()
+ paths[module] = path
+ elif module and line.strip().startswith('url'):
+ url = line.split('=')[1].strip()
+ uris[module] = url
for module in submodules:
- repo_conf = os.path.join(repopath, module, '.git')
- if os.path.exists(repo_conf):
- with open(repo_conf, 'r') as f:
- lines = f.readlines()
- newpath = ''
- for i, line in enumerate(lines):
- if line.startswith('gitdir:'):
- oldpath = line.split(': ')[-1].rstrip()
- if oldpath.startswith('/'):
- newpath = '../' * (module.count('/') + 1) + '.git/modules/' + module
- lines[i] = 'gitdir: %s\n' % newpath
- break
- if newpath:
- with open(repo_conf, 'w') as f:
- for line in lines:
- f.write(line)
-
- repo_conf2 = os.path.join(repopath, '.git', 'modules', module, 'config')
- if os.path.exists(repo_conf2):
- with open(repo_conf2, 'r') as f:
- lines = f.readlines()
- newpath = ''
- for i, line in enumerate(lines):
- if line.lstrip().startswith('worktree = '):
- oldpath = line.split(' = ')[-1].rstrip()
- if oldpath.startswith('/'):
- newpath = '../' * (module.count('/') + 3) + module
- lines[i] = '\tworktree = %s\n' % newpath
- break
- if newpath:
- with open(repo_conf2, 'w') as f:
- for line in lines:
- f.write(line)
+ module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], paths[module]), d, quiet=True, workdir=ud.clonedir)
+ module_hash = module_hash.split()[2]
- def update_submodules(self, ud, d, allow_network):
- # We have to convert bare -> full repo, do the submodule bit, then convert back
- tmpclonedir = ud.clonedir + ".tmp"
- gitdir = tmpclonedir + os.sep + ".git"
- bb.utils.remove(tmpclonedir, True)
- os.mkdir(tmpclonedir)
- os.rename(ud.clonedir, gitdir)
- runfetchcmd("sed " + gitdir + "/config -i -e 's/bare.*=.*true/bare = false/'", d)
- runfetchcmd(ud.basecmd + " reset --hard", d, workdir=tmpclonedir)
- runfetchcmd(ud.basecmd + " checkout -f " + ud.revisions[ud.names[0]], d, workdir=tmpclonedir)
-
- try:
- if allow_network:
- fetch_flags = ""
- else:
- fetch_flags = "--no-fetch"
-
- # The 'git submodule sync' sandwiched between two successive 'git submodule update' commands is
- # intentional. See the notes on the similar construction in download() for an explanation.
- runfetchcmd("%(basecmd)s submodule update --init --recursive %(fetch_flags)s || (%(basecmd)s submodule sync --recursive && %(basecmd)s submodule update --init --recursive %(fetch_flags)s)" % {'basecmd': ud.basecmd, 'fetch_flags' : fetch_flags}, d, workdir=tmpclonedir)
- except bb.fetch.FetchError:
- if allow_network:
- raise
- else:
- # This method was called as a probe to see whether the submodule history
- # is complete enough to allow the current working copy to have its
- # modules filled in. It's not, so swallow up the exception and report
- # the negative result.
- return False
- finally:
- self._set_relative_paths(tmpclonedir)
- runfetchcmd("sed " + gitdir + "/config -i -e 's/bare.*=.*false/bare = true/'", d, workdir=tmpclonedir)
- os.rename(gitdir, ud.clonedir,)
- bb.utils.remove(tmpclonedir, True)
+ try:
+ url = uris[module]
+ if url.startswith('http:'):
+ url = url.replace('http:', 'git:', 1) + ';protocol=http'
+ elif url.startswith('https:'):
+ url = url.replace('https:', 'git:', 1) + ';protocol=https'
+ elif url.startswith('ssh:'):
+ url = url.replace('ssh:', 'git:', 1) + ';protocol=ssh'
+ elif url.startswith('rsync:'):
+ url = url.replace('rsync:', 'git:', 1) + ';protocol=https'
+ url += ";bareclone=1;nocheckout=1;name=%s" % (module)
+ ld = d.createCopy()
+ ld.setVar('SRCREV', module_hash)
+ ld.setVar('SRCPV', d.getVar('SRCPV'))
+ ld.setVar('SRCREV_FORMAT', module)
+ ld.setVar('SRC_URI', url)
+ newfetch = Fetch([url], ld)
+ newfetch.download()
+ local_paths[module] = newfetch.localpath(url)
+
+ # Correct the submodule references to the local download version...
+ bb.warn("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_paths[module]})
+ runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_paths[module]}, d, workdir=ud.clonedir)
+ try:
+ os.mkdir(os.path.join(ud.clonedir, 'modules'))
+ except OSError:
+ pass
+ os.symlink(local_paths[module], os.path.join(ud.clonedir, 'modules', paths[module]))
+
+ except BBFetchException as e:
+ bb.error(str(e))
return True
@@ -155,12 +132,12 @@ class GitSM(Git):
if not ud.shallow or ud.localpath != ud.fullshallow:
submodules = self.uses_submodules(ud, d, ud.clonedir)
if submodules:
- self.update_submodules(ud, d, allow_network=True)
+ self.update_submodules(ud, d, allow_network=False)
def clone_shallow_local(self, ud, dest, d):
super(GitSM, self).clone_shallow_local(ud, dest, d)
- runfetchcmd('cp -fpPRH "%s/modules" "%s/"' % (ud.clonedir, os.path.join(dest, '.git')), d)
+ runfetchcmd('cp -fpLR "%s/modules" "%s/"' % (ud.clonedir, os.path.join(dest, '.git')), d)
def unpack(self, ud, destdir, d):
Git.unpack(self, ud, destdir, d)
@@ -174,29 +151,53 @@ class GitSM(Git):
else:
repo_conf = os.path.join(ud.destdir, '.git')
- if os.path.exists(ud.clonedir):
+ if os.path.exists(os.path.join(ud.clonedir, 'modules')):
# This is not a copy unpacked from a shallow mirror clone. So
# the manual intervention to populate the .git/modules done
# in clone_shallow_local() won't have been done yet.
- runfetchcmd("cp -fpPRH %s %s" % (os.path.join(ud.clonedir, 'modules'), repo_conf), d)
- fetch_flags = "--no-fetch"
+ bb.warn('cp -fpLR')
+ runfetchcmd("cp -fpLR %s %s" % (os.path.join(ud.clonedir, 'modules'), repo_conf), d)
elif os.path.exists(os.path.join(repo_conf, 'modules')):
# Unpacked from a shallow mirror clone. Manual population of
# .git/modules is already done.
- fetch_flags = "--no-fetch"
+ bb.warn('shallow mirror')
+ pass
else:
- # This isn't fatal; git-submodule will just fetch it
- # during do_unpack().
- fetch_flags = ""
- bb.error("submodule history not retrieved during do_fetch()")
-
- # Careful not to hit the network during unpacking; all history should already
- # be fetched.
- #
- # The repeated attempts to do the submodule initialization sandwiched around a sync to
- # install the correct remote URLs into the submodules' .git/config metadata are deliberate.
- # Bad remote URLs are leftover in the modules' .git/config files from the unpack of bare
- # clone tarballs and an initial 'git submodule update' is necessary to prod them back to
- # enough life so that the 'git submodule sync' realizes the existing module .git/config
- # files exist to be updated.
- runfetchcmd("%(basecmd)s submodule update --init --recursive %(fetch_flags)s || (%(basecmd)s submodule sync --recursive && %(basecmd)s submodule update --init --recursive %(fetch_flags)s)" % {'basecmd': ud.basecmd, 'fetch_flags': fetch_flags}, d, workdir=ud.destdir)
+ # This is fatal; git-submodule would fetch it, but that is not allowed
+ raise bb.fetch2.FetchError("submodule contents not retrieved during do_fetch()")
+
+ submodules = []
+ paths = {}
+ uris = {}
+ local_paths = {}
+ for name in ud.names:
+ gitmodules = runfetchcmd("%s show HEAD:.gitmodules" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)
+ bb.warn('gitmodules: %s' % gitmodules)
+
+ module = ""
+ for line in gitmodules.splitlines():
+ if line.startswith('[submodule'):
+ module = line.split('"')[1]
+ submodules.append(module)
+ elif module and line.strip().startswith('path'):
+ path = line.split('=')[1].strip()
+ paths[module] = path
+ elif module and line.strip().startswith('url'):
+ url = line.split('=')[1].strip()
+ uris[module] = url
+
+ for module in submodules:
+ modpath = os.path.join(repo_conf, 'modules', module)
+
+ # Determine (from the submodule) the correct url to reference
+ bb.warn('%s: %s' % (modpath, "%(basecmd)s config remote.origin.url" % {'basecmd': ud.basecmd}))
+ local_paths[module] = runfetchcmd("%(basecmd)s config remote.origin.url" % {'basecmd': ud.basecmd}, d, workdir=modpath)
+
+ # Setup the local URL properly (like git submodule init or sync would do...)
+ runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_paths[module]}, d, workdir=ud.destdir)
+
+ # Ensure the submodule repository is NOT set to bare, since we're checking it out...
+ runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=modpath)
+
+ # Run submodule update, this sets up the directories -- without touching the config
+ runfetchcmd("%s submodule update --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)