diff options
Diffstat (limited to 'lib/bb/fetch2/gitsm.py')
| -rw-r--r-- | lib/bb/fetch2/gitsm.py | 319 |
1 files changed, 230 insertions, 89 deletions
diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py index 0aff1008e..f7f3af721 100644 --- a/lib/bb/fetch2/gitsm.py +++ b/lib/bb/fetch2/gitsm.py @@ -1,5 +1,3 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- """ BitBake 'Fetch' git submodules implementation @@ -16,24 +14,18 @@ NOTE: Switching a SRC_URI from "git://" to "gitsm://" requires a clean of your r # Copyright (C) 2013 Richard Purdie # -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. +# SPDX-License-Identifier: GPL-2.0-only # -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import os import bb +import copy +import shutil +import tempfile from bb.fetch2.git import Git from bb.fetch2 import runfetchcmd from bb.fetch2 import logger +from bb.fetch2 import Fetch class GitSM(Git): def supports(self, ud, d): @@ -42,94 +34,243 @@ class GitSM(Git): """ return ud.type in ['gitsm'] - def uses_submodules(self, ud, d, wd): - for name in ud.names: - try: - runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=wd) - return True - except bb.fetch.FetchError: - pass - return False - - def _set_relative_paths(self, repopath): + def process_submodules(self, ud, workdir, function, d): """ - Fix submodule paths to be relative instead of absolute, - so that when we move the repo it doesn't break - (In Git 1.7.10+ this is done automatically) + Iterate over all of the submodules in this repository and execute + the 'function' for each of them. """ + submodules = [] - with open(os.path.join(repopath, '.gitmodules'), 'r') as f: - for line in f.readlines(): + paths = {} + revision = {} + uris = {} + subrevision = {} + + def parse_gitmodules(gitmodules): + modules = {} + module = "" + for line in gitmodules.splitlines(): if line.startswith('[submodule'): - submodules.append(line.split('"')[1]) + module = line.split('"')[1] + modules[module] = {} + elif module and line.strip().startswith('path'): + path = line.split('=')[1].strip() + modules[module]['path'] = path + elif module and line.strip().startswith('url'): + url = line.split('=')[1].strip() + modules[module]['url'] = url + return modules + + # Collect the defined submodules, and their attributes + for name in ud.names: + try: + gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=workdir) + except: + # No submodules to update + continue + + for m, md in parse_gitmodules(gitmodules).items(): + try: + module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=workdir) + except: + # If the command fails, we don't have a valid file to check. If it doesn't + # fail -- it still might be a failure, see next check... + module_hash = "" + + if not module_hash: + logger.debug("submodule %s is defined, but is not initialized in the repository. Skipping", m) + continue + + submodules.append(m) + paths[m] = md['path'] + revision[m] = ud.revisions[name] + uris[m] = md['url'] + subrevision[m] = module_hash.split()[2] + + # Convert relative to absolute uri based on parent uri + if uris[m].startswith('..') or uris[m].startswith('./'): + newud = copy.copy(ud) + newud.path = os.path.normpath(os.path.join(newud.path, uris[m])) + uris[m] = Git._get_repo_url(self, newud) for module in submodules: - repo_conf = os.path.join(repopath, module, '.git') - if os.path.exists(repo_conf): - with open(repo_conf, 'r') as f: - lines = f.readlines() - newpath = '' - for i, line in enumerate(lines): - if line.startswith('gitdir:'): - oldpath = line.split(': ')[-1].rstrip() - if oldpath.startswith('/'): - newpath = '../' * (module.count('/') + 1) + '.git/modules/' + module - lines[i] = 'gitdir: %s\n' % newpath - break - if newpath: - with open(repo_conf, 'w') as f: - for line in lines: - f.write(line) - - repo_conf2 = os.path.join(repopath, '.git', 'modules', module, 'config') - if os.path.exists(repo_conf2): - with open(repo_conf2, 'r') as f: - lines = f.readlines() - newpath = '' - for i, line in enumerate(lines): - if line.lstrip().startswith('worktree = '): - oldpath = line.split(' = ')[-1].rstrip() - if oldpath.startswith('/'): - newpath = '../' * (module.count('/') + 3) + module - lines[i] = '\tworktree = %s\n' % newpath - break - if newpath: - with open(repo_conf2, 'w') as f: - for line in lines: - f.write(line) - - def update_submodules(self, ud, d): - # We have to convert bare -> full repo, do the submodule bit, then convert back - tmpclonedir = ud.clonedir + ".tmp" - gitdir = tmpclonedir + os.sep + ".git" - bb.utils.remove(tmpclonedir, True) - os.mkdir(tmpclonedir) - os.rename(ud.clonedir, gitdir) - runfetchcmd("sed " + gitdir + "/config -i -e 's/bare.*=.*true/bare = false/'", d) - runfetchcmd(ud.basecmd + " reset --hard", d, workdir=tmpclonedir) - runfetchcmd(ud.basecmd + " checkout -f " + ud.revisions[ud.names[0]], d, workdir=tmpclonedir) - runfetchcmd(ud.basecmd + " submodule update --init --recursive", d, workdir=tmpclonedir) - self._set_relative_paths(tmpclonedir) - runfetchcmd("sed " + gitdir + "/config -i -e 's/bare.*=.*false/bare = true/'", d, workdir=tmpclonedir) - os.rename(gitdir, ud.clonedir,) - bb.utils.remove(tmpclonedir, True) + # Translate the module url into a SRC_URI + + if "://" in uris[module]: + # Properly formated URL already + proto = uris[module].split(':', 1)[0] + url = uris[module].replace('%s:' % proto, 'gitsm:', 1) + else: + if ":" in uris[module]: + # Most likely an SSH style reference + proto = "ssh" + if ":/" in uris[module]: + # Absolute reference, easy to convert.. + url = "gitsm://" + uris[module].replace(':/', '/', 1) + else: + # Relative reference, no way to know if this is right! + logger.warning("Submodule included by %s refers to relative ssh reference %s. References may fail if not absolute." % (ud.url, uris[module])) + url = "gitsm://" + uris[module].replace(':', '/', 1) + else: + # This has to be a file reference + proto = "file" + url = "gitsm://" + uris[module] + if url.endswith("{}{}".format(ud.host, ud.path)): + raise bb.fetch2.FetchError("Submodule refers to the parent repository. This will cause deadlock situation in current version of Bitbake." \ + "Consider using git fetcher instead.") + + url += ';protocol=%s' % proto + url += ";name=%s" % module + url += ";subpath=%s" % module + url += ";nobranch=1" + url += ";lfs=%s" % self._need_lfs(ud) + # Note that adding "user=" here to give credentials to the + # submodule is not supported. Since using SRC_URI to give git:// + # URL a password is not supported, one have to use one of the + # recommended way (eg. ~/.netrc or SSH config) which does specify + # the user (See comment in git.py). + # So, we will not take patches adding "user=" support here. + + ld = d.createCopy() + # Not necessary to set SRC_URI, since we're passing the URI to + # Fetch. + #ld.setVar('SRC_URI', url) + ld.setVar('SRCREV_%s' % module, subrevision[module]) + + # Workaround for issues with SRCPV/SRCREV_FORMAT errors + # error refer to 'multiple' repositories. Only the repository + # in the original SRC_URI actually matters... + ld.setVar('SRCPV', d.getVar('SRCPV')) + ld.setVar('SRCREV_FORMAT', module) + + function(ud, url, module, paths[module], workdir, ld) + + return submodules != [] + + def need_update(self, ud, d): + if Git.need_update(self, ud, d): + return True + + need_update_list = [] + def need_update_submodule(ud, url, module, modpath, workdir, d): + url += ";bareclone=1;nobranch=1" + + try: + newfetch = Fetch([url], d, cache=False) + new_ud = newfetch.ud[url] + if new_ud.method.need_update(new_ud, d): + need_update_list.append(modpath) + except Exception as e: + logger.error('gitsm: submodule update check failed: %s %s' % (type(e).__name__, str(e))) + need_update_result = True + + # If we're using a shallow mirror tarball it needs to be unpacked + # temporarily so that we can examine the .gitmodules file + if ud.shallow and os.path.exists(ud.fullshallow) and not os.path.exists(ud.clonedir): + tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR")) + runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir) + self.process_submodules(ud, tmpdir, need_update_submodule, d) + shutil.rmtree(tmpdir) + else: + self.process_submodules(ud, ud.clonedir, need_update_submodule, d) + + if need_update_list: + logger.debug('gitsm: Submodules requiring update: %s' % (' '.join(need_update_list))) + return True + + return False def download(self, ud, d): - Git.download(self, ud, d) + def download_submodule(ud, url, module, modpath, workdir, d): + url += ";bareclone=1;nobranch=1" - if not ud.shallow or ud.localpath != ud.fullshallow: - submodules = self.uses_submodules(ud, d, ud.clonedir) - if submodules: - self.update_submodules(ud, d) + # Is the following still needed? + #url += ";nocheckout=1" - def clone_shallow_local(self, ud, dest, d): - super(GitSM, self).clone_shallow_local(ud, dest, d) + try: + newfetch = Fetch([url], d, cache=False) + newfetch.download() + except Exception as e: + logger.error('gitsm: submodule download failed: %s %s' % (type(e).__name__, str(e))) + raise + + Git.download(self, ud, d) - runfetchcmd('cp -fpPRH "%s/modules" "%s/"' % (ud.clonedir, os.path.join(dest, '.git')), d) + # If we're using a shallow mirror tarball it needs to be unpacked + # temporarily so that we can examine the .gitmodules file + if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d): + tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR")) + runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir) + self.process_submodules(ud, tmpdir, download_submodule, d) + shutil.rmtree(tmpdir) + else: + self.process_submodules(ud, ud.clonedir, download_submodule, d) def unpack(self, ud, destdir, d): + def unpack_submodules(ud, url, module, modpath, workdir, d): + url += ";bareclone=1;nobranch=1" + + # Figure out where we clone over the bare submodules... + if ud.bareclone: + repo_conf = ud.destdir + else: + repo_conf = os.path.join(ud.destdir, '.git') + + try: + newfetch = Fetch([url], d, cache=False) + # modpath is needed by unpack tracer to calculate submodule + # checkout dir + new_ud = newfetch.ud[url] + new_ud.modpath = modpath + newfetch.unpack(root=os.path.dirname(os.path.join(repo_conf, 'modules', module))) + except Exception as e: + logger.error('gitsm: submodule unpack failed: %s %s' % (type(e).__name__, str(e))) + raise + + local_path = newfetch.localpath(url) + + # Correct the submodule references to the local download version... + runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_path}, d, workdir=ud.destdir) + + if ud.shallow: + runfetchcmd("%(basecmd)s config submodule.%(module)s.shallow true" % {'basecmd': ud.basecmd, 'module': module}, d, workdir=ud.destdir) + + # Ensure the submodule repository is NOT set to bare, since we're checking it out... + try: + runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=os.path.join(repo_conf, 'modules', module)) + except: + logger.error("Unable to set git config core.bare to false for %s" % os.path.join(repo_conf, 'modules', module)) + raise + Git.unpack(self, ud, destdir, d) - if self.uses_submodules(ud, d, ud.destdir): - runfetchcmd(ud.basecmd + " checkout " + ud.revisions[ud.names[0]], d, workdir=ud.destdir) - runfetchcmd(ud.basecmd + " submodule update --init --recursive", d, workdir=ud.destdir) + ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d) + + if not ud.bareclone and ret: + # All submodules should already be downloaded and configured in the tree. This simply + # sets up the configuration and checks out the files. The main project config should + # remain unmodified, and no download from the internet should occur. As such, lfs smudge + # should also be skipped as these files were already smudged in the fetch stage if lfs + # was enabled. + runfetchcmd("GIT_LFS_SKIP_SMUDGE=1 %s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir) + + def implicit_urldata(self, ud, d): + import shutil, subprocess, tempfile + + urldata = [] + def add_submodule(ud, url, module, modpath, workdir, d): + url += ";bareclone=1;nobranch=1" + newfetch = Fetch([url], d, cache=False) + urldata.extend(newfetch.expanded_urldata()) + + # If we're using a shallow mirror tarball it needs to be unpacked + # temporarily so that we can examine the .gitmodules file + if ud.shallow and os.path.exists(ud.fullshallow) and ud.method.need_update(ud, d): + tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR")) + subprocess.check_call("tar -xzf %s" % ud.fullshallow, cwd=tmpdir, shell=True) + self.process_submodules(ud, tmpdir, add_submodule, d) + shutil.rmtree(tmpdir) + else: + self.process_submodules(ud, ud.clonedir, add_submodule, d) + + return urldata |