aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--lib/bb/fetch2/git.py44
-rw-r--r--lib/bb/tests/fetch.py28
2 files changed, 61 insertions, 11 deletions
diff --git a/lib/bb/fetch2/git.py b/lib/bb/fetch2/git.py
index df9538a6..e3ba80a3 100644
--- a/lib/bb/fetch2/git.py
+++ b/lib/bb/fetch2/git.py
@@ -384,6 +384,35 @@ class Git(FetchMethod):
if missing_rev:
raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev)
+ if self._contains_lfs(ud, d, ud.clonedir) and self._need_lfs(ud):
+ # Unpack temporary working copy, use it to run 'git checkout' to force pre-fetching
+ # of all LFS blobs needed at the the srcrev.
+ #
+ # It would be nice to just do this inline here by running 'git-lfs fetch'
+ # on the bare clonedir, but that operation requires a working copy on some
+ # releases of Git LFS.
+ tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
+ try:
+ # Do the checkout. This implicitly involves a Git LFS fetch.
+ self.unpack(ud, tmpdir, d)
+
+ # Scoop up a copy of any stuff that Git LFS downloaded. Merge them into
+ # the bare clonedir.
+ #
+ # As this procedure is invoked repeatedly on incremental fetches as
+ # a recipe's SRCREV is bumped throughout its lifetime, this will
+ # result in a gradual accumulation of LFS blobs in <ud.clonedir>/lfs
+ # corresponding to all the blobs reachable from the different revs
+ # fetched across time.
+ #
+ # Only do this if the unpack resulted in a .git/lfs directory being
+ # created; this only happens if at least one blob needed to be
+ # downloaded.
+ if os.path.exists(os.path.join(tmpdir, "git", ".git", "lfs")):
+ runfetchcmd("tar -cf - lfs | tar -xf - -C %s" % ud.clonedir, d, workdir="%s/git/.git" % tmpdir)
+ finally:
+ bb.utils.remove(tmpdir, recurse=True)
+
def build_mirror_data(self, ud, d):
if ud.shallow and ud.write_shallow_tarballs:
if not os.path.exists(ud.fullshallow):
@@ -479,7 +508,7 @@ class Git(FetchMethod):
if os.path.exists(destdir):
bb.utils.prunedir(destdir)
- need_lfs = ud.parm.get("lfs", "1") == "1"
+ need_lfs = self._need_lfs(ud)
if not need_lfs:
ud.basecmd = "GIT_LFS_SKIP_SMUDGE=1 " + ud.basecmd
@@ -568,6 +597,9 @@ class Git(FetchMethod):
raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output))
return output.split()[0] != "0"
+ def _need_lfs(self, ud):
+ return ud.parm.get("lfs", "1") == "1"
+
def _contains_lfs(self, ud, d, wd):
"""
Check if the repository has 'lfs' (large file) content
@@ -578,8 +610,14 @@ class Git(FetchMethod):
else:
branchname = "master"
- cmd = "%s grep lfs origin/%s:.gitattributes | wc -l" % (
- ud.basecmd, ud.branches[ud.names[0]])
+ # The bare clonedir doesn't use the remote names; it has the branch immediately.
+ if wd == ud.clonedir:
+ refname = ud.branches[ud.names[0]]
+ else:
+ refname = "origin/%s" % ud.branches[ud.names[0]]
+
+ cmd = "%s grep lfs %s:.gitattributes | wc -l" % (
+ ud.basecmd, refname)
try:
output = runfetchcmd(cmd, d, quiet=True, workdir=wd)
diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py
index 20beab38..7b2dac7b 100644
--- a/lib/bb/tests/fetch.py
+++ b/lib/bb/tests/fetch.py
@@ -2108,13 +2108,14 @@ class GitLfsTest(FetcherTest):
cwd = self.gitdir
return bb.process.run(cmd, cwd=cwd)[0]
- def fetch(self, uri=None):
+ def fetch(self, uri=None, download=True):
uris = self.d.getVar('SRC_URI').split()
uri = uris[0]
d = self.d
fetcher = bb.fetch2.Fetch(uris, d)
- fetcher.download()
+ if download:
+ fetcher.download()
ud = fetcher.ud[uri]
return fetcher, ud
@@ -2124,16 +2125,21 @@ class GitLfsTest(FetcherTest):
uri = 'git://%s;protocol=file;subdir=${S};lfs=1' % self.srcdir
self.d.setVar('SRC_URI', uri)
- fetcher, ud = self.fetch()
+ # Careful: suppress initial attempt at downloading until
+ # we know whether git-lfs is installed.
+ fetcher, ud = self.fetch(uri=None, download=False)
self.assertIsNotNone(ud.method._find_git_lfs)
- # If git-lfs can be found, the unpack should be successful
- ud.method._find_git_lfs = lambda d: True
- shutil.rmtree(self.gitdir, ignore_errors=True)
- fetcher.unpack(self.d.getVar('WORKDIR'))
+ # If git-lfs can be found, the unpack should be successful. Only
+ # attempt this with the real live copy of git-lfs installed.
+ if ud.method._find_git_lfs(self.d):
+ fetcher.download()
+ shutil.rmtree(self.gitdir, ignore_errors=True)
+ fetcher.unpack(self.d.getVar('WORKDIR'))
# If git-lfs cannot be found, the unpack should throw an error
with self.assertRaises(bb.fetch2.FetchError):
+ fetcher.download()
ud.method._find_git_lfs = lambda d: False
shutil.rmtree(self.gitdir, ignore_errors=True)
fetcher.unpack(self.d.getVar('WORKDIR'))
@@ -2144,10 +2150,16 @@ class GitLfsTest(FetcherTest):
uri = 'git://%s;protocol=file;subdir=${S};lfs=0' % self.srcdir
self.d.setVar('SRC_URI', uri)
+ # In contrast to test_lfs_enabled(), allow the implicit download
+ # done by self.fetch() to occur here. The point of this test case
+ # is to verify that the fetcher can survive even if the source
+ # repository has Git LFS usage configured.
fetcher, ud = self.fetch()
self.assertIsNotNone(ud.method._find_git_lfs)
- # If git-lfs can be found, the unpack should be successful
+ # If git-lfs can be found, the unpack should be successful. A
+ # live copy of git-lfs is not required for this case, so
+ # unconditionally forge its presence.
ud.method._find_git_lfs = lambda d: True
shutil.rmtree(self.gitdir, ignore_errors=True)
fetcher.unpack(self.d.getVar('WORKDIR'))