diff options
| -rw-r--r-- | lib/bb/data_smart.py | 8 | ||||
| -rw-r--r-- | lib/bb/fetch2/git.py | 45 | ||||
| -rw-r--r-- | lib/bb/parse/__init__.py | 9 | ||||
| -rw-r--r-- | lib/bb/tests/fetch.py | 53 |
4 files changed, 100 insertions, 15 deletions
diff --git a/lib/bb/data_smart.py b/lib/bb/data_smart.py index dd20ca557..111377a75 100644 --- a/lib/bb/data_smart.py +++ b/lib/bb/data_smart.py @@ -261,12 +261,9 @@ class VariableHistory(object): return if 'op' not in loginfo or not loginfo['op']: loginfo['op'] = 'set' - if 'detail' in loginfo: - loginfo['detail'] = str(loginfo['detail']) if 'variable' not in loginfo or 'file' not in loginfo: raise ValueError("record() missing variable or file.") var = loginfo['variable'] - if var not in self.variables: self.variables[var] = [] if not isinstance(self.variables[var], list): @@ -325,7 +322,8 @@ class VariableHistory(object): flag = '[%s] ' % (event['flag']) else: flag = '' - o.write("# %s %s:%s%s\n# %s\"%s\"\n" % (event['op'], event['file'], event['line'], display_func, flag, re.sub('\n', '\n# ', event['detail']))) + o.write("# %s %s:%s%s\n# %s\"%s\"\n" % \ + (event['op'], event['file'], event['line'], display_func, flag, re.sub('\n', '\n# ', str(event['detail'])))) if len(history) > 1: o.write("# pre-expansion value:\n") o.write('# "%s"\n' % (commentVal)) @@ -379,7 +377,7 @@ class VariableHistory(object): if isset and event['op'] == 'set?': continue isset = True - items = d.expand(event['detail']).split() + items = d.expand(str(event['detail'])).split() for item in items: # This is a little crude but is belt-and-braces to avoid us # having to handle every possible operation type specifically diff --git a/lib/bb/fetch2/git.py b/lib/bb/fetch2/git.py index 4d6e57ade..9ecc855af 100644 --- a/lib/bb/fetch2/git.py +++ b/lib/bb/fetch2/git.py @@ -307,7 +307,10 @@ class Git(FetchMethod): return ud.clonedir def need_update(self, ud, d): - return self.clonedir_need_update(ud, d) or self.shallow_tarball_need_update(ud) or self.tarball_need_update(ud) + return self.clonedir_need_update(ud, d) \ + or self.shallow_tarball_need_update(ud) \ + or self.tarball_need_update(ud) \ + or self.lfs_need_update(ud, d) def clonedir_need_update(self, ud, d): if not os.path.exists(ud.clonedir): @@ -319,6 +322,15 @@ class Git(FetchMethod): return True return False + def lfs_need_update(self, ud, d): + if self.clonedir_need_update(ud, d): + return True + + for name in ud.names: + if not self._lfs_objects_downloaded(ud, d, name, ud.clonedir): + return True + return False + def clonedir_need_shallow_revs(self, ud, d): for rev in ud.shallow_revs: try: @@ -406,7 +418,7 @@ class Git(FetchMethod): if missing_rev: raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev) - if self._contains_lfs(ud, d, ud.clonedir) and self._need_lfs(ud): + if self.lfs_need_update(ud, d): # Unpack temporary working copy, use it to run 'git checkout' to force pre-fetching # of all LFS blobs needed at the srcrev. # @@ -649,6 +661,35 @@ class Git(FetchMethod): raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output)) return output.split()[0] != "0" + def _lfs_objects_downloaded(self, ud, d, name, wd): + """ + Verifies whether the LFS objects for requested revisions have already been downloaded + """ + # Bail out early if this repository doesn't use LFS + if not self._need_lfs(ud) or not self._contains_lfs(ud, d, wd): + return True + + # The Git LFS specification specifies ([1]) the LFS folder layout so it should be safe to check for file + # existence. + # [1] https://github.com/git-lfs/git-lfs/blob/main/docs/spec.md#intercepting-git + cmd = "%s lfs ls-files -l %s" \ + % (ud.basecmd, ud.revisions[name]) + output = runfetchcmd(cmd, d, quiet=True, workdir=wd).rstrip() + # Do not do any further matching if no objects are managed by LFS + if not output: + return True + + # Match all lines beginning with the hexadecimal OID + oid_regex = re.compile("^(([a-fA-F0-9]{2})([a-fA-F0-9]{2})[A-Fa-f0-9]+)") + for line in output.split("\n"): + oid = re.search(oid_regex, line) + if not oid: + bb.warn("git lfs ls-files output '%s' did not match expected format." % line) + if not os.path.exists(os.path.join(wd, "lfs", "objects", oid.group(2), oid.group(3), oid.group(1))): + return False + + return True + def _need_lfs(self, ud): return ud.parm.get("lfs", "1") == "1" diff --git a/lib/bb/parse/__init__.py b/lib/bb/parse/__init__.py index 347609513..90b6c5656 100644 --- a/lib/bb/parse/__init__.py +++ b/lib/bb/parse/__init__.py @@ -49,20 +49,23 @@ class SkipPackage(SkipRecipe): __mtime_cache = {} def cached_mtime(f): if f not in __mtime_cache: - __mtime_cache[f] = os.stat(f)[stat.ST_MTIME] + res = os.stat(f) + __mtime_cache[f] = (res.st_mtime_ns, res.st_size, res.st_ino) return __mtime_cache[f] def cached_mtime_noerror(f): if f not in __mtime_cache: try: - __mtime_cache[f] = os.stat(f)[stat.ST_MTIME] + res = os.stat(f) + __mtime_cache[f] = (res.st_mtime_ns, res.st_size, res.st_ino) except OSError: return 0 return __mtime_cache[f] def update_mtime(f): try: - __mtime_cache[f] = os.stat(f)[stat.ST_MTIME] + res = os.stat(f) + __mtime_cache[f] = (res.st_mtime_ns, res.st_size, res.st_ino) except OSError: if f in __mtime_cache: del __mtime_cache[f] diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py index 847a35602..a7d9d5e88 100644 --- a/lib/bb/tests/fetch.py +++ b/lib/bb/tests/fetch.py @@ -6,6 +6,7 @@ # SPDX-License-Identifier: GPL-2.0-only # +import contextlib import unittest import hashlib import tempfile @@ -1334,7 +1335,7 @@ class FetchLatestVersionTest(FetcherTest): # combination version pattern ("sysprof", "git://gitlab.gnome.org/GNOME/sysprof.git;protocol=https;branch=master", "cd44ee6644c3641507fb53b8a2a69137f2971219", "") : "1.2.0", - ("u-boot-mkimage", "git://git.denx.de/u-boot.git;branch=master;protocol=git", "62c175fbb8a0f9a926c88294ea9f7e88eb898f6c", "") + ("u-boot-mkimage", "git://source.denx.de/u-boot/u-boot.git;branch=master;protocol=https", "62c175fbb8a0f9a926c88294ea9f7e88eb898f6c", "") : "2014.01", # version pattern "yyyymmdd" ("mobile-broadband-provider-info", "git://gitlab.gnome.org/GNOME/mobile-broadband-provider-info.git;protocol=https;branch=master", "4ed19e11c2975105b71b956440acdb25d46a347d", "") @@ -2182,10 +2183,14 @@ class GitLfsTest(FetcherTest): bb.utils.mkdirhier(self.srcdir) self.git_init(cwd=self.srcdir) - with open(os.path.join(self.srcdir, '.gitattributes'), 'wt') as attrs: - attrs.write('*.mp3 filter=lfs -text') - self.git(['add', '.gitattributes'], cwd=self.srcdir) - self.git(['commit', '-m', "attributes", '.gitattributes'], cwd=self.srcdir) + self.commit_file('.gitattributes', '*.mp3 filter=lfs -text') + + def commit_file(self, filename, content): + with open(os.path.join(self.srcdir, filename), "w") as f: + f.write(content) + self.git(["add", filename], cwd=self.srcdir) + self.git(["commit", "-m", "Change"], cwd=self.srcdir) + return self.git(["rev-parse", "HEAD"], cwd=self.srcdir).strip() def fetch(self, uri=None, download=True): uris = self.d.getVar('SRC_URI').split() @@ -2206,6 +2211,44 @@ class GitLfsTest(FetcherTest): return unpacked_lfs_file @skipIfNoGitLFS() + def test_fetch_lfs_on_srcrev_change(self): + """Test if fetch downloads missing LFS objects when a different revision within an existing repository is requested""" + self.git(["lfs", "install", "--local"], cwd=self.srcdir) + + @contextlib.contextmanager + def hide_upstream_repository(): + """Hide the upstream repository to make sure that git lfs cannot pull from it""" + temp_name = self.srcdir + ".bak" + os.rename(self.srcdir, temp_name) + try: + yield + finally: + os.rename(temp_name, self.srcdir) + + def fetch_and_verify(revision, filename, content): + self.d.setVar('SRCREV', revision) + fetcher, ud = self.fetch() + + with hide_upstream_repository(): + workdir = self.d.getVar('WORKDIR') + fetcher.unpack(workdir) + + with open(os.path.join(workdir, "git", filename)) as f: + self.assertEqual(f.read(), content) + + commit_1 = self.commit_file("a.mp3", "version 1") + commit_2 = self.commit_file("a.mp3", "version 2") + + self.d.setVar('SRC_URI', "git://%s;protocol=file;lfs=1;branch=master" % self.srcdir) + + # Seed the local download folder by fetching the latest commit and verifying that the LFS contents are + # available even when the upstream repository disappears. + fetch_and_verify(commit_2, "a.mp3", "version 2") + # Verify that even when an older revision is fetched, the needed LFS objects are fetched into the download + # folder. + fetch_and_verify(commit_1, "a.mp3", "version 1") + + @skipIfNoGitLFS() @skipIfNoNetwork() def test_real_git_lfs_repo_succeeds_without_lfs_param(self): self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master") |