From 97f856f0455d014ea34c28b1c25f09e13cdc851b Mon Sep 17 00:00:00 2001 From: Christopher Larson Date: Sat, 13 May 2017 02:46:33 +0500 Subject: fetch/git: add support for removing arbitrary revs for shallow In certain cases, it's valuable to be able to exert more control over what history is removed, beyond srcrev+depth. As one example, you can remove most of the upstream kernel history from a kernel repository, keeping predominently the non-publically-accessible content. If the repository is private, the history in that repo couldn't be restored via `git fetch --unshallow`, but upstream history could be. Example usage: # Remove only these revs, not at a particular depth BB_GIT_SHALLOW_DEPTH_pn-linux-foo = "0" BB_GIT_SHALLOW_REVS_pn-linux-foo = "v4.1" Signed-off-by: Christopher Larson Signed-off-by: Richard Purdie --- lib/bb/fetch2/git.py | 18 +++++++++++++- lib/bb/tests/fetch.py | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 82 insertions(+), 2 deletions(-) diff --git a/lib/bb/fetch2/git.py b/lib/bb/fetch2/git.py index aa972c5cf..534c93d3c 100644 --- a/lib/bb/fetch2/git.py +++ b/lib/bb/fetch2/git.py @@ -196,6 +196,8 @@ class Git(FetchMethod): depth_default = 1 ud.shallow_depths = collections.defaultdict(lambda: depth_default) + revs_default = d.getVar("BB_GIT_SHALLOW_REVS", True) + ud.shallow_revs = [] ud.branches = {} for pos, name in enumerate(ud.names): branch = branches[pos] @@ -213,7 +215,14 @@ class Git(FetchMethod): raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (name, shallow_depth)) ud.shallow_depths[name] = shallow_depth + revs = d.getVar("BB_GIT_SHALLOW_REVS_%s" % name) + if revs is not None: + ud.shallow_revs.extend(revs.split()) + elif revs_default is not None: + ud.shallow_revs.extend(revs_default.split()) + if (ud.shallow and + not ud.shallow_revs and all(ud.shallow_depths[n] == 0 for n in ud.names)): # Shallow disabled for this URL ud.shallow = False @@ -261,6 +270,9 @@ class Git(FetchMethod): if ud.bareclone: tarballname = "%s_bare" % tarballname + if ud.shallow_revs: + tarballname = "%s_%s" % (tarballname, "_".join(sorted(ud.shallow_revs))) + for name, revision in sorted(ud.revisions.items()): tarballname = "%s_%s" % (tarballname, ud.revisions[name][:7]) depth = ud.shallow_depths[name] @@ -413,7 +425,11 @@ class Git(FetchMethod): runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest) # Map srcrev+depths to revisions - shallow_revisions = runfetchcmd("%s rev-parse %s" % (ud.basecmd, " ".join(to_parse)), d, workdir=dest).splitlines() + parsed_depths = runfetchcmd("%s rev-parse %s" % (ud.basecmd, " ".join(to_parse)), d, workdir=dest) + + # Resolve specified revisions + parsed_revs = runfetchcmd("%s rev-parse %s" % (ud.basecmd, " ".join('"%s^{}"' % r for r in ud.shallow_revs)), d, workdir=dest) + shallow_revisions = parsed_depths.splitlines() + parsed_revs.splitlines() # Apply extra ref wildcards all_refs = runfetchcmd('%s for-each-ref "--format=%%(refname)"' % ud.basecmd, diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py index 73f7b3f78..343ae8fe5 100644 --- a/lib/bb/tests/fetch.py +++ b/lib/bb/tests/fetch.py @@ -1252,6 +1252,33 @@ class GitShallowTest(FetcherTest): assert os.path.exists(os.path.join(self.gitdir, 'c')) def test_shallow_multi_one_uri(self): + # Create initial git repo + self.add_empty_file('a') + self.add_empty_file('b') + self.git('checkout -b a_branch', cwd=self.srcdir) + self.add_empty_file('c') + self.add_empty_file('d') + self.git('checkout master', cwd=self.srcdir) + self.git('tag v0.0 a_branch', cwd=self.srcdir) + self.add_empty_file('e') + self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir) + self.add_empty_file('f') + self.assertRevCount(7, cwd=self.srcdir) + + uri = self.d.getVar('SRC_URI', True).split()[0] + uri = '%s;branch=master,a_branch;name=master,a_branch' % uri + + self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') + self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') + self.d.setVar('SRCREV_master', '${AUTOREV}') + self.d.setVar('SRCREV_a_branch', '${AUTOREV}') + + self.fetch_shallow(uri) + + self.assertRevCount(5) + self.assertRefs(['master', 'origin/master', 'origin/a_branch']) + + def test_shallow_multi_one_uri_depths(self): # Create initial git repo self.add_empty_file('a') self.add_empty_file('b') @@ -1375,6 +1402,38 @@ class GitShallowTest(FetcherTest): self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/tags/*') self.fetch() + def test_shallow_remove_revs(self): + # Create initial git repo + self.add_empty_file('a') + self.add_empty_file('b') + self.git('checkout -b a_branch', cwd=self.srcdir) + self.add_empty_file('c') + self.add_empty_file('d') + self.git('checkout master', cwd=self.srcdir) + self.git('tag v0.0 a_branch', cwd=self.srcdir) + self.add_empty_file('e') + self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir) + self.git('branch -d a_branch', cwd=self.srcdir) + self.add_empty_file('f') + self.assertRevCount(7, cwd=self.srcdir) + + self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') + self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') + + self.fetch_shallow() + + self.assertRevCount(5) + + def test_shallow_invalid_revs(self): + self.add_empty_file('a') + self.add_empty_file('b') + + self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') + self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') + + with self.assertRaises(bb.fetch2.FetchError): + self.fetch() + if os.environ.get("BB_SKIP_NETTESTS") == "yes": print("Unset BB_SKIP_NETTESTS to run network tests") else: @@ -1383,11 +1442,16 @@ class GitShallowTest(FetcherTest): self.git('config core.bare true', cwd=self.srcdir) self.git('fetch --tags', cwd=self.srcdir) - self.d.setVar('BB_GIT_SHALLOW_DEPTH', '100') + self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') + # Note that the 1.10.0 tag is annotated, so this also tests + # reference of an annotated vs unannotated tag + self.d.setVar('BB_GIT_SHALLOW_REVS', '1.10.0') self.fetch_shallow() + # Confirm that the history of 1.10.0 was removed orig_revs = len(self.git('rev-list master', cwd=self.srcdir).splitlines()) revs = len(self.git('rev-list master').splitlines()) self.assertNotEqual(orig_revs, revs) self.assertRefs(['master', 'origin/master']) + self.assertRevCount(orig_revs - 1758) -- cgit 1.2.3-korg