summaryrefslogtreecommitdiffstats
path: root/lib/bb/fetch2/git.py
diff options
context:
space:
mode:
authorYu Ke <ke.yu@intel.com>2011-01-10 14:25:04 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2011-01-10 14:25:04 +0000
commit7c567d55f4c44e5086f1580b833bf4cfcec4e202 (patch)
treed85c6ff449484e3a792edee0cde8084cc572c6b5 /lib/bb/fetch2/git.py
parent12b3aeeafcd8b97c162bf236e1e37586ac24db24 (diff)
downloadbitbake-7c567d55f4c44e5086f1580b833bf4cfcec4e202.tar.gz
bitbake: copy bb.fetch to bb.fetch2 as initial code base for fetcher overhaul
Signed-off-by: Yu Ke <ke.yu@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'lib/bb/fetch2/git.py')
-rw-r--r--lib/bb/fetch2/git.py339
1 files changed, 339 insertions, 0 deletions
diff --git a/lib/bb/fetch2/git.py b/lib/bb/fetch2/git.py
new file mode 100644
index 000000000..b37a09743
--- /dev/null
+++ b/lib/bb/fetch2/git.py
@@ -0,0 +1,339 @@
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+"""
+BitBake 'Fetch' git implementation
+
+"""
+
+#Copyright (C) 2005 Richard Purdie
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import os
+import bb
+import bb.persist_data
+from bb import data
+from bb.fetch import Fetch
+from bb.fetch import runfetchcmd
+from bb.fetch import logger
+
+class Git(Fetch):
+ """Class to fetch a module or modules from git repositories"""
+ def init(self, d):
+ #
+ # Only enable _sortable revision if the key is set
+ #
+ if bb.data.getVar("BB_GIT_CLONE_FOR_SRCREV", d, True):
+ self._sortable_buildindex = self._sortable_buildindex_disabled
+ def supports(self, url, ud, d):
+ """
+ Check to see if a given url can be fetched with git.
+ """
+ return ud.type in ['git']
+
+ def localpath(self, url, ud, d):
+
+ if 'protocol' in ud.parm:
+ ud.proto = ud.parm['protocol']
+ elif not ud.host:
+ ud.proto = 'file'
+ else:
+ ud.proto = "rsync"
+
+ ud.branch = ud.parm.get("branch", "master")
+
+ gitsrcname = '%s%s' % (ud.host, ud.path.replace('/', '.'))
+ ud.mirrortarball = 'git_%s.tar.gz' % (gitsrcname)
+ ud.clonedir = os.path.join(data.expand('${GITDIR}', d), gitsrcname)
+
+ tag = Fetch.srcrev_internal_helper(ud, d)
+ if tag is True:
+ ud.tag = self.latest_revision(url, ud, d)
+ elif tag:
+ ud.tag = tag
+
+ if not ud.tag or ud.tag == "master":
+ ud.tag = self.latest_revision(url, ud, d)
+
+ subdir = ud.parm.get("subpath", "")
+ if subdir != "":
+ if subdir.endswith("/"):
+ subdir = subdir[:-1]
+ subdirpath = os.path.join(ud.path, subdir);
+ else:
+ subdirpath = ud.path;
+
+ if 'fullclone' in ud.parm:
+ ud.localfile = ud.mirrortarball
+ else:
+ ud.localfile = data.expand('git_%s%s_%s.tar.gz' % (ud.host, subdirpath.replace('/', '.'), ud.tag), d)
+
+ ud.basecmd = data.getVar("FETCHCMD_git", d, True) or "git"
+
+ if 'noclone' in ud.parm:
+ ud.localfile = None
+ return None
+
+ return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
+
+ def forcefetch(self, url, ud, d):
+ if 'fullclone' in ud.parm:
+ return True
+ if 'noclone' in ud.parm:
+ return False
+ if os.path.exists(ud.localpath):
+ return False
+ if not self._contains_ref(ud.tag, d):
+ return True
+ return False
+
+ def try_premirror(self, u, ud, d):
+ if 'noclone' in ud.parm:
+ return False
+ if os.path.exists(ud.clonedir):
+ return False
+ if os.path.exists(ud.localpath):
+ return False
+
+ return True
+
+ def go(self, loc, ud, d):
+ """Fetch url"""
+
+ if ud.user:
+ username = ud.user + '@'
+ else:
+ username = ""
+
+ repofile = os.path.join(data.getVar("DL_DIR", d, 1), ud.mirrortarball)
+
+
+ coname = '%s' % (ud.tag)
+ codir = os.path.join(ud.clonedir, coname)
+
+ # If we have no existing clone and no mirror tarball, try and obtain one
+ if not os.path.exists(ud.clonedir) and not os.path.exists(repofile):
+ try:
+ Fetch.try_mirrors(ud.mirrortarball)
+ except:
+ pass
+
+ # If the checkout doesn't exist and the mirror tarball does, extract it
+ if not os.path.exists(ud.clonedir) and os.path.exists(repofile):
+ bb.mkdirhier(ud.clonedir)
+ os.chdir(ud.clonedir)
+ runfetchcmd("tar -xzf %s" % (repofile), d)
+
+ # If the repo still doesn't exist, fallback to cloning it
+ if not os.path.exists(ud.clonedir):
+ runfetchcmd("%s clone -n %s://%s%s%s %s" % (ud.basecmd, ud.proto, username, ud.host, ud.path, ud.clonedir), d)
+
+ os.chdir(ud.clonedir)
+ # Update the checkout if needed
+ if not self._contains_ref(ud.tag, d) or 'fullclone' in ud.parm:
+ # Remove all but the .git directory
+ runfetchcmd("rm * -Rf", d)
+ if 'fullclone' in ud.parm:
+ runfetchcmd("%s fetch --all" % (ud.basecmd), d)
+ else:
+ runfetchcmd("%s fetch %s://%s%s%s %s" % (ud.basecmd, ud.proto, username, ud.host, ud.path, ud.branch), d)
+ runfetchcmd("%s fetch --tags %s://%s%s%s" % (ud.basecmd, ud.proto, username, ud.host, ud.path), d)
+ runfetchcmd("%s prune-packed" % ud.basecmd, d)
+ runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d)
+
+ # Generate a mirror tarball if needed
+ os.chdir(ud.clonedir)
+ mirror_tarballs = data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True)
+ if mirror_tarballs != "0" or 'fullclone' in ud.parm:
+ logger.info("Creating tarball of git repository")
+ runfetchcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ), d)
+
+ if 'fullclone' in ud.parm:
+ return
+
+ if os.path.exists(codir):
+ bb.utils.prunedir(codir)
+
+ subdir = ud.parm.get("subpath", "")
+ if subdir != "":
+ if subdir.endswith("/"):
+ subdirbase = os.path.basename(subdir[:-1])
+ else:
+ subdirbase = os.path.basename(subdir)
+ else:
+ subdirbase = ""
+
+ if subdir != "":
+ readpathspec = ":%s" % (subdir)
+ codir = os.path.join(codir, "git")
+ coprefix = os.path.join(codir, subdirbase, "")
+ else:
+ readpathspec = ""
+ coprefix = os.path.join(codir, "git", "")
+
+ scmdata = ud.parm.get("scmdata", "")
+ if scmdata == "keep":
+ runfetchcmd("%s clone -n %s %s" % (ud.basecmd, ud.clonedir, coprefix), d)
+ os.chdir(coprefix)
+ runfetchcmd("%s checkout -q -f %s%s" % (ud.basecmd, ud.tag, readpathspec), d)
+ else:
+ bb.mkdirhier(codir)
+ os.chdir(ud.clonedir)
+ runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.tag, readpathspec), d)
+ runfetchcmd("%s checkout-index -q -f --prefix=%s -a" % (ud.basecmd, coprefix), d)
+
+ os.chdir(codir)
+ logger.info("Creating tarball of git checkout")
+ runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.join(".", "*") ), d)
+
+ os.chdir(ud.clonedir)
+ bb.utils.prunedir(codir)
+
+ def supports_srcrev(self):
+ return True
+
+ def _contains_ref(self, tag, d):
+ basecmd = data.getVar("FETCHCMD_git", d, True) or "git"
+ output = runfetchcmd("%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % (basecmd, tag), d, quiet=True)
+ return output.split()[0] != "0"
+
+ def _revision_key(self, url, ud, d, branch=False):
+ """
+ Return a unique key for the url
+ """
+ key = 'git:' + ud.host + ud.path.replace('/', '.')
+ if branch:
+ return key + ud.branch
+ else:
+ return key
+
+ def generate_revision_key(self, url, ud, d, branch=False):
+ key = self._revision_key(url, ud, d, branch)
+ return "%s-%s" % (key, bb.data.getVar("PN", d, True) or "")
+
+ def _latest_revision(self, url, ud, d):
+ """
+ Compute the HEAD revision for the url
+ """
+ if ud.user:
+ username = ud.user + '@'
+ else:
+ username = ""
+
+ basecmd = data.getVar("FETCHCMD_git", d, True) or "git"
+ cmd = "%s ls-remote %s://%s%s%s %s" % (basecmd, ud.proto, username, ud.host, ud.path, ud.branch)
+ output = runfetchcmd(cmd, d, True)
+ if not output:
+ raise bb.fetch.FetchError("Fetch command %s gave empty output\n" % (cmd))
+ return output.split()[0]
+
+ def latest_revision(self, url, ud, d):
+ """
+ Look in the cache for the latest revision, if not present ask the SCM.
+ """
+ persisted = bb.persist_data.persist(d)
+ revs = persisted['BB_URI_HEADREVS']
+
+ key = self.generate_revision_key(url, ud, d, branch=True)
+ rev = revs[key]
+ if rev is None:
+ # Compatibility with old key format, no branch included
+ oldkey = self.generate_revision_key(url, ud, d, branch=False)
+ rev = revs[oldkey]
+ if rev is not None:
+ del revs[oldkey]
+ else:
+ rev = self._latest_revision(url, ud, d)
+ revs[key] = rev
+
+ return str(rev)
+
+ def sortable_revision(self, url, ud, d):
+ """
+
+ """
+ pd = bb.persist_data.persist(d)
+ localcounts = pd['BB_URI_LOCALCOUNT']
+ key = self.generate_revision_key(url, ud, d, branch=True)
+ oldkey = self.generate_revision_key(url, ud, d, branch=False)
+
+ latest_rev = self._build_revision(url, ud, d)
+ last_rev = localcounts[key + '_rev']
+ if last_rev is None:
+ last_rev = localcounts[oldkey + '_rev']
+ if last_rev is not None:
+ del localcounts[oldkey + '_rev']
+ localcounts[key + '_rev'] = last_rev
+
+ uselocalcount = bb.data.getVar("BB_LOCALCOUNT_OVERRIDE", d, True) or False
+ count = None
+ if uselocalcount:
+ count = Fetch.localcount_internal_helper(ud, d)
+ if count is None:
+ count = localcounts[key + '_count']
+ if count is None:
+ count = localcounts[oldkey + '_count']
+ if count is not None:
+ del localcounts[oldkey + '_count']
+ localcounts[key + '_count'] = count
+
+ if last_rev == latest_rev:
+ return str(count + "+" + latest_rev)
+
+ buildindex_provided = hasattr(self, "_sortable_buildindex")
+ if buildindex_provided:
+ count = self._sortable_buildindex(url, ud, d, latest_rev)
+ if count is None:
+ count = "0"
+ elif uselocalcount or buildindex_provided:
+ count = str(count)
+ else:
+ count = str(int(count) + 1)
+
+ localcounts[key + '_rev'] = latest_rev
+ localcounts[key + '_count'] = count
+
+ return str(count + "+" + latest_rev)
+
+ def _build_revision(self, url, ud, d):
+ return ud.tag
+
+ def _sortable_buildindex_disabled(self, url, ud, d, rev):
+ """
+ Return a suitable buildindex for the revision specified. This is done by counting revisions
+ using "git rev-list" which may or may not work in different circumstances.
+ """
+
+ cwd = os.getcwd()
+
+ # Check if we have the rev already
+
+ if not os.path.exists(ud.clonedir):
+ print("no repo")
+ self.go(None, ud, d)
+ if not os.path.exists(ud.clonedir):
+ logger.error("GIT repository for %s doesn't exist in %s, cannot get sortable buildnumber, using old value", url, ud.clonedir)
+ return None
+
+
+ os.chdir(ud.clonedir)
+ if not self._contains_ref(rev, d):
+ self.go(None, ud, d)
+
+ output = runfetchcmd("%s rev-list %s -- 2> /dev/null | wc -l" % (ud.basecmd, rev), d, quiet=True)
+ os.chdir(cwd)
+
+ buildindex = "%s" % output.split()[0]
+ logger.debug(1, "GIT repository for %s in %s is returning %s revisions in rev-list before %s", url, ud.clonedir, buildindex, rev)
+ return buildindex