diff options
Diffstat (limited to 'lib/bb/fetch2/git.py')
-rw-r--r-- | lib/bb/fetch2/git.py | 339 |
1 files changed, 339 insertions, 0 deletions
diff --git a/lib/bb/fetch2/git.py b/lib/bb/fetch2/git.py new file mode 100644 index 000000000..b37a09743 --- /dev/null +++ b/lib/bb/fetch2/git.py @@ -0,0 +1,339 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake 'Fetch' git implementation + +""" + +#Copyright (C) 2005 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import os +import bb +import bb.persist_data +from bb import data +from bb.fetch import Fetch +from bb.fetch import runfetchcmd +from bb.fetch import logger + +class Git(Fetch): + """Class to fetch a module or modules from git repositories""" + def init(self, d): + # + # Only enable _sortable revision if the key is set + # + if bb.data.getVar("BB_GIT_CLONE_FOR_SRCREV", d, True): + self._sortable_buildindex = self._sortable_buildindex_disabled + def supports(self, url, ud, d): + """ + Check to see if a given url can be fetched with git. + """ + return ud.type in ['git'] + + def localpath(self, url, ud, d): + + if 'protocol' in ud.parm: + ud.proto = ud.parm['protocol'] + elif not ud.host: + ud.proto = 'file' + else: + ud.proto = "rsync" + + ud.branch = ud.parm.get("branch", "master") + + gitsrcname = '%s%s' % (ud.host, ud.path.replace('/', '.')) + ud.mirrortarball = 'git_%s.tar.gz' % (gitsrcname) + ud.clonedir = os.path.join(data.expand('${GITDIR}', d), gitsrcname) + + tag = Fetch.srcrev_internal_helper(ud, d) + if tag is True: + ud.tag = self.latest_revision(url, ud, d) + elif tag: + ud.tag = tag + + if not ud.tag or ud.tag == "master": + ud.tag = self.latest_revision(url, ud, d) + + subdir = ud.parm.get("subpath", "") + if subdir != "": + if subdir.endswith("/"): + subdir = subdir[:-1] + subdirpath = os.path.join(ud.path, subdir); + else: + subdirpath = ud.path; + + if 'fullclone' in ud.parm: + ud.localfile = ud.mirrortarball + else: + ud.localfile = data.expand('git_%s%s_%s.tar.gz' % (ud.host, subdirpath.replace('/', '.'), ud.tag), d) + + ud.basecmd = data.getVar("FETCHCMD_git", d, True) or "git" + + if 'noclone' in ud.parm: + ud.localfile = None + return None + + return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile) + + def forcefetch(self, url, ud, d): + if 'fullclone' in ud.parm: + return True + if 'noclone' in ud.parm: + return False + if os.path.exists(ud.localpath): + return False + if not self._contains_ref(ud.tag, d): + return True + return False + + def try_premirror(self, u, ud, d): + if 'noclone' in ud.parm: + return False + if os.path.exists(ud.clonedir): + return False + if os.path.exists(ud.localpath): + return False + + return True + + def go(self, loc, ud, d): + """Fetch url""" + + if ud.user: + username = ud.user + '@' + else: + username = "" + + repofile = os.path.join(data.getVar("DL_DIR", d, 1), ud.mirrortarball) + + + coname = '%s' % (ud.tag) + codir = os.path.join(ud.clonedir, coname) + + # If we have no existing clone and no mirror tarball, try and obtain one + if not os.path.exists(ud.clonedir) and not os.path.exists(repofile): + try: + Fetch.try_mirrors(ud.mirrortarball) + except: + pass + + # If the checkout doesn't exist and the mirror tarball does, extract it + if not os.path.exists(ud.clonedir) and os.path.exists(repofile): + bb.mkdirhier(ud.clonedir) + os.chdir(ud.clonedir) + runfetchcmd("tar -xzf %s" % (repofile), d) + + # If the repo still doesn't exist, fallback to cloning it + if not os.path.exists(ud.clonedir): + runfetchcmd("%s clone -n %s://%s%s%s %s" % (ud.basecmd, ud.proto, username, ud.host, ud.path, ud.clonedir), d) + + os.chdir(ud.clonedir) + # Update the checkout if needed + if not self._contains_ref(ud.tag, d) or 'fullclone' in ud.parm: + # Remove all but the .git directory + runfetchcmd("rm * -Rf", d) + if 'fullclone' in ud.parm: + runfetchcmd("%s fetch --all" % (ud.basecmd), d) + else: + runfetchcmd("%s fetch %s://%s%s%s %s" % (ud.basecmd, ud.proto, username, ud.host, ud.path, ud.branch), d) + runfetchcmd("%s fetch --tags %s://%s%s%s" % (ud.basecmd, ud.proto, username, ud.host, ud.path), d) + runfetchcmd("%s prune-packed" % ud.basecmd, d) + runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d) + + # Generate a mirror tarball if needed + os.chdir(ud.clonedir) + mirror_tarballs = data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True) + if mirror_tarballs != "0" or 'fullclone' in ud.parm: + logger.info("Creating tarball of git repository") + runfetchcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ), d) + + if 'fullclone' in ud.parm: + return + + if os.path.exists(codir): + bb.utils.prunedir(codir) + + subdir = ud.parm.get("subpath", "") + if subdir != "": + if subdir.endswith("/"): + subdirbase = os.path.basename(subdir[:-1]) + else: + subdirbase = os.path.basename(subdir) + else: + subdirbase = "" + + if subdir != "": + readpathspec = ":%s" % (subdir) + codir = os.path.join(codir, "git") + coprefix = os.path.join(codir, subdirbase, "") + else: + readpathspec = "" + coprefix = os.path.join(codir, "git", "") + + scmdata = ud.parm.get("scmdata", "") + if scmdata == "keep": + runfetchcmd("%s clone -n %s %s" % (ud.basecmd, ud.clonedir, coprefix), d) + os.chdir(coprefix) + runfetchcmd("%s checkout -q -f %s%s" % (ud.basecmd, ud.tag, readpathspec), d) + else: + bb.mkdirhier(codir) + os.chdir(ud.clonedir) + runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.tag, readpathspec), d) + runfetchcmd("%s checkout-index -q -f --prefix=%s -a" % (ud.basecmd, coprefix), d) + + os.chdir(codir) + logger.info("Creating tarball of git checkout") + runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.join(".", "*") ), d) + + os.chdir(ud.clonedir) + bb.utils.prunedir(codir) + + def supports_srcrev(self): + return True + + def _contains_ref(self, tag, d): + basecmd = data.getVar("FETCHCMD_git", d, True) or "git" + output = runfetchcmd("%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % (basecmd, tag), d, quiet=True) + return output.split()[0] != "0" + + def _revision_key(self, url, ud, d, branch=False): + """ + Return a unique key for the url + """ + key = 'git:' + ud.host + ud.path.replace('/', '.') + if branch: + return key + ud.branch + else: + return key + + def generate_revision_key(self, url, ud, d, branch=False): + key = self._revision_key(url, ud, d, branch) + return "%s-%s" % (key, bb.data.getVar("PN", d, True) or "") + + def _latest_revision(self, url, ud, d): + """ + Compute the HEAD revision for the url + """ + if ud.user: + username = ud.user + '@' + else: + username = "" + + basecmd = data.getVar("FETCHCMD_git", d, True) or "git" + cmd = "%s ls-remote %s://%s%s%s %s" % (basecmd, ud.proto, username, ud.host, ud.path, ud.branch) + output = runfetchcmd(cmd, d, True) + if not output: + raise bb.fetch.FetchError("Fetch command %s gave empty output\n" % (cmd)) + return output.split()[0] + + def latest_revision(self, url, ud, d): + """ + Look in the cache for the latest revision, if not present ask the SCM. + """ + persisted = bb.persist_data.persist(d) + revs = persisted['BB_URI_HEADREVS'] + + key = self.generate_revision_key(url, ud, d, branch=True) + rev = revs[key] + if rev is None: + # Compatibility with old key format, no branch included + oldkey = self.generate_revision_key(url, ud, d, branch=False) + rev = revs[oldkey] + if rev is not None: + del revs[oldkey] + else: + rev = self._latest_revision(url, ud, d) + revs[key] = rev + + return str(rev) + + def sortable_revision(self, url, ud, d): + """ + + """ + pd = bb.persist_data.persist(d) + localcounts = pd['BB_URI_LOCALCOUNT'] + key = self.generate_revision_key(url, ud, d, branch=True) + oldkey = self.generate_revision_key(url, ud, d, branch=False) + + latest_rev = self._build_revision(url, ud, d) + last_rev = localcounts[key + '_rev'] + if last_rev is None: + last_rev = localcounts[oldkey + '_rev'] + if last_rev is not None: + del localcounts[oldkey + '_rev'] + localcounts[key + '_rev'] = last_rev + + uselocalcount = bb.data.getVar("BB_LOCALCOUNT_OVERRIDE", d, True) or False + count = None + if uselocalcount: + count = Fetch.localcount_internal_helper(ud, d) + if count is None: + count = localcounts[key + '_count'] + if count is None: + count = localcounts[oldkey + '_count'] + if count is not None: + del localcounts[oldkey + '_count'] + localcounts[key + '_count'] = count + + if last_rev == latest_rev: + return str(count + "+" + latest_rev) + + buildindex_provided = hasattr(self, "_sortable_buildindex") + if buildindex_provided: + count = self._sortable_buildindex(url, ud, d, latest_rev) + if count is None: + count = "0" + elif uselocalcount or buildindex_provided: + count = str(count) + else: + count = str(int(count) + 1) + + localcounts[key + '_rev'] = latest_rev + localcounts[key + '_count'] = count + + return str(count + "+" + latest_rev) + + def _build_revision(self, url, ud, d): + return ud.tag + + def _sortable_buildindex_disabled(self, url, ud, d, rev): + """ + Return a suitable buildindex for the revision specified. This is done by counting revisions + using "git rev-list" which may or may not work in different circumstances. + """ + + cwd = os.getcwd() + + # Check if we have the rev already + + if not os.path.exists(ud.clonedir): + print("no repo") + self.go(None, ud, d) + if not os.path.exists(ud.clonedir): + logger.error("GIT repository for %s doesn't exist in %s, cannot get sortable buildnumber, using old value", url, ud.clonedir) + return None + + + os.chdir(ud.clonedir) + if not self._contains_ref(rev, d): + self.go(None, ud, d) + + output = runfetchcmd("%s rev-list %s -- 2> /dev/null | wc -l" % (ud.basecmd, rev), d, quiet=True) + os.chdir(cwd) + + buildindex = "%s" % output.split()[0] + logger.debug(1, "GIT repository for %s in %s is returning %s revisions in rev-list before %s", url, ud.clonedir, buildindex, rev) + return buildindex |