From 292e3430e5140b602cad86f55b5453e8cebb28a1 Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Sat, 3 Mar 2012 11:25:19 +0000 Subject: bitbake: Drop fetch v1, v2 provides a much better codebase to build from Signed-off-by: Richard Purdie --- lib/bb/__init__.py | 5 +- lib/bb/fetch/__init__.py | 832 ----------------------------------------------- lib/bb/fetch/bzr.py | 148 --------- lib/bb/fetch/cvs.py | 172 ---------- lib/bb/fetch/git.py | 339 ------------------- lib/bb/fetch/hg.py | 180 ---------- lib/bb/fetch/local.py | 73 ----- lib/bb/fetch/osc.py | 143 -------- lib/bb/fetch/perforce.py | 206 ------------ lib/bb/fetch/repo.py | 98 ------ lib/bb/fetch/ssh.py | 118 ------- lib/bb/fetch/svk.py | 104 ------ lib/bb/fetch/svn.py | 204 ------------ lib/bb/fetch/wget.py | 93 ------ 14 files changed, 2 insertions(+), 2713 deletions(-) delete mode 100644 lib/bb/fetch/__init__.py delete mode 100644 lib/bb/fetch/bzr.py delete mode 100644 lib/bb/fetch/cvs.py delete mode 100644 lib/bb/fetch/git.py delete mode 100644 lib/bb/fetch/hg.py delete mode 100644 lib/bb/fetch/local.py delete mode 100644 lib/bb/fetch/osc.py delete mode 100644 lib/bb/fetch/perforce.py delete mode 100644 lib/bb/fetch/repo.py delete mode 100644 lib/bb/fetch/ssh.py delete mode 100644 lib/bb/fetch/svk.py delete mode 100644 lib/bb/fetch/svn.py delete mode 100644 lib/bb/fetch/wget.py diff --git a/lib/bb/__init__.py b/lib/bb/__init__.py index dce561006..8d579c794 100644 --- a/lib/bb/__init__.py +++ b/lib/bb/__init__.py @@ -79,9 +79,8 @@ if "BBDEBUG" in os.environ: if level: bb.msg.set_debug_level(level) -if os.environ.get("BBFETCH2"): - from bb import fetch2 as fetch - sys.modules['bb.fetch'] = sys.modules['bb.fetch2'] +from bb import fetch2 as fetch +sys.modules['bb.fetch'] = sys.modules['bb.fetch2'] # Messaging convenience functions def plain(*args): diff --git a/lib/bb/fetch/__init__.py b/lib/bb/fetch/__init__.py deleted file mode 100644 index 18988646b..000000000 --- a/lib/bb/fetch/__init__.py +++ /dev/null @@ -1,832 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -""" -BitBake 'Fetch' implementations - -Classes for obtaining upstream sources for the -BitBake build tools. -""" - -# Copyright (C) 2003, 2004 Chris Larson -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# Based on functions from the base bb module, Copyright 2003 Holger Schurig - -from __future__ import absolute_import -from __future__ import print_function -import os, re -import logging -import bb -from bb import data -from bb import persist_data -from bb import utils - -__version__ = "1" - -logger = logging.getLogger("BitBake.Fetch") - -class MalformedUrl(Exception): - """Exception raised when encountering an invalid url""" - -class FetchError(Exception): - """Exception raised when a download fails""" - -class NoMethodError(Exception): - """Exception raised when there is no method to obtain a supplied url or set of urls""" - -class MissingParameterError(Exception): - """Exception raised when a fetch method is missing a critical parameter in the url""" - -class ParameterError(Exception): - """Exception raised when a url cannot be proccessed due to invalid parameters.""" - -class MD5SumError(Exception): - """Exception raised when a MD5SUM of a file does not match the expected one""" - -class InvalidSRCREV(Exception): - """Exception raised when an invalid SRCREV is encountered""" - -def decodeurl(url): - """Decodes an URL into the tokens (scheme, network location, path, - user, password, parameters). - """ - - m = re.compile('(?P[^:]*)://((?P.+)@)?(?P[^;]+)(;(?P.*))?').match(url) - if not m: - raise MalformedUrl(url) - - type = m.group('type') - location = m.group('location') - if not location: - raise MalformedUrl(url) - user = m.group('user') - parm = m.group('parm') - - locidx = location.find('/') - if locidx != -1 and type.lower() != 'file': - host = location[:locidx] - path = location[locidx:] - else: - host = "" - path = location - if user: - m = re.compile('(?P[^:]+)(:?(?P.*))').match(user) - if m: - user = m.group('user') - pswd = m.group('pswd') - else: - user = '' - pswd = '' - - p = {} - if parm: - for s in parm.split(';'): - s1, s2 = s.split('=') - p[s1] = s2 - - return (type, host, path, user, pswd, p) - -def encodeurl(decoded): - """Encodes a URL from tokens (scheme, network location, path, - user, password, parameters). - """ - - (type, host, path, user, pswd, p) = decoded - - if not type or not path: - raise MissingParameterError("Type or path url components missing when encoding %s" % decoded) - url = '%s://' % type - if user: - url += "%s" % user - if pswd: - url += ":%s" % pswd - url += "@" - if host: - url += "%s" % host - url += "%s" % path - if p: - for parm in p: - url += ";%s=%s" % (parm, p[parm]) - - return url - -def uri_replace(uri, uri_find, uri_replace, d): - if not uri or not uri_find or not uri_replace: - logger.debug(1, "uri_replace: passed an undefined value, not replacing") - uri_decoded = list(decodeurl(uri)) - uri_find_decoded = list(decodeurl(uri_find)) - uri_replace_decoded = list(decodeurl(uri_replace)) - result_decoded = ['', '', '', '', '', {}] - for i in uri_find_decoded: - loc = uri_find_decoded.index(i) - result_decoded[loc] = uri_decoded[loc] - if isinstance(i, basestring): - if (re.match(i, uri_decoded[loc])): - result_decoded[loc] = re.sub(i, uri_replace_decoded[loc], uri_decoded[loc]) - if uri_find_decoded.index(i) == 2: - if d: - localfn = bb.fetch.localpath(uri, d) - if localfn: - result_decoded[loc] = os.path.join(os.path.dirname(result_decoded[loc]), os.path.basename(bb.fetch.localpath(uri, d))) - else: - return uri - return encodeurl(result_decoded) - -methods = [] -urldata_cache = {} -saved_headrevs = {} - -def fetcher_init(d): - """ - Called to initialize the fetchers once the configuration data is known. - Calls before this must not hit the cache. - """ - # When to drop SCM head revisions controlled by user policy - srcrev_policy = d.getVar('BB_SRCREV_POLICY', True) or "clear" - if srcrev_policy == "cache": - logger.debug(1, "Keeping SRCREV cache due to cache policy of: %s", srcrev_policy) - elif srcrev_policy == "clear": - logger.debug(1, "Clearing SRCREV cache due to cache policy of: %s", srcrev_policy) - revs = persist_data.persist('BB_URI_HEADREVS', d) - try: - bb.fetch.saved_headrevs = revs.items() - except: - pass - revs.clear() - else: - raise FetchError("Invalid SRCREV cache policy of: %s" % srcrev_policy) - - for m in methods: - if hasattr(m, "init"): - m.init(d) - -def fetcher_compare_revisions(d): - """ - Compare the revisions in the persistant cache with current values and - return true/false on whether they've changed. - """ - - data = persist_data.persist('BB_URI_HEADREVS', d).items() - data2 = bb.fetch.saved_headrevs - - changed = False - for key in data: - if key not in data2 or data2[key] != data[key]: - logger.debug(1, "%s changed", key) - changed = True - return True - else: - logger.debug(2, "%s did not change", key) - return False - -# Function call order is usually: -# 1. init -# 2. go -# 3. localpaths -# localpath can be called at any time - -def init(urls, d, setup = True): - urldata = {} - - fn = d.getVar('FILE', True) - if fn in urldata_cache: - urldata = urldata_cache[fn] - - for url in urls: - if url not in urldata: - urldata[url] = FetchData(url, d) - - if setup: - for url in urldata: - if not urldata[url].setup: - urldata[url].setup_localpath(d) - - urldata_cache[fn] = urldata - return urldata - -def mirror_from_string(data): - return [ i.split() for i in (data or "").replace('\\n','\n').split('\n') if i ] - -def verify_checksum(u, ud, d): - """ - verify the MD5 and SHA256 checksum for downloaded src - - return value: - - True: checksum matched - - False: checksum unmatched - - if checksum is missing in recipes file, "BB_STRICT_CHECKSUM" decide the return value. - if BB_STRICT_CHECKSUM = "1" then return false as unmatched, otherwise return true as - matched - """ - - if not ud.type in ["http", "https", "ftp", "ftps"]: - return - - md5data = bb.utils.md5_file(ud.localpath) - sha256data = bb.utils.sha256_file(ud.localpath) - - if (ud.md5_expected == None or ud.sha256_expected == None): - logger.warn('Missing SRC_URI checksum for %s, consider adding to the recipe:\n' - 'SRC_URI[%s] = "%s"\nSRC_URI[%s] = "%s"', - ud.localpath, ud.md5_name, md5data, - ud.sha256_name, sha256data) - if d.getVar("BB_STRICT_CHECKSUM", True) == "1": - raise FetchError("No checksum specified for %s." % u) - return - - if (ud.md5_expected != md5data or ud.sha256_expected != sha256data): - logger.error('The checksums for "%s" did not match.\n' - ' MD5: expected "%s", got "%s"\n' - ' SHA256: expected "%s", got "%s"\n', - ud.localpath, ud.md5_expected, md5data, - ud.sha256_expected, sha256data) - raise FetchError("%s checksum mismatch." % u) - -def go(d, urls = None): - """ - Fetch all urls - init must have previously been called - """ - if not urls: - urls = d.getVar("SRC_URI", True).split() - urldata = init(urls, d, True) - - for u in urls: - ud = urldata[u] - m = ud.method - localpath = "" - - if not ud.localfile: - continue - - lf = bb.utils.lockfile(ud.lockfile) - - if m.try_premirror(u, ud, d): - # First try fetching uri, u, from PREMIRRORS - mirrors = mirror_from_string(d.getVar('PREMIRRORS', True)) - localpath = try_mirrors(d, u, mirrors, False, m.forcefetch(u, ud, d)) - elif os.path.exists(ud.localfile): - localpath = ud.localfile - - # Need to re-test forcefetch() which will return true if our copy is too old - if m.forcefetch(u, ud, d) or not localpath: - # Next try fetching from the original uri, u - try: - m.go(u, ud, d) - localpath = ud.localpath - except FetchError: - # Remove any incomplete file - bb.utils.remove(ud.localpath) - # Finally, try fetching uri, u, from MIRRORS - mirrors = mirror_from_string(d.getVar('MIRRORS', True)) - localpath = try_mirrors (d, u, mirrors) - if not localpath or not os.path.exists(localpath): - raise FetchError("Unable to fetch URL %s from any source." % u) - - ud.localpath = localpath - - if os.path.exists(ud.md5): - # Touch the md5 file to show active use of the download - try: - os.utime(ud.md5, None) - except: - # Errors aren't fatal here - pass - else: - # Only check the checksums if we've not seen this item before - verify_checksum(u, ud, d) - Fetch.write_md5sum(u, ud, d) - - bb.utils.unlockfile(lf) - -def checkstatus(d, urls = None): - """ - Check all urls exist upstream - init must have previously been called - """ - urldata = init([], d, True) - - if not urls: - urls = urldata - - for u in urls: - ud = urldata[u] - m = ud.method - logger.debug(1, "Testing URL %s", u) - # First try checking uri, u, from PREMIRRORS - mirrors = mirror_from_string(d.getVar('PREMIRRORS', True)) - ret = try_mirrors(d, u, mirrors, True) - if not ret: - # Next try checking from the original uri, u - try: - ret = m.checkstatus(u, ud, d) - except: - # Finally, try checking uri, u, from MIRRORS - mirrors = mirror_from_string(d.getVar('MIRRORS', True)) - ret = try_mirrors (d, u, mirrors, True) - - if not ret: - raise FetchError("URL %s doesn't work" % u) - -def localpaths(d): - """ - Return a list of the local filenames, assuming successful fetch - """ - local = [] - urldata = init([], d, True) - - for u in urldata: - ud = urldata[u] - local.append(ud.localpath) - - return local - -srcrev_internal_call = False - -def get_autorev(d): - return get_srcrev(d) - -def get_srcrev(d): - """ - Return the version string for the current package - (usually to be used as PV) - Most packages usually only have one SCM so we just pass on the call. - In the multi SCM case, we build a value based on SRCREV_FORMAT which must - have been set. - """ - - # - # Ugly code alert. localpath in the fetchers will try to evaluate SRCREV which - # could translate into a call to here. If it does, we need to catch this - # and provide some way so it knows get_srcrev is active instead of being - # some number etc. hence the srcrev_internal_call tracking and the magic - # "SRCREVINACTION" return value. - # - # Neater solutions welcome! - # - if bb.fetch.srcrev_internal_call: - return "SRCREVINACTION" - - scms = [] - - # Only call setup_localpath on URIs which supports_srcrev() - urldata = init(d.getVar('SRC_URI', True).split(), d, False) - for u in urldata: - ud = urldata[u] - if ud.method.supports_srcrev(): - if not ud.setup: - ud.setup_localpath(d) - scms.append(u) - - if len(scms) == 0: - logger.error("SRCREV was used yet no valid SCM was found in SRC_URI") - raise ParameterError - - if d.getVar('BB_SRCREV_POLICY', True) != "cache": - d.setVar('__BB_DONT_CACHE', '1') - - if len(scms) == 1: - return urldata[scms[0]].method.sortable_revision(scms[0], urldata[scms[0]], d) - - # - # Mutiple SCMs are in SRC_URI so we resort to SRCREV_FORMAT - # - format = d.getVar('SRCREV_FORMAT', True) - if not format: - logger.error("The SRCREV_FORMAT variable must be set when multiple SCMs are used.") - raise ParameterError - - for scm in scms: - if 'name' in urldata[scm].parm: - name = urldata[scm].parm["name"] - rev = urldata[scm].method.sortable_revision(scm, urldata[scm], d) - format = format.replace(name, rev) - - return format - -def localpath(url, d, cache = True): - """ - Called from the parser with cache=False since the cache isn't ready - at this point. Also called from classed in OE e.g. patch.bbclass - """ - ud = init([url], d) - if ud[url].method: - return ud[url].localpath - return url - -def runfetchcmd(cmd, d, quiet = False): - """ - Run cmd returning the command output - Raise an error if interrupted or cmd fails - Optionally echo command output to stdout - """ - - # Need to export PATH as binary could be in metadata paths - # rather than host provided - # Also include some other variables. - # FIXME: Should really include all export varaiables? - exportvars = ['PATH', 'GIT_PROXY_COMMAND', 'GIT_PROXY_HOST', - 'GIT_PROXY_PORT', 'GIT_CONFIG', 'http_proxy', 'ftp_proxy', - 'https_proxy', 'no_proxy', 'ALL_PROXY', 'all_proxy', - 'KRB5CCNAME', 'SSH_AUTH_SOCK', 'SSH_AGENT_PID', 'HOME'] - - for var in exportvars: - val = data.getVar(var, d, True) - if val: - cmd = 'export ' + var + '=\"%s\"; %s' % (val, cmd) - - logger.debug(1, "Running %s", cmd) - - # redirect stderr to stdout - stdout_handle = os.popen(cmd + " 2>&1", "r") - output = "" - - while True: - line = stdout_handle.readline() - if not line: - break - if not quiet: - print(line, end=' ') - output += line - - status = stdout_handle.close() or 0 - signal = status >> 8 - exitstatus = status & 0xff - - if signal: - raise FetchError("Fetch command %s failed with signal %s, output:\n%s" % (cmd, signal, output)) - elif status != 0: - raise FetchError("Fetch command %s failed with exit code %s, output:\n%s" % (cmd, status, output)) - - return output - -def try_mirrors(d, uri, mirrors, check = False, force = False): - """ - Try to use a mirrored version of the sources. - This method will be automatically called before the fetchers go. - - d Is a bb.data instance - uri is the original uri we're trying to download - mirrors is the list of mirrors we're going to try - """ - fpath = os.path.join(data.getVar("DL_DIR", d, 1), os.path.basename(uri)) - if not check and os.access(fpath, os.R_OK) and not force: - logger.debug(1, "%s already exists, skipping checkout.", fpath) - return fpath - - ld = d.createCopy() - for (find, replace) in mirrors: - newuri = uri_replace(uri, find, replace, ld) - if newuri != uri: - try: - ud = FetchData(newuri, ld) - except bb.fetch.NoMethodError: - logger.debug(1, "No method for %s", uri) - continue - - ud.setup_localpath(ld) - - try: - if check: - found = ud.method.checkstatus(newuri, ud, ld) - if found: - return found - else: - ud.method.go(newuri, ud, ld) - return ud.localpath - except (bb.fetch.MissingParameterError, - bb.fetch.FetchError, - bb.fetch.MD5SumError): - import sys - (type, value, traceback) = sys.exc_info() - logger.debug(2, "Mirror fetch failure: %s", value) - bb.utils.remove(ud.localpath) - continue - return None - - -class FetchData(object): - """ - A class which represents the fetcher state for a given URI. - """ - def __init__(self, url, d): - self.localfile = "" - (self.type, self.host, self.path, self.user, self.pswd, self.parm) = decodeurl(data.expand(url, d)) - self.date = Fetch.getSRCDate(self, d) - self.url = url - if not self.user and "user" in self.parm: - self.user = self.parm["user"] - if not self.pswd and "pswd" in self.parm: - self.pswd = self.parm["pswd"] - self.setup = False - - if "name" in self.parm: - self.md5_name = "%s.md5sum" % self.parm["name"] - self.sha256_name = "%s.sha256sum" % self.parm["name"] - else: - self.md5_name = "md5sum" - self.sha256_name = "sha256sum" - self.md5_expected = d.getVarFlag("SRC_URI", self.md5_name) - self.sha256_expected = d.getVarFlag("SRC_URI", self.sha256_name) - - for m in methods: - if m.supports(url, self, d): - self.method = m - return - raise NoMethodError("Missing implementation for url %s" % url) - - def setup_localpath(self, d): - self.setup = True - if "localpath" in self.parm: - # if user sets localpath for file, use it instead. - self.localpath = self.parm["localpath"] - self.basename = os.path.basename(self.localpath) - else: - premirrors = d.getVar('PREMIRRORS', True) - local = "" - if premirrors and self.url: - aurl = self.url.split(";")[0] - mirrors = mirror_from_string(premirrors) - for (find, replace) in mirrors: - if replace.startswith("file://"): - path = aurl.split("://")[1] - path = path.split(";")[0] - local = replace.split("://")[1] + os.path.basename(path) - if local == aurl or not os.path.exists(local) or os.path.isdir(local): - local = "" - self.localpath = local - if not local: - try: - bb.fetch.srcrev_internal_call = True - self.localpath = self.method.localpath(self.url, self, d) - finally: - bb.fetch.srcrev_internal_call = False - # We have to clear data's internal caches since the cached value of SRCREV is now wrong. - # Horrible... - bb.data.delVar("ISHOULDNEVEREXIST", d) - - if self.localpath is not None: - # Note: These files should always be in DL_DIR whereas localpath may not be. - basepath = bb.data.expand("${DL_DIR}/%s" % os.path.basename(self.localpath), d) - self.md5 = basepath + '.md5' - self.lockfile = basepath + '.lock' - - -class Fetch(object): - """Base class for 'fetch'ing data""" - - def __init__(self, urls = []): - self.urls = [] - - def supports(self, url, urldata, d): - """ - Check to see if this fetch class supports a given url. - """ - return 0 - - def localpath(self, url, urldata, d): - """ - Return the local filename of a given url assuming a successful fetch. - Can also setup variables in urldata for use in go (saving code duplication - and duplicate code execution) - """ - return url - def _strip_leading_slashes(self, relpath): - """ - Remove leading slash as os.path.join can't cope - """ - while os.path.isabs(relpath): - relpath = relpath[1:] - return relpath - - def setUrls(self, urls): - self.__urls = urls - - def getUrls(self): - return self.__urls - - urls = property(getUrls, setUrls, None, "Urls property") - - def forcefetch(self, url, urldata, d): - """ - Force a fetch, even if localpath exists? - """ - return False - - def supports_srcrev(self): - """ - The fetcher supports auto source revisions (SRCREV) - """ - return False - - def go(self, url, urldata, d): - """ - Fetch urls - Assumes localpath was called first - """ - raise NoMethodError("Missing implementation for url") - - def try_premirror(self, url, urldata, d): - """ - Should premirrors be used? - """ - if urldata.method.forcefetch(url, urldata, d): - return True - elif os.path.exists(urldata.md5) and os.path.exists(urldata.localfile): - return False - else: - return True - - def checkstatus(self, url, urldata, d): - """ - Check the status of a URL - Assumes localpath was called first - """ - logger.info("URL %s could not be checked for status since no method exists.", url) - return True - - def getSRCDate(urldata, d): - """ - Return the SRC Date for the component - - d the bb.data module - """ - if "srcdate" in urldata.parm: - return urldata.parm['srcdate'] - - pn = data.getVar("PN", d, 1) - - if pn: - return data.getVar("SRCDATE_%s" % pn, d, 1) or data.getVar("CVSDATE_%s" % pn, d, 1) or data.getVar("SRCDATE", d, 1) or data.getVar("CVSDATE", d, 1) or data.getVar("DATE", d, 1) - - return data.getVar("SRCDATE", d, 1) or data.getVar("CVSDATE", d, 1) or data.getVar("DATE", d, 1) - getSRCDate = staticmethod(getSRCDate) - - def srcrev_internal_helper(ud, d): - """ - Return: - a) a source revision if specified - b) True if auto srcrev is in action - c) False otherwise - """ - - if 'rev' in ud.parm: - return ud.parm['rev'] - - if 'tag' in ud.parm: - return ud.parm['tag'] - - rev = None - if 'name' in ud.parm: - pn = data.getVar("PN", d, 1) - rev = data.getVar("SRCREV_%s_pn-%s" % (ud.parm['name'], pn), d, 1) - if not rev: - rev = data.getVar("SRCREV_pn-%s_%s" % (pn, ud.parm['name']), d, 1) - if not rev: - rev = data.getVar("SRCREV_%s" % (ud.parm['name']), d, 1) - if not rev: - rev = data.getVar("SRCREV", d, 1) - if rev == "INVALID": - raise InvalidSRCREV("Please set SRCREV to a valid value") - if not rev: - return False - if rev == "SRCREVINACTION": - return True - return rev - - srcrev_internal_helper = staticmethod(srcrev_internal_helper) - - def localcount_internal_helper(ud, d): - """ - Return: - a) a locked localcount if specified - b) None otherwise - """ - - localcount = None - if 'name' in ud.parm: - pn = data.getVar("PN", d, 1) - localcount = data.getVar("LOCALCOUNT_" + ud.parm['name'], d, 1) - if not localcount: - localcount = data.getVar("LOCALCOUNT", d, 1) - return localcount - - localcount_internal_helper = staticmethod(localcount_internal_helper) - - def verify_md5sum(ud, got_sum): - """ - Verify the md5sum we wanted with the one we got - """ - wanted_sum = ud.parm.get('md5sum') - if not wanted_sum: - return True - - return wanted_sum == got_sum - verify_md5sum = staticmethod(verify_md5sum) - - def write_md5sum(url, ud, d): - md5data = bb.utils.md5_file(ud.localpath) - # verify the md5sum - if not Fetch.verify_md5sum(ud, md5data): - raise MD5SumError(url) - - md5out = file(ud.md5, 'w') - md5out.write(md5data) - md5out.close() - write_md5sum = staticmethod(write_md5sum) - - def latest_revision(self, url, ud, d): - """ - Look in the cache for the latest revision, if not present ask the SCM. - """ - if not hasattr(self, "_latest_revision"): - raise ParameterError - - revs = persist_data.persist('BB_URI_HEADREVS', d) - key = self.generate_revision_key(url, ud, d) - try: - return revs[key] - except KeyError: - revs[key] = rev = self._latest_revision(url, ud, d) - return rev - - def sortable_revision(self, url, ud, d): - """ - - """ - if hasattr(self, "_sortable_revision"): - return self._sortable_revision(url, ud, d) - - localcounts = persist_data.persist('BB_URI_LOCALCOUNT', d) - key = self.generate_revision_key(url, ud, d) - - latest_rev = self._build_revision(url, ud, d) - last_rev = localcounts.get(key + '_rev') - uselocalcount = d.getVar("BB_LOCALCOUNT_OVERRIDE", True) or False - count = None - if uselocalcount: - count = Fetch.localcount_internal_helper(ud, d) - if count is None: - count = localcounts.get(key + '_count') - - if last_rev == latest_rev: - return str(count + "+" + latest_rev) - - buildindex_provided = hasattr(self, "_sortable_buildindex") - if buildindex_provided: - count = self._sortable_buildindex(url, ud, d, latest_rev) - - if count is None: - count = "0" - elif uselocalcount or buildindex_provided: - count = str(count) - else: - count = str(int(count) + 1) - - localcounts[key + '_rev'] = latest_rev - localcounts[key + '_count'] = count - - return str(count + "+" + latest_rev) - - def generate_revision_key(self, url, ud, d): - key = self._revision_key(url, ud, d) - return "%s-%s" % (key, d.getVar("PN", True) or "") - -from . import cvs -from . import git -from . import local -from . import svn -from . import wget -from . import svk -from . import ssh -from . import perforce -from . import bzr -from . import hg -from . import osc -from . import repo - -methods.append(local.Local()) -methods.append(wget.Wget()) -methods.append(svn.Svn()) -methods.append(git.Git()) -methods.append(cvs.Cvs()) -methods.append(svk.Svk()) -methods.append(ssh.SSH()) -methods.append(perforce.Perforce()) -methods.append(bzr.Bzr()) -methods.append(hg.Hg()) -methods.append(osc.Osc()) -methods.append(repo.Repo()) diff --git a/lib/bb/fetch/bzr.py b/lib/bb/fetch/bzr.py deleted file mode 100644 index 85a92940e..000000000 --- a/lib/bb/fetch/bzr.py +++ /dev/null @@ -1,148 +0,0 @@ -""" -BitBake 'Fetch' implementation for bzr. - -""" - -# Copyright (C) 2007 Ross Burton -# Copyright (C) 2007 Richard Purdie -# -# Classes for obtaining upstream sources for the -# BitBake build tools. -# Copyright (C) 2003, 2004 Chris Larson -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -import os -import sys -import logging -import bb -from bb import data -from bb.fetch import Fetch, FetchError, runfetchcmd, logger - -class Bzr(Fetch): - def supports(self, url, ud, d): - return ud.type in ['bzr'] - - def localpath (self, url, ud, d): - - # Create paths to bzr checkouts - relpath = self._strip_leading_slashes(ud.path) - ud.pkgdir = os.path.join(data.expand('${BZRDIR}', d), ud.host, relpath) - - revision = Fetch.srcrev_internal_helper(ud, d) - if revision is True: - ud.revision = self.latest_revision(url, ud, d) - elif revision: - ud.revision = revision - - if not ud.revision: - ud.revision = self.latest_revision(url, ud, d) - - ud.localfile = data.expand('bzr_%s_%s_%s.tar.gz' % (ud.host, ud.path.replace('/', '.'), ud.revision), d) - - return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile) - - def _buildbzrcommand(self, ud, d, command): - """ - Build up an bzr commandline based on ud - command is "fetch", "update", "revno" - """ - - basecmd = data.expand('${FETCHCMD_bzr}', d) - - proto = ud.parm.get('proto', 'http') - - bzrroot = ud.host + ud.path - - options = [] - - if command == "revno": - bzrcmd = "%s revno %s %s://%s" % (basecmd, " ".join(options), proto, bzrroot) - else: - if ud.revision: - options.append("-r %s" % ud.revision) - - if command == "fetch": - bzrcmd = "%s co %s %s://%s" % (basecmd, " ".join(options), proto, bzrroot) - elif command == "update": - bzrcmd = "%s pull %s --overwrite" % (basecmd, " ".join(options)) - else: - raise FetchError("Invalid bzr command %s" % command) - - return bzrcmd - - def go(self, loc, ud, d): - """Fetch url""" - - if os.access(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir), '.bzr'), os.R_OK): - bzrcmd = self._buildbzrcommand(ud, d, "update") - logger.debug(1, "BZR Update %s", loc) - os.chdir(os.path.join (ud.pkgdir, os.path.basename(ud.path))) - runfetchcmd(bzrcmd, d) - else: - bb.utils.remove(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir)), True) - bzrcmd = self._buildbzrcommand(ud, d, "fetch") - logger.debug(1, "BZR Checkout %s", loc) - bb.utils.mkdirhier(ud.pkgdir) - os.chdir(ud.pkgdir) - logger.debug(1, "Running %s", bzrcmd) - runfetchcmd(bzrcmd, d) - - os.chdir(ud.pkgdir) - - scmdata = ud.parm.get("scmdata", "") - if scmdata == "keep": - tar_flags = "" - else: - tar_flags = "--exclude '.bzr' --exclude '.bzrtags'" - - # tar them up to a defined filename - try: - runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.basename(ud.pkgdir)), d) - except: - t, v, tb = sys.exc_info() - try: - os.unlink(ud.localpath) - except OSError: - pass - raise t, v, tb - - def supports_srcrev(self): - return True - - def _revision_key(self, url, ud, d): - """ - Return a unique key for the url - """ - return "bzr:" + ud.pkgdir - - def _latest_revision(self, url, ud, d): - """ - Return the latest upstream revision number - """ - logger.debug(2, "BZR fetcher hitting network for %s", url) - - output = runfetchcmd(self._buildbzrcommand(ud, d, "revno"), d, True) - - return output.strip() - - def _sortable_revision(self, url, ud, d): - """ - Return a sortable revision number which in our case is the revision number - """ - - return self._build_revision(url, ud, d) - - def _build_revision(self, url, ud, d): - return ud.revision diff --git a/lib/bb/fetch/cvs.py b/lib/bb/fetch/cvs.py deleted file mode 100644 index 64450afc2..000000000 --- a/lib/bb/fetch/cvs.py +++ /dev/null @@ -1,172 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -""" -BitBake 'Fetch' implementations - -Classes for obtaining upstream sources for the -BitBake build tools. - -""" - -# Copyright (C) 2003, 2004 Chris Larson -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -#Based on functions from the base bb module, Copyright 2003 Holger Schurig -# - -import os -import logging -import bb -from bb import data -from bb.fetch import Fetch, FetchError, MissingParameterError, logger - -class Cvs(Fetch): - """ - Class to fetch a module or modules from cvs repositories - """ - def supports(self, url, ud, d): - """ - Check to see if a given url can be fetched with cvs. - """ - return ud.type in ['cvs'] - - def localpath(self, url, ud, d): - if not "module" in ud.parm: - raise MissingParameterError("cvs method needs a 'module' parameter") - ud.module = ud.parm["module"] - - ud.tag = ud.parm.get('tag', "") - - # Override the default date in certain cases - if 'date' in ud.parm: - ud.date = ud.parm['date'] - elif ud.tag: - ud.date = "" - - norecurse = '' - if 'norecurse' in ud.parm: - norecurse = '_norecurse' - - fullpath = '' - if 'fullpath' in ud.parm: - fullpath = '_fullpath' - - ud.localfile = data.expand('%s_%s_%s_%s%s%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.tag, ud.date, norecurse, fullpath), d) - - return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile) - - def forcefetch(self, url, ud, d): - if (ud.date == "now"): - return True - return False - - def go(self, loc, ud, d): - - method = ud.parm.get('method', 'pserver') - localdir = ud.parm.get('localdir', ud.module) - cvs_port = ud.parm.get('port', '') - - cvs_rsh = None - if method == "ext": - if "rsh" in ud.parm: - cvs_rsh = ud.parm["rsh"] - - if method == "dir": - cvsroot = ud.path - else: - cvsroot = ":" + method - cvsproxyhost = data.getVar('CVS_PROXY_HOST', d, True) - if cvsproxyhost: - cvsroot += ";proxy=" + cvsproxyhost - cvsproxyport = data.getVar('CVS_PROXY_PORT', d, True) - if cvsproxyport: - cvsroot += ";proxyport=" + cvsproxyport - cvsroot += ":" + ud.user - if ud.pswd: - cvsroot += ":" + ud.pswd - cvsroot += "@" + ud.host + ":" + cvs_port + ud.path - - options = [] - if 'norecurse' in ud.parm: - options.append("-l") - if ud.date: - # treat YYYYMMDDHHMM specially for CVS - if len(ud.date) == 12: - options.append("-D \"%s %s:%s UTC\"" % (ud.date[0:8], ud.date[8:10], ud.date[10:12])) - else: - options.append("-D \"%s UTC\"" % ud.date) - if ud.tag: - options.append("-r %s" % ud.tag) - - localdata = data.createCopy(d) - data.setVar('OVERRIDES', "cvs:%s" % data.getVar('OVERRIDES', localdata), localdata) - data.update_data(localdata) - - data.setVar('CVSROOT', cvsroot, localdata) - data.setVar('CVSCOOPTS', " ".join(options), localdata) - data.setVar('CVSMODULE', ud.module, localdata) - cvscmd = data.getVar('FETCHCOMMAND', localdata, 1) - cvsupdatecmd = data.getVar('UPDATECOMMAND', localdata, 1) - - if cvs_rsh: - cvscmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvscmd) - cvsupdatecmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvsupdatecmd) - - # create module directory - logger.debug(2, "Fetch: checking for module directory") - pkg = data.expand('${PN}', d) - pkgdir = os.path.join(data.expand('${CVSDIR}', localdata), pkg) - moddir = os.path.join(pkgdir, localdir) - if os.access(os.path.join(moddir, 'CVS'), os.R_OK): - logger.info("Update " + loc) - # update sources there - os.chdir(moddir) - myret = os.system(cvsupdatecmd) - else: - logger.info("Fetch " + loc) - # check out sources there - bb.utils.mkdirhier(pkgdir) - os.chdir(pkgdir) - logger.debug(1, "Running %s", cvscmd) - myret = os.system(cvscmd) - - if myret != 0 or not os.access(moddir, os.R_OK): - try: - os.rmdir(moddir) - except OSError: - pass - raise FetchError(ud.module) - - scmdata = ud.parm.get("scmdata", "") - if scmdata == "keep": - tar_flags = "" - else: - tar_flags = "--exclude 'CVS'" - - # tar them up to a defined filename - if 'fullpath' in ud.parm: - os.chdir(pkgdir) - myret = os.system("tar %s -czf %s %s" % (tar_flags, ud.localpath, localdir)) - else: - os.chdir(moddir) - os.chdir('..') - myret = os.system("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.basename(moddir))) - - if myret != 0: - try: - os.unlink(ud.localpath) - except OSError: - pass - raise FetchError(ud.module) diff --git a/lib/bb/fetch/git.py b/lib/bb/fetch/git.py deleted file mode 100644 index 7160919d5..000000000 --- a/lib/bb/fetch/git.py +++ /dev/null @@ -1,339 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -""" -BitBake 'Fetch' git implementation - -""" - -#Copyright (C) 2005 Richard Purdie -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -import os -import bb -import bb.persist_data -from bb import data -from bb.fetch import Fetch -from bb.fetch import runfetchcmd -from bb.fetch import logger - -class Git(Fetch): - """Class to fetch a module or modules from git repositories""" - def init(self, d): - # - # Only enable _sortable revision if the key is set - # - if d.getVar("BB_GIT_CLONE_FOR_SRCREV", True): - self._sortable_buildindex = self._sortable_buildindex_disabled - def supports(self, url, ud, d): - """ - Check to see if a given url can be fetched with git. - """ - return ud.type in ['git'] - - def localpath(self, url, ud, d): - - if 'protocol' in ud.parm: - ud.proto = ud.parm['protocol'] - elif not ud.host: - ud.proto = 'file' - else: - ud.proto = "rsync" - - ud.branch = ud.parm.get("branch", "master") - - gitsrcname = '%s%s' % (ud.host, ud.path.replace('/', '.')) - ud.mirrortarball = 'git_%s.tar.gz' % (gitsrcname) - ud.clonedir = os.path.join(data.expand('${GITDIR}', d), gitsrcname) - - tag = Fetch.srcrev_internal_helper(ud, d) - if tag is True: - ud.tag = self.latest_revision(url, ud, d) - elif tag: - ud.tag = tag - - if not ud.tag or ud.tag == "master": - ud.tag = self.latest_revision(url, ud, d) - - subdir = ud.parm.get("subpath", "") - if subdir != "": - if subdir.endswith("/"): - subdir = subdir[:-1] - subdirpath = os.path.join(ud.path, subdir); - else: - subdirpath = ud.path; - - if 'fullclone' in ud.parm: - ud.localfile = ud.mirrortarball - else: - ud.localfile = data.expand('git_%s%s_%s.tar.gz' % (ud.host, subdirpath.replace('/', '.'), ud.tag), d) - - ud.basecmd = data.getVar("FETCHCMD_git", d, True) or "git" - - if 'noclone' in ud.parm: - ud.localfile = None - return None - - return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile) - - def forcefetch(self, url, ud, d): - if 'fullclone' in ud.parm: - return True - if 'noclone' in ud.parm: - return False - if os.path.exists(ud.localpath): - return False - if not self._contains_ref(ud.tag, d): - return True - return False - - def try_premirror(self, u, ud, d): - if 'noclone' in ud.parm: - return False - if os.path.exists(ud.clonedir): - return False - if os.path.exists(ud.localpath): - return False - - return True - - def go(self, loc, ud, d): - """Fetch url""" - - if ud.user: - username = ud.user + '@' - else: - username = "" - - repofile = os.path.join(data.getVar("DL_DIR", d, 1), ud.mirrortarball) - - - coname = '%s' % (ud.tag) - codir = os.path.join(ud.clonedir, coname) - - # If we have no existing clone and no mirror tarball, try and obtain one - if not os.path.exists(ud.clonedir) and not os.path.exists(repofile): - try: - Fetch.try_mirrors(ud.mirrortarball) - except: - pass - - # If the checkout doesn't exist and the mirror tarball does, extract it - if not os.path.exists(ud.clonedir) and os.path.exists(repofile): - bb.utils.mkdirhier(ud.clonedir) - os.chdir(ud.clonedir) - runfetchcmd("tar -xzf %s" % (repofile), d) - - # If the repo still doesn't exist, fallback to cloning it - if not os.path.exists(ud.clonedir): - runfetchcmd("%s clone -n %s://%s%s%s %s" % (ud.basecmd, ud.proto, username, ud.host, ud.path, ud.clonedir), d) - - os.chdir(ud.clonedir) - # Update the checkout if needed - if not self._contains_ref(ud.tag, d) or 'fullclone' in ud.parm: - # Remove all but the .git directory - runfetchcmd("rm * -Rf", d) - if 'fullclone' in ud.parm: - runfetchcmd("%s fetch --all" % (ud.basecmd), d) - else: - runfetchcmd("%s fetch %s://%s%s%s %s" % (ud.basecmd, ud.proto, username, ud.host, ud.path, ud.branch), d) - runfetchcmd("%s fetch --tags %s://%s%s%s" % (ud.basecmd, ud.proto, username, ud.host, ud.path), d) - runfetchcmd("%s prune-packed" % ud.basecmd, d) - runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d) - - # Generate a mirror tarball if needed - os.chdir(ud.clonedir) - mirror_tarballs = data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True) - if mirror_tarballs != "0" or 'fullclone' in ud.parm: - logger.info("Creating tarball of git repository") - runfetchcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ), d) - - if 'fullclone' in ud.parm: - return - - if os.path.exists(codir): - bb.utils.prunedir(codir) - - subdir = ud.parm.get("subpath", "") - if subdir != "": - if subdir.endswith("/"): - subdirbase = os.path.basename(subdir[:-1]) - else: - subdirbase = os.path.basename(subdir) - else: - subdirbase = "" - - if subdir != "": - readpathspec = ":%s" % (subdir) - codir = os.path.join(codir, "git") - coprefix = os.path.join(codir, subdirbase, "") - else: - readpathspec = "" - coprefix = os.path.join(codir, "git", "") - - scmdata = ud.parm.get("scmdata", "") - if scmdata == "keep": - runfetchcmd("%s clone -n %s %s" % (ud.basecmd, ud.clonedir, coprefix), d) - os.chdir(coprefix) - runfetchcmd("%s checkout -q -f %s%s" % (ud.basecmd, ud.tag, readpathspec), d) - else: - bb.utils.mkdirhier(codir) - os.chdir(ud.clonedir) - runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.tag, readpathspec), d) - runfetchcmd("%s checkout-index -q -f --prefix=%s -a" % (ud.basecmd, coprefix), d) - - os.chdir(codir) - logger.info("Creating tarball of git checkout") - runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.join(".", "*") ), d) - - os.chdir(ud.clonedir) - bb.utils.prunedir(codir) - - def supports_srcrev(self): - return True - - def _contains_ref(self, tag, d): - basecmd = data.getVar("FETCHCMD_git", d, True) or "git" - output = runfetchcmd("%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % (basecmd, tag), d, quiet=True) - return output.split()[0] != "0" - - def _revision_key(self, url, ud, d, branch=False): - """ - Return a unique key for the url - """ - key = 'git:' + ud.host + ud.path.replace('/', '.') - if branch: - return key + ud.branch - else: - return key - - def generate_revision_key(self, url, ud, d, branch=False): - key = self._revision_key(url, ud, d, branch) - return "%s-%s" % (key, d.getVar("PN", True) or "") - - def _latest_revision(self, url, ud, d): - """ - Compute the HEAD revision for the url - """ - if ud.user: - username = ud.user + '@' - else: - username = "" - - basecmd = data.getVar("FETCHCMD_git", d, True) or "git" - cmd = "%s ls-remote %s://%s%s%s %s" % (basecmd, ud.proto, username, ud.host, ud.path, ud.branch) - output = runfetchcmd(cmd, d, True) - if not output: - raise bb.fetch.FetchError("Fetch command %s gave empty output\n" % (cmd)) - return output.split()[0] - - def latest_revision(self, url, ud, d): - """ - Look in the cache for the latest revision, if not present ask the SCM. - """ - revs = bb.persist_data.persist('BB_URI_HEADREVS', d) - - key = self.generate_revision_key(url, ud, d, branch=True) - - try: - return revs[key] - except KeyError: - # Compatibility with old key format, no branch included - oldkey = self.generate_revision_key(url, ud, d, branch=False) - try: - rev = revs[oldkey] - except KeyError: - rev = self._latest_revision(url, ud, d) - else: - del revs[oldkey] - revs[key] = rev - return rev - - def sortable_revision(self, url, ud, d): - """ - - """ - localcounts = bb.persist_data.persist('BB_URI_LOCALCOUNT', d) - key = self.generate_revision_key(url, ud, d, branch=True) - oldkey = self.generate_revision_key(url, ud, d, branch=False) - - latest_rev = self._build_revision(url, ud, d) - last_rev = localcounts.get(key + '_rev') - if last_rev is None: - last_rev = localcounts.get(oldkey + '_rev') - if last_rev is not None: - del localcounts[oldkey + '_rev'] - localcounts[key + '_rev'] = last_rev - - uselocalcount = d.getVar("BB_LOCALCOUNT_OVERRIDE", True) or False - count = None - if uselocalcount: - count = Fetch.localcount_internal_helper(ud, d) - if count is None: - count = localcounts.get(key + '_count') - if count is None: - count = localcounts.get(oldkey + '_count') - if count is not None: - del localcounts[oldkey + '_count'] - localcounts[key + '_count'] = count - - if last_rev == latest_rev: - return str(count + "+" + latest_rev) - - buildindex_provided = hasattr(self, "_sortable_buildindex") - if buildindex_provided: - count = self._sortable_buildindex(url, ud, d, latest_rev) - if count is None: - count = "0" - elif uselocalcount or buildindex_provided: - count = str(count) - else: - count = str(int(count) + 1) - - localcounts[key + '_rev'] = latest_rev - localcounts[key + '_count'] = count - - return str(count + "+" + latest_rev) - - def _build_revision(self, url, ud, d): - return ud.tag - - def _sortable_buildindex_disabled(self, url, ud, d, rev): - """ - Return a suitable buildindex for the revision specified. This is done by counting revisions - using "git rev-list" which may or may not work in different circumstances. - """ - - cwd = os.getcwd() - - # Check if we have the rev already - - if not os.path.exists(ud.clonedir): - print("no repo") - self.go(None, ud, d) - if not os.path.exists(ud.clonedir): - logger.error("GIT repository for %s doesn't exist in %s, cannot get sortable buildnumber, using old value", url, ud.clonedir) - return None - - - os.chdir(ud.clonedir) - if not self._contains_ref(rev, d): - self.go(None, ud, d) - - output = runfetchcmd("%s rev-list %s -- 2> /dev/null | wc -l" % (ud.basecmd, rev), d, quiet=True) - os.chdir(cwd) - - buildindex = "%s" % output.split()[0] - logger.debug(1, "GIT repository for %s in %s is returning %s revisions in rev-list before %s", url, ud.clonedir, buildindex, rev) - return buildindex diff --git a/lib/bb/fetch/hg.py b/lib/bb/fetch/hg.py deleted file mode 100644 index 2b3aec543..000000000 --- a/lib/bb/fetch/hg.py +++ /dev/null @@ -1,180 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -""" -BitBake 'Fetch' implementation for mercurial DRCS (hg). - -""" - -# Copyright (C) 2003, 2004 Chris Larson -# Copyright (C) 2004 Marcin Juszkiewicz -# Copyright (C) 2007 Robert Schuster -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# Based on functions from the base bb module, Copyright 2003 Holger Schurig - -import os -import sys -import logging -import bb -from bb import data -from bb.fetch import Fetch -from bb.fetch import FetchError -from bb.fetch import MissingParameterError -from bb.fetch import runfetchcmd -from bb.fetch import logger - -class Hg(Fetch): - """Class to fetch from mercurial repositories""" - def supports(self, url, ud, d): - """ - Check to see if a given url can be fetched with mercurial. - """ - return ud.type in ['hg'] - - def forcefetch(self, url, ud, d): - revTag = ud.parm.get('rev', 'tip') - return revTag == "tip" - - def localpath(self, url, ud, d): - if not "module" in ud.parm: - raise MissingParameterError("hg method needs a 'module' parameter") - - ud.module = ud.parm["module"] - - # Create paths to mercurial checkouts - relpath = self._strip_leading_slashes(ud.path) - ud.pkgdir = os.path.join(data.expand('${HGDIR}', d), ud.host, relpath) - ud.moddir = os.path.join(ud.pkgdir, ud.module) - - if 'rev' in ud.parm: - ud.revision = ud.parm['rev'] - else: - tag = Fetch.srcrev_internal_helper(ud, d) - if tag is True: - ud.revision = self.latest_revision(url, ud, d) - elif tag: - ud.revision = tag - else: - ud.revision = self.latest_revision(url, ud, d) - - ud.localfile = data.expand('%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision), d) - - return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile) - - def _buildhgcommand(self, ud, d, command): - """ - Build up an hg commandline based on ud - command is "fetch", "update", "info" - """ - - basecmd = data.expand('${FETCHCMD_hg}', d) - - proto = ud.parm.get('proto', 'http') - - host = ud.host - if proto == "file": - host = "/" - ud.host = "localhost" - - if not ud.user: - hgroot = host + ud.path - else: - hgroot = ud.user + "@" + host + ud.path - - if command is "info": - return "%s identify -i %s://%s/%s" % (basecmd, proto, hgroot, ud.module) - - options = []; - if ud.revision: - options.append("-r %s" % ud.revision) - - if command is "fetch": - cmd = "%s clone %s %s://%s/%s %s" % (basecmd, " ".join(options), proto, hgroot, ud.module, ud.module) - elif command is "pull": - # do not pass options list; limiting pull to rev causes the local - # repo not to contain it and immediately following "update" command - # will crash - cmd = "%s pull" % (basecmd) - elif command is "update": - cmd = "%s update -C %s" % (basecmd, " ".join(options)) - else: - raise FetchError("Invalid hg command %s" % command) - - return cmd - - def go(self, loc, ud, d): - """Fetch url""" - - logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'") - - if os.access(os.path.join(ud.moddir, '.hg'), os.R_OK): - updatecmd = self._buildhgcommand(ud, d, "pull") - logger.info("Update " + loc) - # update sources there - os.chdir(ud.moddir) - logger.debug(1, "Running %s", updatecmd) - runfetchcmd(updatecmd, d) - - else: - fetchcmd = self._buildhgcommand(ud, d, "fetch") - logger.info("Fetch " + loc) - # check out sources there - bb.utils.mkdirhier(ud.pkgdir) - os.chdir(ud.pkgdir) - logger.debug(1, "Running %s", fetchcmd) - runfetchcmd(fetchcmd, d) - - # Even when we clone (fetch), we still need to update as hg's clone - # won't checkout the specified revision if its on a branch - updatecmd = self._buildhgcommand(ud, d, "update") - os.chdir(ud.moddir) - logger.debug(1, "Running %s", updatecmd) - runfetchcmd(updatecmd, d) - - scmdata = ud.parm.get("scmdata", "") - if scmdata == "keep": - tar_flags = "" - else: - tar_flags = "--exclude '.hg' --exclude '.hgrags'" - - os.chdir(ud.pkgdir) - try: - runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, ud.module), d) - except: - t, v, tb = sys.exc_info() - try: - os.unlink(ud.localpath) - except OSError: - pass - raise t, v, tb - - def supports_srcrev(self): - return True - - def _latest_revision(self, url, ud, d): - """ - Compute tip revision for the url - """ - output = runfetchcmd(self._buildhgcommand(ud, d, "info"), d) - return output.strip() - - def _build_revision(self, url, ud, d): - return ud.revision - - def _revision_key(self, url, ud, d): - """ - Return a unique key for the url - """ - return "hg:" + ud.moddir diff --git a/lib/bb/fetch/local.py b/lib/bb/fetch/local.py deleted file mode 100644 index 6aa9e4576..000000000 --- a/lib/bb/fetch/local.py +++ /dev/null @@ -1,73 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -""" -BitBake 'Fetch' implementations - -Classes for obtaining upstream sources for the -BitBake build tools. - -""" - -# Copyright (C) 2003, 2004 Chris Larson -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# Based on functions from the base bb module, Copyright 2003 Holger Schurig - -import os -import bb -import bb.utils -from bb import data -from bb.fetch import Fetch - -class Local(Fetch): - def supports(self, url, urldata, d): - """ - Check to see if a given url represents a local fetch. - """ - return urldata.type in ['file'] - - def localpath(self, url, urldata, d): - """ - Return the local filename of a given url assuming a successful fetch. - """ - path = url.split("://")[1] - path = path.split(";")[0] - newpath = path - if path[0] != "/": - filespath = data.getVar('FILESPATH', d, 1) - if filespath: - newpath = bb.utils.which(filespath, path) - if not newpath: - filesdir = data.getVar('FILESDIR', d, 1) - if filesdir: - newpath = os.path.join(filesdir, path) - # We don't set localfile as for this fetcher the file is already local! - return newpath - - def go(self, url, urldata, d): - """Fetch urls (no-op for Local method)""" - # no need to fetch local files, we'll deal with them in place. - return 1 - - def checkstatus(self, url, urldata, d): - """ - Check the status of the url - """ - if urldata.localpath.find("*") != -1: - logger.info("URL %s looks like a glob and was therefore not checked.", url) - return True - if os.path.exists(urldata.localpath): - return True - return False diff --git a/lib/bb/fetch/osc.py b/lib/bb/fetch/osc.py deleted file mode 100644 index 32237b974..000000000 --- a/lib/bb/fetch/osc.py +++ /dev/null @@ -1,143 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -""" -Bitbake "Fetch" implementation for osc (Opensuse build service client). -Based on the svn "Fetch" implementation. - -""" - -import os -import sys -import logging -import bb -from bb import data -from bb import utils -from bb.fetch import Fetch -from bb.fetch import FetchError -from bb.fetch import MissingParameterError -from bb.fetch import runfetchcmd - -class Osc(Fetch): - """Class to fetch a module or modules from Opensuse build server - repositories.""" - - def supports(self, url, ud, d): - """ - Check to see if a given url can be fetched with osc. - """ - return ud.type in ['osc'] - - def localpath(self, url, ud, d): - if not "module" in ud.parm: - raise MissingParameterError("osc method needs a 'module' parameter.") - - ud.module = ud.parm["module"] - - # Create paths to osc checkouts - relpath = self._strip_leading_slashes(ud.path) - ud.pkgdir = os.path.join(data.expand('${OSCDIR}', d), ud.host) - ud.moddir = os.path.join(ud.pkgdir, relpath, ud.module) - - if 'rev' in ud.parm: - ud.revision = ud.parm['rev'] - else: - pv = data.getVar("PV", d, 0) - rev = Fetch.srcrev_internal_helper(ud, d) - if rev and rev != True: - ud.revision = rev - else: - ud.revision = "" - - ud.localfile = data.expand('%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.path.replace('/', '.'), ud.revision), d) - - return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile) - - def _buildosccommand(self, ud, d, command): - """ - Build up an ocs commandline based on ud - command is "fetch", "update", "info" - """ - - basecmd = data.expand('${FETCHCMD_osc}', d) - - proto = ud.parm.get('proto', 'ocs') - - options = [] - - config = "-c %s" % self.generate_config(ud, d) - - if ud.revision: - options.append("-r %s" % ud.revision) - - coroot = self._strip_leading_slashes(ud.path) - - if command is "fetch": - osccmd = "%s %s co %s/%s %s" % (basecmd, config, coroot, ud.module, " ".join(options)) - elif command is "update": - osccmd = "%s %s up %s" % (basecmd, config, " ".join(options)) - else: - raise FetchError("Invalid osc command %s" % command) - - return osccmd - - def go(self, loc, ud, d): - """ - Fetch url - """ - - logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'") - - if os.access(os.path.join(data.expand('${OSCDIR}', d), ud.path, ud.module), os.R_OK): - oscupdatecmd = self._buildosccommand(ud, d, "update") - logger.info("Update "+ loc) - # update sources there - os.chdir(ud.moddir) - logger.debug(1, "Running %s", oscupdatecmd) - runfetchcmd(oscupdatecmd, d) - else: - oscfetchcmd = self._buildosccommand(ud, d, "fetch") - logger.info("Fetch " + loc) - # check out sources there - bb.utils.mkdirhier(ud.pkgdir) - os.chdir(ud.pkgdir) - logger.debug(1, "Running %s", oscfetchcmd) - runfetchcmd(oscfetchcmd, d) - - os.chdir(os.path.join(ud.pkgdir + ud.path)) - # tar them up to a defined filename - try: - runfetchcmd("tar -czf %s %s" % (ud.localpath, ud.module), d) - except: - t, v, tb = sys.exc_info() - try: - os.unlink(ud.localpath) - except OSError: - pass - raise t, v, tb - - def supports_srcrev(self): - return False - - def generate_config(self, ud, d): - """ - Generate a .oscrc to be used for this run. - """ - - config_path = os.path.join(data.expand('${OSCDIR}', d), "oscrc") - bb.utils.remove(config_path) - - f = open(config_path, 'w') - f.write("[general]\n") - f.write("apisrv = %s\n" % ud.host) - f.write("scheme = http\n") - f.write("su-wrapper = su -c\n") - f.write("build-root = %s\n" % data.expand('${WORKDIR}', d)) - f.write("urllist = http://moblin-obs.jf.intel.com:8888/build/%(project)s/%(repository)s/%(buildarch)s/:full/%(name)s.rpm\n") - f.write("extra-pkgs = gzip\n") - f.write("\n") - f.write("[%s]\n" % ud.host) - f.write("user = %s\n" % ud.parm["user"]) - f.write("pass = %s\n" % ud.parm["pswd"]) - f.close() - - return config_path diff --git a/lib/bb/fetch/perforce.py b/lib/bb/fetch/perforce.py deleted file mode 100644 index e933d2760..000000000 --- a/lib/bb/fetch/perforce.py +++ /dev/null @@ -1,206 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -""" -BitBake 'Fetch' implementations - -Classes for obtaining upstream sources for the -BitBake build tools. - -""" - -# Copyright (C) 2003, 2004 Chris Larson -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# Based on functions from the base bb module, Copyright 2003 Holger Schurig - -from future_builtins import zip -import os -import logging -import bb -from bb import data -from bb.fetch import Fetch -from bb.fetch import FetchError -from bb.fetch import logger - -class Perforce(Fetch): - def supports(self, url, ud, d): - return ud.type in ['p4'] - - def doparse(url, d): - parm = {} - path = url.split("://")[1] - delim = path.find("@"); - if delim != -1: - (user, pswd, host, port) = path.split('@')[0].split(":") - path = path.split('@')[1] - else: - (host, port) = data.getVar('P4PORT', d).split(':') - user = "" - pswd = "" - - if path.find(";") != -1: - keys=[] - values=[] - plist = path.split(';') - for item in plist: - if item.count('='): - (key, value) = item.split('=') - keys.append(key) - values.append(value) - - parm = dict(zip(keys, values)) - path = "//" + path.split(';')[0] - host += ":%s" % (port) - parm["cset"] = Perforce.getcset(d, path, host, user, pswd, parm) - - return host, path, user, pswd, parm - doparse = staticmethod(doparse) - - def getcset(d, depot, host, user, pswd, parm): - p4opt = "" - if "cset" in parm: - return parm["cset"]; - if user: - p4opt += " -u %s" % (user) - if pswd: - p4opt += " -P %s" % (pswd) - if host: - p4opt += " -p %s" % (host) - - p4date = data.getVar("P4DATE", d, 1) - if "revision" in parm: - depot += "#%s" % (parm["revision"]) - elif "label" in parm: - depot += "@%s" % (parm["label"]) - elif p4date: - depot += "@%s" % (p4date) - - p4cmd = data.getVar('FETCHCOMMAND_p4', d, 1) - logger.debug(1, "Running %s%s changes -m 1 %s", p4cmd, p4opt, depot) - p4file = os.popen("%s%s changes -m 1 %s" % (p4cmd, p4opt, depot)) - cset = p4file.readline().strip() - logger.debug(1, "READ %s", cset) - if not cset: - return -1 - - return cset.split(' ')[1] - getcset = staticmethod(getcset) - - def localpath(self, url, ud, d): - - (host, path, user, pswd, parm) = Perforce.doparse(url, d) - - # If a label is specified, we use that as our filename - - if "label" in parm: - ud.localfile = "%s.tar.gz" % (parm["label"]) - return os.path.join(data.getVar("DL_DIR", d, 1), ud.localfile) - - base = path - which = path.find('/...') - if which != -1: - base = path[:which] - - base = self._strip_leading_slashes(base) - - cset = Perforce.getcset(d, path, host, user, pswd, parm) - - ud.localfile = data.expand('%s+%s+%s.tar.gz' % (host, base.replace('/', '.'), cset), d) - - return os.path.join(data.getVar("DL_DIR", d, 1), ud.localfile) - - def go(self, loc, ud, d): - """ - Fetch urls - """ - - (host, depot, user, pswd, parm) = Perforce.doparse(loc, d) - - if depot.find('/...') != -1: - path = depot[:depot.find('/...')] - else: - path = depot - - module = parm.get('module', os.path.basename(path)) - - localdata = data.createCopy(d) - data.setVar('OVERRIDES', "p4:%s" % data.getVar('OVERRIDES', localdata), localdata) - data.update_data(localdata) - - # Get the p4 command - p4opt = "" - if user: - p4opt += " -u %s" % (user) - - if pswd: - p4opt += " -P %s" % (pswd) - - if host: - p4opt += " -p %s" % (host) - - p4cmd = data.getVar('FETCHCOMMAND', localdata, 1) - - # create temp directory - logger.debug(2, "Fetch: creating temporary directory") - bb.utils.mkdirhier(data.expand('${WORKDIR}', localdata)) - data.setVar('TMPBASE', data.expand('${WORKDIR}/oep4.XXXXXX', localdata), localdata) - tmppipe = os.popen(data.getVar('MKTEMPDIRCMD', localdata, 1) or "false") - tmpfile = tmppipe.readline().strip() - if not tmpfile: - logger.error("Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.") - raise FetchError(module) - - if "label" in parm: - depot = "%s@%s" % (depot, parm["label"]) - else: - cset = Perforce.getcset(d, depot, host, user, pswd, parm) - depot = "%s@%s" % (depot, cset) - - os.chdir(tmpfile) - logger.info("Fetch " + loc) - logger.info("%s%s files %s", p4cmd, p4opt, depot) - p4file = os.popen("%s%s files %s" % (p4cmd, p4opt, depot)) - - if not p4file: - logger.error("Fetch: unable to get the P4 files from %s", depot) - raise FetchError(module) - - count = 0 - - for file in p4file: - list = file.split() - - if list[2] == "delete": - continue - - dest = list[0][len(path)+1:] - where = dest.find("#") - - os.system("%s%s print -o %s/%s %s" % (p4cmd, p4opt, module, dest[:where], list[0])) - count = count + 1 - - if count == 0: - logger.error("Fetch: No files gathered from the P4 fetch") - raise FetchError(module) - - myret = os.system("tar -czf %s %s" % (ud.localpath, module)) - if myret != 0: - try: - os.unlink(ud.localpath) - except OSError: - pass - raise FetchError(module) - # cleanup - bb.utils.prunedir(tmpfile) diff --git a/lib/bb/fetch/repo.py b/lib/bb/fetch/repo.py deleted file mode 100644 index 512fffb4c..000000000 --- a/lib/bb/fetch/repo.py +++ /dev/null @@ -1,98 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -""" -BitBake "Fetch" repo (git) implementation - -""" - -# Copyright (C) 2009 Tom Rini -# -# Based on git.py which is: -#Copyright (C) 2005 Richard Purdie -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -import os -import bb -from bb import data -from bb.fetch import Fetch -from bb.fetch import runfetchcmd - -class Repo(Fetch): - """Class to fetch a module or modules from repo (git) repositories""" - def supports(self, url, ud, d): - """ - Check to see if a given url can be fetched with repo. - """ - return ud.type in ["repo"] - - def localpath(self, url, ud, d): - """ - We don"t care about the git rev of the manifests repository, but - we do care about the manifest to use. The default is "default". - We also care about the branch or tag to be used. The default is - "master". - """ - - ud.proto = ud.parm.get('protocol', 'git') - ud.branch = ud.parm.get('branch', 'master') - ud.manifest = ud.parm.get('manifest', 'default.xml') - if not ud.manifest.endswith('.xml'): - ud.manifest += '.xml' - - ud.localfile = data.expand("repo_%s%s_%s_%s.tar.gz" % (ud.host, ud.path.replace("/", "."), ud.manifest, ud.branch), d) - - return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile) - - def go(self, loc, ud, d): - """Fetch url""" - - if os.access(os.path.join(data.getVar("DL_DIR", d, True), ud.localfile), os.R_OK): - logger.debug(1, "%s already exists (or was stashed). Skipping repo init / sync.", ud.localpath) - return - - gitsrcname = "%s%s" % (ud.host, ud.path.replace("/", ".")) - repodir = data.getVar("REPODIR", d, True) or os.path.join(data.getVar("DL_DIR", d, True), "repo") - codir = os.path.join(repodir, gitsrcname, ud.manifest) - - if ud.user: - username = ud.user + "@" - else: - username = "" - - bb.utils.mkdirhier(os.path.join(codir, "repo")) - os.chdir(os.path.join(codir, "repo")) - if not os.path.exists(os.path.join(codir, "repo", ".repo")): - runfetchcmd("repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), d) - - runfetchcmd("repo sync", d) - os.chdir(codir) - - scmdata = ud.parm.get("scmdata", "") - if scmdata == "keep": - tar_flags = "" - else: - tar_flags = "--exclude '.repo' --exclude '.git'" - - # Create a cache - runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.join(".", "*") ), d) - - def supports_srcrev(self): - return False - - def _build_revision(self, url, ud, d): - return ud.manifest - - def _want_sortable_revision(self, url, ud, d): - return False diff --git a/lib/bb/fetch/ssh.py b/lib/bb/fetch/ssh.py deleted file mode 100644 index 86c76f4e4..000000000 --- a/lib/bb/fetch/ssh.py +++ /dev/null @@ -1,118 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -''' -BitBake 'Fetch' implementations - -This implementation is for Secure Shell (SSH), and attempts to comply with the -IETF secsh internet draft: - http://tools.ietf.org/wg/secsh/draft-ietf-secsh-scp-sftp-ssh-uri/ - - Currently does not support the sftp parameters, as this uses scp - Also does not support the 'fingerprint' connection parameter. - -''' - -# Copyright (C) 2006 OpenedHand Ltd. -# -# -# Based in part on svk.py: -# Copyright (C) 2006 Holger Hans Peter Freyther -# Based on svn.py: -# Copyright (C) 2003, 2004 Chris Larson -# Based on functions from the base bb module: -# Copyright 2003 Holger Schurig -# -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -import re, os -from bb import data -from bb.fetch import Fetch -from bb.fetch import FetchError - - -__pattern__ = re.compile(r''' - \s* # Skip leading whitespace - ssh:// # scheme - ( # Optional username/password block - (?P\S+) # username - (:(?P\S+))? # colon followed by the password (optional) - )? - (?P(;[^;]+)*)? # connection parameters block (optional) - @ - (?P\S+?) # non-greedy match of the host - (:(?P[0-9]+))? # colon followed by the port (optional) - / - (?P[^;]+) # path on the remote system, may be absolute or relative, - # and may include the use of '~' to reference the remote home - # directory - (?P(;[^;]+)*)? # parameters block (optional) - $ -''', re.VERBOSE) - -class SSH(Fetch): - '''Class to fetch a module or modules via Secure Shell''' - - def supports(self, url, urldata, d): - return __pattern__.match(url) != None - - def localpath(self, url, urldata, d): - m = __pattern__.match(url) - path = m.group('path') - host = m.group('host') - lpath = os.path.join(data.getVar('DL_DIR', d, True), host, os.path.basename(path)) - return lpath - - def go(self, url, urldata, d): - dldir = data.getVar('DL_DIR', d, 1) - - m = __pattern__.match(url) - path = m.group('path') - host = m.group('host') - port = m.group('port') - user = m.group('user') - password = m.group('pass') - - ldir = os.path.join(dldir, host) - lpath = os.path.join(ldir, os.path.basename(path)) - - if not os.path.exists(ldir): - os.makedirs(ldir) - - if port: - port = '-P %s' % port - else: - port = '' - - if user: - fr = user - if password: - fr += ':%s' % password - fr += '@%s' % host - else: - fr = host - fr += ':%s' % path - - - import commands - cmd = 'scp -B -r %s %s %s/' % ( - port, - commands.mkarg(fr), - commands.mkarg(ldir) - ) - - (exitstatus, output) = commands.getstatusoutput(cmd) - if exitstatus != 0: - print(output) - raise FetchError('Unable to fetch %s' % url) diff --git a/lib/bb/fetch/svk.py b/lib/bb/fetch/svk.py deleted file mode 100644 index dc818d2d6..000000000 --- a/lib/bb/fetch/svk.py +++ /dev/null @@ -1,104 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -""" -BitBake 'Fetch' implementations - -This implementation is for svk. It is based on the svn implementation - -""" - -# Copyright (C) 2006 Holger Hans Peter Freyther -# Copyright (C) 2003, 2004 Chris Larson -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# Based on functions from the base bb module, Copyright 2003 Holger Schurig - -import os -import logging -import bb -from bb import data -from bb.fetch import Fetch -from bb.fetch import FetchError -from bb.fetch import MissingParameterError -from bb.fetch import logger - -class Svk(Fetch): - """Class to fetch a module or modules from svk repositories""" - def supports(self, url, ud, d): - """ - Check to see if a given url can be fetched with svk. - """ - return ud.type in ['svk'] - - def localpath(self, url, ud, d): - if not "module" in ud.parm: - raise MissingParameterError("svk method needs a 'module' parameter") - else: - ud.module = ud.parm["module"] - - ud.revision = ud.parm.get('rev', "") - - ud.localfile = data.expand('%s_%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision, ud.date), d) - - return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile) - - def forcefetch(self, url, ud, d): - return ud.date == "now" - - def go(self, loc, ud, d): - """Fetch urls""" - - svkroot = ud.host + ud.path - - svkcmd = "svk co -r {%s} %s/%s" % (ud.date, svkroot, ud.module) - - if ud.revision: - svkcmd = "svk co -r %s %s/%s" % (ud.revision, svkroot, ud.module) - - # create temp directory - localdata = data.createCopy(d) - data.update_data(localdata) - logger.debug(2, "Fetch: creating temporary directory") - bb.utils.mkdirhier(data.expand('${WORKDIR}', localdata)) - data.setVar('TMPBASE', data.expand('${WORKDIR}/oesvk.XXXXXX', localdata), localdata) - tmppipe = os.popen(data.getVar('MKTEMPDIRCMD', localdata, 1) or "false") - tmpfile = tmppipe.readline().strip() - if not tmpfile: - logger.error("Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.") - raise FetchError(ud.module) - - # check out sources there - os.chdir(tmpfile) - logger.info("Fetch " + loc) - logger.debug(1, "Running %s", svkcmd) - myret = os.system(svkcmd) - if myret != 0: - try: - os.rmdir(tmpfile) - except OSError: - pass - raise FetchError(ud.module) - - os.chdir(os.path.join(tmpfile, os.path.dirname(ud.module))) - # tar them up to a defined filename - myret = os.system("tar -czf %s %s" % (ud.localpath, os.path.basename(ud.module))) - if myret != 0: - try: - os.unlink(ud.localpath) - except OSError: - pass - raise FetchError(ud.module) - # cleanup - bb.utils.prunedir(tmpfile) diff --git a/lib/bb/fetch/svn.py b/lib/bb/fetch/svn.py deleted file mode 100644 index 6c2a1187d..000000000 --- a/lib/bb/fetch/svn.py +++ /dev/null @@ -1,204 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -""" -BitBake 'Fetch' implementation for svn. - -""" - -# Copyright (C) 2003, 2004 Chris Larson -# Copyright (C) 2004 Marcin Juszkiewicz -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# Based on functions from the base bb module, Copyright 2003 Holger Schurig - -import os -import sys -import logging -import bb -from bb import data -from bb.fetch import Fetch -from bb.fetch import FetchError -from bb.fetch import MissingParameterError -from bb.fetch import runfetchcmd -from bb.fetch import logger - -class Svn(Fetch): - """Class to fetch a module or modules from svn repositories""" - def supports(self, url, ud, d): - """ - Check to see if a given url can be fetched with svn. - """ - return ud.type in ['svn'] - - def localpath(self, url, ud, d): - if not "module" in ud.parm: - raise MissingParameterError("svn method needs a 'module' parameter") - - ud.module = ud.parm["module"] - - # Create paths to svn checkouts - relpath = self._strip_leading_slashes(ud.path) - ud.pkgdir = os.path.join(data.expand('${SVNDIR}', d), ud.host, relpath) - ud.moddir = os.path.join(ud.pkgdir, ud.module) - - if 'rev' in ud.parm: - ud.date = "" - ud.revision = ud.parm['rev'] - elif 'date' in ud.date: - ud.date = ud.parm['date'] - ud.revision = "" - else: - # - # ***Nasty hack*** - # If DATE in unexpanded PV, use ud.date (which is set from SRCDATE) - # Should warn people to switch to SRCREV here - # - pv = data.getVar("PV", d, 0) - if "DATE" in pv: - ud.revision = "" - else: - rev = Fetch.srcrev_internal_helper(ud, d) - if rev is True: - ud.revision = self.latest_revision(url, ud, d) - ud.date = "" - elif rev: - ud.revision = rev - ud.date = "" - else: - ud.revision = "" - - ud.localfile = data.expand('%s_%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision, ud.date), d) - - return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile) - - def _buildsvncommand(self, ud, d, command): - """ - Build up an svn commandline based on ud - command is "fetch", "update", "info" - """ - - basecmd = data.expand('${FETCHCMD_svn}', d) - - proto = ud.parm.get('proto', 'svn') - - svn_rsh = None - if proto == "svn+ssh" and "rsh" in ud.parm: - svn_rsh = ud.parm["rsh"] - - svnroot = ud.host + ud.path - - # either use the revision, or SRCDATE in braces, - options = [] - - if ud.user: - options.append("--username %s" % ud.user) - - if ud.pswd: - options.append("--password %s" % ud.pswd) - - if command is "info": - svncmd = "%s info %s %s://%s/%s/" % (basecmd, " ".join(options), proto, svnroot, ud.module) - else: - suffix = "" - if ud.revision: - options.append("-r %s" % ud.revision) - suffix = "@%s" % (ud.revision) - elif ud.date: - options.append("-r {%s}" % ud.date) - - if command is "fetch": - svncmd = "%s co %s %s://%s/%s%s %s" % (basecmd, " ".join(options), proto, svnroot, ud.module, suffix, ud.module) - elif command is "update": - svncmd = "%s update %s" % (basecmd, " ".join(options)) - else: - raise FetchError("Invalid svn command %s" % command) - - if svn_rsh: - svncmd = "svn_RSH=\"%s\" %s" % (svn_rsh, svncmd) - - return svncmd - - def go(self, loc, ud, d): - """Fetch url""" - - logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'") - - if os.access(os.path.join(ud.moddir, '.svn'), os.R_OK): - svnupdatecmd = self._buildsvncommand(ud, d, "update") - logger.info("Update " + loc) - # update sources there - os.chdir(ud.moddir) - logger.debug(1, "Running %s", svnupdatecmd) - runfetchcmd(svnupdatecmd, d) - else: - svnfetchcmd = self._buildsvncommand(ud, d, "fetch") - logger.info("Fetch " + loc) - # check out sources there - bb.utils.mkdirhier(ud.pkgdir) - os.chdir(ud.pkgdir) - logger.debug(1, "Running %s", svnfetchcmd) - runfetchcmd(svnfetchcmd, d) - - scmdata = ud.parm.get("scmdata", "") - if scmdata == "keep": - tar_flags = "" - else: - tar_flags = "--exclude '.svn'" - - os.chdir(ud.pkgdir) - # tar them up to a defined filename - try: - runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, ud.module), d) - except: - t, v, tb = sys.exc_info() - try: - os.unlink(ud.localpath) - except OSError: - pass - raise t, v, tb - - def supports_srcrev(self): - return True - - def _revision_key(self, url, ud, d): - """ - Return a unique key for the url - """ - return "svn:" + ud.moddir - - def _latest_revision(self, url, ud, d): - """ - Return the latest upstream revision number - """ - logger.debug(2, "SVN fetcher hitting network for %s", url) - - output = runfetchcmd("LANG=C LC_ALL=C " + self._buildsvncommand(ud, d, "info"), d, True) - - revision = None - for line in output.splitlines(): - if "Last Changed Rev" in line: - revision = line.split(":")[1].strip() - - return revision - - def _sortable_revision(self, url, ud, d): - """ - Return a sortable revision number which in our case is the revision number - """ - - return self._build_revision(url, ud, d) - - def _build_revision(self, url, ud, d): - return ud.revision diff --git a/lib/bb/fetch/wget.py b/lib/bb/fetch/wget.py deleted file mode 100644 index 4d4bdfd49..000000000 --- a/lib/bb/fetch/wget.py +++ /dev/null @@ -1,93 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -""" -BitBake 'Fetch' implementations - -Classes for obtaining upstream sources for the -BitBake build tools. - -""" - -# Copyright (C) 2003, 2004 Chris Larson -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# Based on functions from the base bb module, Copyright 2003 Holger Schurig - -import os -import logging -import bb -import urllib -from bb import data -from bb.fetch import Fetch, FetchError, encodeurl, decodeurl, logger, runfetchcmd - -class Wget(Fetch): - """Class to fetch urls via 'wget'""" - def supports(self, url, ud, d): - """ - Check to see if a given url can be fetched with wget. - """ - return ud.type in ['http', 'https', 'ftp'] - - def localpath(self, url, ud, d): - - url = encodeurl([ud.type, ud.host, ud.path, ud.user, ud.pswd, {}]) - ud.basename = os.path.basename(ud.path) - ud.localfile = data.expand(urllib.unquote(ud.basename), d) - - return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile) - - def go(self, uri, ud, d, checkonly = False): - """Fetch urls""" - - def fetch_uri(uri, ud, d): - if checkonly: - fetchcmd = data.getVar("CHECKCOMMAND", d, 1) - elif os.path.exists(ud.localpath): - # file exists, but we didnt complete it.. trying again.. - fetchcmd = data.getVar("RESUMECOMMAND", d, 1) - else: - fetchcmd = data.getVar("FETCHCOMMAND", d, 1) - - uri = uri.split(";")[0] - uri_decoded = list(decodeurl(uri)) - uri_type = uri_decoded[0] - uri_host = uri_decoded[1] - - fetchcmd = fetchcmd.replace("${URI}", uri.split(";")[0]) - fetchcmd = fetchcmd.replace("${FILE}", ud.basename) - logger.info("fetch " + uri) - logger.debug(2, "executing " + fetchcmd) - runfetchcmd(fetchcmd, d) - - # Sanity check since wget can pretend it succeed when it didn't - # Also, this used to happen if sourceforge sent us to the mirror page - if not os.path.exists(ud.localpath) and not checkonly: - logger.debug(2, "The fetch command for %s returned success but %s doesn't exist?...", uri, ud.localpath) - return False - - return True - - localdata = data.createCopy(d) - data.setVar('OVERRIDES', "wget:" + data.getVar('OVERRIDES', localdata), localdata) - data.update_data(localdata) - - if fetch_uri(uri, ud, localdata): - return True - - raise FetchError(uri) - - - def checkstatus(self, uri, ud, d): - return self.go(uri, ud, d, True) -- cgit 1.2.3-korg