From ce241d7e6d8e31411867b5f2127b02c776b39d8a Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Fri, 10 Feb 2006 11:43:38 +0000 Subject: Update to 1.3.3 release of bitbake git-svn-id: https://svn.o-hand.com/repos/poky/trunk@269 311d38ba-8fff-0310-9ca6-ca027cbcb966 --- bitbake/ChangeLog | 18 +- bitbake/bin/bitbake | 5 +- bitbake/lib/bb/__init__.py | 2 +- bitbake/lib/bb/fetch.py | 656 --------------------------------------------- 4 files changed, 15 insertions(+), 666 deletions(-) delete mode 100644 bitbake/lib/bb/fetch.py (limited to 'bitbake') diff --git a/bitbake/ChangeLog b/bitbake/ChangeLog index e8a91c1701..c05ff96ab9 100644 --- a/bitbake/ChangeLog +++ b/bitbake/ChangeLog @@ -1,4 +1,4 @@ -Changes in BitBake 1.3.?: +Changes in BitBake 1.3.3: - Create a new Fetcher module to ease the development of new Fetchers. Issue #438 fixed by rpurdie@openedhand.com @@ -7,13 +7,19 @@ Changes in BitBake 1.3.?: Issue #555 fixed by chris@openedhand.com - Expand PREFERRED_PROVIDER properly Issue #436 fixed by rprudie@openedhand.com - - Typo fix for Issue #531 Philipp Zabel for the + - Typo fix for Issue #531 by Philipp Zabel for the BitBake Shell - Introduce a new special variable SRCDATE as - a generic naming to take over CVSDATE. - - Introduce a new keyword 'required' In contrast - to include parsing will fail if a to be included - can not be found. + a generic naming to replace CVSDATE. + - Introduce a new keyword 'required'. In contrast + to 'include' parsing will fail if a to be included + file can not be found. + - Remove hardcoding of the STAMP directory. Patch + courtsey pHilipp Zabel + - Track the RDEPENDS of each package (rpurdie@openedhand.com) + - Introduce BUILD_ALL_DEPS to build all RDEPENDS. E.g + this is used by the OpenEmbedded Meta Packages. + (rpurdie@openedhand.com). Changes in BitBake 1.3.2: - reintegration of make.py into BitBake diff --git a/bitbake/bin/bitbake b/bitbake/bin/bitbake index 09417f6eca..b1fd76a4f1 100755 --- a/bitbake/bin/bitbake +++ b/bitbake/bin/bitbake @@ -31,7 +31,7 @@ import itertools, optparse parsespin = itertools.cycle( r'|/-\\' ) bbdebug = 0 -__version__ = "1.3.2" +__version__ = "1.3.3" #============================================================================# # BBParsingStatus @@ -461,8 +461,7 @@ class BBCooker: pn = bb.data.getVar('PN', the_data, 1) pv = bb.data.getVar('PV', the_data, 1) pr = bb.data.getVar('PR', the_data, 1) - tmpdir = bb.data.getVar('TMPDIR', the_data, 1) - stamp = '%s/stamps/%s-%s-%s.do_populate_staging' % (tmpdir, pn, pv, pr) + stamp = '%s.do_populate_staging' % bb.data.getVar('STAMP', the_data, 1) if os.path.exists(stamp): (newvers, fn) = preferred_versions[pn] if not fn in eligible: diff --git a/bitbake/lib/bb/__init__.py b/bitbake/lib/bb/__init__.py index f27f53b39d..dabe978bf5 100644 --- a/bitbake/lib/bb/__init__.py +++ b/bitbake/lib/bb/__init__.py @@ -23,7 +23,7 @@ this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. """ -__version__ = "1.3.2.1" +__version__ = "1.3.3.0" __all__ = [ diff --git a/bitbake/lib/bb/fetch.py b/bitbake/lib/bb/fetch.py deleted file mode 100644 index 230dd21e2e..0000000000 --- a/bitbake/lib/bb/fetch.py +++ /dev/null @@ -1,656 +0,0 @@ -#!/usr/bin/env python -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -""" -BitBake 'Fetch' implementations - -Classes for obtaining upstream sources for the -BitBake build tools. - -Copyright (C) 2003, 2004 Chris Larson - -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; either version 2 of the License, or (at your option) any later -version. - -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., 59 Temple -Place, Suite 330, Boston, MA 02111-1307 USA. - -Based on functions from the base bb module, Copyright 2003 Holger Schurig -""" - -import os, re -import bb -from bb import data - -class FetchError(Exception): - """Exception raised when a download fails""" - -class NoMethodError(Exception): - """Exception raised when there is no method to obtain a supplied url or set of urls""" - -class MissingParameterError(Exception): - """Exception raised when a fetch method is missing a critical parameter in the url""" - -class MD5SumError(Exception): - """Exception raised when a MD5SUM of a file does not match the expected one""" - -def uri_replace(uri, uri_find, uri_replace, d): -# bb.note("uri_replace: operating on %s" % uri) - if not uri or not uri_find or not uri_replace: - bb.debug(1, "uri_replace: passed an undefined value, not replacing") - uri_decoded = list(bb.decodeurl(uri)) - uri_find_decoded = list(bb.decodeurl(uri_find)) - uri_replace_decoded = list(bb.decodeurl(uri_replace)) - result_decoded = ['','','','','',{}] - for i in uri_find_decoded: - loc = uri_find_decoded.index(i) - result_decoded[loc] = uri_decoded[loc] - import types - if type(i) == types.StringType: - import re - if (re.match(i, uri_decoded[loc])): - result_decoded[loc] = re.sub(i, uri_replace_decoded[loc], uri_decoded[loc]) - if uri_find_decoded.index(i) == 2: - if d: - localfn = bb.fetch.localpath(uri, d) - if localfn: - result_decoded[loc] = os.path.dirname(result_decoded[loc]) + "/" + os.path.basename(bb.fetch.localpath(uri, d)) -# bb.note("uri_replace: matching %s against %s and replacing with %s" % (i, uri_decoded[loc], uri_replace_decoded[loc])) - else: -# bb.note("uri_replace: no match") - return uri -# else: -# for j in i.keys(): -# FIXME: apply replacements against options - return bb.encodeurl(result_decoded) - -methods = [] - -def init(urls = [], d = None): - if d == None: - bb.debug(2,"BUG init called with None as data object!!!") - return - - for m in methods: - m.urls = [] - - for u in urls: - for m in methods: - m.data = d - if m.supports(u, d): - m.urls.append(u) - -def go(d): - """Fetch all urls""" - for m in methods: - if m.urls: - m.go(d) - -def localpaths(d): - """Return a list of the local filenames, assuming successful fetch""" - local = [] - for m in methods: - for u in m.urls: - local.append(m.localpath(u, d)) - return local - -def localpath(url, d): - for m in methods: - if m.supports(url, d): - return m.localpath(url, d) - return url - -class Fetch(object): - """Base class for 'fetch'ing data""" - - def __init__(self, urls = []): - self.urls = [] - for url in urls: - if self.supports(bb.decodeurl(url), d) is 1: - self.urls.append(url) - - def supports(url, d): - """Check to see if this fetch class supports a given url. - Expects supplied url in list form, as outputted by bb.decodeurl(). - """ - return 0 - supports = staticmethod(supports) - - def localpath(url, d): - """Return the local filename of a given url assuming a successful fetch. - """ - return url - localpath = staticmethod(localpath) - - def setUrls(self, urls): - self.__urls = urls - - def getUrls(self): - return self.__urls - - urls = property(getUrls, setUrls, None, "Urls property") - - def setData(self, data): - self.__data = data - - def getData(self): - return self.__data - - data = property(getData, setData, None, "Data property") - - def go(self, urls = []): - """Fetch urls""" - raise NoMethodError("Missing implementation for url") - -class Wget(Fetch): - """Class to fetch urls via 'wget'""" - def supports(url, d): - """Check to see if a given url can be fetched using wget. - Expects supplied url in list form, as outputted by bb.decodeurl(). - """ - (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d)) - return type in ['http','https','ftp'] - supports = staticmethod(supports) - - def localpath(url, d): -# strip off parameters - (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d)) - if "localpath" in parm: -# if user overrides local path, use it. - return parm["localpath"] - url = bb.encodeurl([type, host, path, user, pswd, {}]) - - return os.path.join(data.getVar("DL_DIR", d), os.path.basename(url)) - localpath = staticmethod(localpath) - - def go(self, d, urls = []): - """Fetch urls""" - - def md5_sum(basename, data): - """ - Fast and incomplete OVERRIDE implementation for MD5SUM handling - MD5SUM_basename = "SUM" and fallback to MD5SUM_basename - """ - var = "MD5SUM_%s" % basename - return getVar(var, data) or get("MD5SUM",data) - - - def fetch_uri(uri, basename, dl, md5, parm, d): - if os.path.exists(dl): -# file exists, but we didnt complete it.. trying again.. - fetchcmd = data.getVar("RESUMECOMMAND", d, 1) - else: - fetchcmd = data.getVar("FETCHCOMMAND", d, 1) - - bb.note("fetch " + uri) - fetchcmd = fetchcmd.replace("${URI}", uri) - fetchcmd = fetchcmd.replace("${FILE}", basename) - bb.debug(2, "executing " + fetchcmd) - ret = os.system(fetchcmd) - if ret != 0: - return False - - # check if sourceforge did send us to the mirror page - dl_dir = data.getVar("DL_DIR", d, True) - if not os.path.exists(dl): - os.system("rm %s*" % dl) # FIXME shell quote it - bb.debug(2,"sourceforge.net send us to the mirror on %s" % basename) - return False - -# supposedly complete.. write out md5sum - if bb.which(data.getVar('PATH', d), 'md5sum'): - try: - md5pipe = os.popen('md5sum ' + dl) - md5data = (md5pipe.readline().split() or [ "" ])[0] - md5pipe.close() - except OSError: - md5data = "" - md5out = file(md5, 'w') - md5out.write(md5data) - md5out.close() - else: - md5out = file(md5, 'w') - md5out.write("") - md5out.close() - return True - - if not urls: - urls = self.urls - - localdata = data.createCopy(d) - data.setVar('OVERRIDES', "wget:" + data.getVar('OVERRIDES', localdata), localdata) - data.update_data(localdata) - - for uri in urls: - completed = 0 - (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(uri, localdata)) - basename = os.path.basename(path) - dl = self.localpath(uri, d) - dl = data.expand(dl, localdata) - md5 = dl + '.md5' - - if os.path.exists(md5): -# complete, nothing to see here.. - continue - - premirrors = [ i.split() for i in (data.getVar('PREMIRRORS', localdata, 1) or "").split('\n') if i ] - for (find, replace) in premirrors: - newuri = uri_replace(uri, find, replace, d) - if newuri != uri: - if fetch_uri(newuri, basename, dl, md5, parm, localdata): - completed = 1 - break - - if completed: - continue - - if fetch_uri(uri, basename, dl, md5, parm, localdata): - continue - -# try mirrors - mirrors = [ i.split() for i in (data.getVar('MIRRORS', localdata, 1) or "").split('\n') if i ] - for (find, replace) in mirrors: - newuri = uri_replace(uri, find, replace, d) - if newuri != uri: - if fetch_uri(newuri, basename, dl, md5, parm, localdata): - completed = 1 - break - - if not completed: - raise FetchError(uri) - - del localdata - - -methods.append(Wget()) - -class Cvs(Fetch): - """Class to fetch a module or modules from cvs repositories""" - def supports(url, d): - """Check to see if a given url can be fetched with cvs. - Expects supplied url in list form, as outputted by bb.decodeurl(). - """ - (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d)) - return type in ['cvs', 'pserver'] - supports = staticmethod(supports) - - def localpath(url, d): - (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d)) - if "localpath" in parm: -# if user overrides local path, use it. - return parm["localpath"] - - if not "module" in parm: - raise MissingParameterError("cvs method needs a 'module' parameter") - else: - module = parm["module"] - if 'tag' in parm: - tag = parm['tag'] - else: - tag = "" - if 'date' in parm: - date = parm['date'] - else: - if not tag: - date = data.getVar("CVSDATE", d, 1) or data.getVar("DATE", d, 1) - else: - date = "" - - return os.path.join(data.getVar("DL_DIR", d, 1),data.expand('%s_%s_%s_%s.tar.gz' % ( module.replace('/', '.'), host, tag, date), d)) - localpath = staticmethod(localpath) - - def go(self, d, urls = []): - """Fetch urls""" - if not urls: - urls = self.urls - - localdata = data.createCopy(d) - data.setVar('OVERRIDES', "cvs:%s" % data.getVar('OVERRIDES', localdata), localdata) - data.update_data(localdata) - - for loc in urls: - (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(loc, localdata)) - if not "module" in parm: - raise MissingParameterError("cvs method needs a 'module' parameter") - else: - module = parm["module"] - - dlfile = self.localpath(loc, localdata) - dldir = data.getVar('DL_DIR', localdata, 1) -# if local path contains the cvs -# module, consider the dir above it to be the -# download directory -# pos = dlfile.find(module) -# if pos: -# dldir = dlfile[:pos] -# else: -# dldir = os.path.dirname(dlfile) - -# setup cvs options - options = [] - if 'tag' in parm: - tag = parm['tag'] - else: - tag = "" - - if 'date' in parm: - date = parm['date'] - else: - if not tag: - date = data.getVar("CVSDATE", d, 1) or data.getVar("DATE", d, 1) - else: - date = "" - - if "method" in parm: - method = parm["method"] - else: - method = "pserver" - - if "localdir" in parm: - localdir = parm["localdir"] - else: - localdir = module - - cvs_rsh = None - if method == "ext": - if "rsh" in parm: - cvs_rsh = parm["rsh"] - - tarfn = data.expand('%s_%s_%s_%s.tar.gz' % (module.replace('/', '.'), host, tag, date), localdata) - data.setVar('TARFILES', dlfile, localdata) - data.setVar('TARFN', tarfn, localdata) - - dl = os.path.join(dldir, tarfn) - if os.access(dl, os.R_OK): - bb.debug(1, "%s already exists, skipping cvs checkout." % tarfn) - continue - - pn = data.getVar('PN', d, 1) - cvs_tarball_stash = None - if pn: - cvs_tarball_stash = data.getVar('CVS_TARBALL_STASH_%s' % pn, d, 1) - if cvs_tarball_stash == None: - cvs_tarball_stash = data.getVar('CVS_TARBALL_STASH', d, 1) - if cvs_tarball_stash: - fetchcmd = data.getVar("FETCHCOMMAND_wget", d, 1) - uri = cvs_tarball_stash + tarfn - bb.note("fetch " + uri) - fetchcmd = fetchcmd.replace("${URI}", uri) - ret = os.system(fetchcmd) - if ret == 0: - bb.note("Fetched %s from tarball stash, skipping checkout" % tarfn) - continue - - if date: - options.append("-D %s" % date) - if tag: - options.append("-r %s" % tag) - - olddir = os.path.abspath(os.getcwd()) - os.chdir(data.expand(dldir, localdata)) - -# setup cvsroot - if method == "dir": - cvsroot = path - else: - cvsroot = ":" + method + ":" + user - if pswd: - cvsroot += ":" + pswd - cvsroot += "@" + host + ":" + path - - data.setVar('CVSROOT', cvsroot, localdata) - data.setVar('CVSCOOPTS', " ".join(options), localdata) - data.setVar('CVSMODULE', module, localdata) - cvscmd = data.getVar('FETCHCOMMAND', localdata, 1) - cvsupdatecmd = data.getVar('UPDATECOMMAND', localdata, 1) - - if cvs_rsh: - cvscmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvscmd) - cvsupdatecmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvsupdatecmd) - -# create module directory - bb.debug(2, "Fetch: checking for module directory") - pkg=data.expand('${PN}', d) - pkgdir=os.path.join(data.expand('${CVSDIR}', localdata), pkg) - moddir=os.path.join(pkgdir,localdir) - if os.access(os.path.join(moddir,'CVS'), os.R_OK): - bb.note("Update " + loc) -# update sources there - os.chdir(moddir) - myret = os.system(cvsupdatecmd) - else: - bb.note("Fetch " + loc) -# check out sources there - bb.mkdirhier(pkgdir) - os.chdir(pkgdir) - bb.debug(1, "Running %s" % cvscmd) - myret = os.system(cvscmd) - - if myret != 0: - try: - os.rmdir(moddir) - except OSError: - pass - raise FetchError(module) - - os.chdir(moddir) - os.chdir('..') -# tar them up to a defined filename - myret = os.system("tar -czf %s %s" % (os.path.join(dldir,tarfn), os.path.basename(moddir))) - if myret != 0: - try: - os.unlink(tarfn) - except OSError: - pass - os.chdir(olddir) - del localdata - -methods.append(Cvs()) - -class Bk(Fetch): - def supports(url, d): - """Check to see if a given url can be fetched via bitkeeper. - Expects supplied url in list form, as outputted by bb.decodeurl(). - """ - (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d)) - return type in ['bk'] - supports = staticmethod(supports) - -methods.append(Bk()) - -class Local(Fetch): - def supports(url, d): - """Check to see if a given url can be fetched in the local filesystem. - Expects supplied url in list form, as outputted by bb.decodeurl(). - """ - (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d)) - return type in ['file','patch'] - supports = staticmethod(supports) - - def localpath(url, d): - """Return the local filename of a given url assuming a successful fetch. - """ - path = url.split("://")[1] - newpath = path - if path[0] != "/": - filespath = data.getVar('FILESPATH', d, 1) - if filespath: - newpath = bb.which(filespath, path) - if not newpath: - filesdir = data.getVar('FILESDIR', d, 1) - if filesdir: - newpath = os.path.join(filesdir, path) - return newpath - localpath = staticmethod(localpath) - - def go(self, urls = []): - """Fetch urls (no-op for Local method)""" -# no need to fetch local files, we'll deal with them in place. - return 1 - -methods.append(Local()) - -class Svn(Fetch): - """Class to fetch a module or modules from svn repositories""" - def supports(url, d): - """Check to see if a given url can be fetched with svn. - Expects supplied url in list form, as outputted by bb.decodeurl(). - """ - (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d)) - return type in ['svn'] - supports = staticmethod(supports) - - def localpath(url, d): - (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d)) - if "localpath" in parm: -# if user overrides local path, use it. - return parm["localpath"] - - if not "module" in parm: - raise MissingParameterError("svn method needs a 'module' parameter") - else: - module = parm["module"] - if 'rev' in parm: - revision = parm['rev'] - else: - revision = "" - - date = data.getVar("CVSDATE", d, 1) or data.getVar("DATE", d, 1) - - return os.path.join(data.getVar("DL_DIR", d, 1),data.expand('%s_%s_%s_%s_%s.tar.gz' % ( module.replace('/', '.'), host, path.replace('/','.'), revision, date), d)) - localpath = staticmethod(localpath) - - def go(self, d, urls = []): - """Fetch urls""" - if not urls: - urls = self.urls - - localdata = data.createCopy(d) - data.setVar('OVERRIDES', "svn:%s" % data.getVar('OVERRIDES', localdata), localdata) - data.update_data(localdata) - - for loc in urls: - (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(loc, localdata)) - if not "module" in parm: - raise MissingParameterError("svn method needs a 'module' parameter") - else: - module = parm["module"] - - dlfile = self.localpath(loc, localdata) - dldir = data.getVar('DL_DIR', localdata, 1) -# if local path contains the svn -# module, consider the dir above it to be the -# download directory -# pos = dlfile.find(module) -# if pos: -# dldir = dlfile[:pos] -# else: -# dldir = os.path.dirname(dlfile) - -# setup svn options - options = [] - if 'rev' in parm: - revision = parm['rev'] - else: - revision = "" - - date = data.getVar("CVSDATE", d, 1) or data.getVar("DATE", d, 1) - - if "method" in parm: - method = parm["method"] - else: - method = "pserver" - - if "proto" in parm: - proto = parm["proto"] - else: - proto = "svn" - - svn_rsh = None - if method == "ext": - if "rsh" in parm: - svn_rsh = parm["rsh"] - - tarfn = data.expand('%s_%s_%s_%s_%s.tar.gz' % (module.replace('/', '.'), host, path.replace('/', '.'), revision, date), localdata) - data.setVar('TARFILES', dlfile, localdata) - data.setVar('TARFN', tarfn, localdata) - - dl = os.path.join(dldir, tarfn) - if os.access(dl, os.R_OK): - bb.debug(1, "%s already exists, skipping svn checkout." % tarfn) - continue - - svn_tarball_stash = data.getVar('CVS_TARBALL_STASH', d, 1) - if svn_tarball_stash: - fetchcmd = data.getVar("FETCHCOMMAND_wget", d, 1) - uri = svn_tarball_stash + tarfn - bb.note("fetch " + uri) - fetchcmd = fetchcmd.replace("${URI}", uri) - ret = os.system(fetchcmd) - if ret == 0: - bb.note("Fetched %s from tarball stash, skipping checkout" % tarfn) - continue - - olddir = os.path.abspath(os.getcwd()) - os.chdir(data.expand(dldir, localdata)) - -# setup svnroot -# svnroot = ":" + method + ":" + user -# if pswd: -# svnroot += ":" + pswd - svnroot = host + path - - data.setVar('SVNROOT', svnroot, localdata) - data.setVar('SVNCOOPTS', " ".join(options), localdata) - data.setVar('SVNMODULE', module, localdata) - svncmd = data.getVar('FETCHCOMMAND', localdata, 1) - svncmd = "svn co -r {%s} %s://%s/%s" % (date, proto, svnroot, module) - - if revision: - svncmd = "svn co -r %s %s://%s/%s" % (revision, proto, svnroot, module) - if svn_rsh: - svncmd = "svn_RSH=\"%s\" %s" % (svn_rsh, svncmd) - -# create temp directory - bb.debug(2, "Fetch: creating temporary directory") - bb.mkdirhier(data.expand('${WORKDIR}', localdata)) - data.setVar('TMPBASE', data.expand('${WORKDIR}/oesvn.XXXXXX', localdata), localdata) - tmppipe = os.popen(data.getVar('MKTEMPDIRCMD', localdata, 1) or "false") - tmpfile = tmppipe.readline().strip() - if not tmpfile: - bb.error("Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.") - raise FetchError(module) - -# check out sources there - os.chdir(tmpfile) - bb.note("Fetch " + loc) - bb.debug(1, "Running %s" % svncmd) - myret = os.system(svncmd) - if myret != 0: - try: - os.rmdir(tmpfile) - except OSError: - pass - raise FetchError(module) - - os.chdir(os.path.join(tmpfile, os.path.dirname(module))) -# tar them up to a defined filename - myret = os.system("tar -czf %s %s" % (os.path.join(dldir,tarfn), os.path.basename(module))) - if myret != 0: - try: - os.unlink(tarfn) - except OSError: - pass -# cleanup - os.system('rm -rf %s' % tmpfile) - os.chdir(olddir) - del localdata - -methods.append(Svn()) -- cgit 1.2.3-korg