bitbake/bb/fetch:

    Patch courtsey RP for #438. Move the fetchers
    into a separate module to ease the writing of
    new fetchers and to separate the individual
    implementations.

bitbake/MANIFEST:
    added the new files
    
bitbake/setup.py:
    added the fetch module

11 files changed, 1011 insertions, 669 deletions

diff --git a/ChangeLog b/ChangeLog
index db503ffb1..f4b65ff83 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,7 @@
+Changes in BitBake 1.3.?:
+	- Create a new Fetcher module to ease the
+	  development of new Fetchers.
+
 Changes in BitBake 1.3.2:
 	- reintegration of make.py into BitBake
 	- bbread is gone, use bitbake -e
diff --git a/MANIFEST b/MANIFEST
index 71e3605a4..cf0aac99d 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -9,7 +9,13 @@ lib/bb/build.py
 lib/bb/data.py
 lib/bb/data_smart.py
 lib/bb/event.py
-lib/bb/fetch.py
+lib/bb/fetch/bk.py
+lib/bb/fetch/cvs.py
+lib/bb/fetch/git.py
+lib/bb/fetch/__init__.py
+lib/bb/fetch/local.py
+lib/bb/fetch/svn.py
+lib/bb/fetch/wget.py
 lib/bb/manifest.py
 lib/bb/parse/__init__.py
 lib/bb/parse/parse_py/BBHandler.py
diff --git a/lib/bb/fetch.py b/lib/bb/fetch.py
deleted file mode 100644
index a95c6694e..000000000
--- a/lib/bb/fetch.py
+++ /dev/null
@@ -1,667 +0,0 @@
-#!/usr/bin/env python
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-"""
-BitBake 'Fetch' implementations
-
-Classes for obtaining upstream sources for the
-BitBake build tools.
-
-Copyright (C) 2003, 2004  Chris Larson
-
-This program is free software; you can redistribute it and/or modify it under
-the terms of the GNU General Public License as published by the Free Software
-Foundation; either version 2 of the License, or (at your option) any later
-version.
-
-This program is distributed in the hope that it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along with
-this program; if not, write to the Free Software Foundation, Inc., 59 Temple
-Place, Suite 330, Boston, MA 02111-1307 USA. 
-
-Based on functions from the base bb module, Copyright 2003 Holger Schurig
-"""
-
-import os, re
-import bb
-from   bb import data
-
-class FetchError(Exception):
-    """Exception raised when a download fails"""
-
-class NoMethodError(Exception):
-    """Exception raised when there is no method to obtain a supplied url or set of urls"""
-
-class MissingParameterError(Exception):
-    """Exception raised when a fetch method is missing a critical parameter in the url"""
-
-class MD5SumError(Exception):
-    """Exception raised when a MD5SUM of a file does not match the expected one"""
-
-def uri_replace(uri, uri_find, uri_replace, d):
-#   bb.note("uri_replace: operating on %s" % uri)
-    if not uri or not uri_find or not uri_replace:
-        bb.debug(1, "uri_replace: passed an undefined value, not replacing")
-    uri_decoded = list(bb.decodeurl(uri))
-    uri_find_decoded = list(bb.decodeurl(uri_find))
-    uri_replace_decoded = list(bb.decodeurl(uri_replace))
-    result_decoded = ['','','','','',{}]
-    for i in uri_find_decoded:
-        loc = uri_find_decoded.index(i)
-        result_decoded[loc] = uri_decoded[loc]
-        import types
-        if type(i) == types.StringType:
-            import re
-            if (re.match(i, uri_decoded[loc])):
-                result_decoded[loc] = re.sub(i, uri_replace_decoded[loc], uri_decoded[loc])
-                if uri_find_decoded.index(i) == 2:
-                    if d:
-                        localfn = bb.fetch.localpath(uri, d)
-                        if localfn:
-                            result_decoded[loc] = os.path.dirname(result_decoded[loc]) + "/" + os.path.basename(bb.fetch.localpath(uri, d))
-#                       bb.note("uri_replace: matching %s against %s and replacing with %s" % (i, uri_decoded[loc], uri_replace_decoded[loc]))
-            else:
-#               bb.note("uri_replace: no match")
-                return uri
-#           else:
-#               for j in i.keys():
-#                   FIXME: apply replacements against options
-    return bb.encodeurl(result_decoded)
-
-methods = []
-
-def init(urls = [], d = None):
-    if d == None:
-        bb.debug(2,"BUG init called with None as data object!!!")
-        return
-
-    for m in methods:
-        m.urls = []
-
-    for u in urls:
-        for m in methods:
-            m.data = d
-            if m.supports(u, d):
-                m.urls.append(u)
-
-def go(d):
-    """Fetch all urls"""
-    for m in methods:
-        if m.urls:
-            m.go(d)
-
-def localpaths(d):
-    """Return a list of the local filenames, assuming successful fetch"""
-    local = []
-    for m in methods:
-        for u in m.urls:
-            local.append(m.localpath(u, d))
-    return local
-
-def localpath(url, d):
-    for m in methods:
-        if m.supports(url, d):
-            return m.localpath(url, d)
-    return url
-
-class Fetch(object):
-    """Base class for 'fetch'ing data"""
-
-    def __init__(self, urls = []):
-        self.urls = []
-        for url in urls:
-            if self.supports(bb.decodeurl(url), d) is 1:
-                self.urls.append(url)
-
-    def supports(url, d):
-        """Check to see if this fetch class supports a given url.
-           Expects supplied url in list form, as outputted by bb.decodeurl().
-        """
-        return 0
-    supports = staticmethod(supports)
-
-    def localpath(url, d):
-        """Return the local filename of a given url assuming a successful fetch.
-        """
-        return url
-    localpath = staticmethod(localpath)
-
-    def setUrls(self, urls):
-        self.__urls = urls
-
-    def getUrls(self):
-        return self.__urls
-
-    urls = property(getUrls, setUrls, None, "Urls property")
-
-    def setData(self, data):
-        self.__data = data
-
-    def getData(self):
-        return self.__data
-
-    data = property(getData, setData, None, "Data property")
-
-    def go(self, urls = []):
-        """Fetch urls"""
-        raise NoMethodError("Missing implementation for url")
-
-class Wget(Fetch):
-    """Class to fetch urls via 'wget'"""
-    def supports(url, d):
-        """Check to see if a given url can be fetched using wget.
-           Expects supplied url in list form, as outputted by bb.decodeurl().
-        """
-        (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
-        return type in ['http','https','ftp']
-    supports = staticmethod(supports)
-
-    def localpath(url, d):
-#       strip off parameters
-        (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
-        if "localpath" in parm:
-#           if user overrides local path, use it.
-            return parm["localpath"]
-        url = bb.encodeurl([type, host, path, user, pswd, {}])
-
-        return os.path.join(data.getVar("DL_DIR", d), os.path.basename(url))
-    localpath = staticmethod(localpath)
-
-    def go(self, d, urls = []):
-        """Fetch urls"""
-
-        def md5_sum(basename, d):
-            """
-            Fast and incomplete OVERRIDE implementation for MD5SUM handling
-            MD5SUM_basename = "SUM" and fallback to MD5SUM_basename
-            """
-            var = "MD5SUM_%s" % basename
-            return data.getVar(var, d) or data.getVar("MD5SUM", d)
-
-        def verify_md5sum(wanted_sum, got_sum):
-            """
-            Verify the md5sum we wanted with the one we got
-            """
-            if not wanted_sum:
-                return True
-
-            return wanted_sum == got_sum
-
-        def fetch_uri(uri, basename, dl, md5, parm, d):
-            # the MD5 sum we want to verify
-            wanted_md5sum = md5_sum(basename, d)
-            if os.path.exists(dl):
-#               file exists, but we didnt complete it.. trying again..
-                fetchcmd = data.getVar("RESUMECOMMAND", d, 1)
-            else:
-                fetchcmd = data.getVar("FETCHCOMMAND", d, 1)
-
-            bb.note("fetch " + uri)
-            fetchcmd = fetchcmd.replace("${URI}", uri)
-            fetchcmd = fetchcmd.replace("${FILE}", basename)
-            bb.debug(2, "executing " + fetchcmd)
-            ret = os.system(fetchcmd)
-            if ret != 0:
-                return False
-
-            # check if sourceforge did send us to the mirror page
-            dl_dir = data.getVar("DL_DIR", d, True)
-            if not os.path.exists(dl):
-                os.system("rm %s*" % dl) # FIXME shell quote it
-                bb.debug(2,"sourceforge.net send us to the mirror on %s" % basename)
-                return False
-
-#           supposedly complete.. write out md5sum
-            if bb.which(data.getVar('PATH', d), 'md5sum'):
-                try:
-                    md5pipe = os.popen('md5sum ' + dl)
-                    md5data = (md5pipe.readline().split() or [ "" ])[0]
-                    md5pipe.close()
-                except OSError:
-                    md5data = ""
-
-            # verify the md5sum
-            if not verify_md5sum(wanted_md5sum, md5data):
-                raise MD5SumError(uri)
-
-            md5out = file(md5, 'w')
-            md5out.write(md5data)
-            md5out.close()
-            return True
-
-        if not urls:
-            urls = self.urls
-
-        localdata = data.createCopy(d)
-        data.setVar('OVERRIDES', "wget:" + data.getVar('OVERRIDES', localdata), localdata)
-        data.update_data(localdata)
-
-        for uri in urls:
-            completed = 0
-            (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(uri, localdata))
-            basename = os.path.basename(path)
-            dl = self.localpath(uri, d)
-            dl = data.expand(dl, localdata)
-            md5 = dl + '.md5'
-
-            if os.path.exists(md5):
-#               complete, nothing to see here..
-                continue
-
-            premirrors = [ i.split() for i in (data.getVar('PREMIRRORS', localdata, 1) or "").split('\n') if i ]
-            for (find, replace) in premirrors:
-                newuri = uri_replace(uri, find, replace, d)
-                if newuri != uri:
-                    if fetch_uri(newuri, basename, dl, md5, parm, localdata):
-                        completed = 1
-                        break
-
-            if completed:
-                continue
-
-            if fetch_uri(uri, basename, dl, md5, parm, localdata):
-                continue
-
-#           try mirrors
-            mirrors = [ i.split() for i in (data.getVar('MIRRORS', localdata, 1) or "").split('\n') if i ]
-            for (find, replace) in mirrors:
-                newuri = uri_replace(uri, find, replace, d)
-                if newuri != uri:
-                    if fetch_uri(newuri, basename, dl, md5, parm, localdata):
-                        completed = 1
-                        break
-
-            if not completed:
-                raise FetchError(uri)
-
-        del localdata
-
-
-methods.append(Wget())
-
-class Cvs(Fetch):
-    """Class to fetch a module or modules from cvs repositories"""
-    def supports(url, d):
-        """Check to see if a given url can be fetched with cvs.
-           Expects supplied url in list form, as outputted by bb.decodeurl().
-        """
-        (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
-        return type in ['cvs', 'pserver']
-    supports = staticmethod(supports)
-
-    def localpath(url, d):
-        (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
-        if "localpath" in parm:
-#           if user overrides local path, use it.
-            return parm["localpath"]
-
-        if not "module" in parm:
-            raise MissingParameterError("cvs method needs a 'module' parameter")
-        else:
-            module = parm["module"]
-        if 'tag' in parm:
-            tag = parm['tag']
-        else:
-            tag = ""
-        if 'date' in parm:
-            date = parm['date']
-        else:
-            if not tag:
-                date = data.getVar("CVSDATE", d, 1) or data.getVar("DATE", d, 1)
-            else:
-                date = ""
-
-        return os.path.join(data.getVar("DL_DIR", d, 1),data.expand('%s_%s_%s_%s.tar.gz' % ( module.replace('/', '.'), host, tag, date), d))
-    localpath = staticmethod(localpath)
-
-    def go(self, d, urls = []):
-        """Fetch urls"""
-        if not urls:
-            urls = self.urls
-
-        localdata = data.createCopy(d)
-        data.setVar('OVERRIDES', "cvs:%s" % data.getVar('OVERRIDES', localdata), localdata)
-        data.update_data(localdata)
-
-        for loc in urls:
-            (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(loc, localdata))
-            if not "module" in parm:
-                raise MissingParameterError("cvs method needs a 'module' parameter")
-            else:
-                module = parm["module"]
-
-            dlfile = self.localpath(loc, localdata)
-            dldir = data.getVar('DL_DIR', localdata, 1)
-#           if local path contains the cvs
-#           module, consider the dir above it to be the
-#           download directory
-#           pos = dlfile.find(module)
-#           if pos:
-#               dldir = dlfile[:pos]
-#           else:
-#               dldir = os.path.dirname(dlfile)
-
-#           setup cvs options
-            options = []
-            if 'tag' in parm:
-                tag = parm['tag']
-            else:
-                tag = ""
-
-            if 'date' in parm:
-                date = parm['date']
-            else:
-                if not tag:
-                    date = data.getVar("CVSDATE", d, 1) or data.getVar("DATE", d, 1)
-                else:
-                    date = ""
-
-            if "method" in parm:
-                method = parm["method"]
-            else:
-                method = "pserver"
-
-            if "localdir" in parm:
-                localdir = parm["localdir"]
-            else:
-                localdir = module
-
-            cvs_rsh = None
-            if method == "ext":
-                if "rsh" in parm:
-                    cvs_rsh = parm["rsh"]
-
-            tarfn = data.expand('%s_%s_%s_%s.tar.gz' % (module.replace('/', '.'), host, tag, date), localdata)
-            data.setVar('TARFILES', dlfile, localdata)
-            data.setVar('TARFN', tarfn, localdata)
-
-            dl = os.path.join(dldir, tarfn)
-            if os.access(dl, os.R_OK):
-                bb.debug(1, "%s already exists, skipping cvs checkout." % tarfn)
-                continue
-
-            pn = data.getVar('PN', d, 1)
-            cvs_tarball_stash = None
-            if pn:
-                cvs_tarball_stash = data.getVar('CVS_TARBALL_STASH_%s' % pn, d, 1)
-            if cvs_tarball_stash == None:
-                cvs_tarball_stash = data.getVar('CVS_TARBALL_STASH', d, 1)
-            if cvs_tarball_stash:
-                fetchcmd = data.getVar("FETCHCOMMAND_wget", d, 1)
-                uri = cvs_tarball_stash + tarfn
-                bb.note("fetch " + uri)
-                fetchcmd = fetchcmd.replace("${URI}", uri)
-                ret = os.system(fetchcmd)
-                if ret == 0:
-                    bb.note("Fetched %s from tarball stash, skipping checkout" % tarfn)
-                    continue
-
-            if date:
-                options.append("-D %s" % date)
-            if tag:
-                options.append("-r %s" % tag)
-
-            olddir = os.path.abspath(os.getcwd())
-            os.chdir(data.expand(dldir, localdata))
-
-#           setup cvsroot
-            if method == "dir":
-                cvsroot = path
-            else:
-                cvsroot = ":" + method + ":" + user
-                if pswd:
-                    cvsroot += ":" + pswd
-                cvsroot += "@" + host + ":" + path
-
-            data.setVar('CVSROOT', cvsroot, localdata)
-            data.setVar('CVSCOOPTS', " ".join(options), localdata)
-            data.setVar('CVSMODULE', module, localdata)
-            cvscmd = data.getVar('FETCHCOMMAND', localdata, 1)
-            cvsupdatecmd = data.getVar('UPDATECOMMAND', localdata, 1)
-
-            if cvs_rsh:
-                cvscmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvscmd)
-                cvsupdatecmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvsupdatecmd)
-
-#           create module directory
-            bb.debug(2, "Fetch: checking for module directory")
-            pkg=data.expand('${PN}', d)
-            pkgdir=os.path.join(data.expand('${CVSDIR}', localdata), pkg)
-            moddir=os.path.join(pkgdir,localdir)
-            if os.access(os.path.join(moddir,'CVS'), os.R_OK):
-                bb.note("Update " + loc)
-#               update sources there
-                os.chdir(moddir)
-                myret = os.system(cvsupdatecmd)
-            else:
-                bb.note("Fetch " + loc)
-#               check out sources there
-                bb.mkdirhier(pkgdir)
-                os.chdir(pkgdir)
-                bb.debug(1, "Running %s" % cvscmd)
-                myret = os.system(cvscmd)
-
-            if myret != 0:
-                try:
-                    os.rmdir(moddir)
-                except OSError:
-                    pass
-                raise FetchError(module)
-
-            os.chdir(moddir)
-            os.chdir('..')
-#           tar them up to a defined filename
-            myret = os.system("tar -czf %s %s" % (os.path.join(dldir,tarfn), os.path.basename(moddir)))
-            if myret != 0:
-                try:
-                    os.unlink(tarfn)
-                except OSError:
-                    pass
-            os.chdir(olddir)
-        del localdata
-
-methods.append(Cvs())
-
-class Bk(Fetch):
-    def supports(url, d):
-        """Check to see if a given url can be fetched via bitkeeper.
-           Expects supplied url in list form, as outputted by bb.decodeurl().
-        """
-        (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
-        return type in ['bk']
-    supports = staticmethod(supports)
-
-methods.append(Bk())
-
-class Local(Fetch):
-    def supports(url, d):
-        """Check to see if a given url can be fetched in the local filesystem.
-           Expects supplied url in list form, as outputted by bb.decodeurl().
-        """
-        (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
-        return type in ['file','patch']
-    supports = staticmethod(supports)
-
-    def localpath(url, d):
-        """Return the local filename of a given url assuming a successful fetch.
-        """
-        path = url.split("://")[1]
-        newpath = path
-        if path[0] != "/":
-            filespath = data.getVar('FILESPATH', d, 1)
-            if filespath:
-                newpath = bb.which(filespath, path)
-            if not newpath:
-                filesdir = data.getVar('FILESDIR', d, 1)
-                if filesdir:
-                    newpath = os.path.join(filesdir, path)
-        return newpath
-    localpath = staticmethod(localpath)
-
-    def go(self, urls = []):
-        """Fetch urls (no-op for Local method)"""
-#       no need to fetch local files, we'll deal with them in place.
-        return 1
-
-methods.append(Local())
-
-class Svn(Fetch):
-    """Class to fetch a module or modules from svn repositories"""
-    def supports(url, d):
-        """Check to see if a given url can be fetched with svn.
-           Expects supplied url in list form, as outputted by bb.decodeurl().
-        """
-        (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
-        return type in ['svn']
-    supports = staticmethod(supports)
-
-    def localpath(url, d):
-        (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
-        if "localpath" in parm:
-#           if user overrides local path, use it.
-            return parm["localpath"]
-
-        if not "module" in parm:
-            raise MissingParameterError("svn method needs a 'module' parameter")
-        else:
-            module = parm["module"]
-        if 'rev' in parm:
-            revision = parm['rev']
-        else:
-            revision = ""
-
-        date = data.getVar("CVSDATE", d, 1) or data.getVar("DATE", d, 1)
-
-        return os.path.join(data.getVar("DL_DIR", d, 1),data.expand('%s_%s_%s_%s.tar.gz' % ( module.replace('/', '.'), host, revision, date), d))
-    localpath = staticmethod(localpath)
-
-    def go(self, d, urls = []):
-        """Fetch urls"""
-        if not urls:
-            urls = self.urls
-
-        localdata = data.createCopy(d)
-        data.setVar('OVERRIDES', "svn:%s" % data.getVar('OVERRIDES', localdata), localdata)
-        data.update_data(localdata)
-
-        for loc in urls:
-            (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(loc, localdata))
-            if not "module" in parm:
-                raise MissingParameterError("svn method needs a 'module' parameter")
-            else:
-                module = parm["module"]
-
-            dlfile = self.localpath(loc, localdata)
-            dldir = data.getVar('DL_DIR', localdata, 1)
-#           if local path contains the svn
-#           module, consider the dir above it to be the
-#           download directory
-#           pos = dlfile.find(module)
-#           if pos:
-#               dldir = dlfile[:pos]
-#           else:
-#               dldir = os.path.dirname(dlfile)
-
-#           setup svn options
-            options = []
-            if 'rev' in parm:
-                revision = parm['rev']
-            else:
-                revision = ""
-
-            date = data.getVar("CVSDATE", d, 1) or data.getVar("DATE", d, 1)
-
-            if "method" in parm:
-                method = parm["method"]
-            else:
-                method = "pserver"
-
-            if "proto" in parm:
-                proto = parm["proto"]
-            else:
-                proto = "svn"
-
-            svn_rsh = None
-            if method == "ext":
-                if "rsh" in parm:
-                    svn_rsh = parm["rsh"]
-
-            tarfn = data.expand('%s_%s_%s_%s.tar.gz' % (module.replace('/', '.'), host, revision, date), localdata)
-            data.setVar('TARFILES', dlfile, localdata)
-            data.setVar('TARFN', tarfn, localdata)
-
-            dl = os.path.join(dldir, tarfn)
-            if os.access(dl, os.R_OK):
-                bb.debug(1, "%s already exists, skipping svn checkout." % tarfn)
-                continue
-
-            svn_tarball_stash = data.getVar('CVS_TARBALL_STASH', d, 1)
-            if svn_tarball_stash:
-                fetchcmd = data.getVar("FETCHCOMMAND_wget", d, 1)
-                uri = svn_tarball_stash + tarfn
-                bb.note("fetch " + uri)
-                fetchcmd = fetchcmd.replace("${URI}", uri)
-                ret = os.system(fetchcmd)
-                if ret == 0:
-                    bb.note("Fetched %s from tarball stash, skipping checkout" % tarfn)
-                    continue
-
-            olddir = os.path.abspath(os.getcwd())
-            os.chdir(data.expand(dldir, localdata))
-
-#           setup svnroot
-#            svnroot = ":" + method + ":" + user
-#            if pswd:
-#                svnroot += ":" + pswd
-            svnroot = host + path
-
-            data.setVar('SVNROOT', svnroot, localdata)
-            data.setVar('SVNCOOPTS', " ".join(options), localdata)
-            data.setVar('SVNMODULE', module, localdata)
-            svncmd = data.getVar('FETCHCOMMAND', localdata, 1)
-            svncmd = "svn co %s://%s/%s" % (proto, svnroot, module)
-
-            if revision:
-                svncmd = "svn co -r %s %s://%s/%s" % (revision, proto, svnroot, module)
-            if svn_rsh:
-                svncmd = "svn_RSH=\"%s\" %s" % (svn_rsh, svncmd)
-
-#           create temp directory
-            bb.debug(2, "Fetch: creating temporary directory")
-            bb.mkdirhier(data.expand('${WORKDIR}', localdata))
-            data.setVar('TMPBASE', data.expand('${WORKDIR}/oesvn.XXXXXX', localdata), localdata)
-            tmppipe = os.popen(data.getVar('MKTEMPDIRCMD', localdata, 1) or "false")
-            tmpfile = tmppipe.readline().strip()
-            if not tmpfile:
-                bb.error("Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.")
-                raise FetchError(module)
-
-#           check out sources there
-            os.chdir(tmpfile)
-            bb.note("Fetch " + loc)
-            bb.debug(1, "Running %s" % svncmd)
-            myret = os.system(svncmd)
-            if myret != 0:
-                try:
-                    os.rmdir(tmpfile)
-                except OSError:
-                    pass
-                raise FetchError(module)
-
-            os.chdir(os.path.join(tmpfile, os.path.dirname(module)))
-#           tar them up to a defined filename
-            myret = os.system("tar -czf %s %s" % (os.path.join(dldir,tarfn), os.path.basename(module)))
-            if myret != 0:
-                try:
-                    os.unlink(tarfn)
-                except OSError:
-                    pass
-#           cleanup
-            os.system('rm -rf %s' % tmpfile)
-            os.chdir(olddir)
-        del localdata
-
-methods.append(Svn())
diff --git a/lib/bb/fetch/__init__.py b/lib/bb/fetch/__init__.py
new file mode 100644
index 000000000..d68a1f166
--- /dev/null
+++ b/lib/bb/fetch/__init__.py
@@ -0,0 +1,166 @@
+#!/usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+"""
+BitBake 'Fetch' implementations
+
+Classes for obtaining upstream sources for the
+BitBake build tools.
+
+Copyright (C) 2003, 2004  Chris Larson
+
+This program is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free Software
+Foundation; either version 2 of the License, or (at your option) any later
+version.
+
+This program is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+Place, Suite 330, Boston, MA 02111-1307 USA. 
+
+Based on functions from the base bb module, Copyright 2003 Holger Schurig
+"""
+
+import os, re
+import bb
+from   bb import data
+
+class FetchError(Exception):
+    """Exception raised when a download fails"""
+
+class NoMethodError(Exception):
+    """Exception raised when there is no method to obtain a supplied url or set of urls"""
+
+class MissingParameterError(Exception):
+    """Exception raised when a fetch method is missing a critical parameter in the url"""
+
+class MD5SumError(Exception):
+    """Exception raised when a MD5SUM of a file does not match the expected one"""
+
+def uri_replace(uri, uri_find, uri_replace, d):
+#   bb.note("uri_replace: operating on %s" % uri)
+    if not uri or not uri_find or not uri_replace:
+        bb.debug(1, "uri_replace: passed an undefined value, not replacing")
+    uri_decoded = list(bb.decodeurl(uri))
+    uri_find_decoded = list(bb.decodeurl(uri_find))
+    uri_replace_decoded = list(bb.decodeurl(uri_replace))
+    result_decoded = ['','','','','',{}]
+    for i in uri_find_decoded:
+        loc = uri_find_decoded.index(i)
+        result_decoded[loc] = uri_decoded[loc]
+        import types
+        if type(i) == types.StringType:
+            import re
+            if (re.match(i, uri_decoded[loc])):
+                result_decoded[loc] = re.sub(i, uri_replace_decoded[loc], uri_decoded[loc])
+                if uri_find_decoded.index(i) == 2:
+                    if d:
+                        localfn = bb.fetch.localpath(uri, d)
+                        if localfn:
+                            result_decoded[loc] = os.path.dirname(result_decoded[loc]) + "/" + os.path.basename(bb.fetch.localpath(uri, d))
+#                       bb.note("uri_replace: matching %s against %s and replacing with %s" % (i, uri_decoded[loc], uri_replace_decoded[loc]))
+            else:
+#               bb.note("uri_replace: no match")
+                return uri
+#           else:
+#               for j in i.keys():
+#                   FIXME: apply replacements against options
+    return bb.encodeurl(result_decoded)
+
+methods = []
+
+def init(urls = [], d = None):
+    if d == None:
+        bb.debug(2,"BUG init called with None as data object!!!")
+        return
+
+    for m in methods:
+        m.urls = []
+
+    for u in urls:
+        for m in methods:
+            m.data = d
+            if m.supports(u, d):
+                m.urls.append(u)
+
+def go(d):
+    """Fetch all urls"""
+    for m in methods:
+        if m.urls:
+            m.go(d)
+
+def localpaths(d):
+    """Return a list of the local filenames, assuming successful fetch"""
+    local = []
+    for m in methods:
+        for u in m.urls:
+            local.append(m.localpath(u, d))
+    return local
+
+def localpath(url, d):
+    for m in methods:
+        if m.supports(url, d):
+            return m.localpath(url, d)
+    return url
+
+class Fetch(object):
+    """Base class for 'fetch'ing data"""
+
+    def __init__(self, urls = []):
+        self.urls = []
+        for url in urls:
+            if self.supports(bb.decodeurl(url), d) is 1:
+                self.urls.append(url)
+
+    def supports(url, d):
+        """Check to see if this fetch class supports a given url.
+           Expects supplied url in list form, as outputted by bb.decodeurl().
+        """
+        return 0
+    supports = staticmethod(supports)
+
+    def localpath(url, d):
+        """Return the local filename of a given url assuming a successful fetch.
+        """
+        return url
+    localpath = staticmethod(localpath)
+
+    def setUrls(self, urls):
+        self.__urls = urls
+
+    def getUrls(self):
+        return self.__urls
+
+    urls = property(getUrls, setUrls, None, "Urls property")
+
+    def setData(self, data):
+        self.__data = data
+
+    def getData(self):
+        return self.__data
+
+    data = property(getData, setData, None, "Data property")
+
+    def go(self, urls = []):
+        """Fetch urls"""
+        raise NoMethodError("Missing implementation for url")
+
+#if __name__ == "__main__":
+
+import bk
+import cvs
+import git
+import local
+import svn
+import wget
+
+methods.append(bk.Bk())
+methods.append(cvs.Cvs())
+methods.append(git.Git())
+methods.append(local.Local())
+methods.append(svn.Svn())
+methods.append(wget.Wget())
diff --git a/lib/bb/fetch/bk.py b/lib/bb/fetch/bk.py
new file mode 100644
index 000000000..6bd6c018f
--- /dev/null
+++ b/lib/bb/fetch/bk.py
@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+"""
+BitBake 'Fetch' implementations
+
+Classes for obtaining upstream sources for the
+BitBake build tools.
+
+Copyright (C) 2003, 2004  Chris Larson
+
+This program is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free Software
+Foundation; either version 2 of the License, or (at your option) any later
+version.
+
+This program is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+Place, Suite 330, Boston, MA 02111-1307 USA. 
+
+Based on functions from the base bb module, Copyright 2003 Holger Schurig
+"""
+
+import os, re
+import bb
+from   bb import data
+from   bb.fetch import Fetch
+
+class Bk(Fetch):
+    def supports(url, d):
+        """Check to see if a given url can be fetched via bitkeeper.
+           Expects supplied url in list form, as outputted by bb.decodeurl().
+        """
+        (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
+        return type in ['bk']
+    supports = staticmethod(supports)
diff --git a/lib/bb/fetch/cvs.py b/lib/bb/fetch/cvs.py
new file mode 100644
index 000000000..7935744e7
--- /dev/null
+++ b/lib/bb/fetch/cvs.py
@@ -0,0 +1,214 @@
+#!/usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+"""
+BitBake 'Fetch' implementations
+
+Classes for obtaining upstream sources for the
+BitBake build tools.
+
+Copyright (C) 2003, 2004  Chris Larson
+
+This program is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free Software
+Foundation; either version 2 of the License, or (at your option) any later
+version.
+
+This program is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+Place, Suite 330, Boston, MA 02111-1307 USA. 
+
+Based on functions from the base bb module, Copyright 2003 Holger Schurig
+"""
+
+import os, re
+import bb
+from   bb import data
+from   bb.fetch import Fetch
+from   bb.fetch import FetchError
+from   bb.fetch import MissingParameterError
+
+class Cvs(Fetch):
+    """Class to fetch a module or modules from cvs repositories"""
+    def supports(url, d):
+        """Check to see if a given url can be fetched with cvs.
+           Expects supplied url in list form, as outputted by bb.decodeurl().
+        """
+        (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
+        return type in ['cvs', 'pserver']
+    supports = staticmethod(supports)
+
+    def localpath(url, d):
+        (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
+        if "localpath" in parm:
+#           if user overrides local path, use it.
+            return parm["localpath"]
+
+        if not "module" in parm:
+            raise MissingParameterError("cvs method needs a 'module' parameter")
+        else:
+            module = parm["module"]
+        if 'tag' in parm:
+            tag = parm['tag']
+        else:
+            tag = ""
+        if 'date' in parm:
+            date = parm['date']
+        else:
+            if not tag:
+                date = data.getVar("CVSDATE", d, 1) or data.getVar("DATE", d, 1)
+            else:
+                date = ""
+
+        return os.path.join(data.getVar("DL_DIR", d, 1),data.expand('%s_%s_%s_%s.tar.gz' % ( module.replace('/', '.'), host, tag, date), d))
+    localpath = staticmethod(localpath)
+
+    def go(self, d, urls = []):
+        """Fetch urls"""
+        if not urls:
+            urls = self.urls
+
+        localdata = data.createCopy(d)
+        data.setVar('OVERRIDES', "cvs:%s" % data.getVar('OVERRIDES', localdata), localdata)
+        data.update_data(localdata)
+
+        for loc in urls:
+            (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(loc, localdata))
+            if not "module" in parm:
+                raise MissingParameterError("cvs method needs a 'module' parameter")
+            else:
+                module = parm["module"]
+
+            dlfile = self.localpath(loc, localdata)
+            dldir = data.getVar('DL_DIR', localdata, 1)
+#           if local path contains the cvs
+#           module, consider the dir above it to be the
+#           download directory
+#           pos = dlfile.find(module)
+#           if pos:
+#               dldir = dlfile[:pos]
+#           else:
+#               dldir = os.path.dirname(dlfile)
+
+#           setup cvs options
+            options = []
+            if 'tag' in parm:
+                tag = parm['tag']
+            else:
+                tag = ""
+
+            if 'date' in parm:
+                date = parm['date']
+            else:
+                if not tag:
+                    date = data.getVar("CVSDATE", d, 1) or data.getVar("DATE", d, 1)
+                else:
+                    date = ""
+
+            if "method" in parm:
+                method = parm["method"]
+            else:
+                method = "pserver"
+
+            if "localdir" in parm:
+                localdir = parm["localdir"]
+            else:
+                localdir = module
+
+            cvs_rsh = None
+            if method == "ext":
+                if "rsh" in parm:
+                    cvs_rsh = parm["rsh"]
+
+            tarfn = data.expand('%s_%s_%s_%s.tar.gz' % (module.replace('/', '.'), host, tag, date), localdata)
+            data.setVar('TARFILES', dlfile, localdata)
+            data.setVar('TARFN', tarfn, localdata)
+
+            dl = os.path.join(dldir, tarfn)
+            if os.access(dl, os.R_OK):
+                bb.debug(1, "%s already exists, skipping cvs checkout." % tarfn)
+                continue
+
+            pn = data.getVar('PN', d, 1)
+            cvs_tarball_stash = None
+            if pn:
+                cvs_tarball_stash = data.getVar('CVS_TARBALL_STASH_%s' % pn, d, 1)
+            if cvs_tarball_stash == None:
+                cvs_tarball_stash = data.getVar('CVS_TARBALL_STASH', d, 1)
+            if cvs_tarball_stash:
+                fetchcmd = data.getVar("FETCHCOMMAND_wget", d, 1)
+                uri = cvs_tarball_stash + tarfn
+                bb.note("fetch " + uri)
+                fetchcmd = fetchcmd.replace("${URI}", uri)
+                ret = os.system(fetchcmd)
+                if ret == 0:
+                    bb.note("Fetched %s from tarball stash, skipping checkout" % tarfn)
+                    continue
+
+            if date:
+                options.append("-D %s" % date)
+            if tag:
+                options.append("-r %s" % tag)
+
+            olddir = os.path.abspath(os.getcwd())
+            os.chdir(data.expand(dldir, localdata))
+
+#           setup cvsroot
+            if method == "dir":
+                cvsroot = path
+            else:
+                cvsroot = ":" + method + ":" + user
+                if pswd:
+                    cvsroot += ":" + pswd
+                cvsroot += "@" + host + ":" + path
+
+            data.setVar('CVSROOT', cvsroot, localdata)
+            data.setVar('CVSCOOPTS', " ".join(options), localdata)
+            data.setVar('CVSMODULE', module, localdata)
+            cvscmd = data.getVar('FETCHCOMMAND', localdata, 1)
+            cvsupdatecmd = data.getVar('UPDATECOMMAND', localdata, 1)
+
+            if cvs_rsh:
+                cvscmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvscmd)
+                cvsupdatecmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvsupdatecmd)
+
+#           create module directory
+            bb.debug(2, "Fetch: checking for module directory")
+            pkg=data.expand('${PN}', d)
+            pkgdir=os.path.join(data.expand('${CVSDIR}', localdata), pkg)
+            moddir=os.path.join(pkgdir,localdir)
+            if os.access(os.path.join(moddir,'CVS'), os.R_OK):
+                bb.note("Update " + loc)
+#               update sources there
+                os.chdir(moddir)
+                myret = os.system(cvsupdatecmd)
+            else:
+                bb.note("Fetch " + loc)
+#               check out sources there
+                bb.mkdirhier(pkgdir)
+                os.chdir(pkgdir)
+                bb.debug(1, "Running %s" % cvscmd)
+                myret = os.system(cvscmd)
+
+            if myret != 0:
+                try:
+                    os.rmdir(moddir)
+                except OSError:
+                    pass
+                raise FetchError(module)
+
+            os.chdir(moddir)
+            os.chdir('..')
+#           tar them up to a defined filename
+            myret = os.system("tar -czf %s %s" % (os.path.join(dldir,tarfn), os.path.basename(moddir)))
+            if myret != 0:
+                try:
+                    os.unlink(tarfn)
+                except OSError:
+                    pass
+            os.chdir(olddir)
+        del localdata
diff --git a/lib/bb/fetch/git.py b/lib/bb/fetch/git.py
new file mode 100644
index 000000000..296b92639
--- /dev/null
+++ b/lib/bb/fetch/git.py
@@ -0,0 +1,165 @@
+#!/usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+"""
+BitBake 'Fetch' git implementation
+
+Copyright (C) 2005 Richard Purdie
+
+This program is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free Software
+Foundation; either version 2 of the License, or (at your option) any later
+version.
+
+This program is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+Place, Suite 330, Boston, MA 02111-1307 USA. 
+"""
+
+import os, re
+import bb
+from   bb    import data
+from   bb.fetch import Fetch
+from   bb.fetch import FetchError
+
+def prunedir(topdir):
+    # Delete everything reachable from the directory named in 'topdir'.
+    # CAUTION:  This is dangerous!
+    for root, dirs, files in os.walk(topdir, topdown=False):
+        for name in files:
+            os.remove(os.path.join(root, name))
+        for name in dirs:
+            os.rmdir(os.path.join(root, name))
+
+def rungitcmd(cmd,d):
+
+    bb.debug(1, "Running %s" % cmd)
+
+    # Need to export PATH as git is likely to be in metadata paths 
+    # rather than host provided
+    pathcmd = 'export PATH=%s; %s' % (data.expand('${PATH}', d), cmd)
+
+    myret = os.system(pathcmd)
+
+    if myret != 0:
+        raise FetchError("Git: %s failed" % pathcmd)
+
+def gettag(parm):
+    if 'tag' in parm:
+        tag = parm['tag']
+    else:
+        tag = ""
+    if not tag:
+        tag = "master"
+
+    return tag
+
+class Git(Fetch):
+    """Class to fetch a module or modules from git repositories"""
+    def supports(url, d):
+        """Check to see if a given url can be fetched with cvs.
+           Expects supplied url in list form, as outputted by bb.decodeurl().
+        """
+        (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
+        return type in ['git']
+    supports = staticmethod(supports)
+
+    def localpath(url, d):
+        (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
+
+        #if user sets localpath for file, use it instead.
+        if "localpath" in parm:
+            return parm["localpath"]
+
+        tag = gettag(parm)
+
+        localname = data.expand('git_%s%s_%s.tar.gz' % (host, path.replace('/', '.'), tag), d)
+
+        return os.path.join(data.getVar("DL_DIR", d, 1),data.expand('%s' % (localname), d))
+
+    localpath = staticmethod(localpath)
+
+    def go(self, d, urls = []):
+        """Fetch urls"""
+        if not urls:
+            urls = self.urls
+
+        for loc in urls:
+            (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(loc, d))
+
+            tag = gettag(parm)
+
+            gitsrcname = '%s%s' % (host, path.replace('/', '.'))
+
+            repofile = os.path.join(data.getVar("DL_DIR", d, 1), 'git_%s.tar.gz' % (gitsrcname))
+            repodir = os.path.join(data.expand('${GITDIR}', d), gitsrcname)
+
+            coname = '%s' % (tag)
+            codir = os.path.join(repodir, coname)
+
+            cofile = self.localpath(loc, d)
+
+            # Always update to current if tag=="master"
+            #if os.access(cofile, os.R_OK) and (tag != "master"):
+            if os.access(cofile, os.R_OK):
+                bb.debug(1, "%s already exists, skipping git checkout." % cofile)
+                continue
+
+# Still Need to add GIT_TARBALL_STASH Support...
+#            pn = data.getVar('PN', d, 1)
+#            cvs_tarball_stash = None
+#            if pn:
+#                cvs_tarball_stash = data.getVar('CVS_TARBALL_STASH_%s' % pn, d, 1)
+#            if cvs_tarball_stash == None:
+#                cvs_tarball_stash = data.getVar('CVS_TARBALL_STASH', d, 1)
+#            if cvs_tarball_stash:
+#                fetchcmd = data.getVar("FETCHCOMMAND_wget", d, 1)
+#                uri = cvs_tarball_stash + tarfn
+#                bb.note("fetch " + uri)
+#                fetchcmd = fetchcmd.replace("${URI}", uri)
+#                ret = os.system(fetchcmd)
+#                if ret == 0:
+#                    bb.note("Fetched %s from tarball stash, skipping checkout" % tarfn)
+#                    continue
+         
+            #if os.path.exists(repodir):
+                #prunedir(repodir)
+
+            bb.mkdirhier(repodir)
+            os.chdir(repodir)
+
+            #print("Changing to %s" % repodir)
+
+            if os.access(repofile, os.R_OK):
+                rungitcmd("tar -xzf %s" % (repofile),d)
+            else:
+                rungitcmd("git clone rsync://%s%s %s" % (host, path, repodir),d)
+
+            rungitcmd("rsync -a --verbose --stats --progress rsync://%s%s/ %s" % (host, path, os.path.join(repodir, ".git", "")),d)
+
+            #print("Changing to %s" % repodir)
+            os.chdir(repodir)
+            rungitcmd("git pull rsync://%s%s" % (host, path),d)
+
+            #print("Changing to %s" % repodir)
+            os.chdir(repodir)
+            rungitcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ),d)
+
+            if os.path.exists(codir):
+                prunedir(codir)
+
+            #print("Changing to %s" % repodir)
+            bb.mkdirhier(codir)
+            os.chdir(repodir)
+            rungitcmd("git read-tree %s" % (tag),d)
+
+            rungitcmd("git checkout-index -q -f --prefix=%s -a" % (os.path.join(codir, "git", "")),d)
+
+            #print("Changing to %s" % codir)
+            os.chdir(codir)
+            rungitcmd("tar -czf %s %s" % (cofile, os.path.join(".", "*") ),d)
+
diff --git a/lib/bb/fetch/local.py b/lib/bb/fetch/local.py
new file mode 100644
index 000000000..51938f823
--- /dev/null
+++ b/lib/bb/fetch/local.py
@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+"""
+BitBake 'Fetch' implementations
+
+Classes for obtaining upstream sources for the
+BitBake build tools.
+
+Copyright (C) 2003, 2004  Chris Larson
+
+This program is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free Software
+Foundation; either version 2 of the License, or (at your option) any later
+version.
+
+This program is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+Place, Suite 330, Boston, MA 02111-1307 USA. 
+
+Based on functions from the base bb module, Copyright 2003 Holger Schurig
+"""
+
+import os, re
+import bb
+from   bb import data
+from   bb.fetch import Fetch
+
+class Local(Fetch):
+    def supports(url, d):
+        """Check to see if a given url can be fetched in the local filesystem.
+           Expects supplied url in list form, as outputted by bb.decodeurl().
+        """
+        (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
+        return type in ['file','patch']
+    supports = staticmethod(supports)
+
+    def localpath(url, d):
+        """Return the local filename of a given url assuming a successful fetch.
+        """
+        path = url.split("://")[1]
+        newpath = path
+        if path[0] != "/":
+            filespath = data.getVar('FILESPATH', d, 1)
+            if filespath:
+                newpath = bb.which(filespath, path)
+            if not newpath:
+                filesdir = data.getVar('FILESDIR', d, 1)
+                if filesdir:
+                    newpath = os.path.join(filesdir, path)
+        return newpath
+    localpath = staticmethod(localpath)
+
+    def go(self, urls = []):
+        """Fetch urls (no-op for Local method)"""
+#       no need to fetch local files, we'll deal with them in place.
+        return 1
diff --git a/lib/bb/fetch/svn.py b/lib/bb/fetch/svn.py
new file mode 100644
index 000000000..97672a38e
--- /dev/null
+++ b/lib/bb/fetch/svn.py
@@ -0,0 +1,189 @@
+#!/usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+"""
+BitBake 'Fetch' implementations
+
+Classes for obtaining upstream sources for the
+BitBake build tools.
+
+Copyright (C) 2003, 2004  Chris Larson
+
+This program is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free Software
+Foundation; either version 2 of the License, or (at your option) any later
+version.
+
+This program is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+Place, Suite 330, Boston, MA 02111-1307 USA. 
+
+Based on functions from the base bb module, Copyright 2003 Holger Schurig
+"""
+
+import os, re
+import bb
+from   bb import data
+from   bb.fetch import Fetch
+from   bb.fetch import FetchError
+from   bb.fetch import MissingParameterError
+
+class Svn(Fetch):
+    """Class to fetch a module or modules from svn repositories"""
+    def supports(url, d):
+        """Check to see if a given url can be fetched with svn.
+           Expects supplied url in list form, as outputted by bb.decodeurl().
+        """
+        (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
+        return type in ['svn']
+    supports = staticmethod(supports)
+
+    def localpath(url, d):
+        (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
+        if "localpath" in parm:
+#           if user overrides local path, use it.
+            return parm["localpath"]
+
+        if not "module" in parm:
+            raise MissingParameterError("svn method needs a 'module' parameter")
+        else:
+            module = parm["module"]
+        if 'rev' in parm:
+            revision = parm['rev']
+        else:
+            revision = ""
+
+        date = data.getVar("CVSDATE", d, 1) or data.getVar("DATE", d, 1)
+
+        return os.path.join(data.getVar("DL_DIR", d, 1),data.expand('%s_%s_%s_%s.tar.gz' % ( module.replace('/', '.'), host, revision, date), d))
+    localpath = staticmethod(localpath)
+
+    def go(self, d, urls = []):
+        """Fetch urls"""
+        if not urls:
+            urls = self.urls
+
+        localdata = data.createCopy(d)
+        data.setVar('OVERRIDES', "svn:%s" % data.getVar('OVERRIDES', localdata), localdata)
+        data.update_data(localdata)
+
+        for loc in urls:
+            (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(loc, localdata))
+            if not "module" in parm:
+                raise MissingParameterError("svn method needs a 'module' parameter")
+            else:
+                module = parm["module"]
+
+            dlfile = self.localpath(loc, localdata)
+            dldir = data.getVar('DL_DIR', localdata, 1)
+#           if local path contains the svn
+#           module, consider the dir above it to be the
+#           download directory
+#           pos = dlfile.find(module)
+#           if pos:
+#               dldir = dlfile[:pos]
+#           else:
+#               dldir = os.path.dirname(dlfile)
+
+#           setup svn options
+            options = []
+            if 'rev' in parm:
+                revision = parm['rev']
+            else:
+                revision = ""
+
+            date = data.getVar("CVSDATE", d, 1) or data.getVar("DATE", d, 1)
+
+            if "method" in parm:
+                method = parm["method"]
+            else:
+                method = "pserver"
+
+            if "proto" in parm:
+                proto = parm["proto"]
+            else:
+                proto = "svn"
+
+            svn_rsh = None
+            if method == "ext":
+                if "rsh" in parm:
+                    svn_rsh = parm["rsh"]
+
+            tarfn = data.expand('%s_%s_%s_%s.tar.gz' % (module.replace('/', '.'), host, revision, date), localdata)
+            data.setVar('TARFILES', dlfile, localdata)
+            data.setVar('TARFN', tarfn, localdata)
+
+            dl = os.path.join(dldir, tarfn)
+            if os.access(dl, os.R_OK):
+                bb.debug(1, "%s already exists, skipping svn checkout." % tarfn)
+                continue
+
+            svn_tarball_stash = data.getVar('CVS_TARBALL_STASH', d, 1)
+            if svn_tarball_stash:
+                fetchcmd = data.getVar("FETCHCOMMAND_wget", d, 1)
+                uri = svn_tarball_stash + tarfn
+                bb.note("fetch " + uri)
+                fetchcmd = fetchcmd.replace("${URI}", uri)
+                ret = os.system(fetchcmd)
+                if ret == 0:
+                    bb.note("Fetched %s from tarball stash, skipping checkout" % tarfn)
+                    continue
+
+            olddir = os.path.abspath(os.getcwd())
+            os.chdir(data.expand(dldir, localdata))
+
+#           setup svnroot
+#            svnroot = ":" + method + ":" + user
+#            if pswd:
+#                svnroot += ":" + pswd
+            svnroot = host + path
+
+            data.setVar('SVNROOT', svnroot, localdata)
+            data.setVar('SVNCOOPTS', " ".join(options), localdata)
+            data.setVar('SVNMODULE', module, localdata)
+            svncmd = data.getVar('FETCHCOMMAND', localdata, 1)
+            svncmd = "svn co %s://%s/%s" % (proto, svnroot, module)
+
+            if revision:
+                svncmd = "svn co -r %s %s://%s/%s" % (revision, proto, svnroot, module)
+            if svn_rsh:
+                svncmd = "svn_RSH=\"%s\" %s" % (svn_rsh, svncmd)
+
+#           create temp directory
+            bb.debug(2, "Fetch: creating temporary directory")
+            bb.mkdirhier(data.expand('${WORKDIR}', localdata))
+            data.setVar('TMPBASE', data.expand('${WORKDIR}/oesvn.XXXXXX', localdata), localdata)
+            tmppipe = os.popen(data.getVar('MKTEMPDIRCMD', localdata, 1) or "false")
+            tmpfile = tmppipe.readline().strip()
+            if not tmpfile:
+                bb.error("Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.")
+                raise FetchError(module)
+
+#           check out sources there
+            os.chdir(tmpfile)
+            bb.note("Fetch " + loc)
+            bb.debug(1, "Running %s" % svncmd)
+            myret = os.system(svncmd)
+            if myret != 0:
+                try:
+                    os.rmdir(tmpfile)
+                except OSError:
+                    pass
+                raise FetchError(module)
+
+            os.chdir(os.path.join(tmpfile, os.path.dirname(module)))
+#           tar them up to a defined filename
+            myret = os.system("tar -czf %s %s" % (os.path.join(dldir,tarfn), os.path.basename(module)))
+            if myret != 0:
+                try:
+                    os.unlink(tarfn)
+                except OSError:
+                    pass
+#           cleanup
+            os.system('rm -rf %s' % tmpfile)
+            os.chdir(olddir)
+        del localdata
diff --git a/lib/bb/fetch/wget.py b/lib/bb/fetch/wget.py
new file mode 100644
index 000000000..d9bbdd4b7
--- /dev/null
+++ b/lib/bb/fetch/wget.py
@@ -0,0 +1,164 @@
+#!/usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+"""
+BitBake 'Fetch' implementations
+
+Classes for obtaining upstream sources for the
+BitBake build tools.
+
+Copyright (C) 2003, 2004  Chris Larson
+
+This program is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free Software
+Foundation; either version 2 of the License, or (at your option) any later
+version.
+
+This program is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+Place, Suite 330, Boston, MA 02111-1307 USA. 
+
+Based on functions from the base bb module, Copyright 2003 Holger Schurig
+"""
+
+import os, re
+import bb
+from   bb import data
+from   bb.fetch import Fetch
+from   bb.fetch import FetchError
+from   bb.fetch import MD5SumError
+from   bb.fetch import uri_replace
+
+class Wget(Fetch):
+    """Class to fetch urls via 'wget'"""
+    def supports(url, d):
+        """Check to see if a given url can be fetched using wget.
+           Expects supplied url in list form, as outputted by bb.decodeurl().
+        """
+        (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
+        return type in ['http','https','ftp']
+    supports = staticmethod(supports)
+
+    def localpath(url, d):
+#       strip off parameters
+        (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
+        if "localpath" in parm:
+#           if user overrides local path, use it.
+            return parm["localpath"]
+        url = bb.encodeurl([type, host, path, user, pswd, {}])
+
+        return os.path.join(data.getVar("DL_DIR", d), os.path.basename(url))
+    localpath = staticmethod(localpath)
+
+    def go(self, d, urls = []):
+        """Fetch urls"""
+
+        def md5_sum(basename, d):
+            """
+            Fast and incomplete OVERRIDE implementation for MD5SUM handling
+            MD5SUM_basename = "SUM" and fallback to MD5SUM_basename
+            """
+            var = "MD5SUM_%s" % basename
+            return data.getVar(var, d) or data.getVar("MD5SUM", d)
+
+        def verify_md5sum(wanted_sum, got_sum):
+            """
+            Verify the md5sum we wanted with the one we got
+            """
+            if not wanted_sum:
+                return True
+
+            return wanted_sum == got_sum
+
+        def fetch_uri(uri, basename, dl, md5, parm, d):
+            # the MD5 sum we want to verify
+            wanted_md5sum = md5_sum(basename, d)
+            if os.path.exists(dl):
+#               file exists, but we didnt complete it.. trying again..
+                fetchcmd = data.getVar("RESUMECOMMAND", d, 1)
+            else:
+                fetchcmd = data.getVar("FETCHCOMMAND", d, 1)
+
+            bb.note("fetch " + uri)
+            fetchcmd = fetchcmd.replace("${URI}", uri)
+            fetchcmd = fetchcmd.replace("${FILE}", basename)
+            bb.debug(2, "executing " + fetchcmd)
+            ret = os.system(fetchcmd)
+            if ret != 0:
+                return False
+
+            # check if sourceforge did send us to the mirror page
+            dl_dir = data.getVar("DL_DIR", d, True)
+            if not os.path.exists(dl):
+                os.system("rm %s*" % dl) # FIXME shell quote it
+                bb.debug(2,"sourceforge.net send us to the mirror on %s" % basename)
+                return False
+
+#           supposedly complete.. write out md5sum
+            if bb.which(data.getVar('PATH', d), 'md5sum'):
+                try:
+                    md5pipe = os.popen('md5sum ' + dl)
+                    md5data = (md5pipe.readline().split() or [ "" ])[0]
+                    md5pipe.close()
+                except OSError:
+                    md5data = ""
+
+            # verify the md5sum
+            if not verify_md5sum(wanted_md5sum, md5data):
+                raise MD5SumError(uri)
+
+            md5out = file(md5, 'w')
+            md5out.write(md5data)
+            md5out.close()
+            return True
+
+        if not urls:
+            urls = self.urls
+
+        localdata = data.createCopy(d)
+        data.setVar('OVERRIDES', "wget:" + data.getVar('OVERRIDES', localdata), localdata)
+        data.update_data(localdata)
+
+        for uri in urls:
+            completed = 0
+            (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(uri, localdata))
+            basename = os.path.basename(path)
+            dl = self.localpath(uri, d)
+            dl = data.expand(dl, localdata)
+            md5 = dl + '.md5'
+
+            if os.path.exists(md5):
+#               complete, nothing to see here..
+                continue
+
+            premirrors = [ i.split() for i in (data.getVar('PREMIRRORS', localdata, 1) or "").split('\n') if i ]
+            for (find, replace) in premirrors:
+                newuri = uri_replace(uri, find, replace, d)
+                if newuri != uri:
+                    if fetch_uri(newuri, basename, dl, md5, parm, localdata):
+                        completed = 1
+                        break
+
+            if completed:
+                continue
+
+            if fetch_uri(uri, basename, dl, md5, parm, localdata):
+                continue
+
+#           try mirrors
+            mirrors = [ i.split() for i in (data.getVar('MIRRORS', localdata, 1) or "").split('\n') if i ]
+            for (find, replace) in mirrors:
+                newuri = uri_replace(uri, find, replace, d)
+                if newuri != uri:
+                    if fetch_uri(newuri, basename, dl, md5, parm, localdata):
+                        completed = 1
+                        break
+
+            if not completed:
+                raise FetchError(uri)
+
+        del localdata
diff --git a/setup.py b/setup.py
index a94c967d9..adb14a755 100755
--- a/setup.py
+++ b/setup.py
@@ -55,7 +55,7 @@ setup(name='bitbake',
       long_description='BitBake is a simple tool for the execution of tasks. It is derived from Portage, which is the package management system used by the Gentoo Linux distribution. It is most commonly used to build packages, as it can easily use its rudamentary inheritence to abstract common operations, such as fetching sources, unpacking them, patching them, compiling them, and so on.  It is the basis of the OpenEmbedded project, which is being used for OpenZaurus, Familiar, and a number of other Linux distributions.',
       author='Chris Larson',
       author_email='clarson@elinux.org',
-      packages=['bb', 'bb.parse', 'bb.parse.parse_py'],
+      packages=['bb', 'bb.fetch', 'bb.parse', 'bb.parse.parse_py'],
       package_dir={'bb': os.path.join('lib', 'bb')},
       scripts=[os.path.join('bin', 'bitbake'),
                os.path.join('bin', 'bbimage')],