bitbake/lib/bb/__init__:

    Add a try_mirror method responsible of trying the
    SRC_TARBALL_STASH to get the source from

bitbake/lib/bb/{csv,git,svn}:
    Use this new try_mirror method to get the
    TARBALL_STASH method

bitbake/lib/bb/svk:
    Add a skeleton (copy of svn) svk fetcher

5 files changed, 218 insertions, 42 deletions

diff --git a/lib/bb/fetch/__init__.py b/lib/bb/fetch/__init__.py
index da5b10c4b..f91237b4b 100644
--- a/lib/bb/fetch/__init__.py
+++ b/lib/bb/fetch/__init__.py
@@ -158,7 +158,32 @@ class Fetch(object):
         return data.getVar("SRCDATE", d, 1) or data.getVar("CVSDATE", d, 1) or data.getVar("DATE", d, 1 )
     getSRCDate = staticmethod(getSRCDate)
 
-#if __name__ == "__main__":
+    def try_mirror(d, tarfn):
+        """
+        Try to use a mirrored version of the sources. We do this
+        to avoid massive loads on foreign cvs and svn servers.
+        This method will be used by the different fetcher
+        implementations.
+
+        d Is a bb.data instance
+        tarfn is the name of the tarball
+        """
+        pn = data.getVar('PN', d, True)
+        src_tarball_stash = None
+        if pn:
+            src_tarball_stash = data.getVar('SRC_TARBALL_STASH_%s' % pn, d, True) or data.getVar('CVS_TARBALL_STASH_%s' % pn, d, True) or data.getVar('SRC_TARBALL_STASH', d, True) or data.getVar('CVS_TARBALL_STASH', d, True)
+
+        if src_tarball_stash:
+            fetchcmd = data.getVar("FETCHCOMMAND_mirror", d, True) or data.getVar("FETCHCOMMAND_wget", d, True)
+            uri = src_tarball_stash + tarfn
+            bb.note("fetch " + uri)
+            fetchcmd = fetchcmd.replace("${URI}", uri)
+            ret = os.system(fetchcmd)
+            if ret == 0:
+                bb.note("Fetched %s from tarball stash, skipping checkout" % tarfn)
+                return True
+        return False
+    try_mirror = staticmethod(try_mirror)
 
 import bk
 import cvs
@@ -166,6 +191,7 @@ import git
 import local
 import svn
 import wget
+import svk
 
 methods.append(bk.Bk())
 methods.append(cvs.Cvs())
@@ -173,3 +199,4 @@ methods.append(git.Git())
 methods.append(local.Local())
 methods.append(svn.Svn())
 methods.append(wget.Wget())
+methods.append(svk.Svk())
diff --git a/lib/bb/fetch/cvs.py b/lib/bb/fetch/cvs.py
index 1f75f771a..10ec700dc 100644
--- a/lib/bb/fetch/cvs.py
+++ b/lib/bb/fetch/cvs.py
@@ -133,21 +133,9 @@ class Cvs(Fetch):
                 bb.debug(1, "%s already exists, skipping cvs checkout." % tarfn)
                 continue
 
-            pn = data.getVar('PN', d, 1)
-            cvs_tarball_stash = None
-            if pn:
-                cvs_tarball_stash = data.getVar('CVS_TARBALL_STASH_%s' % pn, d, 1)
-            if cvs_tarball_stash == None:
-                cvs_tarball_stash = data.getVar('CVS_TARBALL_STASH', d, 1)
-            if cvs_tarball_stash:
-                fetchcmd = data.getVar("FETCHCOMMAND_wget", d, 1)
-                uri = cvs_tarball_stash + tarfn
-                bb.note("fetch " + uri)
-                fetchcmd = fetchcmd.replace("${URI}", uri)
-                ret = os.system(fetchcmd)
-                if ret == 0:
-                    bb.note("Fetched %s from tarball stash, skipping checkout" % tarfn)
-                    continue
+            # try to use the tarball stash
+            if Fetch.try_mirror(d, tarfn):
+                continue
 
             if date:
                 options.append("-D %s" % date)
diff --git a/lib/bb/fetch/git.py b/lib/bb/fetch/git.py
index 296b92639..11ca6821a 100644
--- a/lib/bb/fetch/git.py
+++ b/lib/bb/fetch/git.py
@@ -110,22 +110,9 @@ class Git(Fetch):
                 continue
 
 # Still Need to add GIT_TARBALL_STASH Support...
-#            pn = data.getVar('PN', d, 1)
-#            cvs_tarball_stash = None
-#            if pn:
-#                cvs_tarball_stash = data.getVar('CVS_TARBALL_STASH_%s' % pn, d, 1)
-#            if cvs_tarball_stash == None:
-#                cvs_tarball_stash = data.getVar('CVS_TARBALL_STASH', d, 1)
-#            if cvs_tarball_stash:
-#                fetchcmd = data.getVar("FETCHCOMMAND_wget", d, 1)
-#                uri = cvs_tarball_stash + tarfn
-#                bb.note("fetch " + uri)
-#                fetchcmd = fetchcmd.replace("${URI}", uri)
-#                ret = os.system(fetchcmd)
-#                if ret == 0:
-#                    bb.note("Fetched %s from tarball stash, skipping checkout" % tarfn)
-#                    continue
-         
+#           if Fetch.try_mirror(d, tarfn):
+#               continue
+
             #if os.path.exists(repodir):
                 #prunedir(repodir)
 
diff --git a/lib/bb/fetch/svk.py b/lib/bb/fetch/svk.py
new file mode 100644
index 000000000..5caf0d9ec
--- /dev/null
+++ b/lib/bb/fetch/svk.py
@@ -0,0 +1,181 @@
+#!/usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+"""
+BitBake 'Fetch' implementations
+
+This implementation is for svk. It is based on the svn implementation
+
+Copyright (C) 2006 Holger Hans Peter Freyther
+
+GPL and MIT licensed
+
+
+
+Classes for obtaining upstream sources for the
+BitBake build tools.
+
+Copyright (C) 2003, 2004  Chris Larson
+
+This program is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free Software
+Foundation; either version 2 of the License, or (at your option) any later
+version.
+
+This program is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+Place, Suite 330, Boston, MA 02111-1307 USA. 
+
+Based on functions from the base bb module, Copyright 2003 Holger Schurig
+"""
+
+import os, re
+import bb
+from   bb import data
+from   bb.fetch import Fetch
+from   bb.fetch import FetchError
+from   bb.fetch import MissingParameterError
+
+class Svk(Fetch):
+    """Class to fetch a module or modules from svn repositories"""
+    def supports(url, d):
+        """Check to see if a given url can be fetched with svn.
+           Expects supplied url in list form, as outputted by bb.decodeurl().
+        """
+        (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
+        return type in ['svk']
+    supports = staticmethod(supports)
+
+    def localpath(url, d):
+        (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
+        if "localpath" in parm:
+#           if user overrides local path, use it.
+            return parm["localpath"]
+
+        if not "module" in parm:
+            raise MissingParameterError("svn method needs a 'module' parameter")
+        else:
+            module = parm["module"]
+        if 'rev' in parm:
+            revision = parm['rev']
+        else:
+            revision = ""
+
+        date = Fetch.getSRCDate(d)
+
+        return os.path.join(data.getVar("DL_DIR", d, 1),data.expand('%s_%s_%s_%s_%s.tar.gz' % ( module.replace('/', '.'), host, path.replace('/', '.'), revision, date), d))
+    localpath = staticmethod(localpath)
+
+    def go(self, d, urls = []):
+        """Fetch urls"""
+        if not urls:
+            urls = self.urls
+
+        localdata = data.createCopy(d)
+        data.setVar('OVERRIDES', "svn:%s" % data.getVar('OVERRIDES', localdata), localdata)
+        data.update_data(localdata)
+
+        for loc in urls:
+            (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(loc, localdata))
+            if not "module" in parm:
+                raise MissingParameterError("svk method needs a 'module' parameter")
+            else:
+                module = parm["module"]
+
+            dlfile = self.localpath(loc, localdata)
+            dldir = data.getVar('DL_DIR', localdata, 1)
+
+#           setup svn options
+            options = []
+            if 'rev' in parm:
+                revision = parm['rev']
+            else:
+                revision = ""
+
+            date = Fetch.getSRCDate(d)
+
+            if "method" in parm:
+                method = parm["method"]
+            else:
+                method = "pserver"
+
+            if "proto" in parm:
+                proto = parm["proto"]
+            else:
+                proto = "svn"
+
+            svn_rsh = None
+            if method == "ext":
+                if "rsh" in parm:
+                    svn_rsh = parm["rsh"]
+
+            tarfn = data.expand('%s_%s_%s_%s_%s.tar.gz' % (module.replace('/', '.'), host, path.replace('/', '.'), revision, date), localdata)
+            data.setVar('TARFILES', dlfile, localdata)
+            data.setVar('TARFN', tarfn, localdata)
+
+            dl = os.path.join(dldir, tarfn)
+            if os.access(dl, os.R_OK):
+                bb.debug(1, "%s already exists, skipping svn checkout." % tarfn)
+                continue
+
+            if Fetch.try_mirror(d, tarfn):
+                continue
+
+            olddir = os.path.abspath(os.getcwd())
+            os.chdir(data.expand(dldir, localdata))
+
+#           setup svnroot
+#            svnroot = ":" + method + ":" + user
+#            if pswd:
+#                svnroot += ":" + pswd
+            svnroot = host + path
+
+            data.setVar('SVNROOT', svnroot, localdata)
+            data.setVar('SVNCOOPTS', " ".join(options), localdata)
+            data.setVar('SVNMODULE', module, localdata)
+            svncmd = data.getVar('FETCHCOMMAND', localdata, 1)
+            svncmd = "svn co -r {%s} %s://%s/%s" % (date, proto, svnroot, module)
+
+            if revision:
+                svncmd = "svn co -r %s %s://%s/%s" % (revision, proto, svnroot, module)
+            if svn_rsh:
+                svncmd = "svn_RSH=\"%s\" %s" % (svn_rsh, svncmd)
+
+#           create temp directory
+            bb.debug(2, "Fetch: creating temporary directory")
+            bb.mkdirhier(data.expand('${WORKDIR}', localdata))
+            data.setVar('TMPBASE', data.expand('${WORKDIR}/oesvn.XXXXXX', localdata), localdata)
+            tmppipe = os.popen(data.getVar('MKTEMPDIRCMD', localdata, 1) or "false")
+            tmpfile = tmppipe.readline().strip()
+            if not tmpfile:
+                bb.error("Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.")
+                raise FetchError(module)
+
+#           check out sources there
+            os.chdir(tmpfile)
+            bb.note("Fetch " + loc)
+            bb.debug(1, "Running %s" % svncmd)
+            myret = os.system(svncmd)
+            if myret != 0:
+                try:
+                    os.rmdir(tmpfile)
+                except OSError:
+                    pass
+                raise FetchError(module)
+
+            os.chdir(os.path.join(tmpfile, os.path.dirname(module)))
+#           tar them up to a defined filename
+            myret = os.system("tar -czf %s %s" % (os.path.join(dldir,tarfn), os.path.basename(module)))
+            if myret != 0:
+                try:
+                    os.unlink(tarfn)
+                except OSError:
+                    pass
+#           cleanup
+            os.system('rm -rf %s' % tmpfile)
+            os.chdir(olddir)
+        del localdata
diff --git a/lib/bb/fetch/svn.py b/lib/bb/fetch/svn.py
index ac5eebf5c..1d0692399 100644
--- a/lib/bb/fetch/svn.py
+++ b/lib/bb/fetch/svn.py
@@ -122,16 +122,9 @@ class Svn(Fetch):
                 bb.debug(1, "%s already exists, skipping svn checkout." % tarfn)
                 continue
 
-            svn_tarball_stash = data.getVar('CVS_TARBALL_STASH', d, 1)
-            if svn_tarball_stash:
-                fetchcmd = data.getVar("FETCHCOMMAND_wget", d, 1)
-                uri = svn_tarball_stash + tarfn
-                bb.note("fetch " + uri)
-                fetchcmd = fetchcmd.replace("${URI}", uri)
-                ret = os.system(fetchcmd)
-                if ret == 0:
-                    bb.note("Fetched %s from tarball stash, skipping checkout" % tarfn)
-                    continue
+            # try to use the tarball stash
+            if Fetch.try_mirror(d, tarfn):
+                continue
 
             olddir = os.path.abspath(os.getcwd())
             os.chdir(data.expand(dldir, localdata))