toaster: enable SSH-based remote build support

We enable support for starting builds on remote machines
through SSH. The support is limited to poky-based distributions.

We refactor localhost build support and we update
bldcontrol application tests to uniformely test the APIs
of localhost and SSH build controllers.

[YOCTO #6240]

Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>

1 files changed, 12 insertions, 158 deletions

diff --git a/lib/toaster/bldcontrol/bbcontroller.py b/lib/toaster/bldcontrol/bbcontroller.py
index bf9cdf9f67..6812ae3e6e 100644
--- a/lib/toaster/bldcontrol/bbcontroller.py
+++ b/lib/toaster/bldcontrol/bbcontroller.py
@@ -26,10 +26,6 @@ import re
 from django.db import transaction
 from django.db.models import Q
 from bldcontrol.models import BuildEnvironment, BRLayer, BRVariable, BRTarget, BRBitbake
-import subprocess
-
-from toastermain import settings
-
 
 # load Bitbake components
 path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
@@ -72,6 +68,10 @@ def getBuildEnvironmentController(**kwargs):
 
         The return object MUST always be a BuildEnvironmentController.
     """
+
+    from localhostbecontroller import LocalhostBEController
+    from sshbecontroller    import SSHBEController
+
     be = BuildEnvironment.objects.filter(Q(**kwargs))[0]
     if be.betype == BuildEnvironment.TYPE_LOCAL:
         return LocalhostBEController(be)
@@ -81,6 +81,13 @@ def getBuildEnvironmentController(**kwargs):
         raise Exception("FIXME: Implement BEC for type %s" % str(be.betype))
 
 
+def _getgitcheckoutdirectoryname(url):
+    """ Utility that returns the last component of a git path as directory
+    """
+    import re
+    components = re.split(r'[:\.\/]', url)
+    return components[-2] if components[-1] == "git" else components[-1]
+
 
 class BuildEnvironmentController(object):
     """ BuildEnvironmentController (BEC) is the abstract class that defines the operations that MUST
@@ -110,6 +117,7 @@ class BuildEnvironmentController(object):
         self.be = be
         self.connection = None
 
+
     def startBBServer(self):
         """ Starts a  BB server with Toaster toasterui set up to record the builds, an no controlling UI.
             After this method executes, self.be bbaddress/bbport MUST point to a running and free server,
@@ -173,157 +181,3 @@ class ShellCmdException(Exception):
 class BuildSetupException(Exception):
     pass
 
-class LocalhostBEController(BuildEnvironmentController):
-    """ Implementation of the BuildEnvironmentController for the localhost;
-        this controller manages the default build directory,
-        the server setup and system start and stop for the localhost-type build environment
-
-    """
-
-    def __init__(self, be):
-        super(LocalhostBEController, self).__init__(be)
-        self.dburl = settings.getDATABASE_URL()
-        self.pokydirname = None
-
-    def _shellcmd(self, command, cwd = None):
-        if cwd is None:
-            cwd = self.be.sourcedir
-
-        p = subprocess.Popen(command, cwd = cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        (out,err) = p.communicate()
-        if p.returncode:
-            if len(err) == 0:
-                err = "command: %s \n%s" % (command, out)
-            else:
-                err = "command: %s \n%s" % (command, err)
-            raise ShellCmdException(err)
-        else:
-            return out
-
-    def _createdirpath(self, path):
-        from os.path import dirname as DN
-        if not os.path.exists(DN(path)):
-            self._createdirpath(DN(path))
-        if not os.path.exists(path):
-            os.mkdir(path, 0755)
-
-    def _startBE(self):
-        assert self.pokydirname and os.path.exists(self.pokydirname)
-        self._createdirpath(self.be.builddir)
-        self._shellcmd("bash -c \"source %s/oe-init-build-env %s\"" % (self.pokydirname, self.be.builddir))
-
-    def startBBServer(self):
-        assert self.pokydirname and os.path.exists(self.pokydirname)
-        print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb && sleep 1\"" % (self.pokydirname, self.be.builddir, self.dburl))
-        # FIXME unfortunate sleep 1 - we need to make sure that bbserver is started and the toaster ui is connected
-        # but since they start async without any return, we just wait a bit
-        print "Started server"
-        assert self.be.sourcedir and os.path.exists(self.be.builddir)
-        self.be.bbaddress = "localhost"
-        self.be.bbport = "8200"
-        self.be.bbstate = BuildEnvironment.SERVER_STARTED
-        self.be.save()
-
-    def stopBBServer(self):
-        assert self.be.sourcedir
-        print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && %s source toaster stop\"" %
-            (self.be.sourcedir, self.be.builddir, (lambda: "" if self.be.bbtoken is None else "BBTOKEN=%s" % self.be.bbtoken)()))
-        self.be.bbstate = BuildEnvironment.SERVER_STOPPED
-        self.be.save()
-        print "Stopped server"
-
-    def setLayers(self, bitbakes, layers):
-        """ a word of attention: by convention, the first layer for any build will be poky! """
-
-        assert self.be.sourcedir is not None
-        assert len(bitbakes) == 1
-        # set layers in the layersource
-
-        # 1. get a list of repos, and map dirpaths for each layer
-        gitrepos = {}
-        gitrepos[bitbakes[0].giturl] = []
-        gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) )
-        
-        for layer in layers:
-            # we don't process local URLs
-            if layer.giturl.startswith("file://"):
-                continue
-            if not layer.giturl in gitrepos:
-                gitrepos[layer.giturl] = []
-            gitrepos[layer.giturl].append( (layer.name, layer.dirpath, layer.commit))
-        for giturl in gitrepos.keys():
-            commitid = gitrepos[giturl][0][2]
-            for e in gitrepos[giturl]:
-                if commitid != e[2]:
-                    raise BuildSetupException("More than one commit per git url, unsupported configuration")
-
-        def _getgitdirectoryname(url):
-            import re
-            components = re.split(r'[:\.\/]', url)
-            return components[-2] if components[-1] == "git" else components[-1]
-
-        layerlist = []
-
-        # 2. checkout the repositories
-        for giturl in gitrepos.keys():
-            localdirname = os.path.join(self.be.sourcedir, _getgitdirectoryname(giturl))
-            print "DEBUG: giturl ", giturl ,"checking out in current directory", localdirname
-
-            # make sure our directory is a git repository
-            if os.path.exists(localdirname):
-                if not giturl in self._shellcmd("git remote -v", localdirname):
-                    raise BuildSetupException("Existing git repository at %s, but with different remotes (not '%s'). Aborting." % (localdirname, giturl))
-            else:
-                self._shellcmd("git clone \"%s\" \"%s\"" % (giturl, localdirname))
-            # checkout the needed commit
-            commit = gitrepos[giturl][0][2]
-
-            # branch magic name "HEAD" will inhibit checkout
-            if commit != "HEAD":
-                print "DEBUG: checking out commit ", commit, "to", localdirname
-                self._shellcmd("git fetch --all && git checkout \"%s\"" % commit , localdirname)
-
-            # take the localdirname as poky dir if we can find the oe-init-build-env
-            if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")):
-                print "DEBUG: selected poky dir name", localdirname
-                self.pokydirname = localdirname
-
-            # verify our repositories
-            for name, dirpath, commit in gitrepos[giturl]:
-                localdirpath = os.path.join(localdirname, dirpath)
-                if not os.path.exists(localdirpath):
-                    raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit))
-
-                if name != "bitbake":
-                    layerlist.append(localdirpath)
-
-        print "DEBUG: current layer list ", layerlist
-
-        # 3. configure the build environment, so we have a conf/bblayers.conf
-        assert self.pokydirname is not None
-        self._startBE()
-
-        # 4. update the bblayers.conf
-        bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf")
-        if not os.path.exists(bblayerconf):
-            raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf)
-
-        conflines = open(bblayerconf, "r").readlines()
-
-        bblayerconffile = open(bblayerconf, "w")
-        for i in xrange(len(conflines)):
-            if conflines[i].startswith("# line added by toaster"):
-                i += 2
-            else:
-                bblayerconffile.write(conflines[i])
-
-        bblayerconffile.write("\n# line added by toaster build control\nBBLAYERS = \"" + " ".join(layerlist) + "\"")
-        bblayerconffile.close()
-
-        return True
-
-    def release(self):
-        assert self.be.sourcedir and os.path.exists(self.be.builddir)
-        import shutil
-        shutil.rmtree(os.path.join(self.be.sourcedir, "build"))
-        assert not os.path.exists(self.be.builddir)