diff options
| -rw-r--r-- | lib/toaster/bldcontrol/bbcontroller.py | 108 | ||||
| -rw-r--r-- | lib/toaster/bldcontrol/management/commands/runbuilds.py | 23 | ||||
| -rw-r--r-- | lib/toaster/bldcontrol/migrations/0005_auto__add_brerror.py | 112 | ||||
| -rw-r--r-- | lib/toaster/bldcontrol/models.py | 8 |
4 files changed, 226 insertions, 25 deletions
diff --git a/lib/toaster/bldcontrol/bbcontroller.py b/lib/toaster/bldcontrol/bbcontroller.py index c3beba96f..1e58c67fc 100644 --- a/lib/toaster/bldcontrol/bbcontroller.py +++ b/lib/toaster/bldcontrol/bbcontroller.py @@ -126,6 +126,8 @@ class BuildEnvironmentController(object): def setLayers(self,ls): """ Sets the layer variables in the config file, after validating local layer paths. The layer paths must be in a list of BRLayer object + + a word of attention: by convention, the first layer for any build will be poky! """ raise Exception("Must override setLayers") @@ -165,25 +167,31 @@ class BuildEnvironmentController(object): class ShellCmdException(Exception): pass + +class BuildSetupException(Exception): + pass + class LocalhostBEController(BuildEnvironmentController): """ Implementation of the BuildEnvironmentController for the localhost; this controller manages the default build directory, the server setup and system start and stop for the localhost-type build environment """ - from os.path import dirname as DN def __init__(self, be): super(LocalhostBEController, self).__init__(be) - from os.path import dirname as DN self.dburl = settings.getDATABASE_URL() + self.pokydirname = None + + def _shellcmd(self, command, cwd = None): + if cwd is None: + cwd = self.be.sourcedir - def _shellcmd(self, command): - p = subprocess.Popen(command, cwd=self.be.sourcedir, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p = subprocess.Popen(command, cwd = cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (out,err) = p.communicate() if p.returncode: if len(err) == 0: - err = "command: %s" % command + err = "command: %s \n%s" % (command, out) else: err = "command: %s \n%s" % (command, err) raise ShellCmdException(err) @@ -191,22 +199,20 @@ class LocalhostBEController(BuildEnvironmentController): return out def _createdirpath(self, path): + from os.path import dirname as DN if not os.path.exists(DN(path)): self._createdirpath(DN(path)) if not os.path.exists(path): os.mkdir(path, 0755) def _startBE(self): - assert self.be.sourcedir and os.path.exists(self.be.sourcedir) + assert self.pokydirname and os.path.exists(self.pokydirname) self._createdirpath(self.be.builddir) - self._shellcmd("bash -c \"source %s/oe-init-build-env %s\"" % (self.be.sourcedir, self.be.builddir)) + self._shellcmd("bash -c \"source %s/oe-init-build-env %s\"" % (self.pokydirname, self.be.builddir)) def startBBServer(self): - assert self.be.sourcedir and os.path.exists(self.be.sourcedir) - - self._startBE() - - print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb && sleep 1\"" % (self.be.sourcedir, self.be.builddir, self.dburl)) + assert self.pokydirname and os.path.exists(self.pokydirname) + print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb && sleep 1\"" % (self.pokydirname, self.be.builddir, self.dburl)) # FIXME unfortunate sleep 1 - we need to make sure that bbserver is started and the toaster ui is connected # but since they start async without any return, we just wait a bit print "Started server" @@ -225,10 +231,82 @@ class LocalhostBEController(BuildEnvironmentController): print "Stopped server" def setLayers(self, layers): + """ a word of attention: by convention, the first layer for any build will be poky! """ + assert self.be.sourcedir is not None - layerconf = os.path.join(self.be.builddir, "conf/bblayers.conf") - if not os.path.exists(layerconf): - raise Exception("BE is not consistent: bblayers.conf file missing at ", layerconf) + # set layers in the layersource + + # 1. get a list of repos, and map dirpaths for each layer + gitrepos = {} + for layer in layers: + if not layer.giturl in gitrepos: + gitrepos[layer.giturl] = [] + gitrepos[layer.giturl].append( (layer.name, layer.dirpath, layer.commit)) + for giturl in gitrepos.keys(): + commitid = gitrepos[giturl][0][2] + for e in gitrepos[giturl]: + if commitid != e[2]: + raise BuildSetupException("More than one commit per git url, unsupported configuration") + + def _getgitdirectoryname(url): + import re + components = re.split(r'[\.\/]', url) + return components[-2] if components[-1] == "git" else components[-1] + + layerlist = [] + + # 2. checkout the repositories + for giturl in gitrepos.keys(): + localdirname = os.path.join(self.be.sourcedir, _getgitdirectoryname(giturl)) + print "DEBUG: giturl checking out in current directory", localdirname + + # make sure our directory is a git repository + if os.path.exists(localdirname): + if not giturl in self._shellcmd("git remote -v", localdirname): + raise BuildSetupException("Existing git repository at %s, but with different remotes (not '%s'). Aborting." % (localdirname, giturl)) + else: + self._shellcmd("git clone \"%s\" \"%s\"" % (giturl, localdirname)) + # checkout the needed commit + commit = gitrepos[giturl][0][2] + self._shellcmd("git fetch --all && git checkout \"%s\"" % commit , localdirname) + print "DEBUG: checked out commit ", commit, "to", localdirname + + # if this is the first checkout, take the localdirname as poky dir + if self.pokydirname is None: + print "DEBUG: selected poky dir name", localdirname + self.pokydirname = localdirname + + # verify our repositories + for name, dirpath, commit in gitrepos[giturl]: + localdirpath = os.path.join(localdirname, dirpath) + if not os.path.exists(localdirpath): + raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit)) + + layerlist.append(localdirpath) + + print "DEBUG: current layer list ", layerlist + + # 3. configure the build environment, so we have a conf/bblayers.conf + assert self.pokydirname is not None + self._startBE() + + # 4. update the bblayers.conf + bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf") + if not os.path.exists(bblayerconf): + raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf) + + conflines = open(bblayerconf, "r").readlines() + + bblayerconffile = open(bblayerconf, "w") + for i in xrange(len(conflines)): + if conflines[i].startswith("# line added by toaster"): + i += 2 + else: + bblayerconffile.write(conflines[i]) + + bblayerconffile.write("\n# line added by toaster build control\nBBLAYERS = \"" + " ".join(layerlist) + "\"") + bblayerconffile.close() + return True def release(self): diff --git a/lib/toaster/bldcontrol/management/commands/runbuilds.py b/lib/toaster/bldcontrol/management/commands/runbuilds.py index dd8f84b07..fa8c1a990 100644 --- a/lib/toaster/bldcontrol/management/commands/runbuilds.py +++ b/lib/toaster/bldcontrol/management/commands/runbuilds.py @@ -1,8 +1,8 @@ from django.core.management.base import NoArgsCommand, CommandError from django.db import transaction from orm.models import Build -from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException -from bldcontrol.models import BuildRequest, BuildEnvironment +from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException, BuildSetupException +from bldcontrol.models import BuildRequest, BuildEnvironment, BRError import os class Command(NoArgsCommand): @@ -25,6 +25,7 @@ class Command(NoArgsCommand): return br def schedule(self): + import traceback try: br = None try: @@ -63,15 +64,19 @@ class Command(NoArgsCommand): # cleanup to be performed by toaster when the deed is done - except ShellCmdException as e: - import traceback - print " EE Error executing shell command\n", e - traceback.format_exc(e) except Exception as e: - import traceback - traceback.print_exc() - raise e + print " EE Error executing shell command\n", e + traceback.print_exc(e) + BRError.objects.create(req = br, + errtype = str(type(e)), + errmsg = str(e), + traceback = traceback.format_exc(e)) + br.state = BuildRequest.REQ_FAILED + br.save() + bec.be.lock = BuildEnvironment.LOCK_FREE + bec.be.save() + def cleanup(self): from django.utils import timezone diff --git a/lib/toaster/bldcontrol/migrations/0005_auto__add_brerror.py b/lib/toaster/bldcontrol/migrations/0005_auto__add_brerror.py new file mode 100644 index 000000000..98aeb41ce --- /dev/null +++ b/lib/toaster/bldcontrol/migrations/0005_auto__add_brerror.py @@ -0,0 +1,112 @@ +# -*- coding: utf-8 -*- +from south.utils import datetime_utils as datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding model 'BRError' + db.create_table(u'bldcontrol_brerror', ( + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('req', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['bldcontrol.BuildRequest'])), + ('errtype', self.gf('django.db.models.fields.CharField')(max_length=100)), + ('errmsg', self.gf('django.db.models.fields.TextField')()), + ('traceback', self.gf('django.db.models.fields.TextField')()), + )) + db.send_create_signal(u'bldcontrol', ['BRError']) + + + def backwards(self, orm): + # Deleting model 'BRError' + db.delete_table(u'bldcontrol_brerror') + + + models = { + u'bldcontrol.brerror': { + 'Meta': {'object_name': 'BRError'}, + 'errmsg': ('django.db.models.fields.TextField', [], {}), + 'errtype': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}), + 'traceback': ('django.db.models.fields.TextField', [], {}) + }, + u'bldcontrol.brlayer': { + 'Meta': {'object_name': 'BRLayer'}, + 'commit': ('django.db.models.fields.CharField', [], {'max_length': '254'}), + 'dirpath': ('django.db.models.fields.CharField', [], {'max_length': '254'}), + 'giturl': ('django.db.models.fields.CharField', [], {'max_length': '254'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}) + }, + u'bldcontrol.brtarget': { + 'Meta': {'object_name': 'BRTarget'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}), + 'target': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'task': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True'}) + }, + u'bldcontrol.brvariable': { + 'Meta': {'object_name': 'BRVariable'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}), + 'value': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + u'bldcontrol.buildenvironment': { + 'Meta': {'object_name': 'BuildEnvironment'}, + 'address': ('django.db.models.fields.CharField', [], {'max_length': '254'}), + 'bbaddress': ('django.db.models.fields.CharField', [], {'max_length': '254', 'blank': 'True'}), + 'bbport': ('django.db.models.fields.IntegerField', [], {'default': '-1'}), + 'bbstate': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'bbtoken': ('django.db.models.fields.CharField', [], {'max_length': '126', 'blank': 'True'}), + 'betype': ('django.db.models.fields.IntegerField', [], {}), + 'builddir': ('django.db.models.fields.CharField', [], {'max_length': '512', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'lock': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'sourcedir': ('django.db.models.fields.CharField', [], {'max_length': '512', 'blank': 'True'}), + 'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}) + }, + u'bldcontrol.buildrequest': { + 'Meta': {'object_name': 'BuildRequest'}, + 'build': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Build']", 'null': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'project': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Project']"}), + 'state': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}) + }, + u'orm.build': { + 'Meta': {'object_name': 'Build'}, + 'bitbake_version': ('django.db.models.fields.CharField', [], {'max_length': '50'}), + 'build_name': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'completed_on': ('django.db.models.fields.DateTimeField', [], {}), + 'cooker_log_path': ('django.db.models.fields.CharField', [], {'max_length': '500'}), + 'distro': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'distro_version': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'errors_no': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'machine': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'outcome': ('django.db.models.fields.IntegerField', [], {'default': '2'}), + 'project': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Project']", 'null': 'True'}), + 'started_on': ('django.db.models.fields.DateTimeField', [], {}), + 'timespent': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'warnings_no': ('django.db.models.fields.IntegerField', [], {'default': '0'}) + }, + u'orm.project': { + 'Meta': {'object_name': 'Project'}, + 'branch': ('django.db.models.fields.CharField', [], {'max_length': '50'}), + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'short_description': ('django.db.models.fields.CharField', [], {'max_length': '50', 'blank': 'True'}), + 'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'user_id': ('django.db.models.fields.IntegerField', [], {'null': 'True'}) + } + } + + complete_apps = ['bldcontrol']
\ No newline at end of file diff --git a/lib/toaster/bldcontrol/models.py b/lib/toaster/bldcontrol/models.py index 1f253ffb2..8c271ffa9 100644 --- a/lib/toaster/bldcontrol/models.py +++ b/lib/toaster/bldcontrol/models.py @@ -48,12 +48,14 @@ class BuildRequest(models.Model): REQ_QUEUED = 1 REQ_INPROGRESS = 2 REQ_COMPLETED = 3 + REQ_FAILED = 4 REQUEST_STATE = ( (REQ_CREATED, "created"), (REQ_QUEUED, "queued"), (REQ_INPROGRESS, "in progress"), (REQ_COMPLETED, "completed"), + (REQ_FAILED, "failed"), ) project = models.ForeignKey(Project) @@ -84,4 +86,8 @@ class BRTarget(models.Model): target = models.CharField(max_length=100) task = models.CharField(max_length=100, null=True) - +class BRError(models.Model): + req = models.ForeignKey(BuildRequest) + errtype = models.CharField(max_length=100) + errmsg = models.TextField() + traceback = models.TextField() |