diff options
Diffstat (limited to 'lib/toaster/bldcontrol')
| -rw-r--r-- | lib/toaster/bldcontrol/admin.py | 5 | ||||
| -rw-r--r-- | lib/toaster/bldcontrol/bbcontroller.py | 20 | ||||
| -rw-r--r-- | lib/toaster/bldcontrol/localhostbecontroller.py | 237 | ||||
| -rw-r--r-- | lib/toaster/bldcontrol/management/commands/checksettings.py | 11 | ||||
| -rw-r--r-- | lib/toaster/bldcontrol/management/commands/runbuilds.py | 95 | ||||
| -rw-r--r-- | lib/toaster/bldcontrol/migrations/0001_initial.py | 18 | ||||
| -rw-r--r-- | lib/toaster/bldcontrol/migrations/0008_models_bigautofield.py | 48 | ||||
| -rw-r--r-- | lib/toaster/bldcontrol/models.py | 29 | ||||
| -rw-r--r-- | lib/toaster/bldcontrol/views.py | 4 |
9 files changed, 355 insertions, 112 deletions
diff --git a/lib/toaster/bldcontrol/admin.py b/lib/toaster/bldcontrol/admin.py index fcbe5f593..1754bc11c 100644 --- a/lib/toaster/bldcontrol/admin.py +++ b/lib/toaster/bldcontrol/admin.py @@ -1,5 +1,8 @@ +# +# SPDX-License-Identifier: GPL-2.0-only +# + from django.contrib import admin -from django.contrib.admin.filters import RelatedFieldListFilter from .models import BuildEnvironment class BuildEnvironmentAdmin(admin.ModelAdmin): diff --git a/lib/toaster/bldcontrol/bbcontroller.py b/lib/toaster/bldcontrol/bbcontroller.py index 5195600d9..71c288df3 100644 --- a/lib/toaster/bldcontrol/bbcontroller.py +++ b/lib/toaster/bldcontrol/bbcontroller.py @@ -1,31 +1,15 @@ # -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -# # BitBake Toaster Implementation # # Copyright (C) 2014 Intel Corporation # -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# SPDX-License-Identifier: GPL-2.0-only # -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - import os import sys -import re -from django.db import transaction from django.db.models import Q -from bldcontrol.models import BuildEnvironment, BRLayer, BRVariable, BRTarget, BRBitbake +from bldcontrol.models import BuildEnvironment, BRLayer, BRBitbake # load Bitbake components path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) diff --git a/lib/toaster/bldcontrol/localhostbecontroller.py b/lib/toaster/bldcontrol/localhostbecontroller.py index 16c7c8044..577e765f1 100644 --- a/lib/toaster/bldcontrol/localhostbecontroller.py +++ b/lib/toaster/bldcontrol/localhostbecontroller.py @@ -1,44 +1,30 @@ # -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -# # BitBake Toaster Implementation # # Copyright (C) 2014 Intel Corporation # -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# SPDX-License-Identifier: GPL-2.0-only # -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - import os -import sys import re import shutil import time -from django.db import transaction -from django.db.models import Q -from bldcontrol.models import BuildEnvironment, BRLayer, BRVariable, BRTarget, BRBitbake -from orm.models import CustomImageRecipe, Layer, Layer_Version, ProjectLayer, ToasterSetting +from bldcontrol.models import BuildEnvironment, BuildRequest, Build +from orm.models import CustomImageRecipe, Layer, Layer_Version, Project, ToasterSetting +from orm.models import signal_runbuilds import subprocess from toastermain import settings -from bldcontrol.bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException, BitbakeController +from bldcontrol.bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException import logging logger = logging.getLogger("toaster") -from pprint import pprint, pformat +install_dir = os.environ.get('TOASTER_DIR') + +from pprint import pformat class LocalhostBEController(BuildEnvironmentController): """ Implementation of the BuildEnvironmentController for the localhost; @@ -87,10 +73,10 @@ class LocalhostBEController(BuildEnvironmentController): #logger.debug("localhostbecontroller: using HEAD checkout in %s" % local_checkout_path) return local_checkout_path - - def setCloneStatus(self,bitbake,status,total,current): + def setCloneStatus(self,bitbake,status,total,current,repo_name): bitbake.req.build.repos_cloned=current bitbake.req.build.repos_to_clone=total + bitbake.req.build.progress_item=repo_name bitbake.req.build.save() def setLayers(self, bitbake, layers, targets): @@ -100,6 +86,7 @@ class LocalhostBEController(BuildEnvironmentController): layerlist = [] nongitlayerlist = [] + layer_index = 0 git_env = os.environ.copy() # (note: add custom environment settings here) @@ -113,7 +100,7 @@ class LocalhostBEController(BuildEnvironmentController): if bitbake.giturl and bitbake.commit: gitrepos[(bitbake.giturl, bitbake.commit)] = [] gitrepos[(bitbake.giturl, bitbake.commit)].append( - ("bitbake", bitbake.dirpath)) + ("bitbake", bitbake.dirpath, 0)) for layer in layers: # We don't need to git clone the layer for the CustomImageRecipe @@ -124,12 +111,13 @@ class LocalhostBEController(BuildEnvironmentController): # If we have local layers then we don't need clone them # For local layers giturl will be empty if not layer.giturl: - nongitlayerlist.append(layer.layer_version.layer.local_source_dir) + nongitlayerlist.append( "%03d:%s" % (layer_index,layer.local_source_dir) ) continue if not (layer.giturl, layer.commit) in gitrepos: gitrepos[(layer.giturl, layer.commit)] = [] - gitrepos[(layer.giturl, layer.commit)].append( (layer.name, layer.dirpath) ) + gitrepos[(layer.giturl, layer.commit)].append( (layer.name,layer.dirpath,layer_index) ) + layer_index += 1 logger.debug("localhostbecontroller, our git repos are %s" % pformat(gitrepos)) @@ -159,9 +147,9 @@ class LocalhostBEController(BuildEnvironmentController): # 3. checkout the repositories clone_count=0 clone_total=len(gitrepos.keys()) - self.setCloneStatus(bitbake,'Started',clone_total,clone_count) + self.setCloneStatus(bitbake,'Started',clone_total,clone_count,'') for giturl, commit in gitrepos.keys(): - self.setCloneStatus(bitbake,'progress',clone_total,clone_count) + self.setCloneStatus(bitbake,'progress',clone_total,clone_count,gitrepos[(giturl, commit)][0][0]) clone_count += 1 localdirname = os.path.join(self.be.sourcedir, self.getGitCloneDirectory(giturl, commit)) @@ -172,8 +160,11 @@ class LocalhostBEController(BuildEnvironmentController): try: localremotes = self._shellcmd("git remote -v", localdirname,env=git_env) - if not giturl in localremotes and commit != 'HEAD': - raise BuildSetupException("Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something." % (localdirname, ", ".join(localremotes.split("\n")), giturl)) + # NOTE: this nice-to-have check breaks when using git remaping to get past firewall + # Re-enable later with .gitconfig remapping checks + #if not giturl in localremotes and commit != 'HEAD': + # raise BuildSetupException("Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something." % (localdirname, ", ".join(localremotes.split("\n")), giturl)) + pass except ShellCmdException: # our localdirname might not be a git repository #- that's fine @@ -205,21 +196,33 @@ class LocalhostBEController(BuildEnvironmentController): self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbake.commit, bitbake.giturl, os.path.join(self.pokydirname, 'bitbake')),env=git_env) # verify our repositories - for name, dirpath in gitrepos[(giturl, commit)]: + for name, dirpath, index in gitrepos[(giturl, commit)]: localdirpath = os.path.join(localdirname, dirpath) - logger.debug("localhostbecontroller: localdirpath expected '%s'" % localdirpath) + logger.debug("localhostbecontroller: localdirpath expects '%s'" % localdirpath) if not os.path.exists(localdirpath): - raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit)) + raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Exiting." % (localdirpath, giturl, commit)) if name != "bitbake": - layerlist.append(localdirpath.rstrip("/")) + layerlist.append("%03d:%s" % (index,localdirpath.rstrip("/"))) - self.setCloneStatus(bitbake,'complete',clone_total,clone_count) + self.setCloneStatus(bitbake,'complete',clone_total,clone_count,'') logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist)) - if self.pokydirname is None and os.path.exists(os.path.join(self.be.sourcedir, "oe-init-build-env")): - logger.debug("localhostbecontroller: selected poky dir name %s" % self.be.sourcedir) - self.pokydirname = self.be.sourcedir + # Resolve self.pokydirname if not resolved yet, consider the scenario + # where all layers are local, that's the else clause + if self.pokydirname is None: + if os.path.exists(os.path.join(self.be.sourcedir, "oe-init-build-env")): + logger.debug("localhostbecontroller: selected poky dir name %s" % self.be.sourcedir) + self.pokydirname = self.be.sourcedir + else: + # Alternatively, scan local layers for relative "oe-init-build-env" location + for layer in layers: + if os.path.exists(os.path.join(layer.layer_version.layer.local_source_dir,"..","oe-init-build-env")): + logger.debug("localhostbecontroller, setting pokydirname to %s" % (layer.layer_version.layer.local_source_dir)) + self.pokydirname = os.path.join(layer.layer_version.layer.local_source_dir,"..") + break + else: + logger.error("pokydirname is not set, you will run into trouble!") # 5. create custom layer and add custom recipes to it for target in targets: @@ -232,7 +235,7 @@ class LocalhostBEController(BuildEnvironmentController): customrecipe, layers) if os.path.isdir(custom_layer_path): - layerlist.append(custom_layer_path) + layerlist.append("%03d:%s" % (layer_index,custom_layer_path)) except CustomImageRecipe.DoesNotExist: continue # not a custom recipe, skip @@ -240,7 +243,11 @@ class LocalhostBEController(BuildEnvironmentController): layerlist.extend(nongitlayerlist) logger.debug("\n\nset layers gives this list %s" % pformat(layerlist)) self.islayerset = True - return layerlist + + # restore the order of layer list for bblayers.conf + layerlist.sort() + sorted_layerlist = [l[4:] for l in layerlist] + return sorted_layerlist def setup_custom_image_recipe(self, customrecipe, layers): """ Set up toaster-custom-images layer and recipe files """ @@ -310,41 +317,141 @@ class LocalhostBEController(BuildEnvironmentController): def triggerBuild(self, bitbake, layers, variables, targets, brbe): layers = self.setLayers(bitbake, layers, targets) + is_merged_attr = bitbake.req.project.merged_attr + + git_env = os.environ.copy() + # (note: add custom environment settings here) + try: + # insure that the project init/build uses the selected bitbake, and not Toaster's + del git_env['TEMPLATECONF'] + del git_env['BBBASEDIR'] + del git_env['BUILDDIR'] + except KeyError: + pass # init build environment from the clone - builddir = '%s-toaster-%d' % (self.be.builddir, bitbake.req.project.id) + if bitbake.req.project.builddir: + builddir = bitbake.req.project.builddir + else: + builddir = '%s-toaster-%d' % (self.be.builddir, bitbake.req.project.id) oe_init = os.path.join(self.pokydirname, 'oe-init-build-env') # init build environment try: custom_script = ToasterSetting.objects.get(name="CUSTOM_BUILD_INIT_SCRIPT").value custom_script = custom_script.replace("%BUILDDIR%" ,builddir) - self._shellcmd("bash -c 'source %s'" % (custom_script)) + self._shellcmd("bash -c 'source %s'" % (custom_script),env=git_env) except ToasterSetting.DoesNotExist: self._shellcmd("bash -c 'source %s %s'" % (oe_init, builddir), - self.be.sourcedir) + self.be.sourcedir,env=git_env) # update bblayers.conf - bblconfpath = os.path.join(builddir, "conf/toaster-bblayers.conf") - with open(bblconfpath, 'w') as bblayers: - bblayers.write('# line added by toaster build control\n' - 'BBLAYERS = "%s"' % ' '.join(layers)) - - # write configuration file - confpath = os.path.join(builddir, 'conf/toaster.conf') - with open(confpath, 'w') as conf: - for var in variables: - conf.write('%s="%s"\n' % (var.name, var.value)) - conf.write('INHERIT+="toaster buildhistory"') + if not is_merged_attr: + bblconfpath = os.path.join(builddir, "conf/toaster-bblayers.conf") + with open(bblconfpath, 'w') as bblayers: + bblayers.write('# line added by toaster build control\n' + 'BBLAYERS = "%s"' % ' '.join(layers)) + + # write configuration file + confpath = os.path.join(builddir, 'conf/toaster.conf') + with open(confpath, 'w') as conf: + for var in variables: + conf.write('%s="%s"\n' % (var.name, var.value)) + conf.write('INHERIT+="toaster buildhistory"') + else: + # Append the Toaster-specific values directly to the bblayers.conf + bblconfpath = os.path.join(builddir, "conf/bblayers.conf") + bblconfpath_save = os.path.join(builddir, "conf/bblayers.conf.save") + shutil.copyfile(bblconfpath, bblconfpath_save) + with open(bblconfpath) as bblayers: + content = bblayers.readlines() + do_write = True + was_toaster = False + with open(bblconfpath,'w') as bblayers: + for line in content: + #line = line.strip('\n') + if 'TOASTER_CONFIG_PROLOG' in line: + do_write = False + was_toaster = True + elif 'TOASTER_CONFIG_EPILOG' in line: + do_write = True + elif do_write: + bblayers.write(line) + if not was_toaster: + bblayers.write('\n') + bblayers.write('#=== TOASTER_CONFIG_PROLOG ===\n') + bblayers.write('BBLAYERS = "\\\n') + for layer in layers: + bblayers.write(' %s \\\n' % layer) + bblayers.write(' "\n') + bblayers.write('#=== TOASTER_CONFIG_EPILOG ===\n') + # Append the Toaster-specific values directly to the local.conf + bbconfpath = os.path.join(builddir, "conf/local.conf") + bbconfpath_save = os.path.join(builddir, "conf/local.conf.save") + shutil.copyfile(bbconfpath, bbconfpath_save) + with open(bbconfpath) as f: + content = f.readlines() + do_write = True + was_toaster = False + with open(bbconfpath,'w') as conf: + for line in content: + #line = line.strip('\n') + if 'TOASTER_CONFIG_PROLOG' in line: + do_write = False + was_toaster = True + elif 'TOASTER_CONFIG_EPILOG' in line: + do_write = True + elif do_write: + conf.write(line) + if not was_toaster: + conf.write('\n') + conf.write('#=== TOASTER_CONFIG_PROLOG ===\n') + for var in variables: + if (not var.name.startswith("INTERNAL_")) and (not var.name == "BBLAYERS"): + conf.write('%s="%s"\n' % (var.name, var.value)) + conf.write('#=== TOASTER_CONFIG_EPILOG ===\n') + + # If 'target' is just the project preparation target, then we are done + for target in targets: + if "_PROJECT_PREPARE_" == target.target: + logger.debug('localhostbecontroller: Project has been prepared. Done.') + # Update the Build Request and release the build environment + bitbake.req.state = BuildRequest.REQ_COMPLETED + bitbake.req.save() + self.be.lock = BuildEnvironment.LOCK_FREE + self.be.save() + # Close the project build and progress bar + bitbake.req.build.outcome = Build.SUCCEEDED + bitbake.req.build.save() + # Update the project status + bitbake.req.project.set_variable(Project.PROJECT_SPECIFIC_STATUS,Project.PROJECT_SPECIFIC_CLONING_SUCCESS) + signal_runbuilds() + return # clean the Toaster to build environment env_clean = 'unset BBPATH;' # clean BBPATH for <= YP-2.4.0 - # run bitbake server from the clone + # run bitbake server from the clone if available + # otherwise pick it from the PATH bitbake = os.path.join(self.pokydirname, 'bitbake', 'bin', 'bitbake') + if not os.path.exists(bitbake): + logger.info("Bitbake not available under %s, will try to use it from PATH" % + self.pokydirname) + for path in os.environ["PATH"].split(os.pathsep): + if os.path.exists(os.path.join(path, 'bitbake')): + bitbake = os.path.join(path, 'bitbake') + break + else: + logger.error("Looks like Bitbake is not available, please fix your environment") + toasterlayers = os.path.join(builddir,"conf/toaster-bblayers.conf") - self._shellcmd('%s bash -c \"source %s %s; BITBAKE_UI="knotty" %s --read %s --read %s ' - '--server-only -B 0.0.0.0:0\"' % (env_clean, oe_init, - builddir, bitbake, confpath, toasterlayers), self.be.sourcedir) + if not is_merged_attr: + self._shellcmd('%s bash -c \"source %s %s; BITBAKE_UI="knotty" %s --read %s --read %s ' + '--server-only -B 0.0.0.0:0\"' % (env_clean, oe_init, + builddir, bitbake, confpath, toasterlayers), self.be.sourcedir) + else: + self._shellcmd('%s bash -c \"source %s %s; BITBAKE_UI="knotty" %s ' + '--server-only -B 0.0.0.0:0\"' % (env_clean, oe_init, + builddir, bitbake), self.be.sourcedir) # read port number from bitbake.lock self.be.bbport = -1 @@ -360,7 +467,7 @@ class LocalhostBEController(BuildEnvironmentController): logger.debug("localhostbecontroller: waiting for bblock content to appear") time.sleep(1) else: - raise BuildSetupException("Cannot find bitbake server lock file '%s'. Aborting." % bblock) + raise BuildSetupException("Cannot find bitbake server lock file '%s'. Exiting." % bblock) with open(bblock) as fplock: for line in fplock: @@ -390,12 +497,20 @@ class LocalhostBEController(BuildEnvironmentController): log = os.path.join(builddir, 'toaster_ui.log') local_bitbake = os.path.join(os.path.dirname(os.getenv('BBBASEDIR')), 'bitbake') - self._shellcmd(['%s bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:%s" ' + if not is_merged_attr: + self._shellcmd(['%s bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:%s" ' '%s %s -u toasterui --read %s --read %s --token="" >>%s 2>&1;' 'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:%s %s -m)&\"' \ % (env_clean, brbe, self.be.bbport, local_bitbake, bbtargets, confpath, toasterlayers, log, self.be.bbport, bitbake,)], builddir, nowait=True) + else: + self._shellcmd(['%s bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:%s" ' + '%s %s -u toasterui --token="" >>%s 2>&1;' + 'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:%s %s -m)&\"' \ + % (env_clean, brbe, self.be.bbport, local_bitbake, bbtargets, log, + self.be.bbport, bitbake,)], + builddir, nowait=True) logger.debug('localhostbecontroller: Build launched, exiting. ' 'Follow build logs at %s' % log) diff --git a/lib/toaster/bldcontrol/management/commands/checksettings.py b/lib/toaster/bldcontrol/management/commands/checksettings.py index 823c6f154..20f9dce56 100644 --- a/lib/toaster/bldcontrol/management/commands/checksettings.py +++ b/lib/toaster/bldcontrol/management/commands/checksettings.py @@ -1,8 +1,10 @@ -from django.core.management.base import BaseCommand, CommandError -from django.db import transaction +# +# SPDX-License-Identifier: GPL-2.0-only +# + +from django.core.management.base import BaseCommand from django.core.management import call_command -from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException from bldcontrol.models import BuildRequest, BuildEnvironment, BRError from orm.models import ToasterSetting, Build, Layer @@ -74,8 +76,9 @@ class Command(BaseCommand): print("Loading default settings") call_command("loaddata", "settings") template_conf = os.environ.get("TEMPLATECONF", "") + custom_xml_only = os.environ.get("CUSTOM_XML_ONLY") - if ToasterSetting.objects.filter(name='CUSTOM_XML_ONLY').count() > 0: + if ToasterSetting.objects.filter(name='CUSTOM_XML_ONLY').count() > 0 or custom_xml_only is not None: # only use the custom settings pass elif "poky" in template_conf: diff --git a/lib/toaster/bldcontrol/management/commands/runbuilds.py b/lib/toaster/bldcontrol/management/commands/runbuilds.py index 791e53eab..834e32b36 100644 --- a/lib/toaster/bldcontrol/management/commands/runbuilds.py +++ b/lib/toaster/bldcontrol/management/commands/runbuilds.py @@ -1,3 +1,7 @@ +# +# SPDX-License-Identifier: GPL-2.0-only +# + from django.core.management.base import BaseCommand from django.db import transaction from django.db.models import Q @@ -49,7 +53,7 @@ class Command(BaseCommand): # we could not find a BEC; postpone the BR br.state = BuildRequest.REQ_QUEUED br.save() - logger.debug("runbuilds: No build env") + logger.debug("runbuilds: No build env (%s)" % e) return logger.info("runbuilds: starting build %s, environment %s" % @@ -164,17 +168,88 @@ class Command(BaseCommand): try: self.cleanup() except Exception as e: - logger.warn("runbuilds: cleanup exception %s" % str(e)) + logger.warning("runbuilds: cleanup exception %s" % str(e)) try: self.archive() except Exception as e: - logger.warn("runbuilds: archive exception %s" % str(e)) + logger.warning("runbuilds: archive exception %s" % str(e)) try: self.schedule() except Exception as e: - logger.warn("runbuilds: schedule exception %s" % str(e)) + logger.warning("runbuilds: schedule exception %s" % str(e)) + + # Test to see if a build pre-maturely died due to a bitbake crash + def check_dead_builds(self): + do_cleanup = False + try: + for br in BuildRequest.objects.filter(state=BuildRequest.REQ_INPROGRESS): + # Get the build directory + if br.project.builddir: + builddir = br.project.builddir + else: + builddir = '%s-toaster-%d' % (br.environment.builddir,br.project.id) + # Check log to see if there is a recent traceback + toaster_ui_log = os.path.join(builddir, 'toaster_ui.log') + test_file = os.path.join(builddir, '._toaster_check.txt') + os.system("tail -n 50 %s > %s" % (os.path.join(builddir, 'toaster_ui.log'),test_file)) + traceback_text = '' + is_traceback = False + with open(test_file,'r') as test_file_fd: + test_file_tail = test_file_fd.readlines() + for line in test_file_tail: + if line.startswith('Traceback (most recent call last):'): + traceback_text = line + is_traceback = True + elif line.startswith('NOTE: ToasterUI waiting for events'): + # Ignore any traceback before new build start + traceback_text = '' + is_traceback = False + elif line.startswith('Note: Toaster traceback auto-stop'): + # Ignore any traceback before this previous traceback catch + traceback_text = '' + is_traceback = False + elif is_traceback: + traceback_text += line + # Test the results + is_stop = False + if is_traceback: + # Found a traceback + errtype = 'Bitbake crash' + errmsg = 'Bitbake crash\n' + traceback_text + state = BuildRequest.REQ_FAILED + # Clean up bitbake files + bitbake_lock = os.path.join(builddir, 'bitbake.lock') + if os.path.isfile(bitbake_lock): + os.remove(bitbake_lock) + bitbake_sock = os.path.join(builddir, 'bitbake.sock') + if os.path.isfile(bitbake_sock): + os.remove(bitbake_sock) + if os.path.isfile(test_file): + os.remove(test_file) + # Add note to ignore this traceback on next check + os.system('echo "Note: Toaster traceback auto-stop" >> %s' % toaster_ui_log) + is_stop = True + # Add more tests here + #elif ... + # Stop the build request? + if is_stop: + brerror = BRError( + req = br, + errtype = errtype, + errmsg = errmsg, + traceback = traceback_text, + ) + brerror.save() + br.state = state + br.save() + do_cleanup = True + # Do cleanup + if do_cleanup: + self.cleanup() + except Exception as e: + logger.error("runbuilds: Error in check_dead_builds %s" % e) def handle(self, **options): pidfile_path = os.path.join(os.environ.get("BUILDDIR", "."), @@ -183,10 +258,18 @@ class Command(BaseCommand): with open(pidfile_path, 'w') as pidfile: pidfile.write("%s" % os.getpid()) + # Clean up any stale/failed builds from previous Toaster run self.runbuild() signal.signal(signal.SIGUSR1, lambda sig, frame: None) while True: - signal.pause() - self.runbuild() + sigset = signal.sigtimedwait([signal.SIGUSR1], 5) + if sigset: + for sig in sigset: + # Consume each captured pending event + self.runbuild() + else: + # Check for build exceptions + self.check_dead_builds() + diff --git a/lib/toaster/bldcontrol/migrations/0001_initial.py b/lib/toaster/bldcontrol/migrations/0001_initial.py index 67db37856..7ee19f474 100644 --- a/lib/toaster/bldcontrol/migrations/0001_initial.py +++ b/lib/toaster/bldcontrol/migrations/0001_initial.py @@ -37,7 +37,7 @@ class Migration(migrations.Migration): ('giturl', models.CharField(max_length=254)), ('commit', models.CharField(max_length=254)), ('dirpath', models.CharField(max_length=254)), - ('layer_version', models.ForeignKey(to='orm.Layer_Version', null=True)), + ('layer_version', models.ForeignKey(to='orm.Layer_Version', null=True, on_delete=models.CASCADE)), ], ), migrations.CreateModel( @@ -80,34 +80,34 @@ class Migration(migrations.Migration): ('state', models.IntegerField(default=0, choices=[(0, b'created'), (1, b'queued'), (2, b'in progress'), (3, b'completed'), (4, b'failed'), (5, b'deleted'), (6, b'archive')])), ('created', models.DateTimeField(auto_now_add=True)), ('updated', models.DateTimeField(auto_now=True)), - ('build', models.OneToOneField(null=True, to='orm.Build')), - ('environment', models.ForeignKey(to='bldcontrol.BuildEnvironment', null=True)), - ('project', models.ForeignKey(to='orm.Project')), + ('build', models.OneToOneField(null=True, to='orm.Build', on_delete=models.CASCADE)), + ('environment', models.ForeignKey(to='bldcontrol.BuildEnvironment', null=True, on_delete=models.CASCADE)), + ('project', models.ForeignKey(to='orm.Project', on_delete=models.CASCADE)), ], ), migrations.AddField( model_name='brvariable', name='req', - field=models.ForeignKey(to='bldcontrol.BuildRequest'), + field=models.ForeignKey(to='bldcontrol.BuildRequest', on_delete=models.CASCADE), ), migrations.AddField( model_name='brtarget', name='req', - field=models.ForeignKey(to='bldcontrol.BuildRequest'), + field=models.ForeignKey(to='bldcontrol.BuildRequest', on_delete=models.CASCADE), ), migrations.AddField( model_name='brlayer', name='req', - field=models.ForeignKey(to='bldcontrol.BuildRequest'), + field=models.ForeignKey(to='bldcontrol.BuildRequest', on_delete=models.CASCADE), ), migrations.AddField( model_name='brerror', name='req', - field=models.ForeignKey(to='bldcontrol.BuildRequest'), + field=models.ForeignKey(to='bldcontrol.BuildRequest', on_delete=models.CASCADE), ), migrations.AddField( model_name='brbitbake', name='req', - field=models.OneToOneField(to='bldcontrol.BuildRequest'), + field=models.OneToOneField(to='bldcontrol.BuildRequest', on_delete=models.CASCADE), ), ] diff --git a/lib/toaster/bldcontrol/migrations/0008_models_bigautofield.py b/lib/toaster/bldcontrol/migrations/0008_models_bigautofield.py new file mode 100644 index 000000000..45b477d02 --- /dev/null +++ b/lib/toaster/bldcontrol/migrations/0008_models_bigautofield.py @@ -0,0 +1,48 @@ +# Generated by Django 3.2.12 on 2022-03-06 03:28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('bldcontrol', '0007_brlayers_optional_gitinfo'), + ] + + operations = [ + migrations.AlterField( + model_name='brbitbake', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='brerror', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='brlayer', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='brtarget', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='brvariable', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='buildenvironment', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='buildrequest', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + ] diff --git a/lib/toaster/bldcontrol/models.py b/lib/toaster/bldcontrol/models.py index 409614b9e..42750e718 100644 --- a/lib/toaster/bldcontrol/models.py +++ b/lib/toaster/bldcontrol/models.py @@ -1,8 +1,11 @@ +# +# SPDX-License-Identifier: GPL-2.0-only +# + from __future__ import unicode_literals from django.db import models -from django.core.validators import MaxValueValidator, MinValueValidator -from django.utils.encoding import force_text -from orm.models import Project, ProjectLayer, ProjectVariable, ProjectTarget, Build, Layer_Version +from django.utils.encoding import force_str +from orm.models import Project, Build, Layer_Version import logging logger = logging.getLogger("toaster") @@ -82,9 +85,9 @@ class BuildRequest(models.Model): search_allowed_fields = ("brtarget__target", "build__project__name") - project = models.ForeignKey(Project) - build = models.OneToOneField(Build, null = True) # TODO: toasterui should set this when Build is created - environment = models.ForeignKey(BuildEnvironment, null = True) + project = models.ForeignKey(Project, on_delete=models.CASCADE) + build = models.OneToOneField(Build, on_delete=models.CASCADE, null = True) # TODO: toasterui should set this when Build is created + environment = models.ForeignKey(BuildEnvironment, on_delete=models.CASCADE, null = True) state = models.IntegerField(choices = REQUEST_STATE, default = REQ_CREATED) created = models.DateTimeField(auto_now_add = True) updated = models.DateTimeField(auto_now = True) @@ -121,39 +124,39 @@ class BuildRequest(models.Model): return self.brvariable_set.get(name="MACHINE").value def __str__(self): - return force_text('%s %s' % (self.project, self.get_state_display())) + return force_str('%s %s' % (self.project, self.get_state_display())) # These tables specify the settings for running an actual build. # They MUST be kept in sync with the tables in orm.models.Project* class BRLayer(models.Model): - req = models.ForeignKey(BuildRequest) + req = models.ForeignKey(BuildRequest, on_delete=models.CASCADE) name = models.CharField(max_length=100) giturl = models.CharField(max_length=254, null=True) local_source_dir = models.CharField(max_length=254, null=True) commit = models.CharField(max_length=254, null=True) dirpath = models.CharField(max_length=254, null=True) - layer_version = models.ForeignKey(Layer_Version, null=True) + layer_version = models.ForeignKey(Layer_Version, on_delete=models.CASCADE, null=True) class BRBitbake(models.Model): - req = models.OneToOneField(BuildRequest) # only one bitbake for a request + req = models.OneToOneField(BuildRequest, on_delete=models.CASCADE) # only one bitbake for a request giturl = models.CharField(max_length =254) commit = models.CharField(max_length = 254) dirpath = models.CharField(max_length = 254) class BRVariable(models.Model): - req = models.ForeignKey(BuildRequest) + req = models.ForeignKey(BuildRequest, on_delete=models.CASCADE) name = models.CharField(max_length=100) value = models.TextField(blank = True) class BRTarget(models.Model): - req = models.ForeignKey(BuildRequest) + req = models.ForeignKey(BuildRequest, on_delete=models.CASCADE) target = models.CharField(max_length=100) task = models.CharField(max_length=100, null=True) class BRError(models.Model): - req = models.ForeignKey(BuildRequest) + req = models.ForeignKey(BuildRequest, on_delete=models.CASCADE) errtype = models.CharField(max_length=100) errmsg = models.TextField() traceback = models.TextField() diff --git a/lib/toaster/bldcontrol/views.py b/lib/toaster/bldcontrol/views.py index 60f00ef0e..286d88b45 100644 --- a/lib/toaster/bldcontrol/views.py +++ b/lib/toaster/bldcontrol/views.py @@ -1 +1,5 @@ +# +# SPDX-License-Identifier: GPL-2.0-only +# + # Create your views here. |