aboutsummaryrefslogtreecommitdiffstats
path: root/lib/bb/ui/buildinfohelper.py
diff options
context:
space:
mode:
Diffstat (limited to 'lib/bb/ui/buildinfohelper.py')
-rw-r--r--lib/bb/ui/buildinfohelper.py145
1 files changed, 68 insertions, 77 deletions
diff --git a/lib/bb/ui/buildinfohelper.py b/lib/bb/ui/buildinfohelper.py
index 524a5b094..8b212b780 100644
--- a/lib/bb/ui/buildinfohelper.py
+++ b/lib/bb/ui/buildinfohelper.py
@@ -3,18 +3,8 @@
#
# Copyright (C) 2013 Intel Corporation
#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
+# SPDX-License-Identifier: GPL-2.0-only
#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import sys
import bb
@@ -55,7 +45,7 @@ from pprint import pformat
import logging
from datetime import datetime, timedelta
-from django.db import transaction, connection
+from django.db import transaction
# pylint: disable=invalid-name
@@ -158,14 +148,14 @@ class ORMWrapper(object):
buildrequest = None
if brbe is not None:
# Toaster-triggered build
- logger.debug(1, "buildinfohelper: brbe is %s" % brbe)
+ logger.debug("buildinfohelper: brbe is %s" % brbe)
br, _ = brbe.split(":")
buildrequest = BuildRequest.objects.get(pk=br)
prj = buildrequest.project
else:
# CLI build
prj = Project.objects.get_or_create_default_project()
- logger.debug(1, "buildinfohelper: project is not specified, defaulting to %s" % prj)
+ logger.debug("buildinfohelper: project is not specified, defaulting to %s" % prj)
if buildrequest is not None:
# reuse existing Build object
@@ -181,7 +171,7 @@ class ORMWrapper(object):
completed_on=now,
build_name='')
- logger.debug(1, "buildinfohelper: build is created %s" % build)
+ logger.debug("buildinfohelper: build is created %s" % build)
if buildrequest is not None:
buildrequest.build = build
@@ -237,6 +227,12 @@ class ORMWrapper(object):
build.completed_on = timezone.now()
build.outcome = outcome
build.save()
+
+ # We force a sync point here to force the outcome status commit,
+ # which resolves a race condition with the build completion takedown
+ transaction.set_autocommit(True)
+ transaction.set_autocommit(False)
+
signal_runbuilds()
def update_target_set_license_manifest(self, target, license_manifest_path):
@@ -493,14 +489,14 @@ class ORMWrapper(object):
# we already created the root directory, so ignore any
# entry for it
- if len(path) == 0:
+ if not path:
continue
parent_path = "/".join(path.split("/")[:len(path.split("/")) - 1])
- if len(parent_path) == 0:
+ if not parent_path:
parent_path = "/"
parent_obj = self._cached_get(Target_File, target = target_obj, path = parent_path, inodetype = Target_File.ITYPE_DIRECTORY)
- tf_obj = Target_File.objects.create(
+ Target_File.objects.create(
target = target_obj,
path = path,
size = size,
@@ -565,7 +561,7 @@ class ORMWrapper(object):
parent_obj = Target_File.objects.get(target = target_obj, path = parent_path, inodetype = Target_File.ITYPE_DIRECTORY)
- tf_obj = Target_File.objects.create(
+ Target_File.objects.create(
target = target_obj,
path = path,
size = size,
@@ -581,7 +577,7 @@ class ORMWrapper(object):
assert isinstance(build_obj, Build)
assert isinstance(target_obj, Target)
- errormsg = ""
+ errormsg = []
for p in packagedict:
# Search name swtiches round the installed name vs package name
# by default installed name == package name
@@ -643,10 +639,10 @@ class ORMWrapper(object):
packagefile_objects.append(Package_File( package = packagedict[p]['object'],
path = targetpath,
size = targetfilesize))
- if len(packagefile_objects):
+ if packagefile_objects:
Package_File.objects.bulk_create(packagefile_objects)
except KeyError as e:
- errormsg += " stpi: Key error, package %s key %s \n" % ( p, e )
+ errormsg.append(" stpi: Key error, package %s key %s \n" % (p, e))
# save disk installed size
packagedict[p]['object'].installed_size = packagedict[p]['size']
@@ -656,6 +652,9 @@ class ORMWrapper(object):
Target_Installed_Package.objects.create(target = target_obj, package = packagedict[p]['object'])
packagedeps_objs = []
+ pattern_so = re.compile(r'.*\.so(\.\d*)?$')
+ pattern_lib = re.compile(r'.*\-suffix(\d*)?$')
+ pattern_ko = re.compile(r'^kernel-module-.*')
for p in packagedict:
for (px,deptype) in packagedict[p]['depends']:
if deptype == 'depends':
@@ -664,6 +663,13 @@ class ORMWrapper(object):
tdeptype = Package_Dependency.TYPE_TRECOMMENDS
try:
+ # Skip known non-package objects like libraries and kernel modules
+ if pattern_so.match(px) or pattern_lib.match(px):
+ logger.info("Toaster does not add library file dependencies to packages (%s,%s)", p, px)
+ continue
+ if pattern_ko.match(px):
+ logger.info("Toaster does not add kernel module dependencies to packages (%s,%s)", p, px)
+ continue
packagedeps_objs.append(Package_Dependency(
package = packagedict[p]['object'],
depends_on = packagedict[px]['object'],
@@ -673,13 +679,13 @@ class ORMWrapper(object):
logger.warning("Could not add dependency to the package %s "
"because %s is an unknown package", p, px)
- if len(packagedeps_objs) > 0:
+ if packagedeps_objs:
Package_Dependency.objects.bulk_create(packagedeps_objs)
else:
logger.info("No package dependencies created")
- if len(errormsg) > 0:
- logger.warning("buildinfohelper: target_package_info could not identify recipes: \n%s", errormsg)
+ if errormsg:
+ logger.warning("buildinfohelper: target_package_info could not identify recipes: \n%s", "".join(errormsg))
def save_target_image_file_information(self, target_obj, file_name, file_size):
Target_Image_File.objects.create(target=target_obj,
@@ -767,7 +773,7 @@ class ORMWrapper(object):
packagefile_objects.append(Package_File( package = bp_object,
path = path,
size = package_info['FILES_INFO'][path] ))
- if len(packagefile_objects):
+ if packagefile_objects:
Package_File.objects.bulk_create(packagefile_objects)
def _po_byname(p):
@@ -809,7 +815,7 @@ class ORMWrapper(object):
packagedeps_objs.append(Package_Dependency( package = bp_object,
depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RCONFLICTS))
- if len(packagedeps_objs) > 0:
+ if packagedeps_objs:
Package_Dependency.objects.bulk_create(packagedeps_objs)
return bp_object
@@ -826,7 +832,7 @@ class ORMWrapper(object):
desc = vardump[root_var]['doc']
if desc is None:
desc = ''
- if len(desc):
+ if desc:
HelpText.objects.get_or_create(build=build_obj,
area=HelpText.VARIABLE,
key=k, text=desc)
@@ -846,7 +852,7 @@ class ORMWrapper(object):
file_name = vh['file'],
line_number = vh['line'],
operation = vh['op']))
- if len(varhist_objects):
+ if varhist_objects:
VariableHistory.objects.bulk_create(varhist_objects)
@@ -893,9 +899,6 @@ class BuildInfoHelper(object):
self.task_order = 0
self.autocommit_step = 1
self.server = server
- # we use manual transactions if the database doesn't autocommit on us
- if not connection.features.autocommits_when_autocommit_is_off:
- transaction.set_autocommit(False)
self.orm_wrapper = ORMWrapper()
self.has_build_history = has_build_history
self.tmp_dir = self.server.runCommand(["getVariable", "TMPDIR"])[0]
@@ -906,7 +909,7 @@ class BuildInfoHelper(object):
self.project = None
- logger.debug(1, "buildinfohelper: Build info helper inited %s" % vars(self))
+ logger.debug("buildinfohelper: Build info helper inited %s" % vars(self))
###################
@@ -935,7 +938,7 @@ class BuildInfoHelper(object):
# only reset the build name if the one on the server is actually
# a valid value for the build_name field
- if build_name != None:
+ if build_name is not None:
build_info['build_name'] = build_name
changed = True
@@ -1059,27 +1062,6 @@ class BuildInfoHelper(object):
return recipe_info
- def _get_path_information(self, task_object):
- self._ensure_build()
-
- assert isinstance(task_object, Task)
- build_stats_format = "{tmpdir}/buildstats/{buildname}/{package}/"
- build_stats_path = []
-
- for t in self.internal_state['targets']:
- buildname = self.internal_state['build'].build_name
- pe, pv = task_object.recipe.version.split(":",1)
- if len(pe) > 0:
- package = task_object.recipe.name + "-" + pe + "_" + pv
- else:
- package = task_object.recipe.name + "-" + pv
-
- build_stats_path.append(build_stats_format.format(tmpdir=self.tmp_dir,
- buildname=buildname,
- package=package))
-
- return build_stats_path
-
################################
## external available methods to store information
@@ -1194,7 +1176,7 @@ class BuildInfoHelper(object):
evdata = BuildInfoHelper._get_data_from_event(event)
for t in self.internal_state['targets']:
- if t.is_image == True:
+ if t.is_image:
output_files = list(evdata.keys())
for output in output_files:
if t.target in output and 'rootfs' in output and not output.endswith(".manifest"):
@@ -1236,7 +1218,7 @@ class BuildInfoHelper(object):
task_information['outcome'] = Task.OUTCOME_PREBUILT
else:
task_information['task_executed'] = True
- if 'noexec' in vars(event) and event.noexec == True:
+ if 'noexec' in vars(event) and event.noexec:
task_information['task_executed'] = False
task_information['outcome'] = Task.OUTCOME_EMPTY
task_information['script_type'] = Task.CODING_NA
@@ -1313,12 +1295,11 @@ class BuildInfoHelper(object):
task_information['outcome'] = Task.OUTCOME_FAILED
del self.internal_state['taskdata'][identifier]
- if not connection.features.autocommits_when_autocommit_is_off:
- # we force a sync point here, to get the progress bar to show
- if self.autocommit_step % 3 == 0:
- transaction.set_autocommit(True)
- transaction.set_autocommit(False)
- self.autocommit_step += 1
+ # we force a sync point here, to get the progress bar to show
+ if self.autocommit_step % 3 == 0:
+ transaction.set_autocommit(True)
+ transaction.set_autocommit(False)
+ self.autocommit_step += 1
self.orm_wrapper.get_update_task_object(task_information, True) # must exist
@@ -1404,7 +1385,7 @@ class BuildInfoHelper(object):
assert 'pn' in event._depgraph
assert 'tdepends' in event._depgraph
- errormsg = ""
+ errormsg = []
# save layer version priorities
if 'layer-priorities' in event._depgraph.keys():
@@ -1496,7 +1477,7 @@ class BuildInfoHelper(object):
elif dep in self.internal_state['recipes']:
dependency = self.internal_state['recipes'][dep]
else:
- errormsg += " stpd: KeyError saving recipe dependency for %s, %s \n" % (recipe, dep)
+ errormsg.append(" stpd: KeyError saving recipe dependency for %s, %s \n" % (recipe, dep))
continue
recipe_dep = Recipe_Dependency(recipe=target,
depends_on=dependency,
@@ -1537,8 +1518,8 @@ class BuildInfoHelper(object):
taskdeps_objects.append(Task_Dependency( task = target, depends_on = dep ))
Task_Dependency.objects.bulk_create(taskdeps_objects)
- if len(errormsg) > 0:
- logger.warning("buildinfohelper: dependency info not identify recipes: \n%s", errormsg)
+ if errormsg:
+ logger.warning("buildinfohelper: dependency info not identify recipes: \n%s", "".join(errormsg))
def store_build_package_information(self, event):
@@ -1603,14 +1584,14 @@ class BuildInfoHelper(object):
mockevent.lineno = -1
self.store_log_event(mockevent)
- def store_log_event(self, event):
+ def store_log_event(self, event,cli_backlog=True):
self._ensure_build()
if event.levelno < formatter.WARNING:
return
# early return for CLI builds
- if self.brbe is None:
+ if cli_backlog and self.brbe is None:
if not 'backlog' in self.internal_state:
self.internal_state['backlog'] = []
self.internal_state['backlog'].append(event)
@@ -1618,11 +1599,11 @@ class BuildInfoHelper(object):
if 'backlog' in self.internal_state:
# if we have a backlog of events, do our best to save them here
- if len(self.internal_state['backlog']):
+ if self.internal_state['backlog']:
tempevent = self.internal_state['backlog'].pop()
- logger.debug(1, "buildinfohelper: Saving stored event %s "
+ logger.debug("buildinfohelper: Saving stored event %s "
% tempevent)
- self.store_log_event(tempevent)
+ self.store_log_event(tempevent,cli_backlog)
else:
logger.info("buildinfohelper: All events saved")
del self.internal_state['backlog']
@@ -1765,7 +1746,6 @@ class BuildInfoHelper(object):
buildname = self.server.runCommand(['getVariable', 'BUILDNAME'])[0]
machine = self.server.runCommand(['getVariable', 'MACHINE'])[0]
- image_name = self.server.runCommand(['getVariable', 'IMAGE_NAME'])[0]
# location of the manifest files for this build;
# note that this file is only produced if an image is produced
@@ -1776,7 +1756,7 @@ class BuildInfoHelper(object):
image_file_extensions_unique = {}
image_fstypes = self.server.runCommand(
['getVariable', 'IMAGE_FSTYPES'])[0]
- if image_fstypes != None:
+ if image_fstypes is not None:
image_types_str = image_fstypes.strip()
image_file_extensions = re.sub(r' {2,}', ' ', image_types_str)
image_file_extensions_unique = set(image_file_extensions.split(' '))
@@ -1786,6 +1766,18 @@ class BuildInfoHelper(object):
# filter out anything which isn't an image target
image_targets = [target for target in targets if target.is_image]
+ if len(image_targets) > 0:
+ #if there are image targets retrieve image_name
+ image_name = self.server.runCommand(['getVariable', 'IMAGE_NAME'])[0]
+ if not image_name:
+ #When build target is an image and image_name is not found as an environment variable
+ logger.info("IMAGE_NAME not found, extracting from bitbake command")
+ cmd = self.server.runCommand(['getVariable','BB_CMDLINE'])[0]
+ #filter out tokens that are command line options
+ cmd = [token for token in cmd if not token.startswith('-')]
+ image_name = cmd[1].split(':', 1)[0] # remove everything after : in image name
+ logger.info("IMAGE_NAME found as : %s " % image_name)
+
for image_target in image_targets:
# this is set to True if we find at least one file relating to
# this target; if this remains False after the scan, we copy the
@@ -1987,10 +1979,9 @@ class BuildInfoHelper(object):
if 'backlog' in self.internal_state:
# we save missed events in the database for the current build
tempevent = self.internal_state['backlog'].pop()
- self.store_log_event(tempevent)
+ # Do not skip command line build events
+ self.store_log_event(tempevent,False)
- if not connection.features.autocommits_when_autocommit_is_off:
- transaction.set_autocommit(True)
# unset the brbe; this is to prevent subsequent command-line builds
# being incorrectly attached to the previous Toaster-triggered build;