diff options
Diffstat (limited to 'lib/bb/ui/buildinfohelper.py')
-rw-r--r-- | lib/bb/ui/buildinfohelper.py | 145 |
1 files changed, 68 insertions, 77 deletions
diff --git a/lib/bb/ui/buildinfohelper.py b/lib/bb/ui/buildinfohelper.py index 524a5b094..8b212b780 100644 --- a/lib/bb/ui/buildinfohelper.py +++ b/lib/bb/ui/buildinfohelper.py @@ -3,18 +3,8 @@ # # Copyright (C) 2013 Intel Corporation # -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. +# SPDX-License-Identifier: GPL-2.0-only # -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import sys import bb @@ -55,7 +45,7 @@ from pprint import pformat import logging from datetime import datetime, timedelta -from django.db import transaction, connection +from django.db import transaction # pylint: disable=invalid-name @@ -158,14 +148,14 @@ class ORMWrapper(object): buildrequest = None if brbe is not None: # Toaster-triggered build - logger.debug(1, "buildinfohelper: brbe is %s" % brbe) + logger.debug("buildinfohelper: brbe is %s" % brbe) br, _ = brbe.split(":") buildrequest = BuildRequest.objects.get(pk=br) prj = buildrequest.project else: # CLI build prj = Project.objects.get_or_create_default_project() - logger.debug(1, "buildinfohelper: project is not specified, defaulting to %s" % prj) + logger.debug("buildinfohelper: project is not specified, defaulting to %s" % prj) if buildrequest is not None: # reuse existing Build object @@ -181,7 +171,7 @@ class ORMWrapper(object): completed_on=now, build_name='') - logger.debug(1, "buildinfohelper: build is created %s" % build) + logger.debug("buildinfohelper: build is created %s" % build) if buildrequest is not None: buildrequest.build = build @@ -237,6 +227,12 @@ class ORMWrapper(object): build.completed_on = timezone.now() build.outcome = outcome build.save() + + # We force a sync point here to force the outcome status commit, + # which resolves a race condition with the build completion takedown + transaction.set_autocommit(True) + transaction.set_autocommit(False) + signal_runbuilds() def update_target_set_license_manifest(self, target, license_manifest_path): @@ -493,14 +489,14 @@ class ORMWrapper(object): # we already created the root directory, so ignore any # entry for it - if len(path) == 0: + if not path: continue parent_path = "/".join(path.split("/")[:len(path.split("/")) - 1]) - if len(parent_path) == 0: + if not parent_path: parent_path = "/" parent_obj = self._cached_get(Target_File, target = target_obj, path = parent_path, inodetype = Target_File.ITYPE_DIRECTORY) - tf_obj = Target_File.objects.create( + Target_File.objects.create( target = target_obj, path = path, size = size, @@ -565,7 +561,7 @@ class ORMWrapper(object): parent_obj = Target_File.objects.get(target = target_obj, path = parent_path, inodetype = Target_File.ITYPE_DIRECTORY) - tf_obj = Target_File.objects.create( + Target_File.objects.create( target = target_obj, path = path, size = size, @@ -581,7 +577,7 @@ class ORMWrapper(object): assert isinstance(build_obj, Build) assert isinstance(target_obj, Target) - errormsg = "" + errormsg = [] for p in packagedict: # Search name swtiches round the installed name vs package name # by default installed name == package name @@ -643,10 +639,10 @@ class ORMWrapper(object): packagefile_objects.append(Package_File( package = packagedict[p]['object'], path = targetpath, size = targetfilesize)) - if len(packagefile_objects): + if packagefile_objects: Package_File.objects.bulk_create(packagefile_objects) except KeyError as e: - errormsg += " stpi: Key error, package %s key %s \n" % ( p, e ) + errormsg.append(" stpi: Key error, package %s key %s \n" % (p, e)) # save disk installed size packagedict[p]['object'].installed_size = packagedict[p]['size'] @@ -656,6 +652,9 @@ class ORMWrapper(object): Target_Installed_Package.objects.create(target = target_obj, package = packagedict[p]['object']) packagedeps_objs = [] + pattern_so = re.compile(r'.*\.so(\.\d*)?$') + pattern_lib = re.compile(r'.*\-suffix(\d*)?$') + pattern_ko = re.compile(r'^kernel-module-.*') for p in packagedict: for (px,deptype) in packagedict[p]['depends']: if deptype == 'depends': @@ -664,6 +663,13 @@ class ORMWrapper(object): tdeptype = Package_Dependency.TYPE_TRECOMMENDS try: + # Skip known non-package objects like libraries and kernel modules + if pattern_so.match(px) or pattern_lib.match(px): + logger.info("Toaster does not add library file dependencies to packages (%s,%s)", p, px) + continue + if pattern_ko.match(px): + logger.info("Toaster does not add kernel module dependencies to packages (%s,%s)", p, px) + continue packagedeps_objs.append(Package_Dependency( package = packagedict[p]['object'], depends_on = packagedict[px]['object'], @@ -673,13 +679,13 @@ class ORMWrapper(object): logger.warning("Could not add dependency to the package %s " "because %s is an unknown package", p, px) - if len(packagedeps_objs) > 0: + if packagedeps_objs: Package_Dependency.objects.bulk_create(packagedeps_objs) else: logger.info("No package dependencies created") - if len(errormsg) > 0: - logger.warning("buildinfohelper: target_package_info could not identify recipes: \n%s", errormsg) + if errormsg: + logger.warning("buildinfohelper: target_package_info could not identify recipes: \n%s", "".join(errormsg)) def save_target_image_file_information(self, target_obj, file_name, file_size): Target_Image_File.objects.create(target=target_obj, @@ -767,7 +773,7 @@ class ORMWrapper(object): packagefile_objects.append(Package_File( package = bp_object, path = path, size = package_info['FILES_INFO'][path] )) - if len(packagefile_objects): + if packagefile_objects: Package_File.objects.bulk_create(packagefile_objects) def _po_byname(p): @@ -809,7 +815,7 @@ class ORMWrapper(object): packagedeps_objs.append(Package_Dependency( package = bp_object, depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RCONFLICTS)) - if len(packagedeps_objs) > 0: + if packagedeps_objs: Package_Dependency.objects.bulk_create(packagedeps_objs) return bp_object @@ -826,7 +832,7 @@ class ORMWrapper(object): desc = vardump[root_var]['doc'] if desc is None: desc = '' - if len(desc): + if desc: HelpText.objects.get_or_create(build=build_obj, area=HelpText.VARIABLE, key=k, text=desc) @@ -846,7 +852,7 @@ class ORMWrapper(object): file_name = vh['file'], line_number = vh['line'], operation = vh['op'])) - if len(varhist_objects): + if varhist_objects: VariableHistory.objects.bulk_create(varhist_objects) @@ -893,9 +899,6 @@ class BuildInfoHelper(object): self.task_order = 0 self.autocommit_step = 1 self.server = server - # we use manual transactions if the database doesn't autocommit on us - if not connection.features.autocommits_when_autocommit_is_off: - transaction.set_autocommit(False) self.orm_wrapper = ORMWrapper() self.has_build_history = has_build_history self.tmp_dir = self.server.runCommand(["getVariable", "TMPDIR"])[0] @@ -906,7 +909,7 @@ class BuildInfoHelper(object): self.project = None - logger.debug(1, "buildinfohelper: Build info helper inited %s" % vars(self)) + logger.debug("buildinfohelper: Build info helper inited %s" % vars(self)) ################### @@ -935,7 +938,7 @@ class BuildInfoHelper(object): # only reset the build name if the one on the server is actually # a valid value for the build_name field - if build_name != None: + if build_name is not None: build_info['build_name'] = build_name changed = True @@ -1059,27 +1062,6 @@ class BuildInfoHelper(object): return recipe_info - def _get_path_information(self, task_object): - self._ensure_build() - - assert isinstance(task_object, Task) - build_stats_format = "{tmpdir}/buildstats/{buildname}/{package}/" - build_stats_path = [] - - for t in self.internal_state['targets']: - buildname = self.internal_state['build'].build_name - pe, pv = task_object.recipe.version.split(":",1) - if len(pe) > 0: - package = task_object.recipe.name + "-" + pe + "_" + pv - else: - package = task_object.recipe.name + "-" + pv - - build_stats_path.append(build_stats_format.format(tmpdir=self.tmp_dir, - buildname=buildname, - package=package)) - - return build_stats_path - ################################ ## external available methods to store information @@ -1194,7 +1176,7 @@ class BuildInfoHelper(object): evdata = BuildInfoHelper._get_data_from_event(event) for t in self.internal_state['targets']: - if t.is_image == True: + if t.is_image: output_files = list(evdata.keys()) for output in output_files: if t.target in output and 'rootfs' in output and not output.endswith(".manifest"): @@ -1236,7 +1218,7 @@ class BuildInfoHelper(object): task_information['outcome'] = Task.OUTCOME_PREBUILT else: task_information['task_executed'] = True - if 'noexec' in vars(event) and event.noexec == True: + if 'noexec' in vars(event) and event.noexec: task_information['task_executed'] = False task_information['outcome'] = Task.OUTCOME_EMPTY task_information['script_type'] = Task.CODING_NA @@ -1313,12 +1295,11 @@ class BuildInfoHelper(object): task_information['outcome'] = Task.OUTCOME_FAILED del self.internal_state['taskdata'][identifier] - if not connection.features.autocommits_when_autocommit_is_off: - # we force a sync point here, to get the progress bar to show - if self.autocommit_step % 3 == 0: - transaction.set_autocommit(True) - transaction.set_autocommit(False) - self.autocommit_step += 1 + # we force a sync point here, to get the progress bar to show + if self.autocommit_step % 3 == 0: + transaction.set_autocommit(True) + transaction.set_autocommit(False) + self.autocommit_step += 1 self.orm_wrapper.get_update_task_object(task_information, True) # must exist @@ -1404,7 +1385,7 @@ class BuildInfoHelper(object): assert 'pn' in event._depgraph assert 'tdepends' in event._depgraph - errormsg = "" + errormsg = [] # save layer version priorities if 'layer-priorities' in event._depgraph.keys(): @@ -1496,7 +1477,7 @@ class BuildInfoHelper(object): elif dep in self.internal_state['recipes']: dependency = self.internal_state['recipes'][dep] else: - errormsg += " stpd: KeyError saving recipe dependency for %s, %s \n" % (recipe, dep) + errormsg.append(" stpd: KeyError saving recipe dependency for %s, %s \n" % (recipe, dep)) continue recipe_dep = Recipe_Dependency(recipe=target, depends_on=dependency, @@ -1537,8 +1518,8 @@ class BuildInfoHelper(object): taskdeps_objects.append(Task_Dependency( task = target, depends_on = dep )) Task_Dependency.objects.bulk_create(taskdeps_objects) - if len(errormsg) > 0: - logger.warning("buildinfohelper: dependency info not identify recipes: \n%s", errormsg) + if errormsg: + logger.warning("buildinfohelper: dependency info not identify recipes: \n%s", "".join(errormsg)) def store_build_package_information(self, event): @@ -1603,14 +1584,14 @@ class BuildInfoHelper(object): mockevent.lineno = -1 self.store_log_event(mockevent) - def store_log_event(self, event): + def store_log_event(self, event,cli_backlog=True): self._ensure_build() if event.levelno < formatter.WARNING: return # early return for CLI builds - if self.brbe is None: + if cli_backlog and self.brbe is None: if not 'backlog' in self.internal_state: self.internal_state['backlog'] = [] self.internal_state['backlog'].append(event) @@ -1618,11 +1599,11 @@ class BuildInfoHelper(object): if 'backlog' in self.internal_state: # if we have a backlog of events, do our best to save them here - if len(self.internal_state['backlog']): + if self.internal_state['backlog']: tempevent = self.internal_state['backlog'].pop() - logger.debug(1, "buildinfohelper: Saving stored event %s " + logger.debug("buildinfohelper: Saving stored event %s " % tempevent) - self.store_log_event(tempevent) + self.store_log_event(tempevent,cli_backlog) else: logger.info("buildinfohelper: All events saved") del self.internal_state['backlog'] @@ -1765,7 +1746,6 @@ class BuildInfoHelper(object): buildname = self.server.runCommand(['getVariable', 'BUILDNAME'])[0] machine = self.server.runCommand(['getVariable', 'MACHINE'])[0] - image_name = self.server.runCommand(['getVariable', 'IMAGE_NAME'])[0] # location of the manifest files for this build; # note that this file is only produced if an image is produced @@ -1776,7 +1756,7 @@ class BuildInfoHelper(object): image_file_extensions_unique = {} image_fstypes = self.server.runCommand( ['getVariable', 'IMAGE_FSTYPES'])[0] - if image_fstypes != None: + if image_fstypes is not None: image_types_str = image_fstypes.strip() image_file_extensions = re.sub(r' {2,}', ' ', image_types_str) image_file_extensions_unique = set(image_file_extensions.split(' ')) @@ -1786,6 +1766,18 @@ class BuildInfoHelper(object): # filter out anything which isn't an image target image_targets = [target for target in targets if target.is_image] + if len(image_targets) > 0: + #if there are image targets retrieve image_name + image_name = self.server.runCommand(['getVariable', 'IMAGE_NAME'])[0] + if not image_name: + #When build target is an image and image_name is not found as an environment variable + logger.info("IMAGE_NAME not found, extracting from bitbake command") + cmd = self.server.runCommand(['getVariable','BB_CMDLINE'])[0] + #filter out tokens that are command line options + cmd = [token for token in cmd if not token.startswith('-')] + image_name = cmd[1].split(':', 1)[0] # remove everything after : in image name + logger.info("IMAGE_NAME found as : %s " % image_name) + for image_target in image_targets: # this is set to True if we find at least one file relating to # this target; if this remains False after the scan, we copy the @@ -1987,10 +1979,9 @@ class BuildInfoHelper(object): if 'backlog' in self.internal_state: # we save missed events in the database for the current build tempevent = self.internal_state['backlog'].pop() - self.store_log_event(tempevent) + # Do not skip command line build events + self.store_log_event(tempevent,False) - if not connection.features.autocommits_when_autocommit_is_off: - transaction.set_autocommit(True) # unset the brbe; this is to prevent subsequent command-line builds # being incorrectly attached to the previous Toaster-triggered build; |