diff options
Diffstat (limited to 'bitbake')
| -rw-r--r-- | bitbake/lib/bb/ui/buildinfohelper.py | 201 | ||||
| -rw-r--r-- | bitbake/lib/bb/ui/toasterui.py | 6 |
2 files changed, 181 insertions, 26 deletions
diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py index 447670cb8b..8bdc9cc0a7 100644 --- a/bitbake/lib/bb/ui/buildinfohelper.py +++ b/bitbake/lib/bb/ui/buildinfohelper.py @@ -678,6 +678,16 @@ class ORMWrapper(object): file_name = file_name, file_size = file_size) + def save_artifact_information_no_dedupe(self, build_obj, file_name, file_size): + """ + Save artifact information without checking for duplicate paths; + this is used when we are saving data about an artifact which was + generated by a previous build but which is also relevant to this build, + e.g. a bzImage file. + """ + BuildArtifact.objects.create(build=build_obj, file_name=file_name, + file_size=file_size) + def save_artifact_information(self, build_obj, file_name, file_size): # we skip the image files from other builds if Target_Image_File.objects.filter(file_name = file_name).count() > 0: @@ -687,7 +697,8 @@ class ORMWrapper(object): if BuildArtifact.objects.filter(file_name = file_name).count() > 0: return - BuildArtifact.objects.create(build = build_obj, file_name = file_name, file_size = file_size) + self.save_artifact_information_no_dedupe(self, build_obj, file_name, + file_size) def create_logmessage(self, log_information): assert 'build' in log_information @@ -1061,17 +1072,6 @@ class BuildInfoHelper(object): return self.brbe - - def update_target_image_file(self, event): - evdata = BuildInfoHelper._get_data_from_event(event) - - for t in self.internal_state['targets']: - if t.is_image == True: - output_files = list(evdata.keys()) - for output in output_files: - if t.target in output and 'rootfs' in output and not output.endswith(".manifest"): - self.orm_wrapper.save_target_image_file_information(t, output, evdata[output]) - def update_artifact_image_file(self, event): evdata = BuildInfoHelper._get_data_from_event(event) for artifact_path in evdata.keys(): @@ -1081,16 +1081,6 @@ class BuildInfoHelper(object): if 'build' in self.internal_state: self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures) - - def store_license_manifest_path(self, event): - deploy_dir = BuildInfoHelper._get_data_from_event(event)['deploy_dir'] - image_name = BuildInfoHelper._get_data_from_event(event)['image_name'] - path = deploy_dir + "/licenses/" + image_name + "/license.manifest" - for target in self.internal_state['targets']: - if target.target in image_name: - self.orm_wrapper.update_target_set_license_manifest(target, path) - - def store_started_task(self, event): assert isinstance(event, (bb.runqueue.sceneQueueTaskStarted, bb.runqueue.runQueueTaskStarted, bb.runqueue.runQueueTaskSkipped)) assert 'taskfile' in vars(event) @@ -1506,6 +1496,173 @@ class BuildInfoHelper(object): self.orm_wrapper.create_logmessage(log_information) + def _get_files_from_image_license(self, image_license_manifest_path): + """ + Find the FILES line in the image_license.manifest file, + which has the basenames of the bzImage and modules files + in this format: + FILES: bzImage--4.4.11+git0+3a5f494784_53e84104c5-r0-qemux86-20160603165040.bin modules--4.4.11+git0+3a5f494784_53e84104c5-r0-qemux86-20160603165040.tgz + """ + files = [] + with open(image_license_manifest_path) as image_license: + for line in image_license: + if line.startswith('FILES'): + files_str = line.split(':')[1].strip() + files_str = re.sub(r' {2,}', ' ', files_str) + files = files_str.split(' ') + return files + + def _endswith(self, str_to_test, endings): + """ + Returns True if str ends with one of the strings in the list + endings, False otherwise + """ + endswith = False + for ending in endings: + if str_to_test.endswith(ending): + endswith = True + break + return endswith + + def _get_image_files(self, deploy_dir_image, image_name, image_file_extensions): + """ + Find files in deploy_dir_image whose basename starts with the + string image_name and ends with one of the strings in + image_file_extensions. + + Returns a list of file dictionaries like + + [ + { + 'path': '/path/to/image/file', + 'size': <file size in bytes> + } + ] + """ + image_files = [] + + for dirpath, _, filenames in os.walk(deploy_dir_image): + for filename in filenames: + if filename.startswith(image_name) and \ + self._endswith(filename, image_file_extensions): + image_file_path = os.path.join(dirpath, filename) + image_file_size = os.stat(image_file_path).st_size + + image_files.append({ + 'path': image_file_path, + 'size': image_file_size + }) + + return image_files + + def scan_build_artifacts(self): + """ + Scan for build artifacts in DEPLOY_DIR_IMAGE and associate them + with a Target object in self.internal_state['targets']. + + We have two situations to handle: + + 1. This is the first time a target + machine has been built, so + add files from the DEPLOY_DIR_IMAGE to the target. + + OR + + 2. There are no files for the target, so copy them from a + previous build with the same target + machine. + """ + deploy_dir_image = \ + self.server.runCommand(['getVariable', 'DEPLOY_DIR_IMAGE'])[0] + + # if there's no DEPLOY_DIR_IMAGE, there aren't going to be + # any build artifacts, so we can return immediately + if not deploy_dir_image: + return + + buildname = self.server.runCommand(['getVariable', 'BUILDNAME'])[0] + machine = self.server.runCommand(['getVariable', 'MACHINE'])[0] + image_name = self.server.runCommand(['getVariable', 'IMAGE_NAME'])[0] + + # location of the image_license.manifest files for this build; + # note that this file is only produced if an image is produced + license_directory = \ + self.server.runCommand(['getVariable', 'LICENSE_DIRECTORY'])[0] + + # file name extensions for image files + image_file_extensions_unique = {} + image_fstypes = self.server.runCommand( + ['getVariable', 'IMAGE_FSTYPES'])[0] + if image_fstypes != None: + image_types_str = image_fstypes.strip() + image_file_extensions = re.sub(r' {2,}', ' ', image_types_str) + image_file_extensions_unique = set(image_file_extensions.split(' ')) + + targets = self.internal_state['targets'] + image_targets = [target for target in targets if target.is_image] + for target in image_targets: + # this is set to True if we find at least one file relating to + # this target; if this remains False after the scan, we copy the + # files from the most-recent Target with the same target + machine + # onto this Target instead + has_files = False + + # we construct this because by the time we reach + # BuildCompleted, this has reset to + # 'defaultpkgname-<MACHINE>-<BUILDNAME>'; + # we need to change it to + # <TARGET>-<MACHINE>-<BUILDNAME> + real_image_name = re.sub(r'^defaultpkgname', target.target, + image_name) + + image_license_manifest_path = os.path.join( + license_directory, + real_image_name, + 'image_license.manifest') + + # if image_license.manifest exists, we can read the names of bzImage + # and modules files for this build from it, then look for them + # in the DEPLOY_DIR_IMAGE; note that this file is only produced + # if an image file was produced + if os.path.isfile(image_license_manifest_path): + has_files = True + + basenames = self._get_files_from_image_license( + image_license_manifest_path) + + for basename in basenames: + artifact_path = os.path.join(deploy_dir_image, basename) + artifact_size = os.stat(artifact_path).st_size + + self.orm_wrapper.save_artifact_information_no_dedupe( + self.internal_state['build'], artifact_path, + artifact_size) + + # store the license manifest path on the target + # (this file is also created any time an image file is created) + license_path = os.path.join(license_directory, + real_image_name, 'license.manifest') + + self.orm_wrapper.update_target_set_license_manifest(target, + license_path) + + # scan the directory for image files relating to this build + # (via real_image_name); note that we don't have to set + # has_files = True, as searching for the license manifest file + # will already have set it to true if at least one image file was + # produced + image_files = self._get_image_files(deploy_dir_image, + real_image_name, image_file_extensions_unique) + + for image_file in image_files: + self.orm_wrapper.save_target_image_file_information( + target, image_file['path'], image_file['size']) + + if not has_files: + # TODO copy artifact and image files from the + # most-recently-built Target with the same target + machine + # as this Target; also copy the license manifest path, + # as that is treated differently + pass + def close(self, errorcode): if self.brbe is not None: self._store_build_done(errorcode) diff --git a/bitbake/lib/bb/ui/toasterui.py b/bitbake/lib/bb/ui/toasterui.py index 5382935f82..d8bccdb81c 100644 --- a/bitbake/lib/bb/ui/toasterui.py +++ b/bitbake/lib/bb/ui/toasterui.py @@ -363,6 +363,8 @@ def main(server, eventHandler, params): errors += 1 errorcode = 1 logger.error("Command execution failed: %s", event.error) + elif isinstance(event, bb.event.BuildCompleted): + buildinfohelper.scan_build_artifacts() # turn off logging to the current build log _close_build_log(build_log) @@ -410,12 +412,8 @@ def main(server, eventHandler, params): buildinfohelper.store_target_package_data(event) elif event.type == "MissedSstate": buildinfohelper.store_missed_state_tasks(event) - elif event.type == "ImageFileSize": - buildinfohelper.update_target_image_file(event) elif event.type == "ArtifactFileSize": buildinfohelper.update_artifact_image_file(event) - elif event.type == "LicenseManifestPath": - buildinfohelper.store_license_manifest_path(event) elif event.type == "SetBRBE": buildinfohelper.brbe = buildinfohelper._get_data_from_event(event) elif event.type == "OSErrorException": |