From e3e85bdf7100cc5619d7f184dc8d9e2a77a03364 Mon Sep 17 00:00:00 2001 From: Alexandru DAMIAN Date: Thu, 14 May 2015 16:10:50 +0100 Subject: bitbake: toaster logger: refactor recipe and layer file paths This refactoring brings the "local_path" of the layer from the Layer object to the Layer_Version object, which is more appropriate as different checkouts of the same Layer may live in different directories. This enables us to store Recipe file paths relative to a Layer_Version at all times, aleviating the need to store full file paths in the database. We also turn the prefix of the path (e.g. virtual:native path name space) into a pathflag field. In turn, this solves the problem of mis-identification of tasks based on the recipe file paths, since we can also match the namespace of the file paths on the recipe files. [YOCTO #7594] (Bitbake rev: ec43dc569e370767c709dec225cbee0c99151c19) Signed-off-by: Alexandru DAMIAN Signed-off-by: Richard Purdie --- bitbake/lib/bb/ui/buildinfohelper.py | 47 ++++++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 15 deletions(-) (limited to 'bitbake/lib/bb/ui/buildinfohelper.py') diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py index 647d663cb5..6812a52b7b 100644 --- a/bitbake/lib/bb/ui/buildinfohelper.py +++ b/bitbake/lib/bb/ui/buildinfohelper.py @@ -221,12 +221,12 @@ class ORMWrapper(object): def get_update_recipe_object(self, recipe_information, must_exist = False): assert 'layer_version' in recipe_information assert 'file_path' in recipe_information + assert 'pathflags' in recipe_information - if recipe_information['file_path'].startswith(recipe_information['layer_version'].layer.local_path): - recipe_information['file_path'] = recipe_information['file_path'][len(recipe_information['layer_version'].layer.local_path):].lstrip("/") + assert not recipe_information['file_path'].startswith("/") # we should have layer-relative paths at all times recipe_object, created = self._cached_get_or_create(Recipe, layer_version=recipe_information['layer_version'], - file_path=recipe_information['file_path']) + file_path=recipe_information['file_path'], pathflags = recipe_information['pathflags']) if created and must_exist: raise NotExisting("Recipe object created when expected to exist", recipe_information) @@ -247,13 +247,15 @@ class ORMWrapper(object): assert 'branch' in layer_version_information assert 'commit' in layer_version_information assert 'priority' in layer_version_information + assert 'local_path' in layer_version_information layer_version_object, created = Layer_Version.objects.get_or_create( build = build_obj, layer = layer_obj, branch = layer_version_information['branch'], commit = layer_version_information['commit'], - priority = layer_version_information['priority'] + priority = layer_version_information['priority'], + local_path = layer_version_information['local_path'], ) self.layer_version_objects.append(layer_version_object) @@ -262,13 +264,11 @@ class ORMWrapper(object): def get_update_layer_object(self, layer_information, brbe): assert 'name' in layer_information - assert 'local_path' in layer_information assert 'layer_index_url' in layer_information if brbe is None: layer_object, created = Layer.objects.get_or_create( name=layer_information['name'], - local_path=layer_information['local_path'], layer_index_url=layer_information['layer_index_url']) return layer_object else: @@ -297,7 +297,6 @@ class ORMWrapper(object): for pl in buildrequest.project.projectlayer_set.filter(layercommit__layer__name = brl.name): if pl.layercommit.layer.vcs_url == brl.giturl : layer = pl.layercommit.layer - layer.local_path = layer_information['local_path'] layer.save() return layer @@ -687,12 +686,12 @@ class BuildInfoHelper(object): if self.brbe is None: def _slkey_interactive(layer_version): assert isinstance(layer_version, Layer_Version) - return len(layer_version.layer.local_path) + return len(layer_version.local_path) # Heuristics: we always match recipe to the deepest layer path in the discovered layers for lvo in sorted(self.orm_wrapper.layer_version_objects, reverse=True, key=_slkey_interactive): # we can match to the recipe file path - if path.startswith(lvo.layer.local_path): + if path.startswith(lvo.local_path): return lvo else: @@ -721,7 +720,7 @@ class BuildInfoHelper(object): logger.warn("Could not match layer version for recipe path %s : %s" % (path, self.orm_wrapper.layer_version_objects)) #mockup the new layer - unknown_layer, created = Layer.objects.get_or_create(name="__FIXME__unidentified_layer", local_path="/", layer_index_url="") + unknown_layer, created = Layer.objects.get_or_create(name="__FIXME__unidentified_layer", layer_index_url="") unknown_layer_version_obj, created = Layer_Version.objects.get_or_create(layer = unknown_layer, build = self.internal_state['build']) # append it so we don't run into this error again and again @@ -731,11 +730,20 @@ class BuildInfoHelper(object): def _get_recipe_information_from_taskfile(self, taskfile): localfilepath = taskfile.split(":")[-1] + filepath_flags = ":".join(sorted(taskfile.split(":")[:-1])) layer_version_obj = self._get_layer_version_for_path(localfilepath) + + recipe_info = {} recipe_info['layer_version'] = layer_version_obj - recipe_info['file_path'] = taskfile + recipe_info['file_path'] = localfilepath + recipe_info['pathflags'] = filepath_flags + + if recipe_info['file_path'].startswith(recipe_info['layer_version'].local_path): + recipe_info['file_path'] = recipe_info['file_path'][len(recipe_info['layer_version'].local_path):].lstrip("/") + else: + raise RuntimeError("Recipe file path %s is not under layer version at %s" % (recipe_info['file_path'], recipe_info['layer_version'].local_path)) return recipe_info @@ -787,6 +795,7 @@ class BuildInfoHelper(object): for layer in layerinfos: try: self.internal_state['lvs'][self.orm_wrapper.get_update_layer_object(layerinfos[layer], self.brbe)] = layerinfos[layer]['version'] + self.internal_state['lvs'][self.orm_wrapper.get_update_layer_object(layerinfos[layer], self.brbe)]['local_path'] = layerinfos[layer]['local_path'] except NotExisting as nee: logger.warn("buildinfohelper: cannot identify layer exception:%s " % nee) @@ -899,8 +908,8 @@ class BuildInfoHelper(object): recipe_information = self._get_recipe_information_from_taskfile(taskfile) try: - if recipe_information['file_path'].startswith(recipe_information['layer_version'].layer.local_path): - recipe_information['file_path'] = recipe_information['file_path'][len(recipe_information['layer_version'].layer.local_path):].lstrip("/") + if recipe_information['file_path'].startswith(recipe_information['layer_version'].local_path): + recipe_information['file_path'] = recipe_information['file_path'][len(recipe_information['layer_version'].local_path):].lstrip("/") recipe_object = Recipe.objects.get(layer_version = recipe_information['layer_version'], file_path__endswith = recipe_information['file_path'], @@ -1051,8 +1060,9 @@ class BuildInfoHelper(object): self.internal_state['recipes'] = {} for pn in event._depgraph['pn']: - file_name = event._depgraph['pn'][pn]['filename'] - layer_version_obj = self._get_layer_version_for_path(file_name.split(":")[-1]) + file_name = event._depgraph['pn'][pn]['filename'].split(":")[-1] + pathflags = ":".join(sorted(event._depgraph['pn'][pn]['filename'].split(":")[:-1])) + layer_version_obj = self._get_layer_version_for_path(file_name) assert layer_version_obj is not None @@ -1082,6 +1092,13 @@ class BuildInfoHelper(object): recipe_info['bugtracker'] = event._depgraph['pn'][pn]['bugtracker'] recipe_info['file_path'] = file_name + recipe_info['pathflags'] = pathflags + + if recipe_info['file_path'].startswith(recipe_info['layer_version'].local_path): + recipe_info['file_path'] = recipe_info['file_path'][len(recipe_info['layer_version'].local_path):].lstrip("/") + else: + raise RuntimeError("Recipe file path %s is not under layer version at %s" % (recipe_info['file_path'], recipe_info['layer_version'].local_path)) + recipe = self.orm_wrapper.get_update_recipe_object(recipe_info) recipe.is_image = False if 'inherits' in event._depgraph['pn'][pn].keys(): -- cgit 1.2.3-korg