diff options
Diffstat (limited to 'lib/layerindexlib/__init__.py')
| -rw-r--r-- | lib/layerindexlib/__init__.py | 1364 |
1 files changed, 1364 insertions, 0 deletions
diff --git a/lib/layerindexlib/__init__.py b/lib/layerindexlib/__init__.py new file mode 100644 index 000000000..74f3e2e93 --- /dev/null +++ b/lib/layerindexlib/__init__.py @@ -0,0 +1,1364 @@ +# Copyright (C) 2016-2018 Wind River Systems, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +import datetime + +import logging +import imp + +from collections import OrderedDict +from layerindexlib.plugin import LayerIndexPluginUrlError + +logger = logging.getLogger('BitBake.layerindexlib') + +# Exceptions + +class LayerIndexException(Exception): + '''LayerIndex Generic Exception''' + def __init__(self, message): + self.msg = message + Exception.__init__(self, message) + + def __str__(self): + return self.msg + +class LayerIndexUrlError(LayerIndexException): + '''Exception raised when unable to access a URL for some reason''' + def __init__(self, url, message=""): + if message: + msg = "Unable to access layerindex url %s: %s" % (url, message) + else: + msg = "Unable to access layerindex url %s" % url + self.url = url + LayerIndexException.__init__(self, msg) + +class LayerIndexFetchError(LayerIndexException): + '''General layerindex fetcher exception when something fails''' + def __init__(self, url, message=""): + if message: + msg = "Unable to fetch layerindex url %s: %s" % (url, message) + else: + msg = "Unable to fetch layerindex url %s" % url + self.url = url + LayerIndexException.__init__(self, msg) + + +# Interface to the overall layerindex system +# the layer may contain one or more individual indexes +class LayerIndex(): + def __init__(self, d): + if not d: + raise LayerIndexException("Must be initialized with bb.data.") + + self.data = d + + # List of LayerIndexObj + self.indexes = [] + + self.plugins = [] + + import bb.utils + bb.utils.load_plugins(logger, self.plugins, os.path.dirname(__file__)) + for plugin in self.plugins: + if hasattr(plugin, 'init'): + plugin.init(self) + + def __add__(self, other): + newIndex = LayerIndex(self.data) + + if self.__class__ != newIndex.__class__ or \ + other.__class__ != newIndex.__class__: + raise TypeException("Can not add different types.") + + for indexEnt in self.indexes: + newIndex.indexes.append(indexEnt) + + for indexEnt in other.indexes: + newIndex.indexes.append(indexEnt) + + return newIndex + + def _parse_params(self, params): + '''Take a parameter list, return a dictionary of parameters. + + Expected to be called from the data of urllib.parse.urlparse(url).params + + If there are two conflicting parameters, last in wins... + ''' + + param_dict = {} + for param in params.split(';'): + if not param: + continue + item = param.split('=', 1) + logger.debug(1, item) + param_dict[item[0]] = item[1] + + return param_dict + + def _fetch_url(self, url, username=None, password=None, debuglevel=0): + '''Fetch data from a specific URL. + + Fetch something from a specific URL. This is specifically designed to + fetch data from a layerindex-web instance, but may be useful for other + raw fetch actions. + + It is not designed to be used to fetch recipe sources or similar. the + regular fetcher class should used for that. + + It is the responsibility of the caller to check BB_NO_NETWORK and related + BB_ALLOWED_NETWORKS. + ''' + + if not url: + raise LayerIndexUrlError(url, "empty url") + + import urllib + from urllib.request import urlopen, Request + from urllib.parse import urlparse + + up = urlparse(url) + + if username: + logger.debug(1, "Configuring authentication for %s..." % url) + password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm() + password_mgr.add_password(None, "%s://%s" % (up.scheme, up.netloc), username, password) + handler = urllib.request.HTTPBasicAuthHandler(password_mgr) + opener = urllib.request.build_opener(handler, urllib.request.HTTPSHandler(debuglevel=debuglevel)) + else: + opener = urllib.request.build_opener(urllib.request.HTTPSHandler(debuglevel=debuglevel)) + + urllib.request.install_opener(opener) + + logger.debug(1, "Fetching %s (%s)..." % (url, ["without authentication", "with authentication"][bool(username)])) + + try: + res = urlopen(Request(url, headers={'User-Agent': 'Mozilla/5.0 (bitbake/lib/layerindex)'}, unverifiable=True)) + except urllib.error.HTTPError as e: + logger.debug(1, "HTTP Error: %s: %s" % (e.code, e.reason)) + logger.debug(1, " Requested: %s" % (url)) + logger.debug(1, " Actual: %s" % (e.geturl())) + + if e.code == 404: + logger.debug(1, "Request not found.") + raise LayerIndexFetchError(url, e) + else: + logger.debug(1, "Headers:\n%s" % (e.headers)) + raise LayerIndexFetchError(url, e) + except OSError as e: + error = 0 + reason = "" + + # Process base OSError first... + if hasattr(e, 'errno'): + error = e.errno + reason = e.strerror + + # Process gaierror (socket error) subclass if available. + if hasattr(e, 'reason') and hasattr(e.reason, 'errno') and hasattr(e.reason, 'strerror'): + error = e.reason.errno + reason = e.reason.strerror + if error == -2: + raise LayerIndexFetchError(url, "%s: %s" % (e, reason)) + + if error and error != 0: + raise LayerIndexFetchError(url, "Unexpected exception: [Error %s] %s" % (error, reason)) + else: + raise LayerIndexFetchError(url, "Unable to fetch OSError exception: %s" % e) + + finally: + logger.debug(1, "...fetching %s (%s), done." % (url, ["without authentication", "with authentication"][bool(username)])) + + return res + + + def load_layerindex(self, indexURI, load=['layerDependencies', 'recipes', 'machines', 'distros'], reload=False): + '''Load the layerindex. + + indexURI - An index to load. (Use multiple calls to load multiple indexes) + + reload - If reload is True, then any previously loaded indexes will be forgotten. + + load - List of elements to load. Default loads all items. + Note: plugs may ignore this. + +The format of the indexURI: + + <url>;branch=<branch>;cache=<cache>;desc=<description> + + Note: the 'branch' parameter if set can select multiple branches by using + comma, such as 'branch=master,morty,pyro'. However, many operations only look + at the -first- branch specified! + + The cache value may be undefined, in this case a network failure will + result in an error, otherwise the system will look for a file of the cache + name and load that instead. + + For example: + + http://layers.openembedded.org/layerindex/api/;branch=master;desc=OpenEmbedded%20Layer%20Index + cooker:// +''' + if reload: + self.indexes = [] + + logger.debug(1, 'Loading: %s' % indexURI) + + if not self.plugins: + raise LayerIndexException("No LayerIndex Plugins available") + + for plugin in self.plugins: + # Check if the plugin was initialized + logger.debug(1, 'Trying %s' % plugin.__class__) + if not hasattr(plugin, 'type') or not plugin.type: + continue + try: + # TODO: Implement 'cache', for when the network is not available + indexEnt = plugin.load_index(indexURI, load) + break + except LayerIndexPluginUrlError as e: + logger.debug(1, "%s doesn't support %s" % (plugin.type, e.url)) + except NotImplementedError: + pass + else: + logger.debug(1, "No plugins support %s" % indexURI) + raise LayerIndexException("No plugins support %s" % indexURI) + + # Mark CONFIG data as something we've added... + indexEnt.config['local'] = [] + indexEnt.config['local'].append('config') + + # No longer permit changes.. + indexEnt.lockData() + + self.indexes.append(indexEnt) + + def store_layerindex(self, indexURI, index=None): + '''Store one layerindex + +Typically this will be used to create a local cache file of a remote index. + + file://<path>;branch=<branch> + +We can write out in either the restapi or django formats. The split option +will write out the individual elements split by layer and related components. +''' + if not index: + logger.warning('No index to write, nothing to do.') + return + + if not self.plugins: + raise LayerIndexException("No LayerIndex Plugins available") + + for plugin in self.plugins: + # Check if the plugin was initialized + logger.debug(1, 'Trying %s' % plugin.__class__) + if not hasattr(plugin, 'type') or not plugin.type: + continue + try: + plugin.store_index(indexURI, index) + break + except LayerIndexPluginUrlError as e: + logger.debug(1, "%s doesn't support %s" % (plugin.type, e.url)) + except NotImplementedError: + logger.debug(1, "Store not implemented in %s" % plugin.type) + pass + else: + logger.debug(1, "No plugins support %s" % url) + raise LayerIndexException("No plugins support %s" % url) + + + def is_empty(self): + '''Return True or False if the index has any usable data. + +We check the indexes entries to see if they have a branch set, as well as +layerBranches set. If not, they are effectively blank.''' + + found = False + for index in self.indexes: + if index.__bool__(): + found = True + break + return not found + + + def find_vcs_url(self, vcs_url, branch=None): + '''Return the first layerBranch with the given vcs_url + + If a branch has not been specified, we will iterate over the branches in + the default configuration until the first vcs_url/branch match.''' + + for index in self.indexes: + logger.debug(1, ' searching %s' % index.config['DESCRIPTION']) + layerBranch = index.find_vcs_url(vcs_url, [branch]) + if layerBranch: + return layerBranch + return None + + def find_collection(self, collection, version=None, branch=None): + '''Return the first layerBranch with the given collection name + + If a branch has not been specified, we will iterate over the branches in + the default configuration until the first collection/branch match.''' + + logger.debug(1, 'find_collection: %s (%s) %s' % (collection, version, branch)) + + if branch: + branches = [branch] + else: + branches = None + + for index in self.indexes: + logger.debug(1, ' searching %s' % index.config['DESCRIPTION']) + layerBranch = index.find_collection(collection, version, branches) + if layerBranch: + return layerBranch + else: + logger.debug(1, 'Collection %s (%s) not found for branch (%s)' % (collection, version, branch)) + return None + + def find_layerbranch(self, name, branch=None): + '''Return the layerBranch item for a given name and branch + + If a branch has not been specified, we will iterate over the branches in + the default configuration until the first name/branch match.''' + + if branch: + branches = [branch] + else: + branches = None + + for index in self.indexes: + layerBranch = index.find_layerbranch(name, branches) + if layerBranch: + return layerBranch + return None + + def find_dependencies(self, names=None, layerbranches=None, ignores=None): + '''Return a tuple of all dependencies and valid items for the list of (layer) names + + The dependency scanning happens depth-first. The returned + dependencies should be in the best order to define bblayers. + + names - list of layer names (searching layerItems) + branches - when specified (with names) only this list of branches are evaluated + + layerbranches - list of layerbranches to resolve dependencies + + ignores - list of layer names to ignore + + return: (dependencies, invalid) + + dependencies[LayerItem.name] = [ LayerBranch, LayerDependency1, LayerDependency2, ... ] + invalid = [ LayerItem.name1, LayerItem.name2, ... ] + ''' + + invalid = [] + + # Convert name/branch to layerbranches + if layerbranches is None: + layerbranches = [] + + for name in names: + if ignores and name in ignores: + continue + + for index in self.indexes: + layerbranch = index.find_layerbranch(name) + if not layerbranch: + # Not in this index, hopefully it's in another... + continue + layerbranches.append(layerbranch) + break + else: + invalid.append(name) + + + def _resolve_dependencies(layerbranches, ignores, dependencies, invalid): + for layerbranch in layerbranches: + if ignores and layerbranch.layer.name in ignores: + continue + + # Get a list of dependencies and then recursively process them + for layerdependency in layerbranch.index.layerDependencies_layerBranchId[layerbranch.id]: + deplayerbranch = layerdependency.dependency_layerBranch + + if ignores and deplayerbranch.layer.name in ignores: + continue + + # This little block is why we can't re-use the LayerIndexObj version, + # we must be able to satisfy each dependencies across layer indexes and + # use the layer index order for priority. (r stands for replacement below) + + # If this is the primary index, we can fast path and skip this + if deplayerbranch.index != self.indexes[0]: + # Is there an entry in a prior index for this collection/version? + rdeplayerbranch = self.find_collection( + collection=deplayerbranch.collection, + version=deplayerbranch.version + ) + if rdeplayerbranch != deplayerbranch: + logger.debug(1, 'Replaced %s:%s:%s with %s:%s:%s' % \ + (deplayerbranch.index.config['DESCRIPTION'], + deplayerbranch.branch.name, + deplayerbranch.layer.name, + rdeplayerbranch.index.config['DESCRIPTION'], + rdeplayerbranch.branch.name, + rdeplayerbranch.layer.name)) + deplayerbranch = rdeplayerbranch + + # New dependency, we need to resolve it now... depth-first + if deplayerbranch.layer.name not in dependencies: + (dependencies, invalid) = _resolve_dependencies([deplayerbranch], ignores, dependencies, invalid) + + if deplayerbranch.layer.name not in dependencies: + dependencies[deplayerbranch.layer.name] = [deplayerbranch, layerdependency] + else: + if layerdependency not in dependencies[deplayerbranch.layer.name]: + dependencies[deplayerbranch.layer.name].append(layerdependency) + + return (dependencies, invalid) + + # OK, resolve this one... + dependencies = OrderedDict() + (dependencies, invalid) = _resolve_dependencies(layerbranches, ignores, dependencies, invalid) + + for layerbranch in layerbranches: + if layerbranch.layer.name not in dependencies: + dependencies[layerbranch.layer.name] = [layerbranch] + + return (dependencies, invalid) + + + def list_obj(self, object): + '''Print via the plain logger object information + +This function is used to implement debugging and provide the user info. +''' + for lix in self.indexes: + if object not in lix: + continue + + logger.plain ('') + logger.plain ('Index: %s' % lix.config['DESCRIPTION']) + + output = [] + + if object == 'branches': + logger.plain ('%s %s %s' % ('{:26}'.format('branch'), '{:34}'.format('description'), '{:22}'.format('bitbake branch'))) + logger.plain ('{:-^80}'.format("")) + for branchid in lix.branches: + output.append('%s %s %s' % ( + '{:26}'.format(lix.branches[branchid].name), + '{:34}'.format(lix.branches[branchid].short_description), + '{:22}'.format(lix.branches[branchid].bitbake_branch) + )) + for line in sorted(output): + logger.plain (line) + + continue + + if object == 'layerItems': + logger.plain ('%s %s' % ('{:26}'.format('layer'), '{:34}'.format('description'))) + logger.plain ('{:-^80}'.format("")) + for layerid in lix.layerItems: + output.append('%s %s' % ( + '{:26}'.format(lix.layerItems[layerid].name), + '{:34}'.format(lix.layerItems[layerid].summary) + )) + for line in sorted(output): + logger.plain (line) + + continue + + if object == 'layerBranches': + logger.plain ('%s %s %s' % ('{:26}'.format('layer'), '{:34}'.format('description'), '{:19}'.format('collection:version'))) + logger.plain ('{:-^80}'.format("")) + for layerbranchid in lix.layerBranches: + output.append('%s %s %s' % ( + '{:26}'.format(lix.layerBranches[layerbranchid].layer.name), + '{:34}'.format(lix.layerBranches[layerbranchid].layer.summary), + '{:19}'.format("%s:%s" % + (lix.layerBranches[layerbranchid].collection, + lix.layerBranches[layerbranchid].version) + ) + )) + for line in sorted(output): + logger.plain (line) + + continue + + if object == 'layerDependencies': + logger.plain ('%s %s %s %s' % ('{:19}'.format('branch'), '{:26}'.format('layer'), '{:11}'.format('dependency'), '{:26}'.format('layer'))) + logger.plain ('{:-^80}'.format("")) + for layerDependency in lix.layerDependencies: + if not lix.layerDependencies[layerDependency].dependency_layerBranch: + continue + + output.append('%s %s %s %s' % ( + '{:19}'.format(lix.layerDependencies[layerDependency].layerbranch.branch.name), + '{:26}'.format(lix.layerDependencies[layerDependency].layerbranch.layer.name), + '{:11}'.format('requires' if lix.layerDependencies[layerDependency].required else 'recommends'), + '{:26}'.format(lix.layerDependencies[layerDependency].dependency_layerBranch.layer.name) + )) + for line in sorted(output): + logger.plain (line) + + continue + + if object == 'recipes': + logger.plain ('%s %s %s' % ('{:20}'.format('recipe'), '{:10}'.format('version'), 'layer')) + logger.plain ('{:-^80}'.format("")) + output = [] + for recipe in lix.recipes: + output.append('%s %s %s' % ( + '{:30}'.format(lix.recipes[recipe].pn), + '{:30}'.format(lix.recipes[recipe].pv), + lix.recipes[recipe].layer.name + )) + for line in sorted(output): + logger.plain (line) + + continue + + if object == 'machines': + logger.plain ('%s %s %s' % ('{:24}'.format('machine'), '{:34}'.format('description'), '{:19}'.format('layer'))) + logger.plain ('{:-^80}'.format("")) + for machine in lix.machines: + output.append('%s %s %s' % ( + '{:24}'.format(lix.machines[machine].name), + '{:34}'.format(lix.machines[machine].description)[:34], + '{:19}'.format(lix.machines[machine].layerbranch.layer.name) + )) + for line in sorted(output): + logger.plain (line) + + continue + + if object == 'distros': + logger.plain ('%s %s %s' % ('{:24}'.format('distro'), '{:34}'.format('description'), '{:19}'.format('layer'))) + logger.plain ('{:-^80}'.format("")) + for distro in lix.distros: + output.append('%s %s %s' % ( + '{:24}'.format(lix.distros[distro].name), + '{:34}'.format(lix.distros[distro].description)[:34], + '{:19}'.format(lix.distros[distro].layerbranch.layer.name) + )) + for line in sorted(output): + logger.plain (line) + + continue + + logger.plain ('') + + +# This class holds a single layer index instance +# The LayerIndexObj is made up of dictionary of elements, such as: +# index['config'] - configuration data for this index +# index['branches'] - dictionary of Branch objects, by id number +# index['layerItems'] - dictionary of layerItem objects, by id number +# ...etc... (See: http://layers.openembedded.org/layerindex/api/) +# +# The class needs to manage the 'index' entries and allow easily adding +# of new items, as well as simply loading of the items. +class LayerIndexObj(): + def __init__(self): + super().__setattr__('_index', {}) + super().__setattr__('_lock', False) + + def __bool__(self): + '''False if the index is effectively empty + + We check the index to see if it has a branch set, as well as + layerbranches set. If not, it is effectively blank.''' + + if not bool(self._index): + return False + + try: + if self.branches and self.layerBranches: + return True + except AttributeError: + pass + + return False + + def __getattr__(self, name): + if name.startswith('_'): + return super().__getattribute__(name) + + if name not in self._index: + raise AttributeError('%s not in index datastore' % name) + + return self._index[name] + + def __setattr__(self, name, value): + if self.isLocked(): + raise TypeError("Can not set attribute '%s': index is locked" % name) + + if name.startswith('_'): + super().__setattr__(name, value) + return + + self._index[name] = value + + def __delattr__(self, name): + if self.isLocked(): + raise TypeError("Can not delete attribute '%s': index is locked" % name) + + if name.startswith('_'): + super().__delattr__(name) + + self._index.pop(name) + + def lockData(self): + '''Lock data object (make it readonly)''' + super().__setattr__("_lock", True) + + def unlockData(self): + '''unlock data object (make it readonly)''' + super().__setattr__("_lock", False) + + # When the data is unlocked, we have to clear the caches, as + # modification is allowed! + del(self._layerBranches_layerId_branchId) + del(self._layerDependencies_layerBranchId) + del(self._layerBranches_vcsUrl) + + def isLocked(self): + '''Is this object locked (readonly)?''' + return self._lock + + def add_element(self, indexname, objs): + '''Add a layer index object to index.<indexname>''' + if indexname not in self._index: + self._index[indexname] = {} + + for obj in objs: + if obj.id in self._index[indexname]: + if self._index[indexname][obj.id] == obj: + continue + raise LayerIndexError('Conflict adding object %s(%s) to index' % (indexname, obj.id)) + self._index[indexname][obj.id] = obj + + def add_raw_element(self, indexname, objtype, rawobjs): + '''Convert a raw layer index data item to a layer index item object and add to the index''' + objs = [] + for entry in rawobjs: + objs.append(objtype(self, entry)) + self.add_element(indexname, objs) + + # Quick lookup table for searching layerId and branchID combos + @property + def layerBranches_layerId_branchId(self): + def createCache(self): + cache = {} + for layerbranchid in self.layerBranches: + layerbranch = self.layerBranches[layerbranchid] + cache["%s:%s" % (layerbranch.layer_id, layerbranch.branch_id)] = layerbranch + return cache + + if self.isLocked(): + cache = getattr(self, '_layerBranches_layerId_branchId', None) + else: + cache = None + + if not cache: + cache = createCache(self) + + if self.isLocked(): + super().__setattr__('_layerBranches_layerId_branchId', cache) + + return cache + + # Quick lookup table for finding all dependencies of a layerBranch + @property + def layerDependencies_layerBranchId(self): + def createCache(self): + cache = {} + # This ensures empty lists for all branchids + for layerbranchid in self.layerBranches: + cache[layerbranchid] = [] + + for layerdependencyid in self.layerDependencies: + layerdependency = self.layerDependencies[layerdependencyid] + cache[layerdependency.layerbranch_id].append(layerdependency) + return cache + + if self.isLocked(): + cache = getattr(self, '_layerDependencies_layerBranchId', None) + else: + cache = None + + if not cache: + cache = createCache(self) + + if self.isLocked(): + super().__setattr__('_layerDependencies_layerBranchId', cache) + + return cache + + # Quick lookup table for finding all instances of a vcs_url + @property + def layerBranches_vcsUrl(self): + def createCache(self): + cache = {} + for layerbranchid in self.layerBranches: + layerbranch = self.layerBranches[layerbranchid] + if layerbranch.layer.vcs_url not in cache: + cache[layerbranch.layer.vcs_url] = [layerbranch] + else: + cache[layerbranch.layer.vcs_url].append(layerbranch) + return cache + + if self.isLocked(): + cache = getattr(self, '_layerBranches_vcsUrl', None) + else: + cache = None + + if not cache: + cache = createCache(self) + + if self.isLocked(): + super().__setattr__('_layerBranches_vcsUrl', cache) + + return cache + + + def find_vcs_url(self, vcs_url, branches=None): + ''''Return the first layerBranch with the given vcs_url + + If a list of branches has not been specified, we will iterate on + all branches until the first vcs_url is found.''' + + if not self.__bool__(): + return None + + for layerbranch in self.layerBranches_vcsUrl: + if branches and layerbranch.branch.name not in branches: + continue + + return layerbranch + + return None + + + def find_collection(self, collection, version=None, branches=None): + '''Return the first layerBranch with the given collection name + + If a list of branches has not been specified, we will iterate on + all branches until the first collection is found.''' + + if not self.__bool__(): + return None + + for layerbranchid in self.layerBranches: + layerbranch = self.layerBranches[layerbranchid] + if branches and layerbranch.branch.name not in branches: + continue + + if layerbranch.collection == collection and \ + (version is None or version == layerbranch.version): + return layerbranch + + return None + + + def find_layerbranch(self, name, branches=None): + '''Return the first layerbranch whose layer name matches + + If a list of branches has not been specified, we will iterate on + all branches until the first layer with that name is found.''' + + if not self.__bool__(): + return None + + for layerbranchid in self.layerBranches: + layerbranch = self.layerBranches[layerbranchid] + if branches and layerbranch.branch.name not in branches: + continue + + if layerbranch.layer.name == name: + return layerbranch + + return None + + def find_dependencies(self, names=None, branches=None, layerBranches=None, ignores=None): + '''Return a tuple of all dependencies and valid items for the list of (layer) names + + The dependency scanning happens depth-first. The returned + dependencies should be in the best order to define bblayers. + + names - list of layer names (searching layerItems) + branches - when specified (with names) only this list of branches are evaluated + + layerBranches - list of layerBranches to resolve dependencies + + ignores - list of layer names to ignore + + return: (dependencies, invalid) + + dependencies[LayerItem.name] = [ LayerBranch, LayerDependency1, LayerDependency2, ... ] + invalid = [ LayerItem.name1, LayerItem.name2, ... ]''' + + invalid = [] + + # Convert name/branch to layerBranches + if layerbranches is None: + layerbranches = [] + + for name in names: + if ignores and name in ignores: + continue + + layerbranch = self.find_layerbranch(name, branches) + if not layerbranch: + invalid.append(name) + else: + layerbranches.append(layerbranch) + + for layerbranch in layerbranches: + if layerbranch.index != self: + raise LayerIndexException("Can not resolve dependencies across indexes with this class function!") + + def _resolve_dependencies(layerbranches, ignores, dependencies, invalid): + for layerbranch in layerbranches: + if ignores and layerBranch.layer.name in ignores: + continue + + for layerdependency in layerbranch.index.layerDependencies_layerBranchId[layerBranch.id]: + deplayerbranch = layerDependency.dependency_layerBranch + + if ignores and deplayerbranch.layer.name in ignores: + continue + + # New dependency, we need to resolve it now... depth-first + if deplayerbranch.layer.name not in dependencies: + (dependencies, invalid) = _resolve_dependencies([deplayerbranch], ignores, dependencies, invalid) + + if deplayerbranch.layer.name not in dependencies: + dependencies[deplayerbranch.layer.name] = [deplayerbranch, layerdependency] + else: + if layerdependency not in dependencies[deplayerbranch.layer.name]: + dependencies[deplayerbranch.layer.name].append(layerdependency) + + return (dependencies, invalid) + + # OK, resolve this one... + dependencies = OrderedDict() + (dependencies, invalid) = _resolve_dependencies(layerbranches, ignores, dependencies, invalid) + + # Is this item already in the list, if not add it + for layerbranch in layerbranches: + if layerbranch.layer.name not in dependencies: + dependencies[layerbranch.layer.name] = [layerbranch] + + return (dependencies, invalid) + + +# Define a basic LayerIndexItemObj. This object forms the basis for all other +# objects. The raw Layer Index data is stored in the _data element, but we +# do not want users to access data directly. So wrap this and protect it +# from direct manipulation. +# +# It is up to the insantiators of the objects to fill them out, and once done +# lock the objects to prevent further accidently manipulation. +# +# Using the getattr, setattr and properties we can access and manipulate +# the data within the data element. +class LayerIndexItemObj(): + def __init__(self, index, data=None, lock=False): + if data is None: + data = {} + + if type(data) != type(dict()): + raise TypeError('data (%s) is not a dict' % type(data)) + + super().__setattr__('_lock', lock) + super().__setattr__('index', index) + super().__setattr__('_data', data) + + def __eq__(self, other): + if self.__class__ != other.__class__: + return False + res=(self._data == other._data) + return res + + def __bool__(self): + return bool(self._data) + + def __getattr__(self, name): + # These are internal to THIS class, and not part of data + if name == "index" or name.startswith('_'): + return super().__getattribute__(name) + + if name not in self._data: + raise AttributeError('%s not in datastore' % name) + + return self._data[name] + + def _setattr(self, name, value, prop=True): + '''__setattr__ like function, but with control over property object behavior''' + if self.isLocked(): + raise TypeError("Can not set attribute '%s': Object data is locked" % name) + + if name.startswith('_'): + super().__setattr__(name, value) + return + + # Since __setattr__ runs before properties, we need to check if + # there is a setter property and then execute it + # ... or return self._data[name] + propertyobj = getattr(self.__class__, name, None) + if prop and isinstance(propertyobj, property): + if propertyobj.fset: + propertyobj.fset(self, value) + else: + raise AttributeError('Attribute %s is readonly, and may not be set' % name) + else: + self._data[name] = value + + def __setattr__(self, name, value): + self._setattr(name, value, prop=True) + + def _delattr(self, name, prop=True): + # Since __delattr__ runs before properties, we need to check if + # there is a deleter property and then execute it + # ... or we pop it ourselves.. + propertyobj = getattr(self.__class__, name, None) + if prop and isinstance(propertyobj, property): + if propertyobj.fdel: + propertyobj.fdel(self) + else: + raise AttributeError('Attribute %s is readonly, and may not be deleted' % name) + else: + self._data.pop(name) + + def __delattr__(self, name): + self._delattr(name, prop=True) + + def lockData(self): + '''Lock data object (make it readonly)''' + super().__setattr__("_lock", True) + + def unlockData(self): + '''unlock data object (make it readonly)''' + super().__setattr__("_lock", False) + + def isLocked(self): + '''Is this object locked (readonly)?''' + return self._lock + +# Branch object +class Branch(LayerIndexItemObj): + def define_data(self, id, name, bitbake_branch, + short_description=None, sort_priority=1, + updates_enabled=True, updated=None, + update_environment=None): + self.id = id + self.name = name + self.bitbake_branch = bitbake_branch + self.short_description = short_description or name + self.sort_priority = sort_priority + self.updates_enabled = updates_enabled + self.updated = updated or datetime.datetime.today().isoformat() + self.update_environment = update_environment + + @property + def name(self): + return self.__getattr__('name') + + @name.setter + def name(self, value): + self._data['name'] = value + + if self.bitbake_branch == value: + self.bitbake_branch = "" + + @name.deleter + def name(self): + self._delattr('name', prop=False) + + @property + def bitbake_branch(self): + try: + return self.__getattr__('bitbake_branch') + except AttributeError: + return self.name + + @bitbake_branch.setter + def bitbake_branch(self, value): + if self.name == value: + self._data['bitbake_branch'] = "" + else: + self._data['bitbake_branch'] = value + + @bitbake_branch.deleter + def bitbake_branch(self): + self._delattr('bitbake_branch', prop=False) + + +class LayerItem(LayerIndexItemObj): + def define_data(self, id, name, status='P', + layer_type='A', summary=None, + description=None, + vcs_url=None, vcs_web_url=None, + vcs_web_tree_base_url=None, + vcs_web_file_base_url=None, + usage_url=None, + mailing_list_url=None, + index_preference=1, + classic=False, + updated=None): + self.id = id + self.name = name + self.status = status + self.layer_type = layer_type + self.summary = summary or name + self.description = description or summary or name + self.vcs_url = vcs_url + self.vcs_web_url = vcs_web_url + self.vcs_web_tree_base_url = vcs_web_tree_base_url + self.vcs_web_file_base_url = vcs_web_file_base_url + self.index_preference = index_preference + self.classic = classic + self.updated = updated or datetime.datetime.today().isoformat() + + +class LayerBranch(LayerIndexItemObj): + def define_data(self, id, collection, version, layer, branch, + vcs_subdir="", vcs_last_fetch=None, + vcs_last_rev=None, vcs_last_commit=None, + actual_branch="", + updated=None): + self.id = id + self.collection = collection + self.version = version + if type(layer) != type(LayerItem): + self.layer_id = layer + else: + self.layer = layer + + if type(branch) != type(Branch): + self.branch_id = branch + else: + self.branch = branch + + self.vcs_subdir = vcs_subdir + self.vcs_last_fetch = vcs_last_fetch + self.vcs_last_rev = vcs_last_rev + self.vcs_last_commit = vcs_last_commit + self.actual_branch = actual_branch + self.updated = updated or datetime.datetime.today().isoformat() + + # This is a little odd, the _data attribute is 'layer', but it's really + # referring to the layer id.. so lets adjust this to make it useful + @property + def layer_id(self): + return self.__getattr__('layer') + + @layer_id.setter + def layer_id(self, value): + self._setattr('layer', value, prop=False) + + @layer_id.deleter + def layer_id(self): + self._delattr('layer', prop=False) + + @property + def layer(self): + try: + return self.index.layerItems[self.layer_id] + except KeyError: + raise AttributeError('Unable to find layerItems in index to map layer_id %s' % self.layer_id) + except IndexError: + raise AttributeError('Unable to find layer_id %s in index layerItems' % self.layer_id) + + @layer.setter + def layer(self, value): + if type(value) != type(LayerItem): + raise TypeError('value is not a LayerItem') + if self.index != value.index: + raise AttributeError('Object and value do not share the same index and thus key set.') + self.layer_id = value.id + + @layer.deleter + def layer(self): + del self.layer_id + + @property + def branch_id(self): + return self.__getattr__('branch') + + @branch_id.setter + def branch_id(self, value): + self._setattr('branch', value, prop=False) + + @branch_id.deleter + def branch_id(self): + self._delattr('branch', prop=False) + + @property + def branch(self): + try: + logger.debug(1, "Get branch object from branches[%s]" % (self.branch_id)) + return self.index.branches[self.branch_id] + except KeyError: + raise AttributeError('Unable to find branches in index to map branch_id %s' % self.branch_id) + except IndexError: + raise AttributeError('Unable to find branch_id %s in index branches' % self.branch_id) + + @branch.setter + def branch(self, value): + if type(value) != type(LayerItem): + raise TypeError('value is not a LayerItem') + if self.index != value.index: + raise AttributeError('Object and value do not share the same index and thus key set.') + self.branch_id = value.id + + @branch.deleter + def branch(self): + del self.branch_id + + @property + def actual_branch(self): + if self.__getattr__('actual_branch'): + return self.__getattr__('actual_branch') + else: + return self.branch.name + + @actual_branch.setter + def actual_branch(self, value): + logger.debug(1, "Set actual_branch to %s .. name is %s" % (value, self.branch.name)) + if value != self.branch.name: + self._setattr('actual_branch', value, prop=False) + else: + self._setattr('actual_branch', '', prop=False) + + @actual_branch.deleter + def actual_branch(self): + self._delattr('actual_branch', prop=False) + +# Extend LayerIndexItemObj with common LayerBranch manipulations +# All of the remaining LayerIndex objects refer to layerbranch, and it is +# up to the user to follow that back through the LayerBranch object into +# the layer object to get various attributes. So add an intermediate set +# of attributes that can easily get us the layerbranch as well as layer. + +class LayerIndexItemObj_LayerBranch(LayerIndexItemObj): + @property + def layerbranch_id(self): + return self.__getattr__('layerbranch') + + @layerbranch_id.setter + def layerbranch_id(self, value): + self._setattr('layerbranch', value, prop=False) + + @layerbranch_id.deleter + def layerbranch_id(self): + self._delattr('layerbranch', prop=False) + + @property + def layerbranch(self): + try: + return self.index.layerBranches[self.layerbranch_id] + except KeyError: + raise AttributeError('Unable to find layerBranches in index to map layerbranch_id %s' % self.layerbranch_id) + except IndexError: + raise AttributeError('Unable to find layerbranch_id %s in index branches' % self.layerbranch_id) + + @layerbranch.setter + def layerbranch(self, value): + if type(value) != type(LayerBranch): + raise TypeError('value (%s) is not a layerBranch' % type(value)) + if self.index != value.index: + raise AttributeError('Object and value do not share the same index and thus key set.') + self.layerbranch_id = value.id + + @layerbranch.deleter + def layerbranch(self): + del self.layerbranch_id + + @property + def layer_id(self): + return self.layerbranch.layer_id + + # Doesn't make sense to set or delete layer_id + + @property + def layer(self): + return self.layerbranch.layer + + # Doesn't make sense to set or delete layer + + +class LayerDependency(LayerIndexItemObj_LayerBranch): + def define_data(self, id, layerbranch, dependency, required=True): + self.id = id + if type(layerbranch) != type(LayerBranch): + self.layerbranch_id = layerbranch + else: + self.layerbranch = layerbranch + if type(dependency) != type(LayerDependency): + self.dependency_id = dependency + else: + self.dependency = dependency + self.required = required + + @property + def dependency_id(self): + return self.__getattr__('dependency') + + @dependency_id.setter + def dependency_id(self, value): + self._setattr('dependency', value, prop=False) + + @dependency_id.deleter + def dependency_id(self): + self._delattr('dependency', prop=False) + + @property + def dependency(self): + try: + return self.index.layerItems[self.dependency_id] + except KeyError: + raise AttributeError('Unable to find layerItems in index to map layerbranch_id %s' % self.dependency_id) + except IndexError: + raise AttributeError('Unable to find dependency_id %s in index layerItems' % self.dependency_id) + + @dependency.setter + def dependency(self, value): + if type(value) != type(LayerDependency): + raise TypeError('value (%s) is not a dependency' % type(value)) + if self.index != value.index: + raise AttributeError('Object and value do not share the same index and thus key set.') + self.dependency_id = value.id + + @dependency.deleter + def dependency(self): + self._delattr('dependency', prop=False) + + @property + def dependency_layerBranch(self): + layerid = self.dependency_id + branchid = self.layerbranch.branch_id + + try: + return self.index.layerBranches_layerId_branchId["%s:%s" % (layerid, branchid)] + except IndexError: + # layerBranches_layerId_branchId -- but not layerId:branchId + raise AttributeError('Unable to find layerId:branchId %s:%s in index layerBranches_layerId_branchId' % (layerid, branchid)) + except KeyError: + raise AttributeError('Unable to find layerId:branchId %s:%s in layerItems and layerBranches' % (layerid, branchid)) + + # dependency_layerBranch doesn't make sense to set or del + + +class Recipe(LayerIndexItemObj_LayerBranch): + def define_data(self, id, + filename, filepath, pn, pv, layerbranch, + summary="", description="", section="", license="", + homepage="", bugtracker="", provides="", bbclassextend="", + inherits="", blacklisted="", updated=None): + self.id = id + self.filename = filename + self.filepath = filepath + self.pn = pn + self.pv = pv + self.summary = summary + self.description = description + self.section = section + self.license = license + self.homepage = homepage + self.bugtracker = bugtracker + self.provides = provides + self.bbclassextend = bbclassextend + self.inherits = inherits + self.updated = updated or datetime.datetime.today().isoformat() + self.blacklisted = blacklisted + if type(layerbranch) != type(LayerBranch): + self.layerbranch_id = layerbranch + else: + self.layerbranch = layerbranch + + @property + def fullpath(self): + return os.path.join(self.filepath, self.filename) + + # Set would need to understand how to split it + # del would we del both parts? + + @property + def inherits(self): + if 'inherits' not in self._data: + # Older indexes may not have this, so emulate it + if '-image-' in self.pn: + return 'image' + return self.__getattr__('inherits') + + @inherits.setter + def inherits(self, value): + return self._setattr('inherits', value, prop=False) + + @inherits.deleter + def inherits(self): + return self._delattr('inherits', prop=False) + + +class Machine(LayerIndexItemObj_LayerBranch): + def define_data(self, id, + name, description, layerbranch, + updated=None): + self.id = id + self.name = name + self.description = description + if type(layerbranch) != type(LayerBranch): + self.layerbranch_id = layerbranch + else: + self.layerbranch = layerbranch + self.updated = updated or datetime.datetime.today().isoformat() + +class Distro(LayerIndexItemObj_LayerBranch): + def define_data(self, id, + name, description, layerbranch, + updated=None): + self.id = id + self.name = name + self.description = description + if type(layerbranch) != type(LayerBranch): + self.layerbranch_id = layerbranch + else: + self.layerbranch = layerbranch + self.updated = updated or datetime.datetime.today().isoformat() + + +# When performing certain actions, we may need to sort the data. +# This will allow us to keep it consistent from run to run. +def sort_entry(item): + newitem = item + try: + if type(newitem) == type(dict()): + newitem = OrderedDict(sorted(newitem.items(), key=lambda t: t[0])) + for index in newitem: + newitem[index] = sort_entry(newitem[index]) + elif type(newitem) == type(list()): + newitem.sort(key=lambda obj: obj['id']) + for index, _ in enumerate(newitem): + newitem[index] = sort_entry(newitem[index]) + except: + logger.error('Sort failed for item %s' % type(item)) + pass + + return newitem |