diff options
Diffstat (limited to 'lib/layerindexlib/restapi.py')
| -rw-r--r-- | lib/layerindexlib/restapi.py | 398 |
1 files changed, 398 insertions, 0 deletions
diff --git a/lib/layerindexlib/restapi.py b/lib/layerindexlib/restapi.py new file mode 100644 index 000000000..d08eb2055 --- /dev/null +++ b/lib/layerindexlib/restapi.py @@ -0,0 +1,398 @@ +# Copyright (C) 2016-2018 Wind River Systems, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +import logging +import json +from urllib.parse import unquote +from urllib.parse import urlparse + +import layerindexlib +import layerindexlib.plugin + +logger = logging.getLogger('BitBake.layerindexlib.restapi') + +def plugin_init(plugins): + return RestApiPlugin() + +class RestApiPlugin(layerindexlib.plugin.IndexPlugin): + def __init__(self): + self.type = "restapi" + + def load_index(self, url, load): + """ + Fetches layer information from a local or remote layer index. + + The return value is a LayerIndexObj. + + url is the url to the rest api of the layer index, such as: + http://layers.openembedded.org/layerindex/api/ + + Or a local file... + """ + + up = urlparse(url) + + if up.scheme == 'file': + return self.load_index_file(up, url, load) + + if up.scheme == 'http' or up.scheme == 'https': + return self.load_index_web(up, url, load) + + raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url) + + + def load_index_file(self, up, url, load): + """ + Fetches layer information from a local file or directory. + + The return value is a LayerIndexObj. + + ud is the parsed url to the local file or directory. + """ + if not os.path.exists(up.path): + raise FileNotFoundError(up.path) + + index = layerindexlib.LayerIndexObj() + + index.config = {} + index.config['TYPE'] = self.type + index.config['URL'] = url + + params = self.layerindex._parse_params(up.params) + + if 'desc' in params: + index.config['DESCRIPTION'] = unquote(params['desc']) + else: + index.config['DESCRIPTION'] = up.path + + if 'cache' in params: + index.config['CACHE'] = params['cache'] + + if 'branch' in params: + branches = params['branch'].split(',') + index.config['BRANCH'] = branches + else: + branches = ['*'] + + + def load_cache(path, index, branches=[]): + logger.debug(1, 'Loading json file %s' % path) + with open(path, 'rt', encoding='utf-8') as f: + pindex = json.load(f) + + # Filter the branches on loaded files... + newpBranch = [] + for branch in branches: + if branch != '*': + if 'branches' in pindex: + for br in pindex['branches']: + if br['name'] == branch: + newpBranch.append(br) + else: + if 'branches' in pindex: + for br in pindex['branches']: + newpBranch.append(br) + + if newpBranch: + index.add_raw_element('branches', layerindexlib.Branch, newpBranch) + else: + logger.debug(1, 'No matching branches (%s) in index file(s)' % branches) + # No matching branches.. return nothing... + return + + for (lName, lType) in [("layerItems", layerindexlib.LayerItem), + ("layerBranches", layerindexlib.LayerBranch), + ("layerDependencies", layerindexlib.LayerDependency), + ("recipes", layerindexlib.Recipe), + ("machines", layerindexlib.Machine), + ("distros", layerindexlib.Distro)]: + if lName in pindex: + index.add_raw_element(lName, lType, pindex[lName]) + + + if not os.path.isdir(up.path): + load_cache(up.path, index, branches) + return index + + logger.debug(1, 'Loading from dir %s...' % (up.path)) + for (dirpath, _, filenames) in os.walk(up.path): + for filename in filenames: + if not filename.endswith('.json'): + continue + fpath = os.path.join(dirpath, filename) + load_cache(fpath, index, branches) + + return index + + + def load_index_web(self, up, url, load): + """ + Fetches layer information from a remote layer index. + + The return value is a LayerIndexObj. + + ud is the parsed url to the rest api of the layer index, such as: + http://layers.openembedded.org/layerindex/api/ + """ + + def _get_json_response(apiurl=None, username=None, password=None, retry=True): + assert apiurl is not None + + logger.debug(1, "fetching %s" % apiurl) + + up = urlparse(apiurl) + + username=up.username + password=up.password + + # Strip username/password and params + if up.port: + up_stripped = up._replace(params="", netloc="%s:%s" % (up.hostname, up.port)) + else: + up_stripped = up._replace(params="", netloc=up.hostname) + + res = self.layerindex._fetch_url(up_stripped.geturl(), username=username, password=password) + + try: + parsed = json.loads(res.read().decode('utf-8')) + except ConnectionResetError: + if retry: + logger.debug(1, "%s: Connection reset by peer. Retrying..." % url) + parsed = _get_json_response(apiurl=up_stripped.geturl(), username=username, password=password, retry=False) + logger.debug(1, "%s: retry successful.") + else: + raise LayerIndexFetchError('%s: Connection reset by peer. Is there a firewall blocking your connection?' % apiurl) + + return parsed + + index = layerindexlib.LayerIndexObj() + + index.config = {} + index.config['TYPE'] = self.type + index.config['URL'] = url + + params = self.layerindex._parse_params(up.params) + + if 'desc' in params: + index.config['DESCRIPTION'] = unquote(params['desc']) + else: + index.config['DESCRIPTION'] = up.hostname + + if 'cache' in params: + index.config['CACHE'] = params['cache'] + + if 'branch' in params: + branches = params['branch'].split(',') + index.config['BRANCH'] = branches + else: + branches = ['*'] + + try: + index.apilinks = _get_json_response(apiurl=url, username=up.username, password=up.password) + except Exception as e: + raise layerindexlib.LayerIndexFetchError(url, e) + + # Local raw index set... + pindex = {} + + # Load all the requested branches at the same time time, + # a special branch of '*' means load all branches + filter = "" + if "*" not in branches: + filter = "?filter=name:%s" % "OR".join(branches) + + logger.debug(1, "Loading %s from %s" % (branches, index.apilinks['branches'])) + + # The link won't include username/password, so pull it from the original url + pindex['branches'] = _get_json_response(index.apilinks['branches'] + filter, + username=up.username, password=up.password) + if not pindex['branches']: + logger.debug(1, "No valid branches (%s) found at url %s." % (branch, url)) + return index + index.add_raw_element("branches", layerindexlib.Branch, pindex['branches']) + + # Load all of the layerItems (these can not be easily filtered) + logger.debug(1, "Loading %s from %s" % ('layerItems', index.apilinks['layerItems'])) + + + # The link won't include username/password, so pull it from the original url + pindex['layerItems'] = _get_json_response(index.apilinks['layerItems'], + username=up.username, password=up.password) + if not pindex['layerItems']: + logger.debug(1, "No layers were found at url %s." % (url)) + return index + index.add_raw_element("layerItems", layerindexlib.LayerItem, pindex['layerItems']) + + + # From this point on load the contents for each branch. Otherwise we + # could run into a timeout. + for branch in index.branches: + filter = "?filter=branch__name:%s" % index.branches[branch].name + + logger.debug(1, "Loading %s from %s" % ('layerBranches', index.apilinks['layerBranches'])) + + # The link won't include username/password, so pull it from the original url + pindex['layerBranches'] = _get_json_response(index.apilinks['layerBranches'] + filter, + username=up.username, password=up.password) + if not pindex['layerBranches']: + logger.debug(1, "No valid layer branches (%s) found at url %s." % (branches or "*", url)) + return index + index.add_raw_element("layerBranches", layerindexlib.LayerBranch, pindex['layerBranches']) + + + # Load the rest, they all have a similar format + # Note: the layer index has a few more items, we can add them if necessary + # in the future. + filter = "?filter=layerbranch__branch__name:%s" % index.branches[branch].name + for (lName, lType) in [("layerDependencies", layerindexlib.LayerDependency), + ("recipes", layerindexlib.Recipe), + ("machines", layerindexlib.Machine), + ("distros", layerindexlib.Distro)]: + if lName not in load: + continue + logger.debug(1, "Loading %s from %s" % (lName, index.apilinks[lName])) + + # The link won't include username/password, so pull it from the original url + pindex[lName] = _get_json_response(index.apilinks[lName] + filter, + username=up.username, password=up.password) + index.add_raw_element(lName, lType, pindex[lName]) + + return index + + def store_index(self, url, index): + """ + Store layer information into a local file/dir. + + The return value is a dictionary containing API, + layer, branch, dependency, recipe, machine, distro, information. + + ud is a parsed url to a directory or file. If the path is a + directory, we will split the files into one file per layer. + If the path is to a file (exists or not) the entire DB will be + dumped into that one file. + """ + + up = urlparse(url) + + if up.scheme != 'file': + raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url) + + logger.debug(1, "Storing to %s..." % up.path) + + try: + layerbranches = index.layerBranches + except KeyError: + logger.error('No layerBranches to write.') + return + + + def filter_item(layerbranchid, objects): + filtered = [] + for obj in getattr(index, objects, None): + try: + if getattr(index, objects)[obj].layerbranch_id == layerbranchid: + filtered.append(getattr(index, objects)[obj]._data) + except AttributeError: + logger.debug(1, 'No obj.layerbranch_id: %s' % objects) + # No simple filter method, just include it... + try: + filtered.append(getattr(index, objects)[obj]._data) + except AttributeError: + logger.debug(1, 'No obj._data: %s %s' % (objects, type(obj))) + filtered.append(obj) + return filtered + + + # Write out to a single file. + # Filter out unnecessary items, then sort as we write for determinism + if not os.path.isdir(up.path): + pindex = {} + + pindex['branches'] = [] + pindex['layerItems'] = [] + pindex['layerBranches'] = [] + + for layerbranchid in layerbranches: + if layerbranches[layerbranchid].branch._data not in pindex['branches']: + pindex['branches'].append(layerbranches[layerbranchid].branch._data) + + if layerbranches[layerbranchid].layer._data not in pindex['layerItems']: + pindex['layerItems'].append(layerbranches[layerbranchid].layer._data) + + if layerbranches[layerbranchid]._data not in pindex['layerBranches']: + pindex['layerBranches'].append(layerbranches[layerbranchid]._data) + + for entry in index._index: + # Skip local items, apilinks and items already processed + if entry in index.config['local'] or \ + entry == 'apilinks' or \ + entry == 'branches' or \ + entry == 'layerBranches' or \ + entry == 'layerItems': + continue + if entry not in pindex: + pindex[entry] = [] + pindex[entry].extend(filter_item(layerbranchid, entry)) + + bb.debug(1, 'Writing index to %s' % up.path) + with open(up.path, 'wt') as f: + json.dump(layerindexlib.sort_entry(pindex), f, indent=4) + return + + + # Write out to a directory one file per layerBranch + # Prepare all layer related items, to create a minimal file. + # We have to sort the entries as we write so they are deterministic + for layerbranchid in layerbranches: + pindex = {} + + for entry in index._index: + # Skip local items, apilinks and items already processed + if entry in index.config['local'] or \ + entry == 'apilinks' or \ + entry == 'branches' or \ + entry == 'layerBranches' or \ + entry == 'layerItems': + continue + pindex[entry] = filter_item(layerbranchid, entry) + + # Add the layer we're processing as the first one... + pindex['branches'] = [layerbranches[layerbranchid].branch._data] + pindex['layerItems'] = [layerbranches[layerbranchid].layer._data] + pindex['layerBranches'] = [layerbranches[layerbranchid]._data] + + # We also need to include the layerbranch for any dependencies... + for layerdep in pindex['layerDependencies']: + layerdependency = layerindexlib.LayerDependency(index, layerdep) + + layeritem = layerdependency.dependency + layerbranch = layerdependency.dependency_layerBranch + + # We need to avoid duplicates... + if layeritem._data not in pindex['layerItems']: + pindex['layerItems'].append(layeritem._data) + + if layerbranch._data not in pindex['layerBranches']: + pindex['layerBranches'].append(layerbranch._data) + + # apply mirroring adjustments here.... + + fname = index.config['DESCRIPTION'] + '__' + pindex['branches'][0]['name'] + '__' + pindex['layerItems'][0]['name'] + fname = fname.translate(str.maketrans('/ ', '__')) + fpath = os.path.join(up.path, fname) + + bb.debug(1, 'Writing index to %s' % fpath + '.json') + with open(fpath + '.json', 'wt') as f: + json.dump(layerindexlib.sort_entry(pindex), f, indent=4) |