diff options
author | Alexander Kanavin <alex.kanavin@gmail.com> | 2019-01-23 17:17:43 +0100 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2019-01-24 17:45:28 +0000 |
commit | b1d01911fa2a0a4945da071d66fb50e9f14ded81 (patch) | |
tree | c49660da365b1fe6d4c70b601fa74edf874f70cb /meta/lib | |
parent | 7f1f3b45329ea3f43d0ede003e8daa9866bf9f6e (diff) | |
download | openembedded-core-contrib-b1d01911fa2a0a4945da071d66fb50e9f14ded81.tar.gz |
lib/oe/reciputils.py: parallelize upstream version checks
Previously this was done via bitbake tasks, and when this
was rewritten to a for loop, performance sufered significantly:
from 90 seconds to about 12 minutes for oe-core. This change
restores the previous run time, and makes it possible
to perform such checks with command line utilities in an
interactive way.
Implementation note: we have to create a copy of the recipe
data, as Tinfoil API can't be used from multiple threads
and only allows one process to access the data at a time.
Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta/lib')
-rw-r--r-- | meta/lib/oe/recipeutils.py | 77 |
1 files changed, 56 insertions, 21 deletions
diff --git a/meta/lib/oe/recipeutils.py b/meta/lib/oe/recipeutils.py index 39d3de4bb1f..92c0f65257f 100644 --- a/meta/lib/oe/recipeutils.py +++ b/meta/lib/oe/recipeutils.py @@ -1020,8 +1020,54 @@ def get_recipe_upstream_version(rd): return ru +def _get_recipe_upgrade_status(data): + uv = get_recipe_upstream_version(data) + + pn = data.getVar('PN') + cur_ver = uv['current_version'] + + upstream_version_unknown = data.getVar('UPSTREAM_VERSION_UNKNOWN') + if not uv['version']: + status = "UNKNOWN" if upstream_version_unknown else "UNKNOWN_BROKEN" + else: + cmp = vercmp_string(uv['current_version'], uv['version']) + if cmp == -1: + status = "UPDATE" if not upstream_version_unknown else "KNOWN_BROKEN" + elif cmp == 0: + status = "MATCH" if not upstream_version_unknown else "KNOWN_BROKEN" + else: + status = "UNKNOWN" if upstream_version_unknown else "UNKNOWN_BROKEN" + + next_ver = uv['version'] if uv['version'] else "N/A" + revision = uv['revision'] if uv['revision'] else "N/A" + maintainer = data.getVar('RECIPE_MAINTAINER') + no_upgrade_reason = data.getVar('RECIPE_NO_UPDATE_REASON') + + return (pn, status, cur_ver, next_ver, maintainer, revision, no_upgrade_reason) + def get_recipe_upgrade_status(recipes=None): pkgs_list = [] + data_copy_list = [] + copy_vars = ('SRC_URI', + 'PV', + 'GITDIR', + 'DL_DIR', + 'PN', + 'CACHE', + 'PERSISTENT_DIR', + 'BB_URI_HEADREVS', + 'UPSTREAM_CHECK_COMMITS', + 'UPSTREAM_CHECK_GITTAGREGEX', + 'UPSTREAM_CHECK_REGEX', + 'UPSTREAM_CHECK_URI', + 'UPSTREAM_VERSION_UNKNOWN', + 'RECIPE_MAINTAINER', + 'RECIPE_NO_UPDATE_REASON', + 'RECIPE_UPSTREAM_VERSION', + 'RECIPE_UPSTREAM_DATE', + 'CHECK_DATE', + ) + with bb.tinfoil.Tinfoil() as tinfoil: tinfoil.prepare(config_only=False) @@ -1043,28 +1089,17 @@ def get_recipe_upgrade_status(recipes=None): bb.note(" Skip package %s as upstream check unreliable" % pn) continue - uv = get_recipe_upstream_version(data) - - pn = data.getVar('PN') - cur_ver = uv['current_version'] - - upstream_version_unknown = data.getVar('UPSTREAM_VERSION_UNKNOWN') - if not uv['version']: - status = "UNKNOWN" if upstream_version_unknown else "UNKNOWN_BROKEN" - else: - cmp = vercmp_string(uv['current_version'], uv['version']) - if cmp == -1: - status = "UPDATE" if not upstream_version_unknown else "KNOWN_BROKEN" - elif cmp == 0: - status = "MATCH" if not upstream_version_unknown else "KNOWN_BROKEN" - else: - status = "UNKNOWN" if upstream_version_unknown else "UNKNOWN_BROKEN" + data_copy = bb.data.init() + for var in copy_vars: + data_copy.setVar(var, data.getVar(var)) + for k in data: + if k.startswith('SRCREV'): + data_copy.setVar(k, data.getVar(k)) - next_ver = uv['version'] if uv['version'] else "N/A" - revision = uv['revision'] if uv['revision'] else "N/A" - maintainer = data.getVar('RECIPE_MAINTAINER') - no_upgrade_reason = data.getVar('RECIPE_NO_UPDATE_REASON') + data_copy_list.append(data_copy) - pkgs_list.append((pn, status, cur_ver, next_ver, maintainer, revision, no_upgrade_reason)) + from concurrent.futures import ProcessPoolExecutor + with ProcessPoolExecutor(max_workers=utils.cpu_count()) as executor: + pkgs_list = executor.map(_get_recipe_upgrade_status, data_copy_list) return pkgs_list |