summaryrefslogtreecommitdiffstats
path: root/meta/lib
diff options
context:
space:
mode:
authorAlexander Kanavin <alex.kanavin@gmail.com>2019-01-23 17:17:43 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2019-01-24 17:45:28 +0000
commitb1d01911fa2a0a4945da071d66fb50e9f14ded81 (patch)
treec49660da365b1fe6d4c70b601fa74edf874f70cb /meta/lib
parent7f1f3b45329ea3f43d0ede003e8daa9866bf9f6e (diff)
downloadopenembedded-core-contrib-b1d01911fa2a0a4945da071d66fb50e9f14ded81.tar.gz
lib/oe/reciputils.py: parallelize upstream version checks
Previously this was done via bitbake tasks, and when this was rewritten to a for loop, performance sufered significantly: from 90 seconds to about 12 minutes for oe-core. This change restores the previous run time, and makes it possible to perform such checks with command line utilities in an interactive way. Implementation note: we have to create a copy of the recipe data, as Tinfoil API can't be used from multiple threads and only allows one process to access the data at a time. Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta/lib')
-rw-r--r--meta/lib/oe/recipeutils.py77
1 files changed, 56 insertions, 21 deletions
diff --git a/meta/lib/oe/recipeutils.py b/meta/lib/oe/recipeutils.py
index 39d3de4bb1f..92c0f65257f 100644
--- a/meta/lib/oe/recipeutils.py
+++ b/meta/lib/oe/recipeutils.py
@@ -1020,8 +1020,54 @@ def get_recipe_upstream_version(rd):
return ru
+def _get_recipe_upgrade_status(data):
+ uv = get_recipe_upstream_version(data)
+
+ pn = data.getVar('PN')
+ cur_ver = uv['current_version']
+
+ upstream_version_unknown = data.getVar('UPSTREAM_VERSION_UNKNOWN')
+ if not uv['version']:
+ status = "UNKNOWN" if upstream_version_unknown else "UNKNOWN_BROKEN"
+ else:
+ cmp = vercmp_string(uv['current_version'], uv['version'])
+ if cmp == -1:
+ status = "UPDATE" if not upstream_version_unknown else "KNOWN_BROKEN"
+ elif cmp == 0:
+ status = "MATCH" if not upstream_version_unknown else "KNOWN_BROKEN"
+ else:
+ status = "UNKNOWN" if upstream_version_unknown else "UNKNOWN_BROKEN"
+
+ next_ver = uv['version'] if uv['version'] else "N/A"
+ revision = uv['revision'] if uv['revision'] else "N/A"
+ maintainer = data.getVar('RECIPE_MAINTAINER')
+ no_upgrade_reason = data.getVar('RECIPE_NO_UPDATE_REASON')
+
+ return (pn, status, cur_ver, next_ver, maintainer, revision, no_upgrade_reason)
+
def get_recipe_upgrade_status(recipes=None):
pkgs_list = []
+ data_copy_list = []
+ copy_vars = ('SRC_URI',
+ 'PV',
+ 'GITDIR',
+ 'DL_DIR',
+ 'PN',
+ 'CACHE',
+ 'PERSISTENT_DIR',
+ 'BB_URI_HEADREVS',
+ 'UPSTREAM_CHECK_COMMITS',
+ 'UPSTREAM_CHECK_GITTAGREGEX',
+ 'UPSTREAM_CHECK_REGEX',
+ 'UPSTREAM_CHECK_URI',
+ 'UPSTREAM_VERSION_UNKNOWN',
+ 'RECIPE_MAINTAINER',
+ 'RECIPE_NO_UPDATE_REASON',
+ 'RECIPE_UPSTREAM_VERSION',
+ 'RECIPE_UPSTREAM_DATE',
+ 'CHECK_DATE',
+ )
+
with bb.tinfoil.Tinfoil() as tinfoil:
tinfoil.prepare(config_only=False)
@@ -1043,28 +1089,17 @@ def get_recipe_upgrade_status(recipes=None):
bb.note(" Skip package %s as upstream check unreliable" % pn)
continue
- uv = get_recipe_upstream_version(data)
-
- pn = data.getVar('PN')
- cur_ver = uv['current_version']
-
- upstream_version_unknown = data.getVar('UPSTREAM_VERSION_UNKNOWN')
- if not uv['version']:
- status = "UNKNOWN" if upstream_version_unknown else "UNKNOWN_BROKEN"
- else:
- cmp = vercmp_string(uv['current_version'], uv['version'])
- if cmp == -1:
- status = "UPDATE" if not upstream_version_unknown else "KNOWN_BROKEN"
- elif cmp == 0:
- status = "MATCH" if not upstream_version_unknown else "KNOWN_BROKEN"
- else:
- status = "UNKNOWN" if upstream_version_unknown else "UNKNOWN_BROKEN"
+ data_copy = bb.data.init()
+ for var in copy_vars:
+ data_copy.setVar(var, data.getVar(var))
+ for k in data:
+ if k.startswith('SRCREV'):
+ data_copy.setVar(k, data.getVar(k))
- next_ver = uv['version'] if uv['version'] else "N/A"
- revision = uv['revision'] if uv['revision'] else "N/A"
- maintainer = data.getVar('RECIPE_MAINTAINER')
- no_upgrade_reason = data.getVar('RECIPE_NO_UPDATE_REASON')
+ data_copy_list.append(data_copy)
- pkgs_list.append((pn, status, cur_ver, next_ver, maintainer, revision, no_upgrade_reason))
+ from concurrent.futures import ProcessPoolExecutor
+ with ProcessPoolExecutor(max_workers=utils.cpu_count()) as executor:
+ pkgs_list = executor.map(_get_recipe_upgrade_status, data_copy_list)
return pkgs_list