aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorNathan Rossi <nathan@nathanrossi.com>2020-08-10 12:47:36 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2020-08-11 20:35:32 +0100
commitbef4be54e02df5f230d250487f85994a3b7bbd77 (patch)
tree800e08bde158da4717f81ffd0a2f7bcfce5c7ea2
parent53bdf2d20077921bdf83641c365d60d85174f5b5 (diff)
downloadopenembedded-core-contrib-bef4be54e02df5f230d250487f85994a3b7bbd77.tar.gz
openembedded-core-contrib-bef4be54e02df5f230d250487f85994a3b7bbd77.tar.bz2
openembedded-core-contrib-bef4be54e02df5f230d250487f85994a3b7bbd77.zip
python3: Improve handling of python3 manifest generation
Specifically cover detection of modules within a python package that do not import anything within their __init__.py. This is at least the case with the xmlrpc package which is only used via its modules xmlrpc.server and xmlrpc.client. Other important corner cases include ctypes.utils which depends on some modules not used by ctypes. This is implemented by generally assuming that importing all the modules of a package (aka *.py within a package, excluding _*.py) will provide enough information. Also due to this change some modules import sysconfig, resulting in sysconfigdata being imported. Handle the conversion of its path to a wildcard based on the platform dependent name being replaced. Signed-off-by: Nathan Rossi <nathan@nathanrossi.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--meta/recipes-devtools/python/python3/get_module_deps3.py20
-rw-r--r--meta/recipes-devtools/python/python3/python3-manifest.json2
2 files changed, 19 insertions, 3 deletions
diff --git a/meta/recipes-devtools/python/python3/get_module_deps3.py b/meta/recipes-devtools/python/python3/get_module_deps3.py
index fd12baad84e..6806f231721 100644
--- a/meta/recipes-devtools/python/python3/get_module_deps3.py
+++ b/meta/recipes-devtools/python/python3/get_module_deps3.py
@@ -9,6 +9,7 @@
debug=False
import sys
+import os
# We can get a list of the modules which are currently required to run python
# so we run python-core and get its modules, we then import what we need
@@ -48,8 +49,19 @@ current_module = str(sys.argv[1]).rstrip()
if(debug==True):
log = open('log_%s' % current_module,'w')
log.write('Module %s generated the following dependencies:\n' % current_module)
-try:
- importlib.import_module('%s' % current_module)
+try:
+ m = importlib.import_module(current_module)
+ # handle python packages which may not include all modules in the __init__
+ if os.path.basename(m.__file__) == "__init__.py":
+ modulepath = os.path.dirname(m.__file__)
+ for i in os.listdir(modulepath):
+ if i.startswith("_") or not(i.endswith(".py")):
+ continue
+ submodule = "{}.{}".format(current_module, i[:-3])
+ try:
+ importlib.import_module(submodule)
+ except:
+ pass # ignore all import or other exceptions raised during import
except ImportError as e:
if (debug==True):
log.write('Module was not found')
@@ -107,6 +119,8 @@ for item in dif:
dep_path = dep_path.replace(soabi,'*')
print (dep_path)
continue
+ if "_sysconfigdata" in dep_path:
+ dep_path = dep_path.replace(sysconfig._get_sysconfigdata_name(), "_sysconfigdata*")
if (debug==True):
log.write(dep_path+'\n')
@@ -140,6 +154,8 @@ for item in dif:
log.write(cached)
cached = fix_path(cached)
cached = cached.replace(cpython_tag,'*')
+ if "_sysconfigdata" in cached:
+ cached = cached.replace(sysconfig._get_sysconfigdata_name(), "_sysconfigdata*")
print (cached)
if debug==True:
diff --git a/meta/recipes-devtools/python/python3/python3-manifest.json b/meta/recipes-devtools/python/python3/python3-manifest.json
index 3bcc9b8662e..07b084d48c7 100644
--- a/meta/recipes-devtools/python/python3/python3-manifest.json
+++ b/meta/recipes-devtools/python/python3/python3-manifest.json
@@ -324,7 +324,7 @@
"${libdir}/python${PYTHON_MAJMIN}/__pycache__/_compression.*.pyc",
"${libdir}/python${PYTHON_MAJMIN}/__pycache__/_markupbase.*.pyc",
"${libdir}/python${PYTHON_MAJMIN}/__pycache__/_sitebuiltins.*.pyc",
- "${libdir}/python${PYTHON_MAJMIN}/__pycache__/_sysconfigdata.*.pyc",
+ "${libdir}/python${PYTHON_MAJMIN}/__pycache__/_sysconfigdata*.*.pyc",
"${libdir}/python${PYTHON_MAJMIN}/__pycache__/_weakrefset.*.pyc",
"${libdir}/python${PYTHON_MAJMIN}/__pycache__/abc.*.pyc",
"${libdir}/python${PYTHON_MAJMIN}/__pycache__/argparse.*.pyc",