aboutsummaryrefslogtreecommitdiffstats
path: root/scripts/lib/recipetool/create_buildsys_python.py
diff options
context:
space:
mode:
authorChristopher Larson <kergoth@gmail.com>2015-01-19 11:52:31 -0700
committerRichard Purdie <richard.purdie@linuxfoundation.org>2015-01-23 11:35:26 +0000
commitcb093aca3b78f130dc7da820a8710342a12d1231 (patch)
tree8c0fa767de40ae1cd134c34451106389a9760d71 /scripts/lib/recipetool/create_buildsys_python.py
parent6ff693c71d97b4bcfde198c84cf9fac7185cccfd (diff)
downloadopenembedded-core-contrib-cb093aca3b78f130dc7da820a8710342a12d1231.tar.gz
openembedded-core-contrib-cb093aca3b78f130dc7da820a8710342a12d1231.tar.bz2
openembedded-core-contrib-cb093aca3b78f130dc7da820a8710342a12d1231.zip
recipetool: add python dependency scanning support
This uses a standalone python script named `pythondeps` which now lives in scripts. It supports scanning for provided packages and imported modules/packages, the latter via the python ast. It's not perfect, and obviously conditional imports and try/except import blocks are handled naively, listing all the imports even if they aren't all used at once, but it gives the user a solid starting point for the recipe. Currently `python_dir` from setup.py isn't being handled in an ideal way. This is easily seen when testing the python-async package. There, the root of the project is the async package, so the root has __init__.py and friends, and the python provides scanning currently just assumes the basedir of that dir is the package name in this case, which is not correct. Forthcoming patches will resolve this. Signed-off-by: Christopher Larson <kergoth@gmail.com> Signed-off-by: Ross Burton <ross.burton@intel.com>
Diffstat (limited to 'scripts/lib/recipetool/create_buildsys_python.py')
-rw-r--r--scripts/lib/recipetool/create_buildsys_python.py160
1 files changed, 160 insertions, 0 deletions
diff --git a/scripts/lib/recipetool/create_buildsys_python.py b/scripts/lib/recipetool/create_buildsys_python.py
index 9e4e1ebd4c..f4f4212559 100644
--- a/scripts/lib/recipetool/create_buildsys_python.py
+++ b/scripts/lib/recipetool/create_buildsys_python.py
@@ -41,6 +41,13 @@ def tinfoil_init(instance):
class PythonRecipeHandler(RecipeHandler):
+ base_pkgdeps = ['python-core']
+ excluded_pkgdeps = ['python-dbg']
+ # os.path is provided by python-core
+ assume_provided = ['builtins', 'os.path']
+ # Assumes that the host python builtin_module_names is sane for target too
+ assume_provided = assume_provided + list(sys.builtin_module_names)
+
bbvar_map = {
'Name': 'PN',
'Version': 'PV',
@@ -273,6 +280,8 @@ class PythonRecipeHandler(RecipeHandler):
mdinfo.append('{} = "{}"'.format(k, v))
lines_before[src_uri_line-1:src_uri_line-1] = mdinfo
+ mapped_deps, unmapped_deps = self.scan_setup_python_deps(srctree, setup_info, setup_non_literals)
+
extras_req = set()
if 'Extras-require' in info:
extras_req = info['Extras-require']
@@ -284,6 +293,8 @@ class PythonRecipeHandler(RecipeHandler):
lines_after.append('# Uncomment this line to enable all the optional features.')
lines_after.append('#PACKAGECONFIG ?= "{}"'.format(' '.join(k.lower() for k in extras_req.iterkeys())))
for feature, feature_reqs in extras_req.iteritems():
+ unmapped_deps.difference_update(feature_reqs)
+
feature_req_deps = ('python-' + r.replace('.', '-').lower() for r in sorted(feature_reqs))
lines_after.append('PACKAGECONFIG[{}] = ",,,{}"'.format(feature.lower(), ' '.join(feature_req_deps)))
@@ -293,11 +304,34 @@ class PythonRecipeHandler(RecipeHandler):
lines_after.append('')
inst_reqs = info['Install-requires']
if inst_reqs:
+ unmapped_deps.difference_update(inst_reqs)
+
inst_req_deps = ('python-' + r.replace('.', '-').lower() for r in sorted(inst_reqs))
lines_after.append('# WARNING: the following rdepends are from setuptools install_requires. These')
lines_after.append('# upstream names may not correspond exactly to bitbake package names.')
lines_after.append('RDEPENDS_${{PN}} += "{}"'.format(' '.join(inst_req_deps)))
+ if mapped_deps:
+ name = info.get('Name')
+ if name and name[0] in mapped_deps:
+ # Attempt to avoid self-reference
+ mapped_deps.remove(name[0])
+ mapped_deps -= set(self.excluded_pkgdeps)
+ if inst_reqs or extras_req:
+ lines_after.append('')
+ lines_after.append('# WARNING: the following rdepends are determined through basic analysis of the')
+ lines_after.append('# python sources, and might not be 100% accurate.')
+ lines_after.append('RDEPENDS_${{PN}} += "{}"'.format(' '.join(sorted(mapped_deps))))
+
+ unmapped_deps -= set(extensions)
+ unmapped_deps -= set(self.assume_provided)
+ if unmapped_deps:
+ if mapped_deps:
+ lines_after.append('')
+ lines_after.append('# WARNING: We were unable to map the following python package/module')
+ lines_after.append('# dependencies to the bitbake packages which include them:')
+ lines_after.extend('# {}'.format(d) for d in sorted(unmapped_deps))
+
handled.append('buildsystem')
def get_pkginfo(self, pkginfo_fn):
@@ -425,6 +459,132 @@ class PythonRecipeHandler(RecipeHandler):
if value != new_list:
info[variable] = new_list
+ def scan_setup_python_deps(self, srctree, setup_info, setup_non_literals):
+ if 'Package-dir' in setup_info:
+ package_dir = setup_info['Package-dir']
+ else:
+ package_dir = {}
+
+ class PackageDir(distutils.command.build_py.build_py):
+ def __init__(self, package_dir):
+ self.package_dir = package_dir
+
+ pd = PackageDir(package_dir)
+ to_scan = []
+ if not any(v in setup_non_literals for v in ['Py-modules', 'Scripts', 'Packages']):
+ if 'Py-modules' in setup_info:
+ for module in setup_info['Py-modules']:
+ try:
+ package, module = module.rsplit('.', 1)
+ except ValueError:
+ package, module = '.', module
+ module_path = os.path.join(pd.get_package_dir(package), module + '.py')
+ to_scan.append(module_path)
+
+ if 'Packages' in setup_info:
+ for package in setup_info['Packages']:
+ to_scan.append(pd.get_package_dir(package))
+
+ if 'Scripts' in setup_info:
+ to_scan.extend(setup_info['Scripts'])
+ else:
+ logger.info("Scanning the entire source tree, as one or more of the following setup keywords are non-literal: py_modules, scripts, packages.")
+
+ if not to_scan:
+ to_scan = ['.']
+
+ logger.info("Scanning paths for packages & dependencies: %s", ', '.join(to_scan))
+
+ provided_packages = self.parse_pkgdata_for_python_packages()
+ scanned_deps = self.scan_python_dependencies([os.path.join(srctree, p) for p in to_scan])
+ mapped_deps, unmapped_deps = set(self.base_pkgdeps), set()
+ for dep in scanned_deps:
+ mapped = provided_packages.get(dep)
+ if mapped:
+ mapped_deps.add(mapped)
+ else:
+ unmapped_deps.add(dep)
+ return mapped_deps, unmapped_deps
+
+ def scan_python_dependencies(self, paths):
+ deps = set()
+ try:
+ dep_output = self.run_command(['pythondeps', '-d'] + paths)
+ except (OSError, subprocess.CalledProcessError):
+ pass
+ else:
+ for line in dep_output.splitlines():
+ line = line.rstrip()
+ dep, filename = line.split('\t', 1)
+ if filename.endswith('/setup.py'):
+ continue
+ deps.add(dep)
+
+ try:
+ provides_output = self.run_command(['pythondeps', '-p'] + paths)
+ except (OSError, subprocess.CalledProcessError):
+ pass
+ else:
+ provides_lines = (l.rstrip() for l in provides_output.splitlines())
+ provides = set(l for l in provides_lines if l and l != 'setup')
+ deps -= provides
+
+ return deps
+
+ def parse_pkgdata_for_python_packages(self):
+ suffixes = [t[0] for t in imp.get_suffixes()]
+ pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR', True)
+
+ ldata = tinfoil.config_data.createCopy()
+ bb.parse.handle('classes/python-dir.bbclass', ldata, True)
+ python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR', True)
+
+ dynload_dir = os.path.join(os.path.dirname(python_sitedir), 'lib-dynload')
+ python_dirs = [python_sitedir + os.sep,
+ os.path.join(os.path.dirname(python_sitedir), 'dist-packages') + os.sep,
+ os.path.dirname(python_sitedir) + os.sep]
+ packages = {}
+ for pkgdatafile in glob.glob('{}/runtime/*'.format(pkgdata_dir)):
+ files_info = None
+ with open(pkgdatafile, 'r') as f:
+ for line in f.readlines():
+ field, value = line.split(': ', 1)
+ if field == 'FILES_INFO':
+ files_info = ast.literal_eval(value)
+ break
+ else:
+ continue
+
+ for fn in files_info.iterkeys():
+ for suffix in suffixes:
+ if fn.endswith(suffix):
+ break
+ else:
+ continue
+
+ if fn.startswith(dynload_dir + os.sep):
+ base = os.path.basename(fn)
+ provided = base.split('.', 1)[0]
+ packages[provided] = os.path.basename(pkgdatafile)
+ continue
+
+ for python_dir in python_dirs:
+ if fn.startswith(python_dir):
+ relpath = fn[len(python_dir):]
+ relstart, _, relremaining = relpath.partition(os.sep)
+ if relstart.endswith('.egg'):
+ relpath = relremaining
+ base, _ = os.path.splitext(relpath)
+
+ if '/.debug/' in base:
+ continue
+ if os.path.basename(base) == '__init__':
+ base = os.path.dirname(base)
+ base = base.replace(os.sep + os.sep, os.sep)
+ provided = base.replace(os.sep, '.')
+ packages[provided] = os.path.basename(pkgdatafile)
+ return packages
+
@classmethod
def run_command(cls, cmd, **popenargs):
if 'stderr' not in popenargs: