diff options
Diffstat (limited to 'scripts/lib/recipetool/create_buildsys_python.py')
| -rw-r--r-- | scripts/lib/recipetool/create_buildsys_python.py | 198 |
1 files changed, 113 insertions, 85 deletions
diff --git a/scripts/lib/recipetool/create_buildsys_python.py b/scripts/lib/recipetool/create_buildsys_python.py index aff13cf1d2..4675cc68fa 100644 --- a/scripts/lib/recipetool/create_buildsys_python.py +++ b/scripts/lib/recipetool/create_buildsys_python.py @@ -2,23 +2,13 @@ # # Copyright (C) 2015 Mentor Graphics Corporation # -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. +# SPDX-License-Identifier: GPL-2.0-only # -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import ast import codecs import collections -import distutils.command.build_py +import setuptools.command.build_py import email import imp import glob @@ -41,11 +31,11 @@ def tinfoil_init(instance): class PythonRecipeHandler(RecipeHandler): - base_pkgdeps = ['python-core'] - excluded_pkgdeps = ['python-dbg'] - # os.path is provided by python-core + base_pkgdeps = ['python3-core'] + excluded_pkgdeps = ['python3-dbg'] + # os.path is provided by python3-core assume_provided = ['builtins', 'os.path'] - # Assumes that the host python builtin_module_names is sane for target too + # Assumes that the host python3 builtin_module_names is sane for target too assume_provided = assume_provided + list(sys.builtin_module_names) bbvar_map = { @@ -55,14 +45,12 @@ class PythonRecipeHandler(RecipeHandler): 'Summary': 'SUMMARY', 'Description': 'DESCRIPTION', 'License': 'LICENSE', - 'Requires': 'RDEPENDS_${PN}', - 'Provides': 'RPROVIDES_${PN}', - 'Obsoletes': 'RREPLACES_${PN}', + 'Requires': 'RDEPENDS:${PN}', + 'Provides': 'RPROVIDES:${PN}', + 'Obsoletes': 'RREPLACES:${PN}', } # PN/PV are already set by recipetool core & desc can be extremely long excluded_fields = [ - 'Name', - 'Version', 'Description', ] setup_parse_map = { @@ -88,8 +76,11 @@ class PythonRecipeHandler(RecipeHandler): ] setuparg_multi_line_values = ['Description'] replacements = [ + ('License', r' +$', ''), + ('License', r'^ +', ''), ('License', r' ', '-'), - ('License', r'-License$', ''), + ('License', r'^GNU-', ''), + ('License', r'-[Ll]icen[cs]e(,?-[Vv]ersion)?', ''), ('License', r'^UNKNOWN$', ''), # Remove currently unhandled version numbers from these variables @@ -110,30 +101,39 @@ class PythonRecipeHandler(RecipeHandler): 'License :: OSI Approved :: Apple Public Source License': 'APSL', 'License :: OSI Approved :: Artistic License': 'Artistic', 'License :: OSI Approved :: Attribution Assurance License': 'AAL', - 'License :: OSI Approved :: BSD License': 'BSD', + 'License :: OSI Approved :: BSD License': 'BSD-3-Clause', + 'License :: OSI Approved :: Boost Software License 1.0 (BSL-1.0)': 'BSL-1.0', + 'License :: OSI Approved :: CEA CNRS Inria Logiciel Libre License, version 2.1 (CeCILL-2.1)': 'CECILL-2.1', + 'License :: OSI Approved :: Common Development and Distribution License 1.0 (CDDL-1.0)': 'CDDL-1.0', 'License :: OSI Approved :: Common Public License': 'CPL', + 'License :: OSI Approved :: Eclipse Public License 1.0 (EPL-1.0)': 'EPL-1.0', + 'License :: OSI Approved :: Eclipse Public License 2.0 (EPL-2.0)': 'EPL-2.0', 'License :: OSI Approved :: Eiffel Forum License': 'EFL', 'License :: OSI Approved :: European Union Public Licence 1.0 (EUPL 1.0)': 'EUPL-1.0', 'License :: OSI Approved :: European Union Public Licence 1.1 (EUPL 1.1)': 'EUPL-1.1', - 'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)': 'AGPL-3.0+', - 'License :: OSI Approved :: GNU Affero General Public License v3': 'AGPL-3.0', + 'License :: OSI Approved :: European Union Public Licence 1.2 (EUPL 1.2)': 'EUPL-1.2', + 'License :: OSI Approved :: GNU Affero General Public License v3': 'AGPL-3.0-only', + 'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)': 'AGPL-3.0-or-later', 'License :: OSI Approved :: GNU Free Documentation License (FDL)': 'GFDL', 'License :: OSI Approved :: GNU General Public License (GPL)': 'GPL', - 'License :: OSI Approved :: GNU General Public License v2 (GPLv2)': 'GPL-2.0', - 'License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)': 'GPL-2.0+', - 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)': 'GPL-3.0', - 'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)': 'GPL-3.0+', - 'License :: OSI Approved :: GNU Lesser General Public License v2 (LGPLv2)': 'LGPL-2.0', - 'License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)': 'LGPL-2.0+', - 'License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)': 'LGPL-3.0', - 'License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)': 'LGPL-3.0+', + 'License :: OSI Approved :: GNU General Public License v2 (GPLv2)': 'GPL-2.0-only', + 'License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)': 'GPL-2.0-or-later', + 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)': 'GPL-3.0-only', + 'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)': 'GPL-3.0-or-later', + 'License :: OSI Approved :: GNU Lesser General Public License v2 (LGPLv2)': 'LGPL-2.0-only', + 'License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)': 'LGPL-2.0-or-later', + 'License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)': 'LGPL-3.0-only', + 'License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)': 'LGPL-3.0-or-later', 'License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)': 'LGPL', + 'License :: OSI Approved :: Historical Permission Notice and Disclaimer (HPND)': 'HPND', 'License :: OSI Approved :: IBM Public License': 'IPL', 'License :: OSI Approved :: ISC License (ISCL)': 'ISC', 'License :: OSI Approved :: Intel Open Source License': 'Intel', 'License :: OSI Approved :: Jabber Open Source License': 'Jabber', 'License :: OSI Approved :: MIT License': 'MIT', + 'License :: OSI Approved :: MIT No Attribution License (MIT-0)': 'MIT-0', 'License :: OSI Approved :: MITRE Collaborative Virtual Workspace License (CVW)': 'CVWL', + 'License :: OSI Approved :: MirOS License (MirOS)': 'MirOS', 'License :: OSI Approved :: Motosoto License': 'Motosoto', 'License :: OSI Approved :: Mozilla Public License 1.0 (MPL)': 'MPL-1.0', 'License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)': 'MPL-1.1', @@ -141,19 +141,26 @@ class PythonRecipeHandler(RecipeHandler): 'License :: OSI Approved :: Nethack General Public License': 'NGPL', 'License :: OSI Approved :: Nokia Open Source License': 'Nokia', 'License :: OSI Approved :: Open Group Test Suite License': 'OGTSL', + 'License :: OSI Approved :: Open Software License 3.0 (OSL-3.0)': 'OSL-3.0', + 'License :: OSI Approved :: PostgreSQL License': 'PostgreSQL', 'License :: OSI Approved :: Python License (CNRI Python License)': 'CNRI-Python', - 'License :: OSI Approved :: Python Software Foundation License': 'PSF', + 'License :: OSI Approved :: Python Software Foundation License': 'PSF-2.0', 'License :: OSI Approved :: Qt Public License (QPL)': 'QPL', 'License :: OSI Approved :: Ricoh Source Code Public License': 'RSCPL', + 'License :: OSI Approved :: SIL Open Font License 1.1 (OFL-1.1)': 'OFL-1.1', 'License :: OSI Approved :: Sleepycat License': 'Sleepycat', - 'License :: OSI Approved :: Sun Industry Standards Source License (SISSL)': '-- Sun Industry Standards Source License (SISSL)', + 'License :: OSI Approved :: Sun Industry Standards Source License (SISSL)': 'SISSL', 'License :: OSI Approved :: Sun Public License': 'SPL', + 'License :: OSI Approved :: The Unlicense (Unlicense)': 'Unlicense', + 'License :: OSI Approved :: Universal Permissive License (UPL)': 'UPL-1.0', 'License :: OSI Approved :: University of Illinois/NCSA Open Source License': 'NCSA', 'License :: OSI Approved :: Vovida Software License 1.0': 'VSL-1.0', 'License :: OSI Approved :: W3C License': 'W3C', 'License :: OSI Approved :: X.Net License': 'Xnet', 'License :: OSI Approved :: Zope Public License': 'ZPL', 'License :: OSI Approved :: zlib/libpng License': 'Zlib', + 'License :: Other/Proprietary License': 'Proprietary', + 'License :: Public Domain': 'PD', } def __init__(self): @@ -163,8 +170,13 @@ class PythonRecipeHandler(RecipeHandler): if 'buildsystem' in handled: return False - if not RecipeHandler.checkfiles(srctree, ['setup.py']): - return + # Check for non-zero size setup.py files + setupfiles = RecipeHandler.checkfiles(srctree, ['setup.py']) + for fn in setupfiles: + if os.path.getsize(fn): + break + else: + return False # setup.py is always parsed to get at certain required information, such as # distutils vs setuptools @@ -197,6 +209,18 @@ class PythonRecipeHandler(RecipeHandler): continue if line.startswith('['): + # PACKAGECONFIG must not contain expressions or whitespace + line = line.replace(" ", "") + line = line.replace(':', "") + line = line.replace('.', "-dot-") + line = line.replace('"', "") + line = line.replace('<', "-smaller-") + line = line.replace('>', "-bigger-") + line = line.replace('_', "-") + line = line.replace('(', "") + line = line.replace(')', "") + line = line.replace('!', "-not-") + line = line.replace('=', "-equals-") current_feature = line[1:-1] elif current_feature: extras_req[current_feature].append(line) @@ -218,26 +242,48 @@ class PythonRecipeHandler(RecipeHandler): else: info = self.get_setup_args_info(setupscript) + # Grab the license value before applying replacements + license_str = info.get('License', '').strip() + self.apply_info_replacements(info) if uses_setuptools: - classes.append('setuptools') + classes.append('setuptools3') else: - classes.append('distutils') + classes.append('distutils3') + + if license_str: + for i, line in enumerate(lines_before): + if line.startswith('LICENSE = '): + lines_before.insert(i, '# NOTE: License in setup.py/PKGINFO is: %s' % license_str) + break if 'Classifier' in info: + existing_licenses = info.get('License', '') licenses = [] for classifier in info['Classifier']: if classifier in self.classifier_license_map: license = self.classifier_license_map[classifier] + if license == 'Apache' and 'Apache-2.0' in existing_licenses: + license = 'Apache-2.0' + elif license == 'GPL': + if 'GPL-2.0' in existing_licenses or 'GPLv2' in existing_licenses: + license = 'GPL-2.0' + elif 'GPL-3.0' in existing_licenses or 'GPLv3' in existing_licenses: + license = 'GPL-3.0' + elif license == 'LGPL': + if 'LGPL-2.1' in existing_licenses or 'LGPLv2.1' in existing_licenses: + license = 'LGPL-2.1' + elif 'LGPL-2.0' in existing_licenses or 'LGPLv2' in existing_licenses: + license = 'LGPL-2.0' + elif 'LGPL-3.0' in existing_licenses or 'LGPLv3' in existing_licenses: + license = 'LGPL-3.0' licenses.append(license) if licenses: info['License'] = ' & '.join(licenses) - # Map PKG-INFO & setup.py fields to bitbake variables - bbinfo = {} for field, values in info.items(): if field in self.excluded_fields: continue @@ -251,37 +297,8 @@ class PythonRecipeHandler(RecipeHandler): value = ' '.join(str(v) for v in values if v) bbvar = self.bbvar_map[field] - if bbvar not in bbinfo and value: - bbinfo[bbvar] = value - - comment_lic_line = None - for pos, line in enumerate(list(lines_before)): - if line.startswith('#') and 'LICENSE' in line: - comment_lic_line = pos - elif line.startswith('LICENSE =') and 'LICENSE' in bbinfo: - if line in ('LICENSE = "Unknown"', 'LICENSE = "CLOSED"'): - lines_before[pos] = 'LICENSE = "{}"'.format(bbinfo['LICENSE']) - if line == 'LICENSE = "CLOSED"' and comment_lic_line: - lines_before[comment_lic_line:pos] = [ - '# WARNING: the following LICENSE value is a best guess - it is your', - '# responsibility to verify that the value is complete and correct.' - ] - del bbinfo['LICENSE'] - - src_uri_line = None - for pos, line in enumerate(lines_before): - if line.startswith('SRC_URI ='): - src_uri_line = pos - - if bbinfo: - mdinfo = [''] - for k in sorted(bbinfo): - v = bbinfo[k] - mdinfo.append('{} = "{}"'.format(k, v)) - if src_uri_line: - lines_before[src_uri_line-1:src_uri_line-1] = mdinfo - else: - lines_before.extend(mdinfo) + if bbvar not in extravalues and value: + extravalues[bbvar] = value mapped_deps, unmapped_deps = self.scan_setup_python_deps(srctree, setup_info, setup_non_literals) @@ -292,13 +309,14 @@ class PythonRecipeHandler(RecipeHandler): lines_after.append('# The following configs & dependencies are from setuptools extras_require.') lines_after.append('# These dependencies are optional, hence can be controlled via PACKAGECONFIG.') lines_after.append('# The upstream names may not correspond exactly to bitbake package names.') + lines_after.append('# The configs are might not correct, since PACKAGECONFIG does not support expressions as may used in requires.txt - they are just replaced by text.') lines_after.append('#') lines_after.append('# Uncomment this line to enable all the optional features.') lines_after.append('#PACKAGECONFIG ?= "{}"'.format(' '.join(k.lower() for k in extras_req))) for feature, feature_reqs in extras_req.items(): unmapped_deps.difference_update(feature_reqs) - feature_req_deps = ('python-' + r.replace('.', '-').lower() for r in sorted(feature_reqs)) + feature_req_deps = ('python3-' + r.replace('.', '-').lower() for r in sorted(feature_reqs)) lines_after.append('PACKAGECONFIG[{}] = ",,,{}"'.format(feature.lower(), ' '.join(feature_req_deps))) inst_reqs = set() @@ -309,10 +327,10 @@ class PythonRecipeHandler(RecipeHandler): if inst_reqs: unmapped_deps.difference_update(inst_reqs) - inst_req_deps = ('python-' + r.replace('.', '-').lower() for r in sorted(inst_reqs)) + inst_req_deps = ('python3-' + r.replace('.', '-').lower() for r in sorted(inst_reqs)) lines_after.append('# WARNING: the following rdepends are from setuptools install_requires. These') lines_after.append('# upstream names may not correspond exactly to bitbake package names.') - lines_after.append('RDEPENDS_${{PN}} += "{}"'.format(' '.join(inst_req_deps))) + lines_after.append('RDEPENDS:${{PN}} += "{}"'.format(' '.join(inst_req_deps))) if mapped_deps: name = info.get('Name') @@ -324,7 +342,7 @@ class PythonRecipeHandler(RecipeHandler): lines_after.append('') lines_after.append('# WARNING: the following rdepends are determined through basic analysis of the') lines_after.append('# python sources, and might not be 100% accurate.') - lines_after.append('RDEPENDS_${{PN}} += "{}"'.format(' '.join(sorted(mapped_deps)))) + lines_after.append('RDEPENDS:${{PN}} += "{}"'.format(' '.join(sorted(mapped_deps)))) unmapped_deps -= set(extensions) unmapped_deps -= set(self.assume_provided) @@ -362,6 +380,8 @@ class PythonRecipeHandler(RecipeHandler): # Naive mapping of setup() arguments to PKG-INFO field names for d in [info, non_literals]: for key, value in list(d.items()): + if key is None: + continue new_key = _map(key) if new_key != key: del d[key] @@ -370,7 +390,7 @@ class PythonRecipeHandler(RecipeHandler): return info, 'setuptools' in imported_modules, non_literals, extensions def get_setup_args_info(self, setupscript='./setup.py'): - cmd = ['python', setupscript] + cmd = ['python3', setupscript] info = {} keys = set(self.bbvar_map.keys()) keys |= set(self.setuparg_list_fields) @@ -404,7 +424,7 @@ class PythonRecipeHandler(RecipeHandler): def get_setup_byline(self, fields, setupscript='./setup.py'): info = {} - cmd = ['python', setupscript] + cmd = ['python3', setupscript] cmd.extend('--' + self.setuparg_map.get(f, f.lower()) for f in fields) try: info_lines = self.run_command(cmd, cwd=os.path.dirname(setupscript)).splitlines() @@ -468,9 +488,13 @@ class PythonRecipeHandler(RecipeHandler): else: package_dir = {} - class PackageDir(distutils.command.build_py.build_py): + dist = setuptools.Distribution() + + class PackageDir(setuptools.command.build_py.build_py): def __init__(self, package_dir): self.package_dir = package_dir + self.dist = dist + super().__init__(self.dist) pd = PackageDir(package_dir) to_scan = [] @@ -504,8 +528,10 @@ class PythonRecipeHandler(RecipeHandler): for dep in scanned_deps: mapped = provided_packages.get(dep) if mapped: + logger.debug('Mapped %s to %s' % (dep, mapped)) mapped_deps.add(mapped) else: + logger.debug('Could not map %s' % dep) unmapped_deps.add(dep) return mapped_deps, unmapped_deps @@ -516,7 +542,7 @@ class PythonRecipeHandler(RecipeHandler): except (OSError, subprocess.CalledProcessError): pass else: - for line in dep_output.decode('utf-8').splitlines(): + for line in dep_output.splitlines(): line = line.rstrip() dep, filename = line.split('\t', 1) if filename.endswith('/setup.py'): @@ -536,11 +562,11 @@ class PythonRecipeHandler(RecipeHandler): def parse_pkgdata_for_python_packages(self): suffixes = [t[0] for t in imp.get_suffixes()] - pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR', True) + pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR') ldata = tinfoil.config_data.createCopy() - bb.parse.handle('classes/python-dir.bbclass', ldata, True) - python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR', True) + bb.parse.handle('classes-recipe/python3-dir.bbclass', ldata, True) + python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR') dynload_dir = os.path.join(os.path.dirname(python_sitedir), 'lib-dynload') python_dirs = [python_sitedir + os.sep, @@ -552,7 +578,7 @@ class PythonRecipeHandler(RecipeHandler): with open(pkgdatafile, 'r') as f: for line in f.readlines(): field, value = line.split(': ', 1) - if field == 'FILES_INFO': + if field.startswith('FILES_INFO'): files_info = ast.literal_eval(value) break else: @@ -566,6 +592,8 @@ class PythonRecipeHandler(RecipeHandler): continue if fn.startswith(dynload_dir + os.sep): + if '/.debug/' in fn: + continue base = os.path.basename(fn) provided = base.split('.', 1)[0] packages[provided] = os.path.basename(pkgdatafile) @@ -593,7 +621,7 @@ class PythonRecipeHandler(RecipeHandler): if 'stderr' not in popenargs: popenargs['stderr'] = subprocess.STDOUT try: - return subprocess.check_output(cmd, **popenargs) + return subprocess.check_output(cmd, **popenargs).decode('utf-8') except OSError as exc: logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc) raise |