diff options
Diffstat (limited to 'scripts/lib/recipetool/create_npm.py')
| -rw-r--r-- | scripts/lib/recipetool/create_npm.py | 581 |
1 files changed, 281 insertions, 300 deletions
diff --git a/scripts/lib/recipetool/create_npm.py b/scripts/lib/recipetool/create_npm.py index fb57e705f2..113a89f6a6 100644 --- a/scripts/lib/recipetool/create_npm.py +++ b/scripts/lib/recipetool/create_npm.py @@ -1,329 +1,310 @@ -# Recipe creation tool - node.js NPM module support plugin -# # Copyright (C) 2016 Intel Corporation +# Copyright (C) 2020 Savoir-Faire Linux # -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# SPDX-License-Identifier: GPL-2.0-only # -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +"""Recipe creation tool - npm module support plugin""" -import os +import json import logging -import subprocess +import os +import re +import sys import tempfile -import shutil -import json -from recipetool.create import RecipeHandler, split_pkg_licenses, handle_license_vars - +import bb +from bb.fetch2.npm import NpmEnvironment +from bb.fetch2.npm import npm_package +from bb.fetch2.npmsw import foreach_dependencies +from recipetool.create import RecipeHandler +from recipetool.create import get_license_md5sums +from recipetool.create import guess_license +from recipetool.create import split_pkg_licenses logger = logging.getLogger('recipetool') - -tinfoil = None +TINFOIL = None def tinfoil_init(instance): - global tinfoil - tinfoil = instance - + """Initialize tinfoil""" + global TINFOIL + TINFOIL = instance class NpmRecipeHandler(RecipeHandler): - lockdownpath = None + """Class to handle the npm recipe creation""" + + @staticmethod + def _get_registry(lines): + """Get the registry value from the 'npm://registry' url""" + registry = None + + def _handle_registry(varname, origvalue, op, newlines): + nonlocal registry + if origvalue.startswith("npm://"): + registry = re.sub(r"^npm://", "http://", origvalue.split(";")[0]) + return origvalue, None, 0, True + + bb.utils.edit_metadata(lines, ["SRC_URI"], _handle_registry) + + return registry + + @staticmethod + def _ensure_npm(): + """Check if the 'npm' command is available in the recipes""" + if not TINFOIL.recipes_parsed: + TINFOIL.parse_recipes() - def _ensure_npm(self, fixed_setup=False): - if not tinfoil.recipes_parsed: - tinfoil.parse_recipes() try: - rd = tinfoil.parse_recipe('nodejs-native') + d = TINFOIL.parse_recipe("nodejs-native") except bb.providers.NoProvider: - if fixed_setup: - msg = 'nodejs-native is required for npm but is not available within this SDK' - else: - msg = 'nodejs-native is required for npm but is not available - you will likely need to add a layer that provides nodejs' - logger.error(msg) - return None - bindir = rd.getVar('STAGING_BINDIR_NATIVE') - npmpath = os.path.join(bindir, 'npm') + bb.error("Nothing provides 'nodejs-native' which is required for the build") + bb.note("You will likely need to add a layer that provides nodejs") + sys.exit(14) + + bindir = d.getVar("STAGING_BINDIR_NATIVE") + npmpath = os.path.join(bindir, "npm") + if not os.path.exists(npmpath): - tinfoil.build_targets('nodejs-native', 'addto_recipe_sysroot') + TINFOIL.build_targets("nodejs-native", "addto_recipe_sysroot") + if not os.path.exists(npmpath): - logger.error('npm required to process specified source, but nodejs-native did not seem to populate it') - return None + bb.error("Failed to add 'npm' to sysroot") + sys.exit(14) + return bindir - def _handle_license(self, data): - ''' - Handle the license value from an npm package.json file - ''' - license = None - if 'license' in data: - license = data['license'] - if isinstance(license, dict): - license = license.get('type', None) - if license: - if 'OR' in license: - license = license.replace('OR', '|') - license = license.replace('AND', '&') - license = license.replace(' ', '_') - if not license[0] == '(': - license = '(' + license + ')' - else: - license = license.replace('AND', '&') - if license[0] == '(': - license = license[1:] - if license[-1] == ')': - license = license[:-1] - license = license.replace('MIT/X11', 'MIT') - license = license.replace('Public Domain', 'PD') - license = license.replace('SEE LICENSE IN EULA', - 'SEE-LICENSE-IN-EULA') - return license - - def _shrinkwrap(self, srctree, localfilesdir, extravalues, lines_before, d): - try: - runenv = dict(os.environ, PATH=d.getVar('PATH')) - bb.process.run('npm shrinkwrap', cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True) - except bb.process.ExecutionError as e: - logger.warn('npm shrinkwrap failed:\n%s' % e.stdout) - return - - tmpfile = os.path.join(localfilesdir, 'npm-shrinkwrap.json') - shutil.move(os.path.join(srctree, 'npm-shrinkwrap.json'), tmpfile) - extravalues.setdefault('extrafiles', {}) - extravalues['extrafiles']['npm-shrinkwrap.json'] = tmpfile - lines_before.append('NPM_SHRINKWRAP := "${THISDIR}/${PN}/npm-shrinkwrap.json"') - - def _lockdown(self, srctree, localfilesdir, extravalues, lines_before, d): - runenv = dict(os.environ, PATH=d.getVar('PATH')) - if not NpmRecipeHandler.lockdownpath: - NpmRecipeHandler.lockdownpath = tempfile.mkdtemp('recipetool-npm-lockdown') - bb.process.run('npm install lockdown --prefix %s' % NpmRecipeHandler.lockdownpath, - cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True) - relockbin = os.path.join(NpmRecipeHandler.lockdownpath, 'node_modules', 'lockdown', 'relock.js') - if not os.path.exists(relockbin): - logger.warn('Could not find relock.js within lockdown directory; skipping lockdown') - return - try: - bb.process.run('node %s' % relockbin, cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True) - except bb.process.ExecutionError as e: - logger.warn('lockdown-relock failed:\n%s' % e.stdout) - return - - tmpfile = os.path.join(localfilesdir, 'lockdown.json') - shutil.move(os.path.join(srctree, 'lockdown.json'), tmpfile) - extravalues.setdefault('extrafiles', {}) - extravalues['extrafiles']['lockdown.json'] = tmpfile - lines_before.append('NPM_LOCKDOWN := "${THISDIR}/${PN}/lockdown.json"') - - def _handle_dependencies(self, d, deps, optdeps, devdeps, lines_before, srctree): - import scriptutils - # If this isn't a single module we need to get the dependencies - # and add them to SRC_URI - def varfunc(varname, origvalue, op, newlines): - if varname == 'SRC_URI': - if not origvalue.startswith('npm://'): - src_uri = origvalue.split() - deplist = {} - for dep, depver in optdeps.items(): - depdata = self.get_npm_data(dep, depver, d) - if self.check_npm_optional_dependency(depdata): - deplist[dep] = depdata - for dep, depver in devdeps.items(): - depdata = self.get_npm_data(dep, depver, d) - if self.check_npm_optional_dependency(depdata): - deplist[dep] = depdata - for dep, depver in deps.items(): - depdata = self.get_npm_data(dep, depver, d) - deplist[dep] = depdata - - extra_urls = [] - for dep, depdata in deplist.items(): - version = depdata.get('version', None) - if version: - url = 'npm://registry.npmjs.org;name=%s;version=%s;subdir=node_modules/%s' % (dep, version, dep) - extra_urls.append(url) - if extra_urls: - scriptutils.fetch_url(tinfoil, ' '.join(extra_urls), None, srctree, logger) - src_uri.extend(extra_urls) - return src_uri, None, -1, True - return origvalue, None, 0, True - updated, newlines = bb.utils.edit_metadata(lines_before, ['SRC_URI'], varfunc) - if updated: - del lines_before[:] - for line in newlines: - # Hack to avoid newlines that edit_metadata inserts - if line.endswith('\n'): - line = line[:-1] - lines_before.append(line) - return updated + @staticmethod + def _npm_global_configs(dev): + """Get the npm global configuration""" + configs = [] + + if dev: + configs.append(("also", "development")) + else: + configs.append(("only", "production")) + + configs.append(("save", "false")) + configs.append(("package-lock", "false")) + configs.append(("shrinkwrap", "false")) + return configs + + def _run_npm_install(self, d, srctree, registry, dev): + """Run the 'npm install' command without building the addons""" + configs = self._npm_global_configs(dev) + configs.append(("ignore-scripts", "true")) + + if registry: + configs.append(("registry", registry)) + + bb.utils.remove(os.path.join(srctree, "node_modules"), recurse=True) + + env = NpmEnvironment(d, configs=configs) + env.run("npm install", workdir=srctree) + + def _generate_shrinkwrap(self, d, srctree, dev): + """Check and generate the 'npm-shrinkwrap.json' file if needed""" + configs = self._npm_global_configs(dev) + + env = NpmEnvironment(d, configs=configs) + env.run("npm shrinkwrap", workdir=srctree) + + return os.path.join(srctree, "npm-shrinkwrap.json") + + def _handle_licenses(self, srctree, shrinkwrap_file, dev): + """Return the extra license files and the list of packages""" + licfiles = [] + packages = {} + + # Handle the parent package + packages["${PN}"] = "" + + def _licfiles_append_fallback_readme_files(destdir): + """Append README files as fallback to license files if a license files is missing""" + + fallback = True + readmes = [] + basedir = os.path.join(srctree, destdir) + for fn in os.listdir(basedir): + upper = fn.upper() + if upper.startswith("README"): + fullpath = os.path.join(basedir, fn) + readmes.append(fullpath) + if upper.startswith("COPYING") or "LICENCE" in upper or "LICENSE" in upper: + fallback = False + if fallback: + for readme in readmes: + licfiles.append(os.path.relpath(readme, srctree)) + + # Handle the dependencies + def _handle_dependency(name, params, destdir): + deptree = destdir.split('node_modules/') + suffix = "-".join([npm_package(dep) for dep in deptree]) + packages["${PN}" + suffix] = destdir + _licfiles_append_fallback_readme_files(destdir) + + with open(shrinkwrap_file, "r") as f: + shrinkwrap = json.load(f) + + foreach_dependencies(shrinkwrap, _handle_dependency, dev) + + return licfiles, packages + + # Handle the peer dependencies + def _handle_peer_dependency(self, shrinkwrap_file): + """Check if package has peer dependencies and show warning if it is the case""" + with open(shrinkwrap_file, "r") as f: + shrinkwrap = json.load(f) + + packages = shrinkwrap.get("packages", {}) + peer_deps = packages.get("", {}).get("peerDependencies", {}) + + for peer_dep in peer_deps: + peer_dep_yocto_name = npm_package(peer_dep) + bb.warn(peer_dep + " is a peer dependencie of the actual package. " + + "Please add this peer dependencie to the RDEPENDS variable as %s and generate its recipe with devtool" + % peer_dep_yocto_name) + + def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): - import bb.utils - import oe.package - from collections import OrderedDict + """Handle the npm recipe creation""" - if 'buildsystem' in handled: + if "buildsystem" in handled: return False - def read_package_json(fn): - with open(fn, 'r', errors='surrogateescape') as f: - return json.loads(f.read()) + files = RecipeHandler.checkfiles(srctree, ["package.json"]) - files = RecipeHandler.checkfiles(srctree, ['package.json']) - if files: - d = bb.data.createCopy(tinfoil.config_data) - npm_bindir = self._ensure_npm() - if not npm_bindir: - sys.exit(14) - d.prependVar('PATH', '%s:' % npm_bindir) - - data = read_package_json(files[0]) - if 'name' in data and 'version' in data: - extravalues['PN'] = data['name'] - extravalues['PV'] = data['version'] - classes.append('npm') - handled.append('buildsystem') - if 'description' in data: - extravalues['SUMMARY'] = data['description'] - if 'homepage' in data: - extravalues['HOMEPAGE'] = data['homepage'] - - fetchdev = extravalues['fetchdev'] or None - deps, optdeps, devdeps = self.get_npm_package_dependencies(data, fetchdev) - self._handle_dependencies(d, deps, optdeps, devdeps, lines_before, srctree) - - # Shrinkwrap - localfilesdir = tempfile.mkdtemp(prefix='recipetool-npm') - self._shrinkwrap(srctree, localfilesdir, extravalues, lines_before, d) - - # Lockdown - self._lockdown(srctree, localfilesdir, extravalues, lines_before, d) - - # Split each npm module out to is own package - npmpackages = oe.package.npm_split_package_dirs(srctree) - licvalues = None - for item in handled: - if isinstance(item, tuple): - if item[0] == 'license': - licvalues = item[1] - break - if not licvalues: - licvalues = handle_license_vars(srctree, lines_before, handled, extravalues, d) - if licvalues: - # Augment the license list with information we have in the packages - licenses = {} - license = self._handle_license(data) - if license: - licenses['${PN}'] = license - for pkgname, pkgitem in npmpackages.items(): - _, pdata = pkgitem - license = self._handle_license(pdata) - if license: - licenses[pkgname] = license - # Now write out the package-specific license values - # We need to strip out the json data dicts for this since split_pkg_licenses - # isn't expecting it - packages = OrderedDict((x,y[0]) for x,y in npmpackages.items()) - packages['${PN}'] = '' - pkglicenses = split_pkg_licenses(licvalues, packages, lines_after, licenses) - all_licenses = list(set([item.replace('_', ' ') for pkglicense in pkglicenses.values() for item in pkglicense])) - if '&' in all_licenses: - all_licenses.remove('&') - extravalues['LICENSE'] = ' & '.join(all_licenses) - - # Need to move S setting after inherit npm - for i, line in enumerate(lines_before): - if line.startswith('S ='): - lines_before.pop(i) - lines_after.insert(0, '# Must be set after inherit npm since that itself sets S') - lines_after.insert(1, line) - break - - return True - - return False - - # FIXME this is duplicated from lib/bb/fetch2/npm.py - def _parse_view(self, output): - ''' - Parse the output of npm view --json; the last JSON result - is assumed to be the one that we're interested in. - ''' - pdata = None - outdeps = {} - datalines = [] - bracelevel = 0 - for line in output.splitlines(): - if bracelevel: - datalines.append(line) - elif '{' in line: - datalines = [] - datalines.append(line) - bracelevel = bracelevel + line.count('{') - line.count('}') - if datalines: - pdata = json.loads('\n'.join(datalines)) - return pdata - - # FIXME this is effectively duplicated from lib/bb/fetch2/npm.py - # (split out from _getdependencies()) - def get_npm_data(self, pkg, version, d): - import bb.fetch2 - pkgfullname = pkg - if version != '*' and not '/' in version: - pkgfullname += "@'%s'" % version - logger.debug(2, "Calling getdeps on %s" % pkg) - runenv = dict(os.environ, PATH=d.getVar('PATH')) - fetchcmd = "npm view %s --json" % pkgfullname - output, _ = bb.process.run(fetchcmd, stderr=subprocess.STDOUT, env=runenv, shell=True) - data = self._parse_view(output) - return data - - # FIXME this is effectively duplicated from lib/bb/fetch2/npm.py - # (split out from _getdependencies()) - def get_npm_package_dependencies(self, pdata, fetchdev): - dependencies = pdata.get('dependencies', {}) - optionalDependencies = pdata.get('optionalDependencies', {}) - dependencies.update(optionalDependencies) - if fetchdev: - devDependencies = pdata.get('devDependencies', {}) - dependencies.update(devDependencies) - else: - devDependencies = {} - depsfound = {} - optdepsfound = {} - devdepsfound = {} - for dep in dependencies: - if dep in optionalDependencies: - optdepsfound[dep] = dependencies[dep] - elif dep in devDependencies: - devdepsfound[dep] = dependencies[dep] - else: - depsfound[dep] = dependencies[dep] - return depsfound, optdepsfound, devdepsfound - - # FIXME this is effectively duplicated from lib/bb/fetch2/npm.py - # (split out from _getdependencies()) - def check_npm_optional_dependency(self, pdata): - pkg_os = pdata.get('os', None) - if pkg_os: - if not isinstance(pkg_os, list): - pkg_os = [pkg_os] - blacklist = False - for item in pkg_os: - if item.startswith('!'): - blacklist = True - break - if (not blacklist and 'linux' not in pkg_os) or '!linux' in pkg_os: - logger.debug(2, "Skipping %s since it's incompatible with Linux" % pkg) - return False - return True + if not files: + return False + + with open(files[0], "r") as f: + data = json.load(f) + if "name" not in data or "version" not in data: + return False + + extravalues["PN"] = npm_package(data["name"]) + extravalues["PV"] = data["version"] + + if "description" in data: + extravalues["SUMMARY"] = data["description"] + + if "homepage" in data: + extravalues["HOMEPAGE"] = data["homepage"] + + dev = bb.utils.to_boolean(str(extravalues.get("NPM_INSTALL_DEV", "0")), False) + registry = self._get_registry(lines_before) + + bb.note("Checking if npm is available ...") + # The native npm is used here (and not the host one) to ensure that the + # npm version is high enough to ensure an efficient dependency tree + # resolution and avoid issue with the shrinkwrap file format. + # Moreover the native npm is mandatory for the build. + bindir = self._ensure_npm() + + d = bb.data.createCopy(TINFOIL.config_data) + d.prependVar("PATH", bindir + ":") + d.setVar("S", srctree) + + bb.note("Generating shrinkwrap file ...") + # To generate the shrinkwrap file the dependencies have to be installed + # first. During the generation process some files may be updated / + # deleted. By default devtool tracks the diffs in the srctree and raises + # errors when finishing the recipe if some diffs are found. + git_exclude_file = os.path.join(srctree, ".git", "info", "exclude") + if os.path.exists(git_exclude_file): + with open(git_exclude_file, "r+") as f: + lines = f.readlines() + for line in ["/node_modules/", "/npm-shrinkwrap.json"]: + if line not in lines: + f.write(line + "\n") + + lock_file = os.path.join(srctree, "package-lock.json") + lock_copy = lock_file + ".copy" + if os.path.exists(lock_file): + bb.utils.copyfile(lock_file, lock_copy) + + self._run_npm_install(d, srctree, registry, dev) + shrinkwrap_file = self._generate_shrinkwrap(d, srctree, dev) + + with open(shrinkwrap_file, "r") as f: + shrinkwrap = json.load(f) + + if os.path.exists(lock_copy): + bb.utils.movefile(lock_copy, lock_file) + + # Add the shrinkwrap file as 'extrafiles' + shrinkwrap_copy = shrinkwrap_file + ".copy" + bb.utils.copyfile(shrinkwrap_file, shrinkwrap_copy) + extravalues.setdefault("extrafiles", {}) + extravalues["extrafiles"]["npm-shrinkwrap.json"] = shrinkwrap_copy + + url_local = "npmsw://%s" % shrinkwrap_file + url_recipe= "npmsw://${THISDIR}/${BPN}/npm-shrinkwrap.json" + + if dev: + url_local += ";dev=1" + url_recipe += ";dev=1" + + # Add the npmsw url in the SRC_URI of the generated recipe + def _handle_srcuri(varname, origvalue, op, newlines): + """Update the version value and add the 'npmsw://' url""" + value = origvalue.replace("version=" + data["version"], "version=${PV}") + value = value.replace("version=latest", "version=${PV}") + values = [line.strip() for line in value.strip('\n').splitlines()] + if "dependencies" in shrinkwrap.get("packages", {}).get("", {}): + values.append(url_recipe) + return values, None, 4, False + + (_, newlines) = bb.utils.edit_metadata(lines_before, ["SRC_URI"], _handle_srcuri) + lines_before[:] = [line.rstrip('\n') for line in newlines] + + # In order to generate correct licence checksums in the recipe the + # dependencies have to be fetched again using the npmsw url + bb.note("Fetching npm dependencies ...") + bb.utils.remove(os.path.join(srctree, "node_modules"), recurse=True) + fetcher = bb.fetch2.Fetch([url_local], d) + fetcher.download() + fetcher.unpack(srctree) + + bb.note("Handling licences ...") + (licfiles, packages) = self._handle_licenses(srctree, shrinkwrap_file, dev) + + def _guess_odd_license(licfiles): + import bb + + md5sums = get_license_md5sums(d, linenumbers=True) + + chksums = [] + licenses = [] + for licfile in licfiles: + f = os.path.join(srctree, licfile) + md5value = bb.utils.md5_file(f) + (license, beginline, endline, md5) = md5sums.get(md5value, + (None, "", "", "")) + if not license: + license = "Unknown" + logger.info("Please add the following line for '%s' to a " + "'lib/recipetool/licenses.csv' and replace `Unknown`, " + "`X`, `Y` and `MD5` with the license, begin line, " + "end line and partial MD5 checksum:\n" \ + "%s,Unknown,X,Y,MD5" % (licfile, md5value)) + chksums.append("file://%s%s%s;md5=%s" % (licfile, + ";beginline=%s" % (beginline) if beginline else "", + ";endline=%s" % (endline) if endline else "", + md5 if md5 else md5value)) + licenses.append((license, licfile, md5value)) + return (licenses, chksums) + + (licenses, extravalues["LIC_FILES_CHKSUM"]) = _guess_odd_license(licfiles) + split_pkg_licenses([*licenses, *guess_license(srctree, d)], packages, lines_after) + + classes.append("npm") + handled.append("buildsystem") + + # Check if package has peer dependencies and inform the user + self._handle_peer_dependency(shrinkwrap_file) + + return True def register_recipe_handlers(handlers): + """Register the npm handler""" handlers.append((NpmRecipeHandler(), 60)) |