From 54ce2a7cf5d69509c670268e36efe374f6131c47 Mon Sep 17 00:00:00 2001 From: Paul Eggleton Date: Mon, 5 Sep 2016 15:58:01 +1200 Subject: recipetool: create: support node.js code outside of npm If you have your own node.js application you may not publish it (or at least not immediately) in an npm registry - it might just be in a repository on github or on your local machine. Add support to recipetool create for creating recipes to build such applications - extract their dependencies, fetch them, and add corresponding npm:// URLs to SRC_URI, and ensure that LICENSE / LIC_FILES_CHKSUM are updated to match. For example, you can now run: recipetool create https://github.com/diversario/node-ssdp (I had to borrow some code from bitbake/lib/bb/fetch2/npm.py to implement this functionality; this should be refactored out but now isn't the time to do that refactoring.) Part of the fix for [YOCTO #9537]. Signed-off-by: Paul Eggleton --- meta/classes/npm.bbclass | 4 ++ scripts/lib/recipetool/create_npm.py | 111 ++++++++++++++++++++++++++++++++++- 2 files changed, 114 insertions(+), 1 deletion(-) diff --git a/meta/classes/npm.bbclass b/meta/classes/npm.bbclass index 55c7c3e278..fce4c1146f 100644 --- a/meta/classes/npm.bbclass +++ b/meta/classes/npm.bbclass @@ -16,6 +16,10 @@ def npm_oe_arch_map(target_arch, d): NPM_ARCH ?= "${@npm_oe_arch_map(d.getVar('TARGET_ARCH', True), d)}" npm_do_compile() { + # Copy in any additionally fetched modules + if [ -d ${WORKDIR}/node_modules ] ; then + cp -a ${WORKDIR}/node_modules ${S}/ + fi # changing the home directory to the working directory, the .npmrc will # be created in this directory export HOME=${WORKDIR} diff --git a/scripts/lib/recipetool/create_npm.py b/scripts/lib/recipetool/create_npm.py index e5aaa60bf8..e794614978 100644 --- a/scripts/lib/recipetool/create_npm.py +++ b/scripts/lib/recipetool/create_npm.py @@ -21,7 +21,7 @@ import subprocess import tempfile import shutil import json -from recipetool.create import RecipeHandler, split_pkg_licenses +from recipetool.create import RecipeHandler, split_pkg_licenses, handle_license_vars logger = logging.getLogger('recipetool') @@ -83,6 +83,66 @@ class NpmRecipeHandler(RecipeHandler): extravalues['extrafiles']['lockdown.json'] = tmpfile lines_before.append('NPM_LOCKDOWN := "${THISDIR}/${PN}/lockdown.json"') + def _handle_dependencies(self, d, deps, lines_before, srctree): + import scriptutils + # If this isn't a single module we need to get the dependencies + # and add them to SRC_URI + def varfunc(varname, origvalue, op, newlines): + if varname == 'SRC_URI': + if not origvalue.startswith('npm://'): + src_uri = origvalue.split() + changed = False + for dep, depdata in deps.items(): + version = self.get_node_version(dep, depdata, d) + if version: + url = 'npm://registry.npmjs.org;name=%s;version=%s;subdir=node_modules/%s' % (dep, version, dep) + scriptutils.fetch_uri(d, url, srctree) + src_uri.append(url) + changed = True + if changed: + return src_uri, None, -1, True + return origvalue, None, 0, True + updated, newlines = bb.utils.edit_metadata(lines_before, ['SRC_URI'], varfunc) + if updated: + del lines_before[:] + for line in newlines: + # Hack to avoid newlines that edit_metadata inserts + if line.endswith('\n'): + line = line[:-1] + lines_before.append(line) + return updated + + def _replace_license_vars(self, srctree, lines_before, handled, extravalues, d): + for item in handled: + if isinstance(item, tuple): + if item[0] == 'license': + del item + break + + calledvars = [] + def varfunc(varname, origvalue, op, newlines): + if varname in ['LICENSE', 'LIC_FILES_CHKSUM']: + for i, e in enumerate(reversed(newlines)): + if not e.startswith('#'): + stop = i + while stop > 0: + newlines.pop() + stop -= 1 + break + calledvars.append(varname) + if len(calledvars) > 1: + # The second time around, put the new license text in + insertpos = len(newlines) + handle_license_vars(srctree, newlines, handled, extravalues, d) + return None, None, 0, True + return origvalue, None, 0, True + updated, newlines = bb.utils.edit_metadata(lines_before, ['LICENSE', 'LIC_FILES_CHKSUM'], varfunc) + if updated: + del lines_before[:] + lines_before.extend(newlines) + else: + raise Exception('Did not find license variables') + def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): import bb.utils import oe @@ -108,6 +168,12 @@ class NpmRecipeHandler(RecipeHandler): if 'homepage' in data: extravalues['HOMEPAGE'] = data['homepage'] + deps = data.get('dependencies', {}) + updated = self._handle_dependencies(tinfoil.config_data, deps, lines_before, srctree) + if updated: + # We need to redo the license stuff + self._replace_license_vars(srctree, lines_before, handled, extravalues, tinfoil.config_data) + # Shrinkwrap localfilesdir = tempfile.mkdtemp(prefix='recipetool-npm') self._shrinkwrap(srctree, localfilesdir, extravalues, lines_before) @@ -148,9 +214,52 @@ class NpmRecipeHandler(RecipeHandler): lines_before[i] = 'LICENSE = "%s"' % ' & '.join(all_licenses) break + # Need to move S setting after inherit npm + for i, line in enumerate(lines_before): + if line.startswith('S ='): + lines_before.pop(i) + lines_after.insert(0, '# Must be set after inherit npm since that itself sets S') + lines_after.insert(1, line) + break + return True return False + # FIXME this is duplicated from lib/bb/fetch2/npm.py + def _parse_view(self, output): + ''' + Parse the output of npm view --json; the last JSON result + is assumed to be the one that we're interested in. + ''' + pdata = None + outdeps = {} + datalines = [] + bracelevel = 0 + for line in output.splitlines(): + if bracelevel: + datalines.append(line) + elif '{' in line: + datalines = [] + datalines.append(line) + bracelevel = bracelevel + line.count('{') - line.count('}') + if datalines: + pdata = json.loads('\n'.join(datalines)) + return pdata + + # FIXME this is effectively duplicated from lib/bb/fetch2/npm.py + # (split out from _getdependencies()) + def get_node_version(self, pkg, version, d): + import bb.fetch2 + pkgfullname = pkg + if version != '*' and not '/' in version: + pkgfullname += "@'%s'" % version + logger.debug(2, "Calling getdeps on %s" % pkg) + runenv = dict(os.environ, PATH=d.getVar('PATH', True)) + fetchcmd = "npm view %s --json" % pkgfullname + output, _ = bb.process.run(fetchcmd, stderr=subprocess.STDOUT, env=runenv, shell=True) + data = self._parse_view(output) + return data.get('version', None) + def register_recipe_handlers(handlers): handlers.append((NpmRecipeHandler(), 60)) -- cgit 1.2.3-korg