From 3b3fd33190d89c09e62126eea0e45aa84fe5442e Mon Sep 17 00:00:00 2001 From: Paul Eggleton Date: Tue, 22 Dec 2015 17:03:02 +1300 Subject: recipetool: create: support extracting name and version from build scripts Some build systems (notably autotools) support declaring the name and version of the program being built; since we need those for the recipe we can attempt to extract them. It's a little fuzzy as they are often omitted or may not be appropriately formatted for our purposes, but it does work on a reasonable number of software packages to be useful. Signed-off-by: Paul Eggleton Signed-off-by: Richard Purdie --- scripts/lib/recipetool/create_buildsys.py | 282 +++++++++++++++++++++++++----- 1 file changed, 237 insertions(+), 45 deletions(-) (limited to 'scripts/lib/recipetool/create_buildsys.py') diff --git a/scripts/lib/recipetool/create_buildsys.py b/scripts/lib/recipetool/create_buildsys.py index 931ef3b33f..0aff59e229 100644 --- a/scripts/lib/recipetool/create_buildsys.py +++ b/scripts/lib/recipetool/create_buildsys.py @@ -17,7 +17,7 @@ import re import logging -from recipetool.create import RecipeHandler, read_pkgconfig_provides +from recipetool.create import RecipeHandler, read_pkgconfig_provides, validate_pv logger = logging.getLogger('recipetool') @@ -27,13 +27,17 @@ def tinfoil_init(instance): global tinfoil tinfoil = instance + class CmakeRecipeHandler(RecipeHandler): - def process(self, srctree, classes, lines_before, lines_after, handled): + def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): if 'buildsystem' in handled: return False if RecipeHandler.checkfiles(srctree, ['CMakeLists.txt']): classes.append('cmake') + values = CmakeRecipeHandler.extract_cmake_deps(lines_before, srctree, extravalues) + for var, value in values.iteritems(): + lines_before.append('%s = "%s"' % (var, value)) lines_after.append('# Specify any options you want to pass to cmake using EXTRA_OECMAKE:') lines_after.append('EXTRA_OECMAKE = ""') lines_after.append('') @@ -41,8 +45,26 @@ class CmakeRecipeHandler(RecipeHandler): return True return False + @staticmethod + def extract_cmake_deps(outlines, srctree, extravalues, cmakelistsfile=None): + values = {} + + if cmakelistsfile: + srcfiles = [cmakelistsfile] + else: + srcfiles = RecipeHandler.checkfiles(srctree, ['CMakeLists.txt']) + + proj_re = re.compile('project\(([^)]*)\)', re.IGNORECASE) + with open(srcfiles[0], 'r') as f: + for line in f: + res = proj_re.match(line.strip()) + if res: + extravalues['PN'] = res.group(1).split()[0] + + return values + class SconsRecipeHandler(RecipeHandler): - def process(self, srctree, classes, lines_before, lines_after, handled): + def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): if 'buildsystem' in handled: return False @@ -56,7 +78,7 @@ class SconsRecipeHandler(RecipeHandler): return False class QmakeRecipeHandler(RecipeHandler): - def process(self, srctree, classes, lines_before, lines_after, handled): + def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): if 'buildsystem' in handled: return False @@ -67,14 +89,14 @@ class QmakeRecipeHandler(RecipeHandler): return False class AutotoolsRecipeHandler(RecipeHandler): - def process(self, srctree, classes, lines_before, lines_after, handled): + def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): if 'buildsystem' in handled: return False autoconf = False if RecipeHandler.checkfiles(srctree, ['configure.ac', 'configure.in']): autoconf = True - values = AutotoolsRecipeHandler.extract_autotools_deps(lines_before, srctree) + values = AutotoolsRecipeHandler.extract_autotools_deps(lines_before, srctree, extravalues) classes.extend(values.pop('inherit', '').split()) for var, value in values.iteritems(): lines_before.append('%s = "%s"' % (var, value)) @@ -88,6 +110,22 @@ class AutotoolsRecipeHandler(RecipeHandler): autoconf = True break + if autoconf and not ('PV' in extravalues and 'PN' in extravalues): + # Last resort + conffile = RecipeHandler.checkfiles(srctree, ['configure']) + if conffile: + with open(conffile[0], 'r') as f: + for line in f: + line = line.strip() + if line.startswith('VERSION=') or line.startswith('PACKAGE_VERSION='): + pv = line.split('=')[1].strip('"\'') + if pv and not 'PV' in extravalues and validate_pv(pv): + extravalues['PV'] = pv + elif line.startswith('PACKAGE_NAME=') or line.startswith('PACKAGE='): + pn = line.split('=')[1].strip('"\'') + if pn and not 'PN' in extravalues: + extravalues['PN'] = pn + if autoconf: lines_before.append('# NOTE: if this software is not capable of being built in a separate build directory') lines_before.append('# from the source, you should replace autotools with autotools-brokensep in the') @@ -102,7 +140,7 @@ class AutotoolsRecipeHandler(RecipeHandler): return False @staticmethod - def extract_autotools_deps(outlines, srctree, acfile=None): + def extract_autotools_deps(outlines, srctree, extravalues=None, acfile=None): import shlex import oe.package @@ -122,6 +160,9 @@ class AutotoolsRecipeHandler(RecipeHandler): lib_re = re.compile('AC_CHECK_LIB\(\[?([a-zA-Z0-9]*)\]?, .*') progs_re = re.compile('_PROGS?\(\[?[a-zA-Z0-9]*\]?, \[?([^,\]]*)\]?[),].*') dep_re = re.compile('([^ ><=]+)( [<>=]+ [^ ><=]+)?') + ac_init_re = re.compile('AC_INIT\(([^,]+), *([^,]+)[,)].*') + am_init_re = re.compile('AM_INIT_AUTOMAKE\(([^,]+), *([^,]+)[,)].*') + define_re = re.compile(' *(m4_)?define\(([^,]+), *([^,]+)\)') # Build up lib library->package mapping shlib_providers = oe.package.read_shlib_providers(tinfoil.config_data) @@ -157,55 +198,157 @@ class AutotoolsRecipeHandler(RecipeHandler): else: raise + defines = {} + def subst_defines(value): + newvalue = value + for define, defval in defines.iteritems(): + newvalue = newvalue.replace(define, defval) + if newvalue != value: + return subst_defines(newvalue) + return value + + def process_value(value): + value = value.replace('[', '').replace(']', '') + if value.startswith('m4_esyscmd(') or value.startswith('m4_esyscmd_s('): + cmd = subst_defines(value[value.index('(')+1:-1]) + try: + if '|' in cmd: + cmd = 'set -o pipefail; ' + cmd + stdout, _ = bb.process.run(cmd, cwd=srctree, shell=True) + ret = stdout.rstrip() + except bb.process.ExecutionError as e: + ret = '' + elif value.startswith('m4_'): + return None + ret = subst_defines(value) + if ret: + ret = ret.strip('"\'') + return ret + # Since a configure.ac file is essentially a program, this is only ever going to be # a hack unfortunately; but it ought to be enough of an approximation if acfile: srcfiles = [acfile] else: - srcfiles = RecipeHandler.checkfiles(srctree, ['configure.ac', 'configure.in']) + srcfiles = RecipeHandler.checkfiles(srctree, ['acinclude.m4', 'configure.ac', 'configure.in']) + pcdeps = [] deps = [] unmapped = [] unmappedlibs = [] - with open(srcfiles[0], 'r') as f: - for line in f: - if 'PKG_CHECK_MODULES' in line: - res = pkg_re.search(line) - if res: - res = dep_re.findall(res.group(1)) - if res: - pcdeps.extend([x[0] for x in res]) - inherits.append('pkgconfig') - if line.lstrip().startswith('AM_GNU_GETTEXT'): - inherits.append('gettext') - elif 'AC_CHECK_PROG' in line or 'AC_PATH_PROG' in line: - res = progs_re.search(line) + + def process_macro(keyword, value): + if keyword == 'PKG_CHECK_MODULES': + res = pkg_re.search(value) + if res: + res = dep_re.findall(res.group(1)) if res: - for prog in shlex.split(res.group(1)): - prog = prog.split()[0] - progclass = progclassmap.get(prog, None) - if progclass: - inherits.append(progclass) + pcdeps.extend([x[0] for x in res]) + inherits.append('pkgconfig') + elif keyword == 'AM_GNU_GETTEXT': + inherits.append('gettext') + elif keyword == 'AC_CHECK_PROG' or keyword == 'AC_PATH_PROG': + res = progs_re.search(value) + if res: + for prog in shlex.split(res.group(1)): + prog = prog.split()[0] + progclass = progclassmap.get(prog, None) + if progclass: + inherits.append(progclass) + else: + progdep = progmap.get(prog, None) + if progdep: + deps.append(progdep) else: - progdep = progmap.get(prog, None) - if progdep: - deps.append(progdep) - else: - if not prog.startswith('$'): - unmapped.append(prog) - elif 'AC_CHECK_LIB' in line: - res = lib_re.search(line) + if not prog.startswith('$'): + unmapped.append(prog) + elif keyword == 'AC_CHECK_LIB': + res = lib_re.search(value) + if res: + lib = res.group(1) + libdep = recipelibmap.get(lib, None) + if libdep: + deps.append(libdep) + else: + if libdep is None: + if not lib.startswith('$'): + unmappedlibs.append(lib) + elif keyword == 'AC_PATH_X': + deps.append('libx11') + elif keyword == 'AC_INIT': + if extravalues is not None: + res = ac_init_re.match(value) if res: - lib = res.group(1) - libdep = recipelibmap.get(lib, None) - if libdep: - deps.append(libdep) - else: - if libdep is None: - if not lib.startswith('$'): - unmappedlibs.append(lib) - elif 'AC_PATH_X' in line: - deps.append('libx11') + extravalues['PN'] = process_value(res.group(1)) + pv = process_value(res.group(2)) + if validate_pv(pv): + extravalues['PV'] = pv + elif keyword == 'AM_INIT_AUTOMAKE': + if extravalues is not None: + if 'PN' not in extravalues: + res = am_init_re.match(value) + if res: + if res.group(1) != 'AC_PACKAGE_NAME': + extravalues['PN'] = process_value(res.group(1)) + pv = process_value(res.group(2)) + if validate_pv(pv): + extravalues['PV'] = pv + elif keyword == 'define(': + res = define_re.match(value) + if res: + key = res.group(2).strip('[]') + value = process_value(res.group(3)) + if value is not None: + defines[key] = value + + keywords = ['PKG_CHECK_MODULES', + 'AM_GNU_GETTEXT', + 'AC_CHECK_PROG', + 'AC_PATH_PROG', + 'AC_CHECK_LIB', + 'AC_PATH_X', + 'AC_INIT', + 'AM_INIT_AUTOMAKE', + 'define(', + ] + for srcfile in srcfiles: + nesting = 0 + in_keyword = '' + partial = '' + with open(srcfile, 'r') as f: + for line in f: + if in_keyword: + partial += ' ' + line.strip() + if partial.endswith('\\'): + partial = partial[:-1] + nesting = nesting + line.count('(') - line.count(')') + if nesting == 0: + process_macro(in_keyword, partial) + partial = '' + in_keyword = '' + else: + for keyword in keywords: + if keyword in line: + nesting = line.count('(') - line.count(')') + if nesting > 0: + partial = line.strip() + if partial.endswith('\\'): + partial = partial[:-1] + in_keyword = keyword + else: + process_macro(keyword, line.strip()) + break + + if in_keyword: + process_macro(in_keyword, partial) + + if extravalues: + for k,v in extravalues.items(): + if v: + if v.startswith('$') or v.startswith('@') or v.startswith('%'): + del extravalues[k] + else: + extravalues[k] = v.strip('"\'').rstrip('()') if unmapped: outlines.append('# NOTE: the following prog dependencies are unknown, ignoring: %s' % ' '.join(unmapped)) @@ -240,7 +383,7 @@ class AutotoolsRecipeHandler(RecipeHandler): class MakefileRecipeHandler(RecipeHandler): - def process(self, srctree, classes, lines_before, lines_after, handled): + def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): if 'buildsystem' in handled: return False @@ -307,6 +450,53 @@ class MakefileRecipeHandler(RecipeHandler): self.genfunction(lines_after, 'do_install', ['# Specify install commands here']) +class VersionFileRecipeHandler(RecipeHandler): + def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): + if 'PV' not in extravalues: + # Look for a VERSION or version file containing a single line consisting + # only of a version number + filelist = RecipeHandler.checkfiles(srctree, ['VERSION', 'version']) + version = None + for fileitem in filelist: + linecount = 0 + with open(fileitem, 'r') as f: + for line in f: + line = line.rstrip().strip('"\'') + linecount += 1 + if line: + if linecount > 1: + version = None + break + else: + if validate_pv(line): + version = line + if version: + extravalues['PV'] = version + break + + +class SpecFileRecipeHandler(RecipeHandler): + def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): + if 'PV' in extravalues and 'PN' in extravalues: + return + filelist = RecipeHandler.checkfiles(srctree, ['*.spec'], recursive=True) + pn = None + pv = None + for fileitem in filelist: + linecount = 0 + with open(fileitem, 'r') as f: + for line in f: + if line.startswith('Name:') and not pn: + pn = line.split(':')[1].strip() + if line.startswith('Version:') and not pv: + pv = line.split(':')[1].strip() + if pv or pn: + if pv and not 'PV' in extravalues and validate_pv(pv): + extravalues['PV'] = pv + if pn and not 'PN' in extravalues: + extravalues['PN'] = pn + break + def register_recipe_handlers(handlers): # These are in a specific order so that the right one is detected first handlers.append(CmakeRecipeHandler()) @@ -314,3 +504,5 @@ def register_recipe_handlers(handlers): handlers.append(SconsRecipeHandler()) handlers.append(QmakeRecipeHandler()) handlers.append(MakefileRecipeHandler()) + handlers.append((VersionFileRecipeHandler(), -1)) + handlers.append((SpecFileRecipeHandler(), -1)) -- cgit 1.2.3-korg