# Development tool - upgrade command plugin # # Copyright (C) 2014-2015 Intel Corporation # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # """Devtool upgrade plugin""" import os import sys import re import shutil import tempfile import logging import argparse import scriptutils import errno import bb import oe.recipeutils from devtool import standard from devtool import exec_build_env_command, setup_tinfoil, DevtoolError, parse_recipe, use_external_build logger = logging.getLogger('devtool') def plugin_init(pluginlist): """Plugin initialization""" pass def _run(cmd, cwd=''): logger.debug("Running command %s> %s" % (cwd,cmd)) return bb.process.run('%s' % cmd, cwd=cwd) def _get_srctree(tmpdir): srctree = tmpdir dirs = os.listdir(tmpdir) if len(dirs) == 1: srctree = os.path.join(tmpdir, dirs[0]) return srctree def _copy_source_code(orig, dest): for path in standard._ls_tree(orig): dest_dir = os.path.join(dest, os.path.dirname(path)) bb.utils.mkdirhier(dest_dir) dest_path = os.path.join(dest, path) shutil.move(os.path.join(orig, path), dest_path) def _get_checksums(rf): import re checksums = {} with open(rf) as f: for line in f: for cs in ['md5sum', 'sha256sum']: m = re.match("^SRC_URI\[%s\].*=.*\"(.*)\"" % cs, line) if m: checksums[cs] = m.group(1) return checksums def _replace_checksums(rf, md5, sha256): if not md5 and not sha256: return checksums = {'md5sum':md5, 'sha256sum':sha256} with open(rf + ".tmp", "w+") as tmprf: with open(rf) as f: for line in f: m = None for cs in checksums.keys(): m = re.match("^SRC_URI\[%s\].*=.*\"(.*)\"" % cs, line) if m: if checksums[cs]: oldcheck = m.group(1) newcheck = checksums[cs] line = line.replace(oldcheck, newcheck) break tmprf.write(line) os.rename(rf + ".tmp", rf) def _remove_patch_dirs(recipefolder): for root, dirs, files in os.walk(recipefolder): for d in dirs: shutil.rmtree(os.path.join(root,d)) def _recipe_contains(rf, var): import re found = False with open(rf) as f: for line in f: if re.match("^%s.*=.*" % var, line): found = True break return found def _rename_recipe_dirs(oldpv, newpv, path): for root, dirs, files in os.walk(path): for olddir in dirs: if olddir.find(oldpv) != -1: newdir = olddir.replace(oldpv, newpv) if olddir != newdir: _run('mv %s %s' % (olddir, newdir)) def _rename_recipe_file(bpn, oldpv, newpv, path): oldrecipe = "%s_%s.bb" % (bpn, oldpv) newrecipe = "%s_%s.bb" % (bpn, newpv) if os.path.isfile(os.path.join(path, oldrecipe)): if oldrecipe != newrecipe: _run('mv %s %s' % (oldrecipe, newrecipe), cwd=path) else: recipe = "%s_git.bb" % bpn if os.path.isfile(os.path.join(path, recipe)): newrecipe = recipe raise DevtoolError("Original recipe not found on workspace") return os.path.join(path, newrecipe) def _rename_recipe_files(bpn, oldpv, newpv, path): _rename_recipe_dirs(oldpv, newpv, path) return _rename_recipe_file(bpn, oldpv, newpv, path) def _write_append(rc, srctree, same_dir, no_same_dir, rev, workspace, d): """Writes an append file""" if not os.path.exists(rc): raise DevtoolError("bbappend not created because %s does not exist" % rc) appendpath = os.path.join(workspace, 'appends') if not os.path.exists(appendpath): bb.utils.mkdirhier(appendpath) brf = os.path.basename(os.path.splitext(rc)[0]) # rc basename srctree = os.path.abspath(srctree) pn = d.getVar('PN',True) af = os.path.join(appendpath, '%s.bbappend' % brf) with open(af, 'w') as f: f.write('FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n\n') f.write('inherit externalsrc\n') f.write(('# NOTE: We use pn- overrides here to avoid affecting' 'multiple variants in the case where the recipe uses BBCLASSEXTEND\n')) f.write('EXTERNALSRC_pn-%s = "%s"\n' % (pn, srctree)) b_is_s = use_external_build(same_dir, no_same_dir, d) if b_is_s: f.write('EXTERNALSRC_BUILD_pn-%s = "%s"\n' % (pn, srctree)) if rev: f.write('\n# initial_rev: %s\n' % rev) return af def _cleanup_on_error(rf, srctree): rfp = os.path.split(rf)[0] # recipe folder rfpp = os.path.split(rfp)[0] # recipes folder if os.path.exists(rfp): shutil.rmtree(b) if not len(os.listdir(rfpp)): os.rmdir(rfpp) srctree = os.path.abspath(srctree) if os.path.exists(srctree): shutil.rmtree(srctree) def _upgrade_error(e, rf, srctree): if rf: cleanup_on_error(rf, srctree) logger.error(e) raise DevtoolError(e) def _get_uri(rd): srcuris = rd.getVar('SRC_URI', True).split() if not len(srcuris): raise DevtoolError('SRC_URI not found on recipe') srcuri = srcuris[0] # it is assumed, URI is at first position srcrev = '${AUTOREV}' if '://' in srcuri: # Fetch a URL rev_re = re.compile(';rev=([^;]+)') res = rev_re.search(srcuri) if res: srcrev = res.group(1) srcuri = rev_re.sub('', srcuri) return srcuri, srcrev def _extract_new_source(newpv, srctree, no_patch, srcrev, branch, keep_temp, tinfoil, rd): """Extract sources of a recipe with a new version""" def __run(cmd): """Simple wrapper which calls _run with srctree as cwd""" return _run(cmd, srctree) crd = rd.createCopy() pv = crd.getVar('PV', True) crd.setVar('PV', newpv) tmpsrctree = None uri, rev = _get_uri(crd) if srcrev: rev = srcrev if uri.startswith('git://'): __run('git checkout %s' % rev) __run('git tag -f devtool-base-new') md5 = None sha256 = None else: __run('git checkout -b devtool-%s' % newpv) tmpdir = tempfile.mkdtemp(prefix='devtool') try: md5, sha256 = scriptutils.fetch_uri(tinfoil.config_data, uri, tmpdir, rev) except bb.fetch2.FetchError as e: raise DevtoolError(e) tmpsrctree = _get_srctree(tmpdir) scrtree = os.path.abspath(srctree) _copy_source_code(tmpsrctree, srctree) (stdout,_) = __run('git ls-files --modified --others --exclude-standard') for f in stdout.splitlines(): __run('git add "%s"' % f) __run('git commit -q -m "Commit of upstream changes at version %s" --allow-empty' % newpv) __run('git tag -f devtool-base-%s' % newpv) (stdout, _) = __run('git rev-parse HEAD') rev = stdout.rstrip() if no_patch: patches = oe.recipeutils.get_recipe_patches(crd) if len(patches): logger.warn('By user choice, the following patches will NOT be applied') for patch in patches: logger.warn("%s" % os.path.basename(patch)) else: try: __run('git checkout devtool-patched -b %s' % branch) __run('git rebase %s' % rev) if uri.startswith('git://'): suffix = 'new' else: suffix = newpv __run('git tag -f devtool-patched-%s' % suffix) except bb.process.ExecutionError as e: logger.warn('Command \'%s\' failed:\n%s' % (e.command, e.stdout)) if tmpsrctree: if keep_temp: logger.info('Preserving temporary directory %s' % tmpsrctree) else: shutil.rmtree(tmpsrctree) return (rev, md5, sha256) def _create_new_recipe(newpv, md5, sha256, workspace, rd): """Creates the new recipe under workspace""" crd = rd.createCopy() bpn = crd.getVar('BPN', True) path = os.path.join(workspace, 'recipes', bpn) bb.utils.mkdirhier(path) oe.recipeutils.copy_recipe_files(crd, path) oldpv = crd.getVar('PV', True) if not newpv: newpv = oldpv fullpath = _rename_recipe_files(bpn, oldpv, newpv, path) if _recipe_contains(fullpath, 'PV') and newpv != oldpv: oe.recipeutils.patch_recipe(d, fullpath, {'PV':newpv}) if md5 and sha256: # Unfortunately, oe.recipeutils.patch_recipe cannot update flags. # once the latter feature is implemented, we should call patch_recipe # instead of the following function _replace_checksums(fullpath, md5, sha256) return fullpath def upgrade(args, config, basepath, workspace): """Entry point for the devtool 'upgrade' subcommand""" if args.recipename in workspace: raise DevtoolError("recipe %s is already in your workspace" % args.recipename) if not args.version and not args.srcrev: raise DevtoolError("You must provide a version using the --version/-V option, or for recipes that fetch from an SCM such as git, the --srcrev/-S option") reason = oe.recipeutils.validate_pn(args.recipename) if reason: raise DevtoolError(reason) tinfoil = setup_tinfoil(basepath=basepath) rd = parse_recipe(config, tinfoil, args.recipename, True) if not rd: return 1 pn = rd.getVar('PN', True) if pn != args.recipename: logger.info('Mapping %s to %s' % (args.recipename, pn)) if pn in workspace: raise DevtoolError("recipe %s is already in your workspace" % pn) standard._check_compatible_recipe(pn, rd) if rd.getVar('PV', True) == args.version and rd.getVar('SRCREV', True) == args.srcrev: raise DevtoolError("Current and upgrade versions are the same version" % version) rf = None try: rev1 = standard._extract_source(args.srctree, False, 'devtool-orig', False, rd) rev2, md5, sha256 = _extract_new_source(args.version, args.srctree, args.no_patch, args.srcrev, args.branch, args.keep_temp, tinfoil, rd) rf = _create_new_recipe(args.version, md5, sha256, config.workspace_path, rd) except bb.process.CmdError as e: _upgrade_error(e, rf, args.srctree) except DevtoolError as e: _upgrade_error(e, rf, args.srctree) standard._add_md5(config, pn, os.path.dirname(rf)) af = _write_append(rf, args.srctree, args.same_dir, args.no_same_dir, rev2, config.workspace_path, rd) standard._add_md5(config, pn, af) logger.info('Upgraded source extracted to %s' % args.srctree) return 0 def register_commands(subparsers, context): """Register devtool subcommands from this plugin""" parser_upgrade = subparsers.add_parser('upgrade', help='Upgrade an existing recipe', description='Upgrades an existing recipe to a new upstream version') parser_upgrade.add_argument('recipename', help='Name for recipe to extract the source for') parser_upgrade.add_argument('srctree', help='Path to where to extract the source tree') parser_upgrade.add_argument('--version', '-V', help='Version to upgrade to (PV)') parser_upgrade.add_argument('--srcrev', '-S', help='Source revision to upgrade to (if fetching from an SCM such as git)') parser_upgrade.add_argument('--branch', '-b', default="devtool", help='Name for new development branch to checkout (default "%(default)s")') parser_upgrade.add_argument('--no-patch', action="store_true", help='Do not apply patches from the recipe to the new source code') group = parser_upgrade.add_mutually_exclusive_group() group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true") group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true") parser_upgrade.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') parser_upgrade.set_defaults(func=upgrade)