scripts/devtool: add development helper tool

Provides an easy means to work on developing applications and system
components with the build system.

For example to "modify" the source for an existing recipe:

  $ devtool modify -x pango /home/projects/pango
  Parsing recipes..done.
  NOTE: Fetching pango...
  NOTE: Unpacking...
  NOTE: Patching...
  NOTE: Source tree extracted to /home/projects/pango
  NOTE: Recipe pango now set up to build from /home/paul/projects/pango

The pango source is now extracted to /home/paul/projects/pango, managed
in git, with each patch as a commit, and a bbappend is created in the
workspace layer to use the source in /home/paul/projects/pango when
building.

Additionally, you can add a new piece of software:

  $ devtool add pv /home/projects/pv
  NOTE: Recipe /path/to/workspace/recipes/pv/pv.bb has been
  automatically created; further editing may be required to make it
  fully functional

The latter uses recipetool to create a skeleton recipe and again sets up
a bbappend to use the source in /home/projects/pv when building.

Having done a "devtool modify", can also write any changes to the
external git repository back as patches next to the recipe:

  $ devtool update-recipe mdadm
  Parsing recipes..done.
  NOTE: Removing patch mdadm-3.2.2_fix_for_x32.patch
  NOTE: Removing patch gcc-4.9.patch
  NOTE: Updating recipe mdadm_3.3.1.bb

[YOCTO #6561]
[YOCTO #6653]
[YOCTO #6656]

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

1 files changed, 255 insertions, 0 deletions

diff --git a/scripts/devtool b/scripts/devtool
new file mode 100755
index 0000000000..d6e1b9710d
--- /dev/null
+++ b/scripts/devtool
@@ -0,0 +1,255 @@
+#!/usr/bin/env python
+
+# OpenEmbedded Development tool
+#
+# Copyright (C) 2014 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import sys
+import os
+import argparse
+import glob
+import re
+import ConfigParser
+import subprocess
+import logging
+
+basepath = ''
+workspace = {}
+config = None
+context = None
+
+
+scripts_path = os.path.dirname(os.path.realpath(__file__))
+lib_path = scripts_path + '/lib'
+sys.path = sys.path + [lib_path]
+import scriptutils
+logger = scriptutils.logger_create('devtool')
+
+plugins = []
+
+
+class ConfigHandler(object):
+    config_file = ''
+    config_obj = None
+    init_path = ''
+    workspace_path = ''
+
+    def __init__(self, filename):
+        self.config_file = filename
+        self.config_obj = ConfigParser.SafeConfigParser()
+
+    def get(self, section, option, default=None):
+        try:
+            ret = self.config_obj.get(section, option)
+        except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
+            if default != None:
+                ret = default
+            else:
+                raise
+        return ret
+
+    def read(self):
+        if os.path.exists(self.config_file):
+            self.config_obj.read(self.config_file)
+
+            if self.config_obj.has_option('General', 'init_path'):
+                pth = self.get('General', 'init_path')
+                self.init_path = os.path.join(basepath, pth)
+                if not os.path.exists(self.init_path):
+                    logger.error('init_path %s specified in config file cannot be found' % pth)
+                    return False
+        else:
+            self.config_obj.add_section('General')
+
+        self.workspace_path = self.get('General', 'workspace_path', os.path.join(basepath, 'workspace'))
+        return True
+
+
+    def write(self):
+        logger.debug('writing to config file %s' % self.config_file)
+        self.config_obj.set('General', 'workspace_path', self.workspace_path)
+        with open(self.config_file, 'w') as f:
+            self.config_obj.write(f)
+
+class Context:
+    def __init__(self, **kwargs):
+        self.__dict__.update(kwargs)
+
+
+def read_workspace():
+    global workspace
+    workspace = {}
+    if not os.path.exists(os.path.join(config.workspace_path, 'conf', 'layer.conf')):
+        if context.fixed_setup:
+            logger.error("workspace layer not set up")
+            sys.exit(1)
+        else:
+            logger.info('Creating workspace layer in %s' % config.workspace_path)
+            _create_workspace(config.workspace_path, config, basepath)
+
+    logger.debug('Reading workspace in %s' % config.workspace_path)
+    externalsrc_re = re.compile(r'^EXTERNALSRC(_pn-[a-zA-Z0-9-]*)? =.*$')
+    for fn in glob.glob(os.path.join(config.workspace_path, 'appends', '*.bbappend')):
+        pn = os.path.splitext(os.path.basename(fn))[0].split('_')[0]
+        with open(fn, 'r') as f:
+            for line in f:
+                if externalsrc_re.match(line.rstrip()):
+                    splitval = line.split('=', 2)
+                    workspace[pn] = splitval[1].strip('" \n\r\t')
+                    break
+
+def create_workspace(args, config, basepath, workspace):
+    if args.directory:
+        workspacedir = os.path.abspath(args.directory)
+    else:
+        workspacedir = os.path.abspath(os.path.join(basepath, 'workspace'))
+    _create_workspace(workspacedir, config, basepath, args.create_only)
+
+def _create_workspace(workspacedir, config, basepath, create_only=False):
+    import bb
+
+    confdir = os.path.join(workspacedir, 'conf')
+    if os.path.exists(os.path.join(confdir, 'layer.conf')):
+        logger.info('Specified workspace already set up, leaving as-is')
+    else:
+        # Add a config file
+        bb.utils.mkdirhier(confdir)
+        with open(os.path.join(confdir, 'layer.conf'), 'w') as f:
+            f.write('# ### workspace layer auto-generated by devtool ###\n')
+            f.write('BBPATH =. "$' + '{LAYERDIR}:"\n')
+            f.write('BBFILES += "$' + '{LAYERDIR}/recipes/*/*.bb \\\n')
+            f.write('            $' + '{LAYERDIR}/appends/*.bbappend"\n')
+            f.write('BBFILE_COLLECTIONS += "workspacelayer"\n')
+            f.write('BBFILE_PATTERN_workspacelayer = "^$' + '{LAYERDIR}/"\n')
+            f.write('BBFILE_PATTERN_IGNORE_EMPTY_workspacelayer = "1"\n')
+            f.write('BBFILE_PRIORITY_workspacelayer = "99"\n')
+        # Add a README file
+        with open(os.path.join(workspacedir, 'README'), 'w') as f:
+            f.write('This layer was created by the OpenEmbedded devtool utility in order to\n')
+            f.write('contain recipes and bbappends. In most instances you should use the\n')
+            f.write('devtool utility to manage files within it rather than modifying files\n')
+            f.write('directly (although recipes added with "devtool add" will often need\n')
+            f.write('direct modification.)\n')
+            f.write('\nIf you no longer need to use devtool you can remove the path to this\n')
+            f.write('workspace layer from your conf/bblayers.conf file (and then delete the\n')
+            f.write('layer, if you wish).\n')
+    if not create_only:
+        # Add the workspace layer to bblayers.conf
+        bblayers_conf = os.path.join(basepath, 'conf', 'bblayers.conf')
+        if not os.path.exists(bblayers_conf):
+            logger.error('Unable to find bblayers.conf')
+            return -1
+        bb.utils.edit_bblayers_conf(bblayers_conf, workspacedir, config.workspace_path)
+        if config.workspace_path != workspacedir:
+            # Update our config to point to the new location
+            config.workspace_path = workspacedir
+            config.write()
+
+
+def main():
+    global basepath
+    global config
+    global context
+
+    context = Context(fixed_setup=False)
+
+    # Default basepath
+    basepath = os.path.dirname(os.path.abspath(__file__))
+    pth = basepath
+    while pth != '' and pth != os.sep:
+        if os.path.exists(os.path.join(pth, '.devtoolbase')):
+            context.fixed_setup = True
+            basepath = pth
+            break
+        pth = os.path.dirname(pth)
+
+    parser = argparse.ArgumentParser(description="OpenEmbedded development tool",
+                                     epilog="Use %(prog)s <command> --help to get help on a specific command")
+    parser.add_argument('--basepath', help='Base directory of SDK / build directory')
+    parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true')
+    parser.add_argument('-q', '--quiet', help='Print only errors', action='store_true')
+    parser.add_argument('--color', help='Colorize output', choices=['auto', 'always', 'never'], default='auto')
+
+    subparsers = parser.add_subparsers(dest="subparser_name")
+
+    if not context.fixed_setup:
+        parser_create_workspace = subparsers.add_parser('create-workspace', help='Set up a workspace')
+        parser_create_workspace.add_argument('directory', nargs='?', help='Directory for the workspace')
+        parser_create_workspace.add_argument('--create-only', action="store_true", help='Only create the workspace, do not alter configuration')
+        parser_create_workspace.set_defaults(func=create_workspace)
+
+    scriptutils.load_plugins(logger, plugins, os.path.join(scripts_path, 'lib', 'devtool'))
+    for plugin in plugins:
+        if hasattr(plugin, 'register_commands'):
+            plugin.register_commands(subparsers, context)
+
+    args = parser.parse_args()
+
+    if args.debug:
+        logger.setLevel(logging.DEBUG)
+    elif args.quiet:
+        logger.setLevel(logging.ERROR)
+
+    if args.basepath:
+        # Override
+        basepath = args.basepath
+    elif not context.fixed_setup:
+        basepath = os.environ.get('BUILDDIR')
+        if not basepath:
+            logger.error("This script can only be run after initialising the build environment (e.g. by using oe-init-build-env)")
+            sys.exit(1)
+
+    logger.debug('Using basepath %s' % basepath)
+
+    config = ConfigHandler(os.path.join(basepath, 'conf', 'devtool.conf'))
+    if not config.read():
+        return -1
+
+    bitbake_subdir = config.get('General', 'bitbake_subdir', '')
+    if bitbake_subdir:
+        # Normally set for use within the SDK
+        logger.debug('Using bitbake subdir %s' % bitbake_subdir)
+        sys.path.insert(0, os.path.join(basepath, bitbake_subdir, 'lib'))
+        core_meta_subdir = config.get('General', 'core_meta_subdir')
+        sys.path.insert(0, os.path.join(basepath, core_meta_subdir, 'lib'))
+    else:
+        # Standard location
+        import scriptpath
+        bitbakepath = scriptpath.add_bitbake_lib_path()
+        if not bitbakepath:
+            logger.error("Unable to find bitbake by searching parent directory of this script or PATH")
+            sys.exit(1)
+        logger.debug('Using standard bitbake path %s' % bitbakepath)
+        scriptpath.add_oe_lib_path()
+
+    scriptutils.logger_setup_color(logger, args.color)
+
+    if args.subparser_name != 'create-workspace':
+        read_workspace()
+
+    ret = args.func(args, config, basepath, workspace)
+
+    return ret
+
+
+if __name__ == "__main__":
+    try:
+        ret = main()
+    except Exception:
+        ret = 1
+        import traceback
+        traceback.print_exc(5)
+    sys.exit(ret)