devtool: finish: add dry-run option

If you're not sure what changes devtool finish is going to make, or
you're not sure you're finished with your modifications, it is useful to
be able to see what devtool finish is going to do beforehand, so add
a -N/--dry-run option to make that possible.

(It's also very useful for debugging devtool finish itself.)

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>

1 files changed, 199 insertions, 71 deletions

diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index 9d9031c2b3..d55d504a14 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -341,19 +341,45 @@ def _check_compatible_recipe(pn, d):
                            "from working. You will need to disable this "
                            "first." % pn)
 
-def _move_file(src, dst):
-    """Move a file. Creates all the directory components of destination path."""
+def _dry_run_copy(src, dst, dry_run_outdir, base_outdir):
+    """Common function for copying a file to the dry run output directory"""
+    relpath = os.path.relpath(dst, base_outdir)
+    if relpath.startswith('..'):
+        raise Exception('Incorrect base path %s for path %s' % (base_outdir, dst))
+    dst = os.path.join(dry_run_outdir, relpath)
     dst_d = os.path.dirname(dst)
     if dst_d:
         bb.utils.mkdirhier(dst_d)
-    shutil.move(src, dst)
+    # Don't overwrite existing files, otherwise in the case of an upgrade
+    # the dry-run written out recipe will be overwritten with an unmodified
+    # version
+    if not os.path.exists(dst):
+        shutil.copy(src, dst)
+
+def _move_file(src, dst, dry_run_outdir=None, base_outdir=None):
+    """Move a file. Creates all the directory components of destination path."""
+    dry_run_suffix = ' (dry-run)' if dry_run_outdir else ''
+    logger.debug('Moving %s to %s%s' % (src, dst, dry_run_suffix))
+    if dry_run_outdir:
+        # We want to copy here, not move
+        _dry_run_copy(src, dst, dry_run_outdir, base_outdir)
+    else:
+        dst_d = os.path.dirname(dst)
+        if dst_d:
+            bb.utils.mkdirhier(dst_d)
+        shutil.move(src, dst)
 
-def _copy_file(src, dst):
+def _copy_file(src, dst, dry_run_outdir=None):
     """Copy a file. Creates all the directory components of destination path."""
-    dst_d = os.path.dirname(dst)
-    if dst_d:
-        bb.utils.mkdirhier(dst_d)
-    shutil.copy(src, dst)
+    dry_run_suffix = ' (dry-run)' if dry_run_outdir else ''
+    logger.debug('Copying %s to %s%s' % (src, dst, dry_run_suffix))
+    if dry_run_outdir:
+        _dry_run_copy(src, dst, dry_run_outdir, base_outdir)
+    else:
+        dst_d = os.path.dirname(dst)
+        if dst_d:
+            bb.utils.mkdirhier(dst_d)
+        shutil.copy(src, dst)
 
 def _git_ls_tree(repodir, treeish='HEAD', recursive=False):
     """List contents of a git treeish"""
@@ -1054,8 +1080,11 @@ def _replace_srcuri_entry(srcuri, filename, newentry):
             srcuri.insert(i, newentry)
             break
 
-def _remove_source_files(append, files, destpath, no_report_remove=False):
+def _remove_source_files(append, files, destpath, no_report_remove=False, dry_run=False):
     """Unlink existing patch files"""
+
+    dry_run_suffix = ' (dry-run)' if dry_run else ''
+
     for path in files:
         if append:
             if not destpath:
@@ -1064,19 +1093,20 @@ def _remove_source_files(append, files, destpath, no_report_remove=False):
 
         if os.path.exists(path):
             if not no_report_remove:
-                logger.info('Removing file %s' % path)
-            # FIXME "git rm" here would be nice if the file in question is
-            #       tracked
-            # FIXME there's a chance that this file is referred to by
-            #       another recipe, in which case deleting wouldn't be the
-            #       right thing to do
-            os.remove(path)
-            # Remove directory if empty
-            try:
-                os.rmdir(os.path.dirname(path))
-            except OSError as ose:
-                if ose.errno != errno.ENOTEMPTY:
-                    raise
+                logger.info('Removing file %s%s' % (path, dry_run_suffix))
+            if not dry_run:
+                # FIXME "git rm" here would be nice if the file in question is
+                #       tracked
+                # FIXME there's a chance that this file is referred to by
+                #       another recipe, in which case deleting wouldn't be the
+                #       right thing to do
+                os.remove(path)
+                # Remove directory if empty
+                try:
+                    os.rmdir(os.path.dirname(path))
+                except OSError as ose:
+                    if ose.errno != errno.ENOTEMPTY:
+                        raise
 
 
 def _export_patches(srctree, rd, start_rev, destdir, changed_revs=None):
@@ -1277,13 +1307,16 @@ def _determine_files_dir(rd):
     return os.path.join(recipedir, rd.getVar('BPN'))
 
 
-def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove):
+def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir=None):
     """Implement the 'srcrev' mode of update-recipe"""
     import bb
     import oe.recipeutils
 
+    dry_run_suffix = ' (dry-run)' if dry_run_outdir else ''
+
     recipefile = rd.getVar('FILE')
-    logger.info('Updating SRCREV in recipe %s' % os.path.basename(recipefile))
+    recipedir = os.path.basename(recipefile)
+    logger.info('Updating SRCREV in recipe %s%s' % (recipedir, dry_run_suffix))
 
     # Get HEAD revision
     try:
@@ -1303,6 +1336,7 @@ def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wi
     srcuri = orig_src_uri.split()
     tempdir = tempfile.mkdtemp(prefix='devtool')
     update_srcuri = False
+    appendfile = None
     try:
         local_files_dir = tempfile.mkdtemp(dir=tempdir)
         srctreebase = workspace[recipename]['srctreebase']
@@ -1328,29 +1362,36 @@ def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wi
             if update_srcuri:
                 removevalues  = {'SRC_URI': removedentries}
                 patchfields['SRC_URI'] = '\\\n    '.join(srcuri)
-            _, destpath = oe.recipeutils.bbappend_recipe(
-                    rd, appendlayerdir, files, wildcardver=wildcard_version,
-                    extralines=patchfields, removevalues=removevalues)
+            if dry_run_outdir:
+                logger.info('Creating bbappend (dry-run)')
+            else:
+                appendfile, destpath = oe.recipeutils.bbappend_recipe(
+                        rd, appendlayerdir, files, wildcardver=wildcard_version,
+                        extralines=patchfields, removevalues=removevalues,
+                        redirect_output=dry_run_outdir)
         else:
             files_dir = _determine_files_dir(rd)
             for basepath, path in upd_f.items():
-                logger.info('Updating file %s' % basepath)
+                logger.info('Updating file %s%s' % (basepath, dry_run_suffix))
                 if os.path.isabs(basepath):
                     # Original file (probably with subdir pointing inside source tree)
                     # so we do not want to move it, just copy
-                    _copy_file(basepath, path)
+                    _copy_file(basepath, path, dry_run_outdir=dry_run_outdir, base_outdir=recipedir)
                 else:
-                    _move_file(os.path.join(local_files_dir, basepath), path)
+                    _move_file(os.path.join(local_files_dir, basepath), path,
+                               dry_run_outdir=dry_run_outdir, base_outdir=recipedir)
                 update_srcuri= True
             for basepath, path in new_f.items():
-                logger.info('Adding new file %s' % basepath)
+                logger.info('Adding new file %s%s' % (basepath, dry_run_suffix))
                 _move_file(os.path.join(local_files_dir, basepath),
-                           os.path.join(files_dir, basepath))
+                           os.path.join(files_dir, basepath),
+                           dry_run_outdir=dry_run_outdir,
+                           base_outdir=recipedir)
                 srcuri.append('file://%s' % basepath)
                 update_srcuri = True
             if update_srcuri:
                 patchfields['SRC_URI'] = ' '.join(srcuri)
-            oe.recipeutils.patch_recipe(rd, recipefile, patchfields)
+            ret = oe.recipeutils.patch_recipe(rd, recipefile, patchfields, redirect_output=dry_run_outdir)
     finally:
         shutil.rmtree(tempdir)
     if not 'git://' in orig_src_uri:
@@ -1358,15 +1399,16 @@ def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wi
                     'point to a git repository where you have pushed your '
                     'changes')
 
-    _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove)
-    return True
+    _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove, dry_run=dry_run_outdir)
+    return True, appendfile, remove_files
 
-def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev):
+def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir=None):
     """Implement the 'patch' mode of update-recipe"""
     import bb
     import oe.recipeutils
 
     recipefile = rd.getVar('FILE')
+    recipedir = os.path.dirname(recipefile)
     append = workspace[recipename]['bbappend']
     if not os.path.exists(append):
         raise DevtoolError('unable to find workspace bbappend for recipe %s' %
@@ -1377,10 +1419,13 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
         raise DevtoolError('Unable to find initial revision - please specify '
                            'it with --initial-rev')
 
+    appendfile = None
     dl_dir = rd.getVar('DL_DIR')
     if not dl_dir.endswith('/'):
         dl_dir += '/'
 
+    dry_run_suffix = ' (dry-run)' if dry_run_outdir else ''
+
     tempdir = tempfile.mkdtemp(prefix='devtool')
     try:
         local_files_dir = tempfile.mkdtemp(dir=tempdir)
@@ -1418,10 +1463,11 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
                         remaining = ['file://' + os.path.basename(item) for
                                      item in remaining]
                         removevalues = {'SRC_URI': removedentries + remaining}
-                _, destpath = oe.recipeutils.bbappend_recipe(
+                appendfile, destpath = oe.recipeutils.bbappend_recipe(
                                 rd, appendlayerdir, files,
                                 wildcardver=wildcard_version,
-                                removevalues=removevalues)
+                                removevalues=removevalues,
+                                redirect_output=dry_run_outdir)
             else:
                 logger.info('No patches or local source files needed updating')
         else:
@@ -1432,9 +1478,11 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
                 if os.path.isabs(basepath):
                     # Original file (probably with subdir pointing inside source tree)
                     # so we do not want to move it, just copy
-                    _copy_file(basepath, path)
+                    _copy_file(basepath, path,
+                               dry_run_outdir=dry_run_outdir, base_outdir=recipedir)
                 else:
-                    _move_file(os.path.join(local_files_dir, basepath), path)
+                    _move_file(os.path.join(local_files_dir, basepath), path,
+                               dry_run_outdir=dry_run_outdir, base_outdir=recipedir)
                 updatefiles = True
             for basepath, path in upd_p.items():
                 patchfn = os.path.join(patches_dir, basepath)
@@ -1446,39 +1494,45 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
                     _replace_srcuri_entry(srcuri, basepath, 'file://%s' % basepath)
                     updaterecipe = True
                 else:
-                    logger.info('Updating patch %s' % basepath)
-                logger.debug('Moving new patch %s to %s' % (patchfn, path))
-                _move_file(patchfn, path)
+                    logger.info('Updating patch %s%s' % (basepath, dry_run_suffix))
+                _move_file(patchfn, path,
+                           dry_run_outdir=dry_run_outdir, base_outdir=recipedir)
                 updatefiles = True
             # Add any new files
             for basepath, path in new_f.items():
-                logger.info('Adding new file %s' % basepath)
+                logger.info('Adding new file %s%s' % (basepath, dry_run_suffix))
                 _move_file(os.path.join(local_files_dir, basepath),
-                           os.path.join(files_dir, basepath))
+                           os.path.join(files_dir, basepath),
+                           dry_run_outdir=dry_run_outdir,
+                           base_outdir=recipedir)
                 srcuri.append('file://%s' % basepath)
                 updaterecipe = True
             for basepath, path in new_p.items():
-                logger.info('Adding new patch %s' % basepath)
+                logger.info('Adding new patch %s%s' % (basepath, dry_run_suffix))
                 _move_file(os.path.join(patches_dir, basepath),
-                           os.path.join(files_dir, basepath))
+                           os.path.join(files_dir, basepath),
+                           dry_run_outdir=dry_run_outdir,
+                           base_outdir=recipedir)
                 srcuri.append('file://%s' % basepath)
                 updaterecipe = True
             # Update recipe, if needed
             if _remove_file_entries(srcuri, remove_files)[0]:
                 updaterecipe = True
             if updaterecipe:
-                logger.info('Updating recipe %s' % os.path.basename(recipefile))
-                oe.recipeutils.patch_recipe(rd, recipefile,
-                                            {'SRC_URI': ' '.join(srcuri)})
+                if not dry_run_outdir:
+                    logger.info('Updating recipe %s' % os.path.basename(recipefile))
+                ret = oe.recipeutils.patch_recipe(rd, recipefile,
+                                                  {'SRC_URI': ' '.join(srcuri)},
+                                                  redirect_output=dry_run_outdir)
             elif not updatefiles:
                 # Neither patches nor recipe were updated
                 logger.info('No patches or files need updating')
-                return False
+                return False, None, []
     finally:
         shutil.rmtree(tempdir)
 
-    _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove)
-    return True
+    _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove, dry_run=dry_run_outdir)
+    return True, appendfile, remove_files
 
 def _guess_recipe_update_mode(srctree, rdata):
     """Guess the recipe update mode to use"""
@@ -1502,18 +1556,18 @@ def _guess_recipe_update_mode(srctree, rdata):
 
     return 'patch'
 
-def _update_recipe(recipename, workspace, rd, mode, appendlayerdir, wildcard_version, no_remove, initial_rev, no_report_remove=False):
+def _update_recipe(recipename, workspace, rd, mode, appendlayerdir, wildcard_version, no_remove, initial_rev, no_report_remove=False, dry_run_outdir=None):
     srctree = workspace[recipename]['srctree']
     if mode == 'auto':
         mode = _guess_recipe_update_mode(srctree, rd)
 
     if mode == 'srcrev':
-        updated = _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove)
+        updated, appendfile, removed = _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir)
     elif mode == 'patch':
-        updated = _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev)
+        updated, appendfile, removed = _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir)
     else:
         raise DevtoolError('update_recipe: invalid mode %s' % mode)
-    return updated
+    return updated, appendfile, removed
 
 def update_recipe(args, config, basepath, workspace):
     """Entry point for the devtool 'update-recipe' subcommand"""
@@ -1534,7 +1588,12 @@ def update_recipe(args, config, basepath, workspace):
         if not rd:
             return 1
 
-        updated = _update_recipe(args.recipename, workspace, rd, args.mode, args.append, args.wildcard_version, args.no_remove, args.initial_rev)
+        dry_run_output = None
+        dry_run_outdir = None
+        if args.dry_run:
+            dry_run_output = tempfile.TemporaryDirectory(prefix='devtool')
+            dry_run_outdir = dry_run_output.name
+        updated, _, _ = _update_recipe(args.recipename, workspace, rd, args.mode, args.append, args.wildcard_version, args.no_remove, args.initial_rev, dry_run_outdir=dry_run_outdir)
 
         if updated:
             rf = rd.getVar('FILE')
@@ -1680,6 +1739,8 @@ def finish(args, config, basepath, workspace):
 
     check_workspace_recipe(workspace, args.recipename)
 
+    dry_run_suffix = ' (dry-run)' if args.dry_run else ''
+
     # Grab the equivalent of COREBASE without having to initialise tinfoil
     corebasedir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..'))
 
@@ -1700,7 +1761,9 @@ def finish(args, config, basepath, workspace):
             return 1
 
         destlayerdir = _get_layer(args.destination, tinfoil.config_data)
-        origlayerdir = oe.recipeutils.find_layerdir(rd.getVar('FILE'))
+        recipefile = rd.getVar('FILE')
+        recipedir = os.path.dirname(recipefile)
+        origlayerdir = oe.recipeutils.find_layerdir(recipefile)
 
         if not os.path.isdir(destlayerdir):
             raise DevtoolError('Unable to find layer or directory matching "%s"' % args.destination)
@@ -1748,38 +1811,101 @@ def finish(args, config, basepath, workspace):
 
         # Actually update the recipe / bbappend
         removing_original = (origpath and origfilelist and oe.recipeutils.find_layerdir(origpath) == destlayerbasedir)
-        _update_recipe(args.recipename, workspace, rd, args.mode, appendlayerdir, wildcard_version=True, no_remove=False, initial_rev=args.initial_rev, no_report_remove=removing_original)
+        dry_run_output = None
+        dry_run_outdir = None
+        if args.dry_run:
+            dry_run_output = tempfile.TemporaryDirectory(prefix='devtool')
+            dry_run_outdir = dry_run_output.name
+        updated, appendfile, removed = _update_recipe(args.recipename, workspace, rd, args.mode, appendlayerdir, wildcard_version=True, no_remove=False, no_report_remove=removing_original, initial_rev=args.initial_rev, dry_run_outdir=dry_run_outdir)
+        removed = [os.path.relpath(pth, recipedir) for pth in removed]
 
         # Remove any old files in the case of an upgrade
-        recipedir = os.path.dirname(rd.getVar('FILE'))
         if removing_original:
             for fn in origfilelist:
                 fnp = os.path.join(origpath, fn)
-                if not os.path.exists(os.path.join(recipedir, fn)):
-                    logger.info('Removing file %s' % fnp)
-                try:
-                    os.remove(fnp)
-                except FileNotFoundError:
-                    pass
+                if fn in removed or not os.path.exists(os.path.join(recipedir, fn)):
+                    logger.info('Removing file %s%s' % (fnp, dry_run_suffix))
+                if not args.dry_run:
+                    try:
+                        os.remove(fnp)
+                    except FileNotFoundError:
+                        pass
 
         if origlayerdir == config.workspace_path and destpath:
             # Recipe file itself is in the workspace - need to move it and any
             # associated files to the specified layer
             no_clean = True
-            logger.info('Moving recipe file to %s' % destpath)
+            logger.info('Moving recipe file to %s%s' % (destpath, dry_run_suffix))
             for root, _, files in os.walk(recipedir):
                 for fn in files:
                     srcpath = os.path.join(root, fn)
                     relpth = os.path.relpath(os.path.dirname(srcpath), recipedir)
                     destdir = os.path.abspath(os.path.join(destpath, relpth))
-                    bb.utils.mkdirhier(destdir)
-                    shutil.move(srcpath, os.path.join(destdir, fn))
+                    destfp = os.path.join(destdir, fn)
+                    _move_file(srcpath, destfp, dry_run_outdir=dry_run_outdir, base_outdir=destpath)
 
+        if dry_run_outdir:
+            import difflib
+            comparelist = []
+            for root, _, files in os.walk(dry_run_outdir):
+                for fn in files:
+                    outf = os.path.join(root, fn)
+                    relf = os.path.relpath(outf, dry_run_outdir)
+                    logger.debug('dry-run: output file %s' % relf)
+                    if fn.endswith('.bb'):
+                        if origfilelist and origpath and destpath:
+                            # Need to match this up with the pre-upgrade recipe file
+                            for origf in origfilelist:
+                                if origf.endswith('.bb'):
+                                    comparelist.append((os.path.abspath(os.path.join(origpath, origf)),
+                                                        outf,
+                                                        os.path.abspath(os.path.join(destpath, relf))))
+                                    break
+                        else:
+                            # Compare to the existing recipe
+                            comparelist.append((recipefile, outf, recipefile))
+                    elif fn.endswith('.bbappend'):
+                        if appendfile:
+                            if os.path.exists(appendfile):
+                                comparelist.append((appendfile, outf, appendfile))
+                            else:
+                                comparelist.append((None, outf, appendfile))
+                    else:
+                        if destpath:
+                            recipedest = destpath
+                        elif appendfile:
+                            recipedest = os.path.dirname(appendfile)
+                        else:
+                            recipedest = os.path.dirname(recipefile)
+                        destfp = os.path.join(recipedest, relf)
+                        if os.path.exists(destfp):
+                            comparelist.append((destfp, outf, destfp))
+            output = ''
+            for oldfile, newfile, newfileshow in comparelist:
+                if oldfile:
+                    with open(oldfile, 'r') as f:
+                        oldlines = f.readlines()
+                else:
+                    oldfile = '/dev/null'
+                    oldlines = []
+                with open(newfile, 'r') as f:
+                    newlines = f.readlines()
+                if not newfileshow:
+                    newfileshow = newfile
+                diff = difflib.unified_diff(oldlines, newlines, oldfile, newfileshow)
+                difflines = list(diff)
+                if difflines:
+                    output += ''.join(difflines)
+            if output:
+                logger.info('Diff of changed files:\n%s' % output)
     finally:
         tinfoil.shutdown()
 
     # Everything else has succeeded, we can now reset
-    _reset([args.recipename], no_clean=no_clean, config=config, basepath=basepath, workspace=workspace)
+    if args.dry_run:
+        logger.info('Resetting recipe (dry-run)')
+    else:
+        _reset([args.recipename], no_clean=no_clean, config=config, basepath=basepath, workspace=workspace)
 
     return 0
 
@@ -1873,6 +1999,7 @@ def register_commands(subparsers, context):
     parser_update_recipe.add_argument('--append', '-a', help='Write changes to a bbappend in the specified layer instead of the recipe', metavar='LAYERDIR')
     parser_update_recipe.add_argument('--wildcard-version', '-w', help='In conjunction with -a/--append, use a wildcard to make the bbappend apply to any recipe version', action='store_true')
     parser_update_recipe.add_argument('--no-remove', '-n', action="store_true", help='Don\'t remove patches, only add or update')
+    parser_update_recipe.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)')
     parser_update_recipe.set_defaults(func=update_recipe)
 
     parser_status = subparsers.add_parser('status', help='Show workspace status',
@@ -1896,4 +2023,5 @@ def register_commands(subparsers, context):
     parser_finish.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE')
     parser_finish.add_argument('--initial-rev', help='Override starting revision for patches')
     parser_finish.add_argument('--force', '-f', action="store_true", help='Force continuing even if there are uncommitted changes in the source tree repository')
+    parser_finish.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)')
     parser_finish.set_defaults(func=finish)