summaryrefslogtreecommitdiffstats
path: root/scripts/test-reexec
blob: 9eaa96e754b6aa384aef53f27a994c26d3c30088 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
#!/bin/bash

# Test Script for task re-execution
# 
# Copyright 2012 Intel Corporation
# All rights reserved.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
# DESCRIPTION
# This script is intended to address issues for re-execution of 
# tasks. The test results are saved in ./reexeclogs. Force build
# logs are saved with prefix "force". Build failure logs are saved with
# prefix "failed". Log files with prefix "initial" are used to save
# initial build logs for each recipe. Log files with prefix "clean" are
# used to save logs of clean task after testing for a recipe is finished.
#

targets=`bitbake -s | cut -d " " -f 1`

LOGS=./reexeclogs

mkdir -p $LOGS

# Clear sstate files for specified recipe
function clearsstate {
	target=$1

	sstate_dir=`bitbake $target -e | grep "^SSTATE_DIR" | cut -d "\"" -f 2`
	sstate_pkgspec=`bitbake $target -e | grep "^SSTATE_PKGSPEC" | cut -d "\"" -f 2`
	sstasks=`bitbake $target -e | grep "^SSTATETASKS" | cut -d "\"" -f 2`

	for sstask in $sstasks
	do
		sstask=${sstask:3}
		case $sstask in
			populate_sysroot) sstask="populate-sysroot"
			;;
			populate_lic) sstask="populate-lic"
			;;
			package_write_ipk) sstask="deploy-ipk"
			;;
			package_write_deb) sstask="deploy-deb"
			;;
			package_write_rpm) sstask="deploy-rpm"
			;;
			package) sstask="package"
			;;
			deploy) sstask="deploy"
			;;
			*)
			;;
		esac

		echo "Removing ${sstate_dir}/${sstate_pkgspec}*_${sstask}.tgz* for $target"
		rm -rf ${sstate_dir}/${sstate_pkgspec}*_${sstask}.tgz*
	done
}

# Function to re-execute specified task of recipe
function testit {
	target=$1
	task=$2

	task=`echo $task | sed 's/_setscene//'`

	if [ -f $LOGS/force.$target.$task ]; then
		return
	fi

	case $task in
		clean|build|cleansstate|cleanall|package|cleansstate2|package_write|package_write_ipk|package_write_rpm|package_write_deb|fetch|populate_lic) return;;
		fetchall|devshell|buildall|listtasks|checkuri|checkuriall) return;;
	esac

	echo "Attempting target $target, task $task"
	echo "Initial build"
	bitbake $target -c cleansstate > $LOGS/initial.$target.$task
	bitbake $target >> $LOGS/initial.$target.$task
	clearsstate $target >> $LOGS/initial.$target.$task
	echo "Re-execution build"
	bitbake $target -c $task -f  > $LOGS/force.$target.$task
	if [ "$?" != 0 ]; then
		echo "FAILURE for $target $task"
		cp $LOGS/force.$target.$task $LOGS/failed.$target.$task
		bitbake $target -c clean > $LOGS/clean.$target.$task
	else
		bitbake $target >> $LOGS/force.$target.$task
		if [ "$?" != 0 ]; then
			echo "FAILURE2 for $target $task"
			cp $LOGS/force.$target.$task $LOGS/failed.$target.$task
			bitbake $target -c clean > $LOGS/clean.$target.$task
		fi
	fi
	echo "Done"
}

# Go through the recipe list and these recipes' task list
# Then re-execute them
for target in $targets; do
	# Remove log messages from bitbake output
	case $target in
		Summary*|WARNING*|Loading*|Loaded*|Package*|=====*) continue;;
	esac
	tasks=`bitbake $target -c listtasks | grep ^do_ | sed s/do_//`
	for task in $tasks; do
		testit $target $task
	done
done
Literal.String.Single */ .highlight .ss { color: #e6db74 } /* Literal.String.Symbol */ .highlight .bp { color: #f8f8f2 } /* Name.Builtin.Pseudo */ .highlight .fm { color: #a6e22e } /* Name.Function.Magic */ .highlight .vc { color: #f8f8f2 } /* Name.Variable.Class */ .highlight .vg { color: #f8f8f2 } /* Name.Variable.Global */ .highlight .vi { color: #f8f8f2 } /* Name.Variable.Instance */ .highlight .vm { color: #f8f8f2 } /* Name.Variable.Magic */ .highlight .il { color: #ae81ff } /* Literal.Number.Integer.Long */ } @media (prefers-color-scheme: light) { .highlight .hll { background-color: #ffffcc } .highlight .c { color: #888888 } /* Comment */ .highlight .err { color: #a61717; background-color: #e3d2d2 } /* Error */ .highlight .k { color: #008800; font-weight: bold } /* Keyword */ .highlight .ch { color: #888888 } /* Comment.Hashbang */ .highlight .cm { color: #888888 } /* Comment.Multiline */ .highlight .cp { color: #cc0000; font-weight: bold } /* Comment.Preproc */ .highlight .cpf { color: #888888 } /* Comment.PreprocFile */ .highlight .c1 { color: #888888 } /* Comment.Single */ .highlight .cs { color: #cc0000; font-weight: bold; background-color: #fff0f0 } /* Comment.Special */ .highlight .gd { color: #000000; background-color: #ffdddd } /* Generic.Deleted */ .highlight .ge { font-style: italic } /* Generic.Emph */ .highlight .gr { color: #aa0000 } /* Generic.Error */ .highlight .gh { color: #333333 } /* Generic.Heading */ .highlight .gi { color: #000000; background-color: #ddffdd } /* Generic.Inserted */ .highlight .go { color: #888888 } /* Generic.Output */ .highlight .gp { color: #555555 } /* Generic.Prompt */ .highlight .gs { font-weight: bold } /* Generic.Strong */ .highlight .gu { color: #666666 } /* Generic.Subheading */ .highlight .gt { color: #aa0000 } /* Generic.Traceback */ .highlight .kc { color: #008800; font-weight: bold } /* Keyword.Constant */ .highlight .kd { color: #008800; font-weight: bold } /* Keyword.Declaration */ .highlight .kn { color: #008800; font-weight: bold } /* Keyword.Namespace */ .highlight .kp { color: #008800 } /* Keyword.Pseudo */ .highlight .kr { color: #008800; font-weight: bold } /* Keyword.Reserved */ .highlight .kt { color: #888888; font-weight: bold } /* Keyword.Type */ .highlight .m { color: #0000DD; font-weight: bold } /* Literal.Number */ .highlight .s { color: #dd2200; background-color: #fff0f0 } /* Literal.String */ .highlight .na { color: #336699 } /* Name.Attribute */ .highlight .nb { color: #003388 } /* Name.Builtin */ .highlight .nc { color: #bb0066; font-weight: bold } /* Name.Class */ .highlight .no { color: #003366; font-weight: bold } /* Name.Constant */ .highlight .nd { color: #555555 } /* Name.Decorator */ .highlight .ne { color: #bb0066; font-weight: bold } /* Name.Exception */ .highlight .nf { color: #0066bb; font-weight: bold } /* Name.Function */ .highlight .nl { color: #336699; font-style: italic } /* Name.Label */ .highlight .nn { color: #bb0066; font-weight: bold } /* Name.Namespace */ .highlight .py { color: #336699; font-weight: bold } /* Name.Property */ .highlight .nt { color: #bb0066; font-weight: bold } /* Name.Tag */ .highlight .nv { color: #336699 } /* Name.Variable */ .highlight .ow { color: #008800 } /* Operator.Word */ .highlight .w { color: #bbbbbb } /* Text.Whitespace */ .highlight .mb { color: #0000DD; font-weight: bold } /* Literal.Number.Bin */ .highlight .mf { color: #0000DD; font-weight: bold } /* Literal.Number.Float */ .highlight .mh { color: #0000DD; font-weight: bold } /* Literal.Number.Hex */ .highlight .mi { color: #0000DD; font-weight: bold } /* Literal.Number.Integer */ .highlight .mo { color: #0000DD; font-weight: bold } /* Literal.Number.Oct */ .highlight .sa { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Affix */ .highlight .sb { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Backtick */ .highlight .sc { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Char */ .highlight .dl { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Delimiter */ .highlight .sd { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Doc */ .highlight .s2 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Double */ .highlight .se { color: #0044dd; background-color: #fff0f0 } /* Literal.String.Escape */ .highlight .sh { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Heredoc */ .highlight .si { color: #3333bb; background-color: #fff0f0 } /* Literal.String.Interpol */ .highlight .sx { color: #22bb22; background-color: #f0fff0 } /* Literal.String.Other */ .highlight .sr { color: #008800; background-color: #fff0ff } /* Literal.String.Regex */ .highlight .s1 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Single */ .highlight .ss { color: #aa6600; background-color: #fff0f0 } /* Literal.String.Symbol */ .highlight .bp { color: #003388 } /* Name.Builtin.Pseudo */ .highlight .fm { color: #0066bb; font-weight: bold } /* Name.Function.Magic */ .highlight .vc { color: #336699 } /* Name.Variable.Class */ .highlight .vg { color: #dd7700 } /* Name.Variable.Global */ .highlight .vi { color: #3333bb } /* Name.Variable.Instance */ .highlight .vm { color: #336699 } /* Name.Variable.Magic */ .highlight .il { color: #0000DD; font-weight: bold } /* Literal.Number.Integer.Long */ }
#
# Copyright (c) 2013, Intel Corporation.
#
# SPDX-License-Identifier: GPL-2.0-only
#
# DESCRIPTION

# This module implements the image creation engine used by 'wic' to
# create images.  The engine parses through the OpenEmbedded kickstart
# (wks) file specified and generates images that can then be directly
# written onto media.
#
# AUTHORS
# Tom Zanussi <tom.zanussi (at] linux.intel.com>
#

import logging
import os
import tempfile
import json
import subprocess
import shutil
import re

from collections import namedtuple, OrderedDict

from wic import WicError
from wic.filemap import sparse_copy
from wic.pluginbase import PluginMgr
from wic.misc import get_bitbake_var, exec_cmd

logger = logging.getLogger('wic')

def verify_build_env():
    """
    Verify that the build environment is sane.

    Returns True if it is, false otherwise
    """
    if not os.environ.get("BUILDDIR"):
        raise WicError("BUILDDIR not found, exiting. (Did you forget to source oe-init-build-env?)")

    return True


CANNED_IMAGE_DIR = "lib/wic/canned-wks" # relative to scripts
SCRIPTS_CANNED_IMAGE_DIR = "scripts/" + CANNED_IMAGE_DIR
WIC_DIR = "wic"

def build_canned_image_list(path):
    layers_path = get_bitbake_var("BBLAYERS")
    canned_wks_layer_dirs = []

    if layers_path is not None:
        for layer_path in layers_path.split():
            for wks_path in (WIC_DIR, SCRIPTS_CANNED_IMAGE_DIR):
                cpath = os.path.join(layer_path, wks_path)
                if os.path.isdir(cpath):
                    canned_wks_layer_dirs.append(cpath)

    cpath = os.path.join(path, CANNED_IMAGE_DIR)
    canned_wks_layer_dirs.append(cpath)

    return canned_wks_layer_dirs

def find_canned_image(scripts_path, wks_file):
    """
    Find a .wks file with the given name in the canned files dir.

    Return False if not found
    """
    layers_canned_wks_dir = build_canned_image_list(scripts_path)

    for canned_wks_dir in layers_canned_wks_dir:
        for root, dirs, files in os.walk(canned_wks_dir):
            for fname in files:
                if fname.endswith("~") or fname.endswith("#"):
                    continue
                if ((fname.endswith(".wks") and wks_file + ".wks" == fname) or \
                   (fname.endswith(".wks.in") and wks_file + ".wks.in" == fname)):
                    fullpath = os.path.join(canned_wks_dir, fname)
                    return fullpath
    return None


def list_canned_images(scripts_path):
    """
    List the .wks files in the canned image dir, minus the extension.
    """
    layers_canned_wks_dir = build_canned_image_list(scripts_path)

    for canned_wks_dir in layers_canned_wks_dir:
        for root, dirs, files in os.walk(canned_wks_dir):
            for fname in files:
                if fname.endswith("~") or fname.endswith("#"):
                    continue
                if fname.endswith(".wks") or fname.endswith(".wks.in"):
                    fullpath = os.path.join(canned_wks_dir, fname)
                    with open(fullpath) as wks:
                        for line in wks:
                            desc = ""
                            idx = line.find("short-description:")
                            if idx != -1:
                                desc = line[idx + len("short-description:"):].strip()
                                break
                    basename = fname.split('.')[0]
                    print("  %s\t\t%s" % (basename.ljust(30), desc))


def list_canned_image_help(scripts_path, fullpath):
    """
    List the help and params in the specified canned image.
    """
    found = False
    with open(fullpath) as wks:
        for line in wks:
            if not found:
                idx = line.find("long-description:")
                if idx != -1:
                    print()
                    print(line[idx + len("long-description:"):].strip())
                    found = True
                continue
            if not line.strip():
                break
            idx = line.find("#")
            if idx != -1:
                print(line[idx + len("#:"):].rstrip())
            else:
                break


def list_source_plugins():
    """
    List the available source plugins i.e. plugins available for --source.
    """
    plugins = PluginMgr.get_plugins('source')

    for plugin in plugins:
        print("  %s" % plugin)


def wic_create(wks_file, rootfs_dir, bootimg_dir, kernel_dir,
               native_sysroot, options):
    """
    Create image

    wks_file - user-defined OE kickstart file
    rootfs_dir - absolute path to the build's /rootfs dir
    bootimg_dir - absolute path to the build's boot artifacts directory
    kernel_dir - absolute path to the build's kernel directory
    native_sysroot - absolute path to the build's native sysroots dir
    image_output_dir - dirname to create for image
    options - wic command line options (debug, bmap, etc)

    Normally, the values for the build artifacts values are determined
    by 'wic -e' from the output of the 'bitbake -e' command given an
    image name e.g. 'core-image-minimal' and a given machine set in
    local.conf.  If that's the case, the variables get the following
    values from the output of 'bitbake -e':

    rootfs_dir:        IMAGE_ROOTFS
    kernel_dir:        DEPLOY_DIR_IMAGE
    native_sysroot:    STAGING_DIR_NATIVE

    In the above case, bootimg_dir remains unset and the
    plugin-specific image creation code is responsible for finding the
    bootimg artifacts.

    In the case where the values are passed in explicitly i.e 'wic -e'
    is not used but rather the individual 'wic' options are used to
    explicitly specify these values.
    """
    try:
        oe_builddir = os.environ["BUILDDIR"]
    except KeyError:
        raise WicError("BUILDDIR not found, exiting. (Did you forget to source oe-init-build-env?)")

    if not os.path.exists(options.outdir):
        os.makedirs(options.outdir)

    pname = options.imager
    plugin_class = PluginMgr.get_plugins('imager').get(pname)
    if not plugin_class:
        raise WicError('Unknown plugin: %s' % pname)

    plugin = plugin_class(wks_file, rootfs_dir, bootimg_dir, kernel_dir,
                          native_sysroot, oe_builddir, options)

    plugin.do_create()

    logger.info("The image(s) were created using OE kickstart file:\n  %s", wks_file)


def wic_list(args, scripts_path):
    """
    Print the list of images or source plugins.
    """
    if args.list_type is None:
        return False

    if args.list_type == "images":

        list_canned_images(scripts_path)
        return True
    elif args.list_type == "source-plugins":
        list_source_plugins()
        return True
    elif len(args.help_for) == 1 and args.help_for[0] == 'help':
        wks_file = args.list_type
        fullpath = find_canned_image(scripts_path, wks_file)
        if not fullpath:
            raise WicError("No image named %s found, exiting. "
                           "(Use 'wic list images' to list available images, "
                           "or specify a fully-qualified OE kickstart (.wks) "
                           "filename)" % wks_file)

        list_canned_image_help(scripts_path, fullpath)
        return True

    return False


class Disk:
    def __init__(self, imagepath, native_sysroot, fstypes=('fat', 'ext')):
        self.imagepath = imagepath
        self.native_sysroot = native_sysroot
        self.fstypes = fstypes
        self._partitions = None
        self._partimages = {}
        self._lsector_size = None
        self._psector_size = None
        self._ptable_format = None

        # find parted
        # read paths from $PATH environment variable
        # if it fails, use hardcoded paths
        pathlist = "/bin:/usr/bin:/usr/sbin:/sbin/"
        try:
            self.paths = os.environ['PATH'] + ":" + pathlist
        except KeyError:
            self.paths = pathlist

        if native_sysroot:
            for path in pathlist.split(':'):
                self.paths = "%s%s:%s" % (native_sysroot, path, self.paths)

        self.parted = shutil.which("parted", path=self.paths)
        if not self.parted:
            raise WicError("Can't find executable parted")

        self.partitions = self.get_partitions()

    def __del__(self):
        for path in self._partimages.values():
            os.unlink(path)

    def get_partitions(self):
        if self._partitions is None:
            self._partitions = OrderedDict()
            out = exec_cmd("%s -sm %s unit B print" % (self.parted, self.imagepath))
            parttype = namedtuple("Part", "pnum start end size fstype")
            splitted = out.splitlines()
            # skip over possible errors in exec_cmd output
            try:
                idx =splitted.index("BYT;")
            except ValueError:
                raise WicError("Error getting partition information from %s" % (self.parted))
            lsector_size, psector_size, self._ptable_format = splitted[idx + 1].split(":")[3:6]
            self._lsector_size = int(lsector_size)
            self._psector_size = int(psector_size)
            for line in splitted[idx + 2:]:
                pnum, start, end, size, fstype = line.split(':')[:5]
                partition = parttype(int(pnum), int(start[:-1]), int(end[:-1]),
                                     int(size[:-1]), fstype)
                self._partitions[pnum] = partition

        return self._partitions

    def __getattr__(self, name):
        """Get path to the executable in a lazy way."""
        if name in ("mdir", "mcopy", "mdel", "mdeltree", "sfdisk", "e2fsck",
                    "resize2fs", "mkswap", "mkdosfs", "debugfs"):
            aname = "_%s" % name
            if aname not in self.__dict__:
                setattr(self, aname, shutil.which(name, path=self.paths))
                if aname not in self.__dict__ or self.__dict__[aname] is None:
                    raise WicError("Can't find executable '{}'".format(name))
            return self.__dict__[aname]
        return self.__dict__[name]

    def _get_part_image(self, pnum):
        if pnum not in self.partitions:
            raise WicError("Partition %s is not in the image" % pnum)
        part = self.partitions[pnum]
        # check if fstype is supported
        for fstype in self.fstypes:
            if part.fstype.startswith(fstype):
                break
        else:
            raise WicError("Not supported fstype: {}".format(part.fstype))
        if pnum not in self._partimages:
            tmpf = tempfile.NamedTemporaryFile(prefix="wic-part")
            dst_fname = tmpf.name
            tmpf.close()
            sparse_copy(self.imagepath, dst_fname, skip=part.start, length=part.size)
            self._partimages[pnum] = dst_fname

        return self._partimages[pnum]

    def _put_part_image(self, pnum):
        """Put partition image into partitioned image."""
        sparse_copy(self._partimages[pnum], self.imagepath,
                    seek=self.partitions[pnum].start)

    def dir(self, pnum, path):
        if pnum not in self.partitions:
            raise WicError("Partition %s is not in the image" % pnum)

        if self.partitions[pnum].fstype.startswith('ext'):
            return exec_cmd("{} {} -R 'ls -l {}'".format(self.debugfs,
                                                         self._get_part_image(pnum),
                                                         path), as_shell=True)
        else: # fat
            return exec_cmd("{} -i {} ::{}".format(self.mdir,
                                                   self._get_part_image(pnum),
                                                   path))

    def copy(self, src, dest):
        """Copy partition image into wic image."""
        pnum =  dest.part if isinstance(src, str) else src.part

        if self.partitions[pnum].fstype.startswith('ext'):
            if isinstance(src, str):
                cmd = "printf 'cd {}\nwrite {} {}\n' | {} -w {}".\
                      format(os.path.dirname(dest.path), src, os.path.basename(src),
                             self.debugfs, self._get_part_image(pnum))
            else: # copy from wic
                # run both dump and rdump to support both files and directory
                cmd = "printf 'cd {}\ndump /{} {}\nrdump /{} {}\n' | {} {}".\
                      format(os.path.dirname(src.path), src.path,
                             dest, src.path, dest, self.debugfs,
                             self._get_part_image(pnum))
        else: # fat
            if isinstance(src, str):
                cmd = "{} -i {} -snop {} ::{}".format(self.mcopy,
                                                  self._get_part_image(pnum),
                                                  src, dest.path)
            else:
                cmd = "{} -i {} -snop ::{} {}".format(self.mcopy,
                                                  self._get_part_image(pnum),
                                                  src.path, dest)

        exec_cmd(cmd, as_shell=True)
        self._put_part_image(pnum)

    def remove_ext(self, pnum, path, recursive):
        """
        Remove files/dirs and their contents from the partition.
        This only applies to ext* partition.
        """
        abs_path = re.sub('\/\/+', '/', path)
        cmd = "{} {} -wR 'rm \"{}\"'".format(self.debugfs,
                                            self._get_part_image(pnum),
                                            abs_path)
        out = exec_cmd(cmd , as_shell=True)
        for line in out.splitlines():
            if line.startswith("rm:"):
                if "file is a directory" in line:
                    if recursive:
                        # loop through content and delete them one by one if
                        # flaged with -r
                        subdirs = iter(self.dir(pnum, abs_path).splitlines())
                        next(subdirs)
                        for subdir in subdirs:
                            dir = subdir.split(':')[1].split(" ", 1)[1]
                            if not dir == "." and not dir == "..":
                                self.remove_ext(pnum, "%s/%s" % (abs_path, dir), recursive)

                    rmdir_out = exec_cmd("{} {} -wR 'rmdir \"{}\"'".format(self.debugfs,
                                                    self._get_part_image(pnum),
                                                    abs_path.rstrip('/'))
                                                    , as_shell=True)

                    for rmdir_line in rmdir_out.splitlines():
                        if "directory not empty" in rmdir_line:
                            raise WicError("Could not complete operation: \n%s \n"
                                            "use -r to remove non-empty directory" % rmdir_line)
                        if rmdir_line.startswith("rmdir:"):
                            raise WicError("Could not complete operation: \n%s "
                                            "\n%s" % (str(line), rmdir_line))

                else:
                    raise WicError("Could not complete operation: \n%s "
                                    "\nUnable to remove %s" % (str(line), abs_path))

    def remove(self, pnum, path, recursive):
        """Remove files/dirs from the partition."""
        partimg = self._get_part_image(pnum)
        if self.partitions[pnum].fstype.startswith('ext'):
            self.remove_ext(pnum, path, recursive)

        else: # fat
            cmd = "{} -i {} ::{}".format(self.mdel, partimg, path)
            try:
                exec_cmd(cmd)
            except WicError as err:
                if "not found" in str(err) or "non empty" in str(err):
                    # mdel outputs 'File ... not found' or 'directory .. non empty"
                    # try to use mdeltree as path could be a directory
                    cmd = "{} -i {} ::{}".format(self.mdeltree,
                                                 partimg, path)
                    exec_cmd(cmd)
                else:
                    raise err
        self._put_part_image(pnum)

    def write(self, target, expand):
        """Write disk image to the media or file."""
        def write_sfdisk_script(outf, parts):
            for key, val in parts['partitiontable'].items():
                if key in ("partitions", "device", "firstlba", "lastlba"):
                    continue
                if key == "id":
                    key = "label-id"
                outf.write("{}: {}\n".format(key, val))
            outf.write("\n")
            for part in parts['partitiontable']['partitions']:
                line = ''
                for name in ('attrs', 'name', 'size', 'type', 'uuid'):
                    if name == 'size' and part['type'] == 'f':
                        # don't write size for extended partition
                        continue
                    val = part.get(name)
                    if val:
                        line += '{}={}, '.format(name, val)
                if line:
                    line = line[:-2] # strip ', '
                if part.get('bootable'):
                    line += ' ,bootable'
                outf.write("{}\n".format(line))
            outf.flush()

        def read_ptable(path):
            out = exec_cmd("{} -J {}".format(self.sfdisk, path))
            return json.loads(out)

        def write_ptable(parts, target):
            with tempfile.NamedTemporaryFile(prefix="wic-sfdisk-", mode='w') as outf:
                write_sfdisk_script(outf, parts)
                cmd = "{} --no-reread {} < {} ".format(self.sfdisk, target, outf.name)
                exec_cmd(cmd, as_shell=True)

        if expand is None:
            sparse_copy(self.imagepath, target)
        else:
            # copy first sectors that may contain bootloader
            sparse_copy(self.imagepath, target, length=2048 * self._lsector_size)

            # copy source partition table to the target
            parts = read_ptable(self.imagepath)
            write_ptable(parts, target)

            # get size of unpartitioned space
            free = None
            for line in exec_cmd("{} -F {}".format(self.sfdisk, target)).splitlines():
                if line.startswith("Unpartitioned space ") and line.endswith("sectors"):
                    free = int(line.split()[-2])
                    # Align free space to a 2048 sector boundary. YOCTO #12840.
                    free = free - (free % 2048)
            if free is None:
                raise WicError("Can't get size of unpartitioned space")

            # calculate expanded partitions sizes
            sizes = {}
            num_auto_resize = 0
            for num, part in enumerate(parts['partitiontable']['partitions'], 1):
                if num in expand:
                    if expand[num] != 0: # don't resize partition if size is set to 0
                        sectors = expand[num] // self._lsector_size
                        free -= sectors - part['size']
                        part['size'] = sectors
                        sizes[num] = sectors
                elif part['type'] != 'f':
                    sizes[num] = -1
                    num_auto_resize += 1

            for num, part in enumerate(parts['partitiontable']['partitions'], 1):
                if sizes.get(num) == -1:
                    part['size'] += free // num_auto_resize

            # write resized partition table to the target
            write_ptable(parts, target)

            # read resized partition table
            parts = read_ptable(target)

            # copy partitions content
            for num, part in enumerate(parts['partitiontable']['partitions'], 1):
                pnum = str(num)
                fstype = self.partitions[pnum].fstype

                # copy unchanged partition
                if part['size'] == self.partitions[pnum].size // self._lsector_size:
                    logger.info("copying unchanged partition {}".format(pnum))
                    sparse_copy(self._get_part_image(pnum), target, seek=part['start'] * self._lsector_size)
                    continue

                # resize or re-create partitions
                if fstype.startswith('ext') or fstype.startswith('fat') or \
                   fstype.startswith('linux-swap'):

                    partfname = None
                    with tempfile.NamedTemporaryFile(prefix="wic-part{}-".format(pnum)) as partf:
                        partfname = partf.name

                    if fstype.startswith('ext'):
                        logger.info("resizing ext partition {}".format(pnum))
                        partimg = self._get_part_image(pnum)
                        sparse_copy(partimg, partfname)
                        exec_cmd("{} -pf {}".format(self.e2fsck, partfname))
                        exec_cmd("{} {} {}s".format(\
                                 self.resize2fs, partfname, part['size']))
                    elif fstype.startswith('fat'):
                        logger.info("copying content of the fat partition {}".format(pnum))
                        with tempfile.TemporaryDirectory(prefix='wic-fatdir-') as tmpdir:
                            # copy content to the temporary directory
                            cmd = "{} -snompi {} :: {}".format(self.mcopy,
                                                               self._get_part_image(pnum),
                                                               tmpdir)
                            exec_cmd(cmd)
                            # create new msdos partition
                            label = part.get("name")
                            label_str = "-n {}".format(label) if label else ''

                            cmd = "{} {} -C {} {}".format(self.mkdosfs, label_str, partfname,
                                                          part['size'])
                            exec_cmd(cmd)
                            # copy content from the temporary directory to the new partition
                            cmd = "{} -snompi {} {}/* ::".format(self.mcopy, partfname, tmpdir)
                            exec_cmd(cmd, as_shell=True)
                    elif fstype.startswith('linux-swap'):
                        logger.info("creating swap partition {}".format(pnum))
                        label = part.get("name")
                        label_str = "-L {}".format(label) if label else ''
                        uuid = part.get("uuid")
                        uuid_str = "-U {}".format(uuid) if uuid else ''
                        with open(partfname, 'w') as sparse:
                            os.ftruncate(sparse.fileno(), part['size'] * self._lsector_size)
                        exec_cmd("{} {} {} {}".format(self.mkswap, label_str, uuid_str, partfname))
                    sparse_copy(partfname, target, seek=part['start'] * self._lsector_size)
                    os.unlink(partfname)
                elif part['type'] != 'f':
                    logger.warning("skipping partition {}: unsupported fstype {}".format(pnum, fstype))

def wic_ls(args, native_sysroot):
    """List contents of partitioned image or vfat partition."""
    disk = Disk(args.path.image, native_sysroot)
    if not args.path.part:
        if disk.partitions:
            print('Num     Start        End          Size      Fstype')
            for part in disk.partitions.values():
                print("{:2d}  {:12d} {:12d} {:12d}  {}".format(\
                          part.pnum, part.start, part.end,
                          part.size, part.fstype))
    else:
        path = args.path.path or '/'
        print(disk.dir(args.path.part, path))

def wic_cp(args, native_sysroot):
    """
    Copy file or directory to/from the vfat/ext partition of
    partitioned image.
    """
    if isinstance(args.dest, str):
        disk = Disk(args.src.image, native_sysroot)
    else:
        disk = Disk(args.dest.image, native_sysroot)
    disk.copy(args.src, args.dest)


def wic_rm(args, native_sysroot):
    """
    Remove files or directories from the vfat partition of
    partitioned image.
    """
    disk = Disk(args.path.image, native_sysroot)
    disk.remove(args.path.part, args.path.path, args.recursive_delete)

def wic_write(args, native_sysroot):
    """
    Write image to a target device.
    """
    disk = Disk(args.image, native_sysroot, ('fat', 'ext', 'linux-swap'))
    disk.write(args.target, args.expand)

def find_canned(scripts_path, file_name):
    """
    Find a file either by its path or by name in the canned files dir.

    Return None if not found
    """
    if os.path.exists(file_name):
        return file_name

    layers_canned_wks_dir = build_canned_image_list(scripts_path)
    for canned_wks_dir in layers_canned_wks_dir:
        for root, dirs, files in os.walk(canned_wks_dir):
            for fname in files:
                if fname == file_name:
                    fullpath = os.path.join(canned_wks_dir, fname)
                    return fullpath

def get_custom_config(boot_file):
    """
    Get the custom configuration to be used for the bootloader.

    Return None if the file can't be found.
    """
    # Get the scripts path of poky
    scripts_path = os.path.abspath("%s/../.." % os.path.dirname(__file__))

    cfg_file = find_canned(scripts_path, boot_file)
    if cfg_file:
        with open(cfg_file, "r") as f:
            config = f.read()
        return config