path: root/scripts/lib/mic/utils/misc.py

#!/usr/bin/python -tt
#
# Copyright (c) 2010, 2011 Intel Inc.
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation; version 2 of the License
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
# for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc., 59
# Temple Place - Suite 330, Boston, MA 02111-1307, USA.

import os
import sys
import time
import tempfile
import re
import shutil
import glob
import hashlib
import subprocess
import platform
import traceback


try:
    import sqlite3 as sqlite
except ImportError:
    import sqlite

try:
    from xml.etree import cElementTree
except ImportError:
    import cElementTree
xmlparse = cElementTree.parse

from mic import msger
from mic.utils.errors import CreatorError, SquashfsError
from mic.utils.fs_related import find_binary_path, makedirs
from mic.utils.grabber import myurlgrab
from mic.utils.proxy import get_proxy_for
from mic.utils import runner
from mic.utils import rpmmisc


RPM_RE  = re.compile("(.*)\.(.*) (.*)-(.*)")
RPM_FMT = "%(name)s.%(arch)s %(version)s-%(release)s"
SRPM_RE = re.compile("(.*)-(\d+.*)-(\d+\.\d+).src.rpm")


def build_name(kscfg, release=None, prefix = None, suffix = None):
    """Construct and return an image name string.

    This is a utility function to help create sensible name and fslabel
    strings. The name is constructed using the sans-prefix-and-extension
    kickstart filename and the supplied prefix and suffix.

    kscfg -- a path to a kickstart file
    release --  a replacement to suffix for image release
    prefix -- a prefix to prepend to the name; defaults to None, which causes
              no prefix to be used
    suffix -- a suffix to append to the name; defaults to None, which causes
              a YYYYMMDDHHMM suffix to be used

    Note, if maxlen is less then the len(suffix), you get to keep both pieces.

    """
    name = os.path.basename(kscfg)
    idx = name.rfind('.')
    if idx >= 0:
        name = name[:idx]

    if release is not None:
        suffix = ""
    if prefix is None:
        prefix = ""
    if suffix is None:
        suffix = time.strftime("%Y%m%d%H%M")

    if name.startswith(prefix):
        name = name[len(prefix):]

    prefix = "%s-" % prefix if prefix else ""
    suffix = "-%s" % suffix if suffix else ""

    ret = prefix + name + suffix
    return ret

def get_distro():
    """Detect linux distribution, support "meego"
    """

    support_dists = ('SuSE',
                     'debian',
                     'fedora',
                     'redhat',
                     'centos',
                     'meego',
                     'moblin',
                     'tizen')
    try:
        (dist, ver, id) = platform.linux_distribution( \
                              supported_dists = support_dists)
    except:
        (dist, ver, id) = platform.dist( \
                              supported_dists = support_dists)

    return (dist, ver, id)

def get_distro_str():
    """Get composited string for current linux distribution
    """
    (dist, ver, id) = get_distro()

    if not dist:
        return 'Unknown Linux Distro'
    else:
        distro_str = ' '.join(map(str.strip, (dist, ver, id)))
        return distro_str.strip()

_LOOP_RULE_PTH = None

def hide_loopdev_presentation():
    udev_rules = "80-prevent-loop-present.rules"
    udev_rules_dir = [
                       '/usr/lib/udev/rules.d/',
                       '/lib/udev/rules.d/',
                       '/etc/udev/rules.d/'
                     ]

    global _LOOP_RULE_PTH

    for rdir in udev_rules_dir:
        if os.path.exists(rdir):
            _LOOP_RULE_PTH = os.path.join(rdir, udev_rules)

    if not _LOOP_RULE_PTH:
        return

    try:
        with open(_LOOP_RULE_PTH, 'w') as wf:
            wf.write('KERNEL=="loop*", ENV{UDISKS_PRESENTATION_HIDE}="1"')

        runner.quiet('udevadm trigger')
    except:
        pass

def unhide_loopdev_presentation():
    global _LOOP_RULE_PTH

    if not _LOOP_RULE_PTH:
        return

    try:
        os.unlink(_LOOP_RULE_PTH)
        runner.quiet('udevadm trigger')
    except:
        pass

def extract_rpm(rpmfile, targetdir):
    rpm2cpio = find_binary_path("rpm2cpio")
    cpio = find_binary_path("cpio")

    olddir = os.getcwd()
    os.chdir(targetdir)

    msger.verbose("Extract rpm file with cpio: %s" % rpmfile)
    p1 = subprocess.Popen([rpm2cpio, rpmfile], stdout=subprocess.PIPE)
    p2 = subprocess.Popen([cpio, "-idv"], stdin=p1.stdout,
                          stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    (sout, serr) = p2.communicate()
    msger.verbose(sout or serr)

    os.chdir(olddir)

def compressing(fpath, method):
    comp_map = {
        "gz": "gzip",
        "bz2": "bzip2"
    }
    if method not in comp_map:
        raise CreatorError("Unsupport compress format: %s, valid values: %s"
                           % (method, ','.join(comp_map.keys())))
    cmd = find_binary_path(comp_map[method])
    rc = runner.show([cmd, "-f", fpath])
    if rc:
        raise CreatorError("Failed to %s file: %s" % (comp_map[method], fpath))

def taring(dstfile, target):
    import tarfile
    basen, ext = os.path.splitext(dstfile)
    comp = {".tar": None,
            ".gz": "gz", # for .tar.gz
            ".bz2": "bz2", # for .tar.bz2
            ".tgz": "gz",
            ".tbz": "bz2"}[ext]

    # specify tarball file path
    if not comp:
        tarpath = dstfile
    elif basen.endswith(".tar"):
        tarpath = basen
    else:
        tarpath = basen + ".tar"
    wf = tarfile.open(tarpath, 'w')

    if os.path.isdir(target):
        for item in os.listdir(target):
            wf.add(os.path.join(target, item), item)
    else:
        wf.add(target, os.path.basename(target))
    wf.close()

    if comp:
        compressing(tarpath, comp)
        # when dstfile ext is ".tgz" and ".tbz", should rename
        if not basen.endswith(".tar"):
            shutil.move("%s.%s" % (tarpath, comp), dstfile)

def ziping(dstfile, target):
    import zipfile
    wf = zipfile.ZipFile(dstfile, 'w', compression=zipfile.ZIP_DEFLATED)
    if os.path.isdir(target):
        for item in os.listdir(target):
            fpath = os.path.join(target, item)
            if not os.path.isfile(fpath):
                continue
            wf.write(fpath, item, zipfile.ZIP_DEFLATED)
    else:
        wf.write(target, os.path.basename(target), zipfile.ZIP_DEFLATED)
    wf.close()

pack_formats = {
    ".tar": taring,
    ".tar.gz": taring,
    ".tar.bz2": taring,
    ".tgz": taring,
    ".tbz": taring,
    ".zip": ziping,
}

def packing(dstfile, target):
    (base, ext) = os.path.splitext(dstfile)
    if ext in (".gz", ".bz2") and base.endswith(".tar"):
        ext = ".tar" + ext
    if ext not in pack_formats:
        raise CreatorError("Unsupport pack format: %s, valid values: %s"
                           % (ext, ','.join(pack_formats.keys())))
    func = pack_formats[ext]
    # func should be callable
    func(dstfile, target)

def human_size(size):
    """Return human readable string for Bytes size
    """

    if size <= 0:
        return "0M"
    import math
    measure = ['B', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y']
    expo = int(math.log(size, 1024))
    mant = float(size/math.pow(1024, expo))
    return "{0:.1f}{1:s}".format(mant, measure[expo])

def get_block_size(file_obj):
    """ Returns block size for file object 'file_obj'. Errors are indicated by
    the 'IOError' exception. """

    from fcntl import ioctl
    import struct

    # Get the block size of the host file-system for the image file by calling
    # the FIGETBSZ ioctl (number 2).
    binary_data = ioctl(file_obj, 2, struct.pack('I', 0))
    return struct.unpack('I', binary_data)[0]

def check_space_pre_cp(src, dst):
    """Check whether disk space is enough before 'cp' like
    operations, else exception will be raised.
    """

    srcsize  = get_file_size(src) * 1024 * 1024
    freesize = get_filesystem_avail(dst)
    if srcsize > freesize:
        raise CreatorError("space on %s(%s) is not enough for about %s files"
                           % (dst, human_size(freesize), human_size(srcsize)))

def calc_hashes(file_path, hash_names, start = 0, end = None):
    """ Calculate hashes for a file. The 'file_path' argument is the file
    to calculate hash functions for, 'start' and 'end' are the starting and
    ending file offset to calculate the has functions for. The 'hash_names'
    argument is a list of hash names to calculate. Returns the the list
    of calculated hash values in the hexadecimal form in the same order
    as 'hash_names'.
    """
    if end == None:
        end = os.path.getsize(file_path)

    chunk_size = 65536
    to_read = end - start
    read = 0

    hashes = []
    for hash_name in hash_names:
        hashes.append(hashlib.new(hash_name))

    with open(file_path, "rb") as f:
        f.seek(start)

        while read < to_read:
            if read + chunk_size > to_read:
                chunk_size = to_read - read
            chunk = f.read(chunk_size)
            for hash_obj in hashes:
                hash_obj.update(chunk)
            read += chunk_size

    result = []
    for hash_obj in hashes:
        result.append(hash_obj.hexdigest())

    return result

def get_md5sum(fpath):
    return calc_hashes(fpath, ('md5', ))[0]


def normalize_ksfile(ksconf, release, arch):
    '''
    Return the name of a normalized ks file in which macro variables
    @BUILD_ID@ and @ARCH@ are replace with real values.

    The original ks file is returned if no special macro is used, otherwise
    a temp file is created and returned, which will be deleted when program
    exits normally.
    '''

    if not release:
        release = "latest"
    if not arch or re.match(r'i.86', arch):
        arch = "ia32"

    with open(ksconf) as f:
        ksc = f.read()

    if "@ARCH@" not in ksc and "@BUILD_ID@" not in ksc:
        return ksconf

    msger.info("Substitute macro variable @BUILD_ID@/@ARCH@ in ks: %s" % ksconf)
    ksc = ksc.replace("@ARCH@", arch)
    ksc = ksc.replace("@BUILD_ID@", release)

    fd, ksconf = tempfile.mkstemp(prefix=os.path.basename(ksconf))
    os.write(fd, ksc)
    os.close(fd)

    msger.debug('normalized ks file:%s' % ksconf)

    def remove_temp_ks():
        try:
            os.unlink(ksconf)
        except OSError, err:
            msger.warning('Failed to remove temp ks file:%s:%s' % (ksconf, err))

    import atexit
    atexit.register(remove_temp_ks)

    return ksconf


def _check_mic_chroot(rootdir):
    def _path(path):
        return rootdir.rstrip('/') + path

    release_files = map(_path, [ "/etc/moblin-release",
                                 "/etc/meego-release",
                                 "/etc/tizen-release"])

    if not any(map(os.path.exists, release_files)):
        msger.warning("Dir %s is not a MeeGo/Tizen chroot env" % rootdir)

    if not glob.glob(rootdir + "/boot/vmlinuz-*"):
        msger.warning("Failed to find kernel module under %s" % rootdir)

    return

def selinux_check(arch, fstypes):
    try:
        getenforce = find_binary_path('getenforce')
    except CreatorError:
        return

    selinux_status = runner.outs([getenforce])
    if arch and arch.startswith("arm") and selinux_status == "Enforcing":
        raise CreatorError("Can't create arm image if selinux is enabled, "
                           "please run 'setenforce 0' to disable selinux")

    use_btrfs = filter(lambda typ: typ == 'btrfs', fstypes)
    if use_btrfs and selinux_status == "Enforcing":
        raise CreatorError("Can't create btrfs image if selinux is enabled,"
                           " please run 'setenforce 0' to disable selinux")

def get_image_type(path):
    def _get_extension_name(path):
        match = re.search("(?<=\.)\w+$", path)
        if match:
            return match.group(0)
        else:
            return None

    if os.path.isdir(path):
        _check_mic_chroot(path)
        return "fs"

    maptab = {
              "tar": "loop",
              "raw":"raw",
              "vmdk":"vmdk",
              "vdi":"vdi",
              "iso":"livecd",
              "usbimg":"liveusb",
             }

    extension = _get_extension_name(path)
    if extension in maptab:
        return maptab[extension]

    fd = open(path, "rb")
    file_header = fd.read(1024)
    fd.close()
    vdi_flag = "<<< Sun VirtualBox Disk Image >>>"
    if file_header[0:len(vdi_flag)] == vdi_flag:
        return maptab["vdi"]

    output = runner.outs(['file', path])
    isoptn = re.compile(r".*ISO 9660 CD-ROM filesystem.*(bootable).*")
    usbimgptn = re.compile(r".*x86 boot sector.*active.*")
    rawptn = re.compile(r".*x86 boot sector.*")
    vmdkptn = re.compile(r".*VMware. disk image.*")
    ext3fsimgptn = re.compile(r".*Linux.*ext3 filesystem data.*")
    ext4fsimgptn = re.compile(r".*Linux.*ext4 filesystem data.*")
    btrfsimgptn = re.compile(r".*BTRFS.*")
    if isoptn.match(output):
        return maptab["iso"]
    elif usbimgptn.match(output):
        return maptab["usbimg"]
    elif rawptn.match(output):
        return maptab["raw"]
    elif vmdkptn.match(output):
        return maptab["vmdk"]
    elif ext3fsimgptn.match(output):
        return "ext3fsimg"
    elif ext4fsimgptn.match(output):
        return "ext4fsimg"
    elif btrfsimgptn.match(output):
        return "btrfsimg"
    else:
        raise CreatorError("Cannot detect the type of image: %s" % path)


def get_file_size(filename):
    """ Return size in MB unit """
    cmd = ['du', "-s", "-b", "-B", "1M", filename]
    rc, duOutput  = runner.runtool(cmd)
    if rc != 0:
        raise CreatorError("Failed to run: %s" % ' '.join(cmd))
    size1 = int(duOutput.split()[0])

    cmd = ['du', "-s", "-B", "1M", filename]
    rc, duOutput = runner.runtool(cmd)
    if rc != 0:
        raise CreatorError("Failed to run: %s" % ' '.join(cmd))

    size2 = int(duOutput.split()[0])
    return max(size1, size2)


def get_filesystem_avail(fs):
    vfstat = os.statvfs(fs)
    return vfstat.f_bavail * vfstat.f_bsize

def convert_image(srcimg, srcfmt, dstimg, dstfmt):
    #convert disk format
    if dstfmt != "raw":
        raise CreatorError("Invalid destination image format: %s" % dstfmt)
    msger.debug("converting %s image to %s" % (srcimg, dstimg))
    if srcfmt == "vmdk":
        path = find_binary_path("qemu-img")
        argv = [path, "convert", "-f", "vmdk", srcimg, "-O", dstfmt,  dstimg]
    elif srcfmt == "vdi":
        path = find_binary_path("VBoxManage")
        argv = [path, "internalcommands", "converttoraw", srcimg, dstimg]
    else:
        raise CreatorError("Invalid soure image format: %s" % srcfmt)

    rc = runner.show(argv)
    if rc == 0:
        msger.debug("convert successful")
    if rc != 0:
        raise CreatorError("Unable to convert disk to %s" % dstfmt)

def uncompress_squashfs(squashfsimg, outdir):
    """Uncompress file system from squshfs image"""
    unsquashfs = find_binary_path("unsquashfs")
    args = [ unsquashfs, "-d", outdir, squashfsimg ]
    rc = runner.show(args)
    if (rc != 0):
        raise SquashfsError("Failed to uncompress %s." % squashfsimg)

def mkdtemp(dir = "/var/tmp", prefix = "wic-tmp-"):
    """ FIXME: use the dir in wic.conf instead """

    makedirs(dir)
    return tempfile.mkdtemp(dir = dir, prefix = prefix)

def get_repostrs_from_ks(ks):
    def _get_temp_reponame(baseurl):
        md5obj = hashlib.md5(baseurl)
        tmpreponame = "%s" % md5obj.hexdigest()
        return tmpreponame

    kickstart_repos = []

    for repodata in ks.handler.repo.repoList:
        repo = {}
        for attr in ('name',
                     'baseurl',
                     'mirrorlist',
                     'includepkgs', # val is list
                     'excludepkgs', # val is list
                     'cost',    # int
                     'priority',# int
                     'save',
                     'proxy',
                     'proxyuser',
                     'proxypasswd',
                     'proxypasswd',
                     'debuginfo',
                     'source',
                     'gpgkey',
                     'ssl_verify'):
            if hasattr(repodata, attr) and getattr(repodata, attr):
                repo[attr] = getattr(repodata, attr)

        if 'name' not in repo:
            repo['name'] = _get_temp_reponame(repodata.baseurl)

        kickstart_repos.append(repo)

    return kickstart_repos

def _get_uncompressed_data_from_url(url, filename, proxies):
    filename = myurlgrab(url, filename, proxies)
    suffix = None
    if filename.endswith(".gz"):
        suffix = ".gz"
        runner.quiet(['gunzip', "-f", filename])
    elif filename.endswith(".bz2"):
        suffix = ".bz2"
        runner.quiet(['bunzip2', "-f", filename])
    if suffix:
        filename = filename.replace(suffix, "")
    return filename

def _get_metadata_from_repo(baseurl, proxies, cachedir, reponame, filename,
                            sumtype=None, checksum=None):
    url = os.path.join(baseurl, filename)
    filename_tmp = str("%s/%s/%s" % (cachedir, reponame, os.path.basename(filename)))
    if os.path.splitext(filename_tmp)[1] in (".gz", ".bz2"):
        filename = os.path.splitext(filename_tmp)[0]
    else:
        filename = filename_tmp
    if sumtype and checksum and os.path.exists(filename):
        try:
            sumcmd = find_binary_path("%ssum" % sumtype)
        except:
            file_checksum = None
        else:
            file_checksum = runner.outs([sumcmd, filename]).split()[0]

        if file_checksum and file_checksum == checksum:
            return filename

    return _get_uncompressed_data_from_url(url,filename_tmp,proxies)

def get_metadata_from_repos(repos, cachedir):
    my_repo_metadata = []
    for repo in repos:
        reponame = repo['name']
        baseurl  = repo['baseurl']


        if 'proxy' in repo:
            proxy = repo['proxy']
        else:
            proxy = get_proxy_for(baseurl)

        proxies = None
        if proxy:
            proxies = {str(baseurl.split(":")[0]):str(proxy)}

        makedirs(os.path.join(cachedir, reponame))
        url = os.path.join(baseurl, "repodata/repomd.xml")
        filename = os.path.join(cachedir, reponame, 'repomd.xml')
        repomd = myurlgrab(url, filename, proxies)
        try:
            root = xmlparse(repomd)
        except SyntaxError:
            raise CreatorError("repomd.xml syntax error.")

        ns = root.getroot().tag
        ns = ns[0:ns.rindex("}")+1]

        filepaths = {}
        checksums = {}
        sumtypes = {}

        for elm in root.getiterator("%sdata" % ns):
            if elm.attrib["type"] == "patterns":
                filepaths['patterns'] = elm.find("%slocation" % ns).attrib['href']
                checksums['patterns'] = elm.find("%sopen-checksum" % ns).text
                sumtypes['patterns'] = elm.find("%sopen-checksum" % ns).attrib['type']
                break

        for elm in root.getiterator("%sdata" % ns):
            if elm.attrib["type"] in ("group_gz", "group"):
                filepaths['comps'] = elm.find("%slocation" % ns).attrib['href']
                checksums['comps'] = elm.find("%sopen-checksum" % ns).text
                sumtypes['comps'] = elm.find("%sopen-checksum" % ns).attrib['type']
                break

        primary_type = None
        for elm in root.getiterator("%sdata" % ns):
            if elm.attrib["type"] in ("primary_db", "primary"):
                primary_type = elm.attrib["type"]
                filepaths['primary'] = elm.find("%slocation" % ns).attrib['href']
                checksums['primary'] = elm.find("%sopen-checksum" % ns).text
                sumtypes['primary'] = elm.find("%sopen-checksum" % ns).attrib['type']
                break

        if not primary_type:
            continue

        for item in ("primary", "patterns", "comps"):
            if item not in filepaths:
                filepaths[item] = None
                continue
            if not filepaths[item]:
                continue
            filepaths[item] = _get_metadata_from_repo(baseurl,
                                                      proxies,
                                                      cachedir,
                                                      reponame,
                                                      filepaths[item],
                                                      sumtypes[item],
                                                      checksums[item])

        """ Get repo key """
        try:
            repokey = _get_metadata_from_repo(baseurl,
                                              proxies,
                                              cachedir,
                                              reponame,
                                              "repodata/repomd.xml.key")
        except CreatorError:
            repokey = None
            msger.debug("\ncan't get %s/%s" % (baseurl, "repodata/repomd.xml.key"))

        my_repo_metadata.append({"name":reponame,
                                 "baseurl":baseurl,
                                 "repomd":repomd,
                                 "primary":filepaths['primary'],
                                 "cachedir":cachedir,
                                 "proxies":proxies,
                                 "patterns":filepaths['patterns'],
                                 "comps":filepaths['comps'],
                                 "repokey":repokey})

    return my_repo_metadata

def get_rpmver_in_repo(repometadata):
    for repo in repometadata:
        if repo["primary"].endswith(".xml"):
            root = xmlparse(repo["primary"])
            ns = root.getroot().tag
            ns = ns[0:ns.rindex("}")+1]

            versionlist = []
            for elm in root.getiterator("%spackage" % ns):
                if elm.find("%sname" % ns).text == 'rpm':
                    for node in elm.getchildren():
                        if node.tag == "%sversion" % ns:
                            versionlist.append(node.attrib['ver'])

            if versionlist:
                return reversed(
                         sorted(
                           versionlist,
                           key = lambda ver: map(int, ver.split('.')))).next()

        elif repo["primary"].endswith(".sqlite"):
            con = sqlite.connect(repo["primary"])
            for row in con.execute("select version from packages where "
                                   "name=\"rpm\" ORDER by version DESC"):
                con.close()
                return row[0]

    return None

def get_arch(repometadata):
    archlist = []
    for repo in repometadata:
        if repo["primary"].endswith(".xml"):
            root = xmlparse(repo["primary"])
            ns = root.getroot().tag
            ns = ns[0:ns.rindex("}")+1]
            for elm in root.getiterator("%spackage" % ns):
                if elm.find("%sarch" % ns).text not in ("noarch", "src"):
                    arch = elm.find("%sarch" % ns).text
                    if arch not in archlist:
                        archlist.append(arch)
        elif repo["primary"].endswith(".sqlite"):
            con = sqlite.connect(repo["primary"])
            for row in con.execute("select arch from packages where arch not in (\"src\", \"noarch\")"):
                if row[0] not in archlist:
                    archlist.append(row[0])

            con.close()

    uniq_arch = []
    for i in range(len(archlist)):
        if archlist[i] not in rpmmisc.archPolicies.keys():
            continue
        need_append = True
        j = 0
        while j < len(uniq_arch):
            if archlist[i] in rpmmisc.archPolicies[uniq_arch[j]].split(':'):
                need_append = False
                break
            if uniq_arch[j] in rpmmisc.archPolicies[archlist[i]].split(':'):
                if need_append:
                    uniq_arch[j] = archlist[i]
                    need_append = False
                else:
                    uniq_arch.remove(uniq_arch[j])
                    continue
            j += 1
        if need_append:
             uniq_arch.append(archlist[i])

    return uniq_arch, archlist

def get_package(pkg, repometadata, arch = None):
    ver = ""
    target_repo = None
    if not arch:
        arches = []
    elif arch not in rpmmisc.archPolicies:
        arches = [arch]
    else:
        arches = rpmmisc.archPolicies[arch].split(':')
        arches.append('noarch')

    for repo in repometadata:
        if repo["primary"].endswith(".xml"):
            root = xmlparse(repo["primary"])
            ns = root.getroot().tag
            ns = ns[0:ns.rindex("}")+1]
            for elm in root.getiterator("%spackage" % ns):
                if elm.find("%sname" % ns).text == pkg:
                    if elm.find("%sarch" % ns).text in arches:
                        version = elm.find("%sversion" % ns)
                        tmpver = "%s-%s" % (version.attrib['ver'], version.attrib['rel'])
                        if tmpver > ver:
                            ver = tmpver
                            location = elm.find("%slocation" % ns)
                            pkgpath = "%s" % location.attrib['href']
                            target_repo = repo
                        break
        if repo["primary"].endswith(".sqlite"):
            con = sqlite.connect(repo["primary"])
            if arch:
                sql = 'select version, release, location_href from packages ' \
                      'where name = "%s" and arch IN ("%s")' % \
                      (pkg, '","'.join(arches))
                for row in con.execute(sql):
                    tmpver = "%s-%s" % (row[0], row[1])
                    if tmpver > ver:
                        ver = tmpver
                        pkgpath = "%s" % row[2]
                        target_repo = repo
                    break
            else:
                sql = 'select version, release, location_href from packages ' \
                      'where name = "%s"' % pkg
                for row in con.execute(sql):
                    tmpver = "%s-%s" % (row[0], row[1])
                    if tmpver > ver:
                        ver = tmpver
                        pkgpath = "%s" % row[2]
                        target_repo = repo
                    break
            con.close()
    if target_repo:
        makedirs("%s/packages/%s" % (target_repo["cachedir"], target_repo["name"]))
        url = os.path.join(target_repo["baseurl"], pkgpath)
        filename = str("%s/packages/%s/%s" % (target_repo["cachedir"], target_repo["name"], os.path.basename(pkgpath)))
        if os.path.exists(filename):
            ret = rpmmisc.checkRpmIntegrity('rpm', filename)
            if ret == 0:
                return filename

            msger.warning("package %s is damaged: %s" %
                          (os.path.basename(filename), filename))
            os.unlink(filename)

        pkg = myurlgrab(str(url), filename, target_repo["proxies"])
        return pkg
    else:
        return None

def get_source_name(pkg, repometadata):

    def get_bin_name(pkg):
        m = RPM_RE.match(pkg)
        if m:
            return m.group(1)
        return None

    def get_src_name(srpm):
        m = SRPM_RE.match(srpm)
        if m:
            return m.group(1)
        return None

    ver = ""
    target_repo = None

    pkg_name = get_bin_name(pkg)
    if not pkg_name:
        return None

    for repo in repometadata:
        if repo["primary"].endswith(".xml"):
            root = xmlparse(repo["primary"])
            ns = root.getroot().tag
            ns = ns[0:ns.rindex("}")+1]
            for elm in root.getiterator("%spackage" % ns):
                if elm.find("%sname" % ns).text == pkg_name:
                    if elm.find("%sarch" % ns).text != "src":
                        version = elm.find("%sversion" % ns)
                        tmpver = "%s-%s" % (version.attrib['ver'], version.attrib['rel'])
                        if tmpver > ver:
                            ver = tmpver
                            fmt = elm.find("%sformat" % ns)
                            if fmt:
                                fns = fmt.getchildren()[0].tag
                                fns = fns[0:fns.rindex("}")+1]
                                pkgpath = fmt.find("%ssourcerpm" % fns).text
                                target_repo = repo
                        break

        if repo["primary"].endswith(".sqlite"):
            con = sqlite.connect(repo["primary"])
            for row in con.execute("select version, release, rpm_sourcerpm from packages where name = \"%s\" and arch != \"src\"" % pkg_name):
                tmpver = "%s-%s" % (row[0], row[1])
                if tmpver > ver:
                    pkgpath = "%s" % row[2]
                    target_repo = repo
                break
            con.close()
    if target_repo:
        return get_src_name(pkgpath)
    else:
        return None

def get_pkglist_in_patterns(group, patterns):
    found = False
    pkglist = []
    try:
        root = xmlparse(patterns)
    except SyntaxError:
        raise SyntaxError("%s syntax error." % patterns)

    for elm in list(root.getroot()):
        ns = elm.tag
        ns = ns[0:ns.rindex("}")+1]
        name = elm.find("%sname" % ns)
        summary = elm.find("%ssummary" % ns)
        if name.text == group or summary.text == group:
            found = True
            break

    if not found:
        return pkglist

    found = False
    for requires in list(elm):
        if requires.tag.endswith("requires"):
            found = True
            break

    if not found:
        return pkglist

    for pkg in list(requires):
        pkgname = pkg.attrib["name"]
        if pkgname not in pkglist:
            pkglist.append(pkgname)

    return pkglist

def get_pkglist_in_comps(group, comps):
    found = False
    pkglist = []
    try:
        root = xmlparse(comps)
    except SyntaxError:
        raise SyntaxError("%s syntax error." % comps)

    for elm in root.getiterator("group"):
        id = elm.find("id")
        name = elm.find("name")
        if id.text == group or name.text == group:
            packagelist = elm.find("packagelist")
            found = True
            break

    if not found:
        return pkglist

    for require in elm.getiterator("packagereq"):
        if require.tag.endswith("packagereq"):
            pkgname = require.text
        if pkgname not in pkglist:
            pkglist.append(pkgname)

    return pkglist

def is_statically_linked(binary):
    return ", statically linked, " in runner.outs(['file', binary])

def setup_qemu_emulator(rootdir, arch):
    # mount binfmt_misc if it doesn't exist
    if not os.path.exists("/proc/sys/fs/binfmt_misc"):
        modprobecmd = find_binary_path("modprobe")
        runner.show([modprobecmd, "binfmt_misc"])
    if not os.path.exists("/proc/sys/fs/binfmt_misc/register"):
        mountcmd = find_binary_path("mount")
        runner.show([mountcmd, "-t", "binfmt_misc", "none", "/proc/sys/fs/binfmt_misc"])

    # qemu_emulator is a special case, we can't use find_binary_path
    # qemu emulator should be a statically-linked executable file
    qemu_emulator = "/usr/bin/qemu-arm"
    if not os.path.exists(qemu_emulator) or not is_statically_linked(qemu_emulator):
        qemu_emulator = "/usr/bin/qemu-arm-static"
    if not os.path.exists(qemu_emulator):
        raise CreatorError("Please install a statically-linked qemu-arm")

    # qemu emulator version check
    armv7_list = [arch for arch in rpmmisc.archPolicies.keys() if arch.startswith('armv7')]
    if arch in armv7_list:  # need qemu (>=0.13.0)
        qemuout = runner.outs([qemu_emulator, "-h"])
        m = re.search("version\s*([.\d]+)", qemuout)
        if m:
            qemu_version = m.group(1)
            if qemu_version < "0.13":
                raise CreatorError("Requires %s version >=0.13 for %s" % (qemu_emulator, arch))
        else:
            msger.warning("Can't get version info of %s, please make sure it's higher than 0.13.0" % qemu_emulator)

    if not os.path.exists(rootdir + "/usr/bin"):
        makedirs(rootdir + "/usr/bin")
    shutil.copy(qemu_emulator, rootdir + "/usr/bin/qemu-arm-static")
    qemu_emulator = "/usr/bin/qemu-arm-static"

    # disable selinux, selinux will block qemu emulator to run
    if os.path.exists("/usr/sbin/setenforce"):
        msger.info('Try to disable selinux')
        runner.show(["/usr/sbin/setenforce", "0"])

    # unregister it if it has been registered and is a dynamically-linked executable
    node = "/proc/sys/fs/binfmt_misc/arm"
    if os.path.exists(node):
        qemu_unregister_string = "-1\n"
        fd = open("/proc/sys/fs/binfmt_misc/arm", "w")
        fd.write(qemu_unregister_string)
        fd.close()

    # register qemu emulator for interpreting other arch executable file
    if not os.path.exists(node):
        qemu_arm_string = ":arm:M::\\x7fELF\\x01\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x28\\x00:\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x00\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xfa\\xff\\xff\\xff:%s:\n" % qemu_emulator
        fd = open("/proc/sys/fs/binfmt_misc/register", "w")
        fd.write(qemu_arm_string)
        fd.close()

    return qemu_emulator

def SrcpkgsDownload(pkgs, repometadata, instroot, cachedir):
    def get_source_repometadata(repometadata):
        src_repometadata=[]
        for repo in repometadata:
            if repo["name"].endswith("-source"):
                src_repometadata.append(repo)
        if src_repometadata:
            return src_repometadata
        return None

    def get_src_name(srpm):
        m = SRPM_RE.match(srpm)
        if m:
            return m.group(1)
        return None

    src_repometadata = get_source_repometadata(repometadata)

    if not src_repometadata:
        msger.warning("No source repo found")
        return None

    src_pkgs = []
    lpkgs_dict = {}
    lpkgs_path = []
    for repo in src_repometadata:
        cachepath = "%s/%s/packages/*.src.rpm" %(cachedir, repo["name"])
        lpkgs_path += glob.glob(cachepath)

    for lpkg in lpkgs_path:
        lpkg_name = get_src_name(os.path.basename(lpkg))
        lpkgs_dict[lpkg_name] = lpkg
    localpkgs = lpkgs_dict.keys()

    cached_count = 0
    destdir = instroot+'/usr/src/SRPMS'
    if not os.path.exists(destdir):
        os.makedirs(destdir)

    srcpkgset = set()
    for _pkg in pkgs:
        srcpkg_name = get_source_name(_pkg, repometadata)
        if not srcpkg_name:
            continue
        srcpkgset.add(srcpkg_name)

    for pkg in list(srcpkgset):
        if pkg in localpkgs:
            cached_count += 1
            shutil.copy(lpkgs_dict[pkg], destdir)
            src_pkgs.append(os.path.basename(lpkgs_dict[pkg]))
        else:
            src_pkg = get_package(pkg, src_repometadata, 'src')
            if src_pkg:
                shutil.copy(src_pkg, destdir)
                src_pkgs.append(src_pkg)
    msger.info("%d source packages gotten from cache" % cached_count)

    return src_pkgs

def strip_end(text, suffix):
    if not text.endswith(suffix):
        return text
    return text[:-len(suffix)]