diff options
Diffstat (limited to 'scripts')
29 files changed, 294 insertions, 248 deletions
diff --git a/scripts/buildhistory-diff b/scripts/buildhistory-diff index 833f7c33a5..02eedafd6e 100755 --- a/scripts/buildhistory-diff +++ b/scripts/buildhistory-diff @@ -11,7 +11,6 @@ import sys import os import argparse -from distutils.version import LooseVersion # Ensure PythonGit is installed (buildhistory_analysis needs it) try: @@ -71,10 +70,6 @@ def main(): parser = get_args_parser() args = parser.parse_args() - if LooseVersion(git.__version__) < '0.3.1': - sys.stderr.write("Version of GitPython is too old, please install GitPython (python-git) 0.3.1 or later in order to use this script\n") - sys.exit(1) - if len(args.revisions) > 2: sys.stderr.write('Invalid argument(s) specified: %s\n\n' % ' '.join(args.revisions[2:])) parser.print_help() diff --git a/scripts/contrib/build-perf-test-wrapper.sh b/scripts/contrib/build-perf-test-wrapper.sh index fa71d4a2e9..0a85e6e708 100755 --- a/scripts/contrib/build-perf-test-wrapper.sh +++ b/scripts/contrib/build-perf-test-wrapper.sh @@ -87,21 +87,10 @@ if [ $# -ne 0 ]; then exit 1 fi -if [ -n "$email_to" ]; then - if ! [ -x "$(command -v phantomjs)" ]; then - echo "ERROR: Sending email needs phantomjs." - exit 1 - fi - if ! [ -x "$(command -v optipng)" ]; then - echo "ERROR: Sending email needs optipng." - exit 1 - fi -fi - # Open a file descriptor for flock and acquire lock LOCK_FILE="/tmp/oe-build-perf-test-wrapper.lock" if ! exec 3> "$LOCK_FILE"; then - echo "ERROR: Unable to open lock file" + echo "ERROR: Unable to open loemack file" exit 1 fi if ! flock -n 3; then @@ -226,7 +215,7 @@ if [ -n "$results_repo" ]; then if [ -n "$email_to" ]; then echo "Emailing test report" os_name=`get_os_release_var PRETTY_NAME` - "$script_dir"/oe-build-perf-report-email.py --to "$email_to" --subject "Build Perf Test Report for $os_name" --text $report_txt --html $report_html "${OE_BUILD_PERF_REPORT_EMAIL_EXTRA_ARGS[@]}" + "$script_dir"/oe-build-perf-report-email.py --to "$email_to" --subject "Build Perf Test Report for $os_name" --text $report_txt "${OE_BUILD_PERF_REPORT_EMAIL_EXTRA_ARGS[@]}" fi # Upload report files, unless we're on detached head diff --git a/scripts/contrib/convert-srcuri.py b/scripts/contrib/convert-srcuri.py new file mode 100755 index 0000000000..5b362ea2e8 --- /dev/null +++ b/scripts/contrib/convert-srcuri.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +# +# Conversion script to update SRC_URI to add branch to git urls +# +# Copyright (C) 2021 Richard Purdie +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import re +import os +import sys +import tempfile +import shutil +import mimetypes + +if len(sys.argv) < 2: + print("Please specify a directory to run the conversion script against.") + sys.exit(1) + +def processfile(fn): + def matchline(line): + if "MIRROR" in line or ".*" in line or "GNOME_GIT" in line: + return False + return True + print("processing file '%s'" % fn) + try: + if "distro_alias.inc" in fn or "linux-yocto-custom.bb" in fn: + return + fh, abs_path = tempfile.mkstemp() + modified = False + with os.fdopen(fh, 'w') as new_file: + with open(fn, "r") as old_file: + for line in old_file: + if ("git://" in line or "gitsm://" in line) and "branch=" not in line and matchline(line): + if line.endswith('"\n'): + line = line.replace('"\n', ';branch=master"\n') + elif line.endswith(" \\\n"): + line = line.replace(' \\\n', ';branch=master \\\n') + modified = True + if ("git://" in line or "gitsm://" in line) and "github.com" in line and "protocol=https" not in line and matchline(line): + if "protocol=git" in line: + line = line.replace('protocol=git', 'protocol=https') + elif line.endswith('"\n'): + line = line.replace('"\n', ';protocol=https"\n') + elif line.endswith(" \\\n"): + line = line.replace(' \\\n', ';protocol=https \\\n') + modified = True + new_file.write(line) + if modified: + shutil.copymode(fn, abs_path) + os.remove(fn) + shutil.move(abs_path, fn) + except UnicodeDecodeError: + pass + +ourname = os.path.basename(sys.argv[0]) +ourversion = "0.1" + +if os.path.isfile(sys.argv[1]): + processfile(sys.argv[1]) + sys.exit(0) + +for targetdir in sys.argv[1:]: + print("processing directory '%s'" % targetdir) + for root, dirs, files in os.walk(targetdir): + for name in files: + if name == ourname: + continue + fn = os.path.join(root, name) + if os.path.islink(fn): + continue + if "/.git/" in fn or fn.endswith(".html") or fn.endswith(".patch") or fn.endswith(".m4") or fn.endswith(".diff"): + continue + processfile(fn) + +print("All files processed with version %s" % ourversion) diff --git a/scripts/contrib/oe-build-perf-report-email.py b/scripts/contrib/oe-build-perf-report-email.py index de3862c897..7192113c28 100755 --- a/scripts/contrib/oe-build-perf-report-email.py +++ b/scripts/contrib/oe-build-perf-report-email.py @@ -19,8 +19,6 @@ import socket import subprocess import sys import tempfile -from email.mime.image import MIMEImage -from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText @@ -29,30 +27,6 @@ logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") log = logging.getLogger('oe-build-perf-report') -# Find js scaper script -SCRAPE_JS = os.path.join(os.path.dirname(__file__), '..', 'lib', 'build_perf', - 'scrape-html-report.js') -if not os.path.isfile(SCRAPE_JS): - log.error("Unableto find oe-build-perf-report-scrape.js") - sys.exit(1) - - -class ReportError(Exception): - """Local errors""" - pass - - -def check_utils(): - """Check that all needed utils are installed in the system""" - missing = [] - for cmd in ('phantomjs', 'optipng'): - if not shutil.which(cmd): - missing.append(cmd) - if missing: - log.error("The following tools are missing: %s", ' '.join(missing)) - sys.exit(1) - - def parse_args(argv): """Parse command line arguments""" description = """Email build perf test report""" @@ -77,137 +51,19 @@ def parse_args(argv): "the email parts") parser.add_argument('--text', help="Plain text message") - parser.add_argument('--html', - help="HTML peport generated by oe-build-perf-report") - parser.add_argument('--phantomjs-args', action='append', - help="Extra command line arguments passed to PhantomJS") args = parser.parse_args(argv) - if not args.html and not args.text: - parser.error("Please specify --html and/or --text") + if not args.text: + parser.error("Please specify --text") return args -def decode_png(infile, outfile): - """Parse/decode/optimize png data from a html element""" - with open(infile) as f: - raw_data = f.read() - - # Grab raw base64 data - b64_data = re.sub('^.*href="data:image/png;base64,', '', raw_data, 1) - b64_data = re.sub('">.+$', '', b64_data, 1) - - # Replace file with proper decoded png - with open(outfile, 'wb') as f: - f.write(base64.b64decode(b64_data)) - - subprocess.check_output(['optipng', outfile], stderr=subprocess.STDOUT) - - -def mangle_html_report(infile, outfile, pngs): - """Mangle html file into a email compatible format""" - paste = True - png_dir = os.path.dirname(outfile) - with open(infile) as f_in: - with open(outfile, 'w') as f_out: - for line in f_in.readlines(): - stripped = line.strip() - # Strip out scripts - if stripped == '<!--START-OF-SCRIPTS-->': - paste = False - elif stripped == '<!--END-OF-SCRIPTS-->': - paste = True - elif paste: - if re.match('^.+href="data:image/png;base64', stripped): - # Strip out encoded pngs (as they're huge in size) - continue - elif 'www.gstatic.com' in stripped: - # HACK: drop references to external static pages - continue - - # Replace charts with <img> elements - match = re.match('<div id="(?P<id>\w+)"', stripped) - if match and match.group('id') in pngs: - f_out.write('<img src="cid:{}"\n'.format(match.group('id'))) - else: - f_out.write(line) - - -def scrape_html_report(report, outdir, phantomjs_extra_args=None): - """Scrape html report into a format sendable by email""" - tmpdir = tempfile.mkdtemp(dir='.') - log.debug("Using tmpdir %s for phantomjs output", tmpdir) - - if not os.path.isdir(outdir): - os.mkdir(outdir) - if os.path.splitext(report)[1] not in ('.html', '.htm'): - raise ReportError("Invalid file extension for report, needs to be " - "'.html' or '.htm'") - - try: - log.info("Scraping HTML report with PhangomJS") - extra_args = phantomjs_extra_args if phantomjs_extra_args else [] - subprocess.check_output(['phantomjs', '--debug=true'] + extra_args + - [SCRAPE_JS, report, tmpdir], - stderr=subprocess.STDOUT) - - pngs = [] - images = [] - for fname in os.listdir(tmpdir): - base, ext = os.path.splitext(fname) - if ext == '.png': - log.debug("Decoding %s", fname) - decode_png(os.path.join(tmpdir, fname), - os.path.join(outdir, fname)) - pngs.append(base) - images.append(fname) - elif ext in ('.html', '.htm'): - report_file = fname - else: - log.warning("Unknown file extension: '%s'", ext) - #shutil.move(os.path.join(tmpdir, fname), outdir) - - log.debug("Mangling html report file %s", report_file) - mangle_html_report(os.path.join(tmpdir, report_file), - os.path.join(outdir, report_file), pngs) - return (os.path.join(outdir, report_file), - [os.path.join(outdir, i) for i in images]) - finally: - shutil.rmtree(tmpdir) - -def send_email(text_fn, html_fn, image_fns, subject, recipients, copy=[], - blind_copy=[]): - """Send email""" +def send_email(text_fn, subject, recipients, copy=[], blind_copy=[]): # Generate email message - text_msg = html_msg = None - if text_fn: - with open(text_fn) as f: - text_msg = MIMEText("Yocto build performance test report.\n" + - f.read(), 'plain') - if html_fn: - html_msg = msg = MIMEMultipart('related') - with open(html_fn) as f: - html_msg.attach(MIMEText(f.read(), 'html')) - for img_fn in image_fns: - # Expect that content id is same as the filename - cid = os.path.splitext(os.path.basename(img_fn))[0] - with open(img_fn, 'rb') as f: - image_msg = MIMEImage(f.read()) - image_msg['Content-ID'] = '<{}>'.format(cid) - html_msg.attach(image_msg) - - if text_msg and html_msg: - msg = MIMEMultipart('alternative') - msg.attach(text_msg) - msg.attach(html_msg) - elif text_msg: - msg = text_msg - elif html_msg: - msg = html_msg - else: - raise ReportError("Neither plain text nor html body specified") + with open(text_fn) as f: + msg = MIMEText("Yocto build performance test report.\n" + f.read(), 'plain') pw_data = pwd.getpwuid(os.getuid()) full_name = pw_data.pw_gecos.split(',')[0] @@ -234,8 +90,6 @@ def main(argv=None): if args.debug: log.setLevel(logging.DEBUG) - check_utils() - if args.outdir: outdir = args.outdir if not os.path.exists(outdir): @@ -245,25 +99,16 @@ def main(argv=None): try: log.debug("Storing email parts in %s", outdir) - html_report = images = None - if args.html: - html_report, images = scrape_html_report(args.html, outdir, - args.phantomjs_args) - if args.to: log.info("Sending email to %s", ', '.join(args.to)) if args.cc: log.info("Copying to %s", ', '.join(args.cc)) if args.bcc: log.info("Blind copying to %s", ', '.join(args.bcc)) - send_email(args.text, html_report, images, args.subject, - args.to, args.cc, args.bcc) + send_email(args.text, args.subject, args.to, args.cc, args.bcc) except subprocess.CalledProcessError as err: log.error("%s, with output:\n%s", str(err), err.output.decode()) return 1 - except ReportError as err: - log.error(err) - return 1 finally: if not args.outdir: log.debug("Wiping %s", outdir) diff --git a/scripts/create-pull-request b/scripts/create-pull-request index 8eefcf63a5..2f91a355b0 100755 --- a/scripts/create-pull-request +++ b/scripts/create-pull-request @@ -128,7 +128,7 @@ PROTO_RE="[a-z][a-z+]*://" GIT_RE="\(^\($PROTO_RE\)\?\)\($USER_RE@\)\?\([^:/]*\)[:/]\(.*\)" REMOTE_URL=${REMOTE_URL%.git} REMOTE_REPO=$(echo $REMOTE_URL | sed "s#$GIT_RE#\5#") -REMOTE_URL=$(echo $REMOTE_URL | sed "s#$GIT_RE#git://\4/\5#") +REMOTE_URL=$(echo $REMOTE_URL | sed "s#$GIT_RE#https://\4/\5#") if [ -z "$BRANCH" ]; then BRANCH=$(git branch | grep -e "^\* " | cut -d' ' -f2) diff --git a/scripts/git b/scripts/git new file mode 100755 index 0000000000..644055e540 --- /dev/null +++ b/scripts/git @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +# +# Wrapper around 'git' that doesn't think we are root + +import os +import shutil +import sys + +os.environ['PSEUDO_UNLOAD'] = '1' + +# calculate path to the real 'git' +path = os.environ['PATH'] +# we need to remove our path but also any other copy of this script which +# may be present, e.g. eSDK. +replacements = [os.path.dirname(sys.argv[0])] +for p in path.split(":"): + if p.endswith("/scripts"): + replacements.append(p) +for r in replacements: + path = path.replace(r, '/ignoreme') +real_git = shutil.which('git', path=path) + +if len(sys.argv) == 1: + os.execl(real_git, 'git') + +os.execv(real_git, sys.argv) diff --git a/scripts/lib/buildstats.py b/scripts/lib/buildstats.py index c69b5bf4d7..3b76286ba5 100644 --- a/scripts/lib/buildstats.py +++ b/scripts/lib/buildstats.py @@ -8,7 +8,7 @@ import json import logging import os import re -from collections import namedtuple,OrderedDict +from collections import namedtuple from statistics import mean @@ -238,7 +238,7 @@ class BuildStats(dict): subdirs = os.listdir(path) for dirname in subdirs: recipe_dir = os.path.join(path, dirname) - if not os.path.isdir(recipe_dir): + if dirname == "reduced_proc_pressure" or not os.path.isdir(recipe_dir): continue name, epoch, version, revision = cls.split_nevr(dirname) bsrecipe = BSRecipe(name, epoch, version, revision) diff --git a/scripts/lib/checklayer/cases/common.py b/scripts/lib/checklayer/cases/common.py index b82304e361..4495f71b24 100644 --- a/scripts/lib/checklayer/cases/common.py +++ b/scripts/lib/checklayer/cases/common.py @@ -14,7 +14,7 @@ class CommonCheckLayer(OECheckLayerTestCase): # The top-level README file may have a suffix (like README.rst or README.txt). readme_files = glob.glob(os.path.join(self.tc.layer['path'], '[Rr][Ee][Aa][Dd][Mm][Ee]*')) self.assertTrue(len(readme_files) > 0, - msg="Layer doesn't contains README file.") + msg="Layer doesn't contain a README file.") # There might be more than one file matching the file pattern above # (for example, README.rst and README-COPYING.rst). The one with the shortest diff --git a/scripts/lib/devtool/deploy.py b/scripts/lib/devtool/deploy.py index d802b22e8f..b4f9fbfe45 100644 --- a/scripts/lib/devtool/deploy.py +++ b/scripts/lib/devtool/deploy.py @@ -170,7 +170,7 @@ def deploy(args, config, basepath, workspace): srcdir = recipe_outdir recipe_outdir = os.path.join(rd.getVar('WORKDIR'), 'devtool-deploy-target-stripped') if os.path.isdir(recipe_outdir): - bb.utils.remove(recipe_outdir, True) + exec_fakeroot(rd, "rm -rf %s" % recipe_outdir, shell=True) exec_fakeroot(rd, "cp -af %s %s" % (os.path.join(srcdir, '.'), recipe_outdir), shell=True) os.environ['PATH'] = ':'.join([os.environ['PATH'], rd.getVar('PATH') or '']) oe.package.strip_execs(args.recipename, recipe_outdir, rd.getVar('STRIP'), rd.getVar('libdir'), @@ -201,9 +201,9 @@ def deploy(args, config, basepath, workspace): print(' %s' % item) return 0 - extraoptions = '' + extraoptions = '-o HostKeyAlgorithms=+ssh-rsa' if args.no_host_check: - extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' + extraoptions += ' -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' if not args.show_status: extraoptions += ' -q' @@ -274,9 +274,9 @@ def undeploy(args, config, basepath, workspace): elif not args.recipename and not args.all: raise argparse_oe.ArgumentUsageError('If you don\'t specify a recipe, you must specify -a/--all', 'undeploy-target') - extraoptions = '' + extraoptions = '-o HostKeyAlgorithms=+ssh-rsa' if args.no_host_check: - extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' + extraoptions += ' -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' if not args.show_status: extraoptions += ' -q' diff --git a/scripts/lib/devtool/menuconfig.py b/scripts/lib/devtool/menuconfig.py index 95384c5333..ff9227035d 100644 --- a/scripts/lib/devtool/menuconfig.py +++ b/scripts/lib/devtool/menuconfig.py @@ -43,7 +43,7 @@ def menuconfig(args, config, basepath, workspace): return 1 check_workspace_recipe(workspace, args.component) - pn = rd.getVar('PN', True) + pn = rd.getVar('PN') if not rd.getVarFlag('do_menuconfig','task'): raise DevtoolError("This recipe does not support menuconfig option") diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py index f364a45283..cfa88616af 100644 --- a/scripts/lib/devtool/standard.py +++ b/scripts/lib/devtool/standard.py @@ -357,7 +357,7 @@ def _move_file(src, dst, dry_run_outdir=None, base_outdir=None): bb.utils.mkdirhier(dst_d) shutil.move(src, dst) -def _copy_file(src, dst, dry_run_outdir=None): +def _copy_file(src, dst, dry_run_outdir=None, base_outdir=None): """Copy a file. Creates all the directory components of destination path.""" dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' logger.debug('Copying %s to %s%s' % (src, dst, dry_run_suffix)) diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py index 566c75369a..a2c6d052a6 100644 --- a/scripts/lib/recipetool/create.py +++ b/scripts/lib/recipetool/create.py @@ -435,7 +435,7 @@ def create_recipe(args): if args.binary: # Assume the archive contains the directory structure verbatim # so we need to extract to a subdirectory - fetchuri += ';subdir=${BP}' + fetchuri += ';subdir=${BPN}' srcuri = fetchuri rev_re = re.compile(';rev=([^;]+)') res = rev_re.search(srcuri) @@ -478,6 +478,9 @@ def create_recipe(args): storeTagName = params['tag'] params['nobranch'] = '1' del params['tag'] + # Assume 'master' branch if not set + if scheme in ['git', 'gitsm'] and 'branch' not in params and 'nobranch' not in params: + params['branch'] = 'master' fetchuri = bb.fetch2.encodeurl((scheme, network, path, user, passwd, params)) tmpparent = tinfoil.config_data.getVar('BASE_WORKDIR') @@ -527,10 +530,9 @@ def create_recipe(args): # Remove HEAD reference point and drop remote prefix get_branch = [x.split('/', 1)[1] for x in get_branch if not x.startswith('origin/HEAD')] if 'master' in get_branch: - # If it is master, we do not need to append 'branch=master' as this is default. # Even with the case where get_branch has multiple objects, if 'master' is one # of them, we should default take from 'master' - srcbranch = '' + srcbranch = 'master' elif len(get_branch) == 1: # If 'master' isn't in get_branch and get_branch contains only ONE object, then store result into 'srcbranch' srcbranch = get_branch[0] @@ -543,8 +545,8 @@ def create_recipe(args): # Since we might have a value in srcbranch, we need to # recontruct the srcuri to include 'branch' in params. scheme, network, path, user, passwd, params = bb.fetch2.decodeurl(srcuri) - if srcbranch: - params['branch'] = srcbranch + if scheme in ['git', 'gitsm']: + params['branch'] = srcbranch or 'master' if storeTagName and scheme in ['git', 'gitsm']: # Check srcrev using tag and check validity of the tag @@ -603,7 +605,7 @@ def create_recipe(args): splitline = line.split() if len(splitline) > 1: if splitline[0] == 'origin' and scriptutils.is_src_url(splitline[1]): - srcuri = reformat_git_uri(splitline[1]) + srcuri = reformat_git_uri(splitline[1]) + ';branch=master' srcsubdir = 'git' break @@ -743,6 +745,10 @@ def create_recipe(args): for handler in handlers: handler.process(srctree_use, classes, lines_before, lines_after, handled, extravalues) + # native and nativesdk classes are special and must be inherited last + # If present, put them at the end of the classes list + classes.sort(key=lambda c: c in ("native", "nativesdk")) + extrafiles = extravalues.pop('extrafiles', {}) extra_pn = extravalues.pop('PN', None) extra_pv = extravalues.pop('PV', None) diff --git a/scripts/lib/resulttool/report.py b/scripts/lib/resulttool/report.py index f0ca50ebe2..a349510ab8 100644 --- a/scripts/lib/resulttool/report.py +++ b/scripts/lib/resulttool/report.py @@ -176,7 +176,10 @@ class ResultsTextReport(object): vals['sort'] = line['testseries'] + "_" + line['result_id'] vals['failed_testcases'] = line['failed_testcases'] for k in cols: - vals[k] = "%d (%s%%)" % (line[k], format(line[k] / total_tested * 100, '.0f')) + if total_tested: + vals[k] = "%d (%s%%)" % (line[k], format(line[k] / total_tested * 100, '.0f')) + else: + vals[k] = "0 (0%)" for k in maxlen: if k in vals and len(vals[k]) > maxlen[k]: maxlen[k] = len(vals[k]) diff --git a/scripts/lib/resulttool/resultutils.py b/scripts/lib/resulttool/resultutils.py index 8917022d36..c5521d81bd 100644 --- a/scripts/lib/resulttool/resultutils.py +++ b/scripts/lib/resulttool/resultutils.py @@ -58,7 +58,11 @@ def append_resultsdata(results, f, configmap=store_map, configvars=extra_configv testseries = posixpath.basename(posixpath.dirname(url.path)) else: with open(f, "r") as filedata: - data = json.load(filedata) + try: + data = json.load(filedata) + except json.decoder.JSONDecodeError: + print("Cannot decode {}. Possible corruption. Skipping.".format(f)) + data = "" testseries = os.path.basename(os.path.dirname(f)) else: data = f @@ -142,7 +146,7 @@ def generic_get_log(sectionname, results, section): return decode_log(ptest['log']) def ptestresult_get_log(results, section): - return generic_get_log('ptestresuls.sections', results, section) + return generic_get_log('ptestresult.sections', results, section) def generic_get_rawlogs(sectname, results): if sectname not in results: diff --git a/scripts/lib/scriptutils.py b/scripts/lib/scriptutils.py index f92255d8dc..47a08194d0 100644 --- a/scripts/lib/scriptutils.py +++ b/scripts/lib/scriptutils.py @@ -18,7 +18,8 @@ import sys import tempfile import threading import importlib -from importlib import machinery +import importlib.machinery +import importlib.util class KeepAliveStreamHandler(logging.StreamHandler): def __init__(self, keepalive=True, **kwargs): @@ -82,7 +83,9 @@ def load_plugins(logger, plugins, pluginpath): logger.debug('Loading plugin %s' % name) spec = importlib.machinery.PathFinder.find_spec(name, path=[pluginpath] ) if spec: - return spec.loader.load_module() + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod def plugin_name(filename): return os.path.splitext(os.path.basename(filename))[0] @@ -215,7 +218,8 @@ def fetch_url(tinfoil, srcuri, srcrev, destdir, logger, preserve_tmp=False, mirr pathvars = ['T', 'RECIPE_SYSROOT', 'RECIPE_SYSROOT_NATIVE'] for pathvar in pathvars: path = rd.getVar(pathvar) - shutil.rmtree(path) + if os.path.exists(path): + shutil.rmtree(path) finally: if fetchrecipe: try: diff --git a/scripts/lib/wic/engine.py b/scripts/lib/wic/engine.py index 9ff4394757..7dbde85696 100644 --- a/scripts/lib/wic/engine.py +++ b/scripts/lib/wic/engine.py @@ -19,10 +19,10 @@ import os import tempfile import json import subprocess +import shutil import re from collections import namedtuple, OrderedDict -from distutils.spawn import find_executable from wic import WicError from wic.filemap import sparse_copy @@ -245,7 +245,7 @@ class Disk: for path in pathlist.split(':'): self.paths = "%s%s:%s" % (native_sysroot, path, self.paths) - self.parted = find_executable("parted", self.paths) + self.parted = shutil.which("parted", path=self.paths) if not self.parted: raise WicError("Can't find executable parted") @@ -283,7 +283,7 @@ class Disk: "resize2fs", "mkswap", "mkdosfs", "debugfs"): aname = "_%s" % name if aname not in self.__dict__: - setattr(self, aname, find_executable(name, self.paths)) + setattr(self, aname, shutil.which(name, path=self.paths)) if aname not in self.__dict__ or self.__dict__[aname] is None: raise WicError("Can't find executable '{}'".format(name)) return self.__dict__[aname] diff --git a/scripts/lib/wic/help.py b/scripts/lib/wic/help.py index 62a2a90e79..fcace95ff4 100644 --- a/scripts/lib/wic/help.py +++ b/scripts/lib/wic/help.py @@ -840,8 +840,8 @@ DESCRIPTION meanings. The commands are based on the Fedora kickstart documentation but with modifications to reflect wic capabilities. - http://fedoraproject.org/wiki/Anaconda/Kickstart#part_or_partition - http://fedoraproject.org/wiki/Anaconda/Kickstart#bootloader + https://pykickstart.readthedocs.io/en/latest/kickstart-docs.html#part-or-partition + https://pykickstart.readthedocs.io/en/latest/kickstart-docs.html#bootloader Commands diff --git a/scripts/lib/wic/misc.py b/scripts/lib/wic/misc.py index 8fb508dd39..3e11822996 100644 --- a/scripts/lib/wic/misc.py +++ b/scripts/lib/wic/misc.py @@ -16,9 +16,9 @@ import logging import os import re import subprocess +import shutil from collections import defaultdict -from distutils import spawn from wic import WicError @@ -46,7 +46,8 @@ NATIVE_RECIPES = {"bmaptool": "bmap-tools", "parted": "parted", "sfdisk": "util-linux", "sgdisk": "gptfdisk", - "syslinux": "syslinux" + "syslinux": "syslinux", + "tar": "tar" } def runtool(cmdln_or_args): @@ -113,6 +114,15 @@ def exec_cmd(cmd_and_args, as_shell=False): """ return _exec_cmd(cmd_and_args, as_shell)[1] +def find_executable(cmd, paths): + recipe = cmd + if recipe in NATIVE_RECIPES: + recipe = NATIVE_RECIPES[recipe] + provided = get_bitbake_var("ASSUME_PROVIDED") + if provided and "%s-native" % recipe in provided: + return True + + return shutil.which(cmd, path=paths) def exec_native_cmd(cmd_and_args, native_sysroot, pseudo=""): """ @@ -141,7 +151,7 @@ def exec_native_cmd(cmd_and_args, native_sysroot, pseudo=""): logger.debug("exec_native_cmd: %s", native_cmd_and_args) # If the command isn't in the native sysroot say we failed. - if spawn.find_executable(args[0], native_paths): + if find_executable(args[0], native_paths): ret, out = _exec_cmd(native_cmd_and_args, True) else: ret = 127 diff --git a/scripts/lib/wic/pluginbase.py b/scripts/lib/wic/pluginbase.py index d9b4e57747..b64568339b 100644 --- a/scripts/lib/wic/pluginbase.py +++ b/scripts/lib/wic/pluginbase.py @@ -9,9 +9,11 @@ __all__ = ['ImagerPlugin', 'SourcePlugin'] import os import logging +import types from collections import defaultdict -from importlib.machinery import SourceFileLoader +import importlib +import importlib.util from wic import WicError from wic.misc import get_bitbake_var @@ -54,7 +56,9 @@ class PluginMgr: mname = fname[:-3] mpath = os.path.join(ppath, fname) logger.debug("loading plugin module %s", mpath) - SourceFileLoader(mname, mpath).load_module() + spec = importlib.util.spec_from_file_location(mname, mpath) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) return PLUGINS.get(ptype) diff --git a/scripts/lib/wic/plugins/imager/direct.py b/scripts/lib/wic/plugins/imager/direct.py index 7e1c1c03ab..42704d1e10 100644 --- a/scripts/lib/wic/plugins/imager/direct.py +++ b/scripts/lib/wic/plugins/imager/direct.py @@ -115,7 +115,7 @@ class DirectPlugin(ImagerPlugin): updated = False for part in self.parts: if not part.realnum or not part.mountpoint \ - or part.mountpoint == "/": + or part.mountpoint == "/" or not (part.mountpoint.startswith('/') or part.mountpoint == "swap"): continue if part.use_uuid: diff --git a/scripts/lib/wic/plugins/source/bootimg-efi.py b/scripts/lib/wic/plugins/source/bootimg-efi.py index 2cfdc10ecd..05e8471116 100644 --- a/scripts/lib/wic/plugins/source/bootimg-efi.py +++ b/scripts/lib/wic/plugins/source/bootimg-efi.py @@ -277,6 +277,13 @@ class BootimgEFIPlugin(SourcePlugin): logger.debug("Added %d extra blocks to %s to get to %d total blocks", extra_blocks, part.mountpoint, blocks) + # required for compatibility with certain devices expecting file system + # block count to be equal to partition block count + if blocks < part.fixed_size: + blocks = part.fixed_size + logger.debug("Overriding %s to %d total blocks for compatibility", + part.mountpoint, blocks) + # dosfs image, created by mkdosfs bootimg = "%s/boot.img" % cr_workdir diff --git a/scripts/nativesdk-intercept/chgrp b/scripts/nativesdk-intercept/chgrp new file mode 100755 index 0000000000..30cc417d3a --- /dev/null +++ b/scripts/nativesdk-intercept/chgrp @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +# +# Wrapper around 'chgrp' that redirects to root in all cases + +import os +import shutil +import sys + +# calculate path to the real 'chgrp' +path = os.environ['PATH'] +path = path.replace(os.path.dirname(sys.argv[0]), '') +real_chgrp = shutil.which('chgrp', path=path) + +args = list() + +found = False +for i in sys.argv: + if i.startswith("-"): + args.append(i) + continue + if not found: + args.append("root") + found = True + else: + args.append(i) + +os.execv(real_chgrp, args) diff --git a/scripts/nativesdk-intercept/chown b/scripts/nativesdk-intercept/chown new file mode 100755 index 0000000000..3914b3e384 --- /dev/null +++ b/scripts/nativesdk-intercept/chown @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +# +# Wrapper around 'chown' that redirects to root in all cases + +import os +import shutil +import sys + +# calculate path to the real 'chown' +path = os.environ['PATH'] +path = path.replace(os.path.dirname(sys.argv[0]), '') +real_chown = shutil.which('chown', path=path) + +args = list() + +found = False +for i in sys.argv: + if i.startswith("-"): + args.append(i) + continue + if not found: + args.append("root:root") + found = True + else: + args.append(i) + +os.execv(real_chown, args) diff --git a/scripts/oe-depends-dot b/scripts/oe-depends-dot index 5eb3e12769..1c2d51c6ec 100755 --- a/scripts/oe-depends-dot +++ b/scripts/oe-depends-dot @@ -15,7 +15,7 @@ class Dot(object): def __init__(self): parser = argparse.ArgumentParser( description="Analyse recipe-depends.dot generated by bitbake -g", - epilog="Use %(prog)s --help to get help") + formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("dotfile", help = "Specify the dotfile", nargs = 1, action='store', default='') parser.add_argument("-k", "--key", @@ -32,6 +32,21 @@ class Dot(object): " For example, A->B, B->C, A->C, then A->C can be removed.", action="store_true", default=False) + parser.epilog = """ +Examples: +First generate the .dot file: + bitbake -g core-image-minimal + +To find out why a package is being built: + %(prog)s -k <package> -w ./task-depends.dot + +To find out what a package depends on: + %(prog)s -k <package> -d ./task-depends.dot + +Reduce the .dot file packages only, no tasks: + %(prog)s -r ./task-depends.dot +""" + self.args = parser.parse_args() if len(sys.argv) != 3 and len(sys.argv) < 5: @@ -99,6 +114,10 @@ class Dot(object): if key == "meta-world-pkgdata": continue dep = m.group(2) + key = key.split('.')[0] + dep = dep.split('.')[0] + if key == dep: + continue if key in depends: if not key in depends[key]: depends[key].add(dep) diff --git a/scripts/oe-pkgdata-browser b/scripts/oe-pkgdata-browser index 8d223185a4..65a6ee956e 100755 --- a/scripts/oe-pkgdata-browser +++ b/scripts/oe-pkgdata-browser @@ -236,6 +236,8 @@ class PkgUi(): update_deps("RPROVIDES", "Provides: ", self.provides_label, clickable=False) def load_recipes(self): + if not os.path.exists(pkgdata): + sys.exit("Error: Please ensure %s exists by generating packages before using this tool." % pkgdata) for recipe in sorted(os.listdir(pkgdata)): if os.path.isfile(os.path.join(pkgdata, recipe)): self.recipe_iters[recipe] = self.recipe_store.append([recipe]) diff --git a/scripts/pybootchartgui/pybootchartgui/parsing.py b/scripts/pybootchartgui/pybootchartgui/parsing.py index b42dac6b88..9d6787ec5a 100644 --- a/scripts/pybootchartgui/pybootchartgui/parsing.py +++ b/scripts/pybootchartgui/pybootchartgui/parsing.py @@ -128,7 +128,7 @@ class Trace: def compile(self, writer): def find_parent_id_for(pid): - if pid is 0: + if pid == 0: return 0 ppid = self.parent_map.get(pid) if ppid: diff --git a/scripts/relocate_sdk.py b/scripts/relocate_sdk.py index 8c0fdb986a..8079d13750 100755 --- a/scripts/relocate_sdk.py +++ b/scripts/relocate_sdk.py @@ -97,11 +97,12 @@ def change_interpreter(elf_file_name): if (len(new_dl_path) >= p_filesz): print("ERROR: could not relocate %s, interp size = %i and %i is needed." \ % (elf_file_name, p_memsz, len(new_dl_path) + 1)) - break + return False dl_path = new_dl_path + b("\0") * (p_filesz - len(new_dl_path)) f.seek(p_offset) f.write(dl_path) break + return True def change_dl_sysdirs(elf_file_name): if arch == 32: @@ -215,6 +216,7 @@ else: executables_list = sys.argv[3:] +errors = False for e in executables_list: perms = os.stat(e)[stat.ST_MODE] if os.access(e, os.W_OK|os.R_OK): @@ -240,7 +242,8 @@ for e in executables_list: arch = get_arch() if arch: parse_elf_header() - change_interpreter(e) + if not change_interpreter(e): + errors = True change_dl_sysdirs(e) """ change permissions back """ @@ -253,3 +256,6 @@ for e in executables_list: print("New file size for %s is different. Looks like a relocation error!", e) sys.exit(-1) +if errors: + print("Relocation of one or more executables failed.") + sys.exit(-1) diff --git a/scripts/runqemu b/scripts/runqemu index 10880ba6bb..4dfc0e2d38 100755 --- a/scripts/runqemu +++ b/scripts/runqemu @@ -974,17 +974,14 @@ class BaseConfig(object): else: self.nfs_server = '192.168.7.1' - # Figure out a new nfs_instance to allow multiple qemus running. - ps = subprocess.check_output(("ps", "auxww")).decode('utf-8') - pattern = '/bin/unfsd .* -i .*\.pid -e .*/exports([0-9]+) ' - all_instances = re.findall(pattern, ps, re.M) - if all_instances: - all_instances.sort(key=int) - self.nfs_instance = int(all_instances.pop()) + 1 - - nfsd_port = 3049 + 2 * self.nfs_instance - mountd_port = 3048 + 2 * self.nfs_instance + nfsd_port = 3048 + self.nfs_instance + lockdir = "/tmp/qemu-port-locks" + self.make_lock_dir(lockdir) + while not self.check_free_port('localhost', nfsd_port, lockdir): + self.nfs_instance += 1 + nfsd_port += 1 + mountd_port = nfsd_port # Export vars for runqemu-export-rootfs export_dict = { 'NFS_INSTANCE': self.nfs_instance, @@ -1034,6 +1031,17 @@ class BaseConfig(object): self.set('NETWORK_CMD', '-netdev bridge,br=%s,id=net0,helper=%s -device virtio-net-pci,netdev=net0 ' % ( self.net_bridge, os.path.join(self.bindir_native, 'qemu-oe-bridge-helper'))) + def make_lock_dir(self, lockdir): + if not os.path.exists(lockdir): + # There might be a race issue when multi runqemu processess are + # running at the same time. + try: + os.mkdir(lockdir) + os.chmod(lockdir, 0o777) + except FileExistsError: + pass + return + def setup_slirp(self): """Setup user networking""" @@ -1052,14 +1060,7 @@ class BaseConfig(object): mac = 2 lockdir = "/tmp/qemu-port-locks" - if not os.path.exists(lockdir): - # There might be a race issue when multi runqemu processess are - # running at the same time. - try: - os.mkdir(lockdir) - os.chmod(lockdir, 0o777) - except FileExistsError: - pass + self.make_lock_dir(lockdir) # Find a free port to avoid conflicts for p in ports[:]: @@ -1099,14 +1100,7 @@ class BaseConfig(object): logger.error("ip: %s" % ip) raise OEPathError("runqemu-ifup, runqemu-ifdown or ip not found") - if not os.path.exists(lockdir): - # There might be a race issue when multi runqemu processess are - # running at the same time. - try: - os.mkdir(lockdir) - os.chmod(lockdir, 0o777) - except FileExistsError: - pass + self.make_lock_dir(lockdir) cmd = (ip, 'link') logger.debug('Running %s...' % str(cmd)) @@ -1423,13 +1417,13 @@ class BaseConfig(object): logger.debug('Running %s' % str(cmd)) subprocess.check_call(cmd) self.release_taplock() - self.release_portlock() if self.nfs_running: logger.info("Shutting down the userspace NFS server...") cmd = ("runqemu-export-rootfs", "stop", self.rootfs) logger.debug('Running %s' % str(cmd)) subprocess.check_call(cmd) + self.release_portlock() if self.saved_stty: subprocess.check_call(("stty", self.saved_stty)) @@ -1516,7 +1510,8 @@ def main(): def sigterm_handler(signum, frame): logger.info("SIGTERM received") - os.kill(config.qemupid, signal.SIGTERM) + if config.qemupid: + os.kill(config.qemupid, signal.SIGTERM) config.cleanup() # Deliberately ignore the return code of 'tput smam'. subprocess.call(["tput", "smam"]) diff --git a/scripts/wic b/scripts/wic index a741aed364..99a8a97ccb 100755 --- a/scripts/wic +++ b/scripts/wic @@ -22,9 +22,9 @@ import sys import argparse import logging import subprocess +import shutil from collections import namedtuple -from distutils import spawn # External modules scripts_path = os.path.dirname(os.path.realpath(__file__)) @@ -47,7 +47,7 @@ if os.environ.get('SDKTARGETSYSROOT'): break sdkroot = os.path.dirname(sdkroot) -bitbake_exe = spawn.find_executable('bitbake') +bitbake_exe = shutil.which('bitbake') if bitbake_exe: bitbake_path = scriptpath.add_bitbake_lib_path() import bb @@ -206,7 +206,7 @@ def wic_create_subcommand(options, usage_str): logger.info(" (Please check that the build artifacts for the machine") logger.info(" selected in local.conf actually exist and that they") logger.info(" are the correct artifacts for the image (.wks file)).\n") - raise WicError("The artifact that couldn't be found was %s:\n %s", not_found, not_found_dir) + raise WicError("The artifact that couldn't be found was %s:\n %s" % (not_found, not_found_dir)) krootfs_dir = options.rootfs_dir if krootfs_dir is None: |