From 2d0aea6a73c427ce6aa17dc71e0783977a52bb2b Mon Sep 17 00:00:00 2001 From: Tim Orling Date: Wed, 25 Mar 2020 21:26:55 -0700 Subject: scripts/install-buildtools: add helper script to install buildtools For distros such as CentOS-7 where the default buildtools are too old we need to make it easy for users to install a pre-built SDK with all of "build-essentials" included. Other uses may include building older Yocto Project releases with a distro where buildtools are too new. For convenience, the standard buildtools installation is also supported. NOTE: extended buildtools is the default, e.g. --with-extended-buildtools is on by default Example usage (extended buildtools from milestone): (1) using --url and --filename $ install-buildtools \ --url http://downloads.yoctoproject.org/releases/yocto/milestones/yocto-3.1_M2/buildtools \ --filename x86_64-buildtools-extended-nativesdk-standalone-3.0+snapshot-20200122.sh (2) using --base-url, --release, --installer-version and --build-date $ install-buildtools \ --base-url http://downloads.yoctoproject.org/releases/yocto \ --release yocto-3.1_M2 \ --install-version 3.0+snapshot --build-date 202000122 Example usage (standard buildtools from release): (3) using --url and --filename $ install-buildtools --without-extended-buildtools \ --url http://downloads.yoctoproject.org/releases/yocto/yocto-3.0.2/buildtools \ --filename x86_64-buildtools-nativesdk-standalone-3.0.2.sh (4) using --base-url, --release and --installer-version $ install-buildtools --without-extended-buildtools \ --base-url http://downloads.yoctoproject.org/releases/yocto \ --release yocto-3.0.2 \ --install-version 3.0.2 [YOCTO #13832] Signed-off-by: Tim Orling Signed-off-by: Richard Purdie --- scripts/install-buildtools | 270 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 270 insertions(+) create mode 100755 scripts/install-buildtools diff --git a/scripts/install-buildtools b/scripts/install-buildtools new file mode 100755 index 0000000000..0947e9c4d6 --- /dev/null +++ b/scripts/install-buildtools @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 + +# Buildtools and buildtools extended installer helper script +# +# Copyright (C) 2017-2020 Intel Corporation +# +# SPDX-License-Identifier: MIT +# +# NOTE: --with-extended-buildtools is on by default +# +# Example usage (extended buildtools from milestone): +# (1) using --url and --filename +# $ install-buildtools \ +# --url http://downloads.yoctoproject.org/releases/yocto/milestones/yocto-3.1_M2/buildtools \ +# --filename x86_64-buildtools-extended-nativesdk-standalone-3.0+snapshot-20200122.sh +# (2) using --base-url, --release, --installer-version and --build-date +# $ install-buildtools \ +# --base-url http://downloads.yoctoproject.org/releases/yocto \ +# --release yocto-3.1_M2 \ +# --install-version 3.0+snapshot +# --build-date 202000122 +# +# Example usage (standard buildtools from release): +# (3) using --url and --filename +# $ install-buildtools --without-extended-buildtools \ +# --url http://downloads.yoctoproject.org/releases/yocto/yocto-3.0.2/buildtools \ +# --filename x86_64-buildtools-nativesdk-standalone-3.0.2.sh +# (4) using --base-url, --release and --installer-version +# $ install-buildtools --without-extended-buildtools \ +# --base-url http://downloads.yoctoproject.org/releases/yocto \ +# --release yocto-3.0.2 \ +# --install-version 3.0.2 +# + +import argparse +import logging +import os +import re +import shutil +import stat +import subprocess +import sys +import tempfile +from urllib.parse import quote + +scripts_path = os.path.dirname(os.path.realpath(__file__)) +lib_path = scripts_path + '/lib' +sys.path = sys.path + [lib_path] +import scriptutils +import scriptpath + +# Figure out where is the bitbake/lib/bb since we need bb.utils.md5_file +bitbakepath = scriptpath.add_bitbake_lib_path() +if not bitbakepath: + sys.stderr.write("Unable to find bitbake by searching parent directory " + "of this script or PATH\n") + sys.exit(1) + +PROGNAME = 'install-buildtools' +logger = scriptutils.logger_create(PROGNAME, stream=sys.stdout) + +DEFAULT_BASE_URL: str = 'http://downloads.yoctoproject.org/releases/yocto' +DEFAULT_RELEASE: str = 'yocto-3.1_M2' +DEFAULT_INSTALLER_VERSION: str = '3.0+snapshot' +DEFAULT_BUILDDATE: str = "20200122" + + +def main(): + global DEFAULT_BASE_URL + global DEFAULT_RELEASE + global DEFAULT_INSTALLER_VERSION + global DEFAULT_BUILDDATE + filename: str = "" + release: str = "" + buildtools_url: str = "" + + parser = argparse.ArgumentParser( + description="Buildtools installation helper", + add_help=False) + parser.add_argument('-u', '--url', + help='URL from where to fetch buildtools SDK installer, not ' + 'including filename (optional)\n' + 'Requires --filename.', + action='store') + parser.add_argument('-f', '--filename', + help='filename for the buildtools SDK installer to be installed ' + '(optional)\nRequires --url', + action='store') + parser.add_argument('-d', '--directory', + help='directory where buildtools SDK will be installed (optional)', + action='store') + parser.add_argument('-r', '--release', + default=DEFAULT_RELEASE, + help='Yocto Project release string for SDK which will be ' + 'installed (optional)', + action='store') + parser.add_argument('-V', '--installer-version', + default=DEFAULT_INSTALLER_VERSION, + help='version string for the SDK to be installed (optional)', + action='store') + parser.add_argument('-b', '--base-url', + default=DEFAULT_BASE_URL, + help='base URL from which to fetch SDK (optional)', action='store') + parser.add_argument('-t', '--build-date', + default=DEFAULT_BUILDDATE, + help='Build date of pre-release SDK (optional)', action='store') + group = parser.add_mutually_exclusive_group() + group.add_argument('--with-extended-buildtools', action='store_true', + dest='with_extended_buildtools', + default=True, + help='enable extended buildtools tarball (on by default)') + group.add_argument('--without-extended-buildtools', action='store_false', + dest='with_extended_buildtools', + help='disable extended buildtools (traditional buildtools tarball)') + parser.add_argument('-c', '--check', help='enable md5 checksum checking', + default=True, + action='store_true') + parser.add_argument('-D', '--debug', help='enable debug output', + action='store_true') + parser.add_argument('-q', '--quiet', help='print only errors', + action='store_true') + + parser.add_argument('-h', '--help', action='help', + default=argparse.SUPPRESS, + help='show this help message and exit') + + args = parser.parse_args() + + if args.debug: + logger.setLevel(logging.DEBUG) + elif args.quiet: + logger.setLevel(logging.ERROR) + + if args.url and args.filename: + logger.debug("--url and --filename detected. Ignoring --base-url " + "--release --installer-version arguments.") + filename = args.filename + buildtools_url = "%s/%s" % (args.url, filename) + else: + if args.base_url: + base_url = args.base_url + else: + base_url = DEFAULT_BASE_URL + if args.release: + # check if this is a pre-release "milestone" SDK + m = re.search(r"^(?P[a-zA-Z\-]+)(?P[0-9.]+)(?P_M[1-9])$", + args.release) + logger.debug("milestone regex: %s" % m) + if m and m.group('milestone'): + logger.debug("release[distro]: %s" % m.group('distro')) + logger.debug("release[version]: %s" % m.group('version')) + logger.debug("release[milestone]: %s" % m.group('milestone')) + if not args.build_date: + logger.error("Milestone installers require --build-date") + else: + if args.with_extended_buildtools: + filename = "x86_64-buildtools-extended-nativesdk-standalone-%s-%s.sh" % ( + args.installer_version, args.build_date) + else: + filename = "x86_64-buildtools-nativesdk-standalone-%s-%s.sh" % ( + args.installer_version, args.build_date) + safe_filename = quote(filename) + buildtools_url = "%s/milestones/%s/buildtools/%s" % (base_url, args.release, safe_filename) + # regular release SDK + else: + if args.with_extended_buildtools: + filename = "x86_64-buildtools-extended-nativesdk-standalone-%s.sh" % args.installer_version + else: + filename = "x86_64-buildtools-nativesdk-standalone-%s.sh" % args.installer_version + safe_filename = quote(filename) + buildtools_url = "%s/%s/buildtools/%s" % (base_url, args.release, safe_filename) + + tmpsdk_dir = tempfile.mkdtemp() + try: + # Fetch installer + logger.info("Fetching buildtools installer") + tmpbuildtools = os.path.join(tmpsdk_dir, filename) + ret = subprocess.call("wget -q -O %s %s" % + (tmpbuildtools, buildtools_url), shell=True) + if ret != 0: + logger.error("Could not download file from %s" % buildtools_url) + return ret + + # Verify checksum + if args.check: + import bb + logger.info("Fetching buildtools installer checksum") + check_url = "{}.md5sum".format(buildtools_url) + checksum_filename = "%s.md5sum" % filename + tmpbuildtools_checksum = os.path.join(tmpsdk_dir, checksum_filename) + ret = subprocess.call("wget -q -O %s %s" % + (tmpbuildtools_checksum, check_url), shell=True) + if ret != 0: + logger.error("Could not download file from %s" % check_url) + return ret + regex = re.compile(r"^(?P[0-9a-f]+)\s\s(?P.*/)(?P.*)$") + with open(tmpbuildtools_checksum, 'rb') as f: + original = f.read() + m = re.search(regex, original.decode("utf-8")) + logger.debug("md5sum: %s" % m.group('md5sum')) + logger.debug("path: %s" % m.group('path')) + logger.debug("filename: %s" % m.group('filename')) + if filename != m.group('filename'): + logger.error("Filename does not match name in checksum") + return 1 + md5sum = m.group('md5sum') + md5value = bb.utils.md5_file(tmpbuildtools) + if md5sum == md5value: + logger.info("Checksum success") + else: + logger.error("Checksum %s expected. Actual checksum is %s." % + (md5sum, md5value)) + + # Make installer executable + logger.info("Making installer executable") + st = os.stat(tmpbuildtools) + os.chmod(tmpbuildtools, st.st_mode | stat.S_IEXEC) + logger.debug(os.stat(tmpbuildtools)) + install_dir = "/opt/poky/%s" % args.installer_version + if args.directory: + install_dir = args.directory + ret = subprocess.call("%s -d %s -y" % + (tmpbuildtools, install_dir), shell=True) + else: + ret = subprocess.call("%s -y" % tmpbuildtools, shell=True) + if ret != 0: + logger.error("Could not run buildtools installer") + + # Test installation + logger.info("Testing installation") + tool = "" + m = re.search("extended", tmpbuildtools) + logger.debug("extended regex: %s" % m) + if args.with_extended_buildtools and not m: + logger.info("Ignoring --with-extended-buildtools as filename " + "does not contain 'extended'") + if args.with_extended_buildtools and m: + tool = 'gcc' + else: + tool = 'tar' + proc = subprocess.run("source %s/environment-setup-x86_64-pokysdk-linux && which %s" % + (install_dir, tool), + shell=True, stdout=subprocess.PIPE) + which_tool = proc.stdout.decode("utf-8") + logger.debug("which %s: %s" % (tool, which_tool)) + ret = proc.returncode + if not which_tool.startswith(install_dir): + logger.error("Something went wrong: %s not found in %s" % + (tool, install_dir)) + if ret != 0: + logger.error("Something went wrong: installation failed") + else: + logger.info("Installation successful. Remember to source the " + "environment setup script now and in any new session.") + return ret + + finally: + # cleanup tmp directory + shutil.rmtree(tmpsdk_dir) + + +if __name__ == '__main__': + try: + ret = main() + except Exception: + ret = 1 + import traceback + + traceback.print_exc() + sys.exit(ret) -- cgit 1.2.3-korg