diff options
Diffstat (limited to 'meta/lib/oeqa/selftest/cases/devtool.py')
-rw-r--r-- | meta/lib/oeqa/selftest/cases/devtool.py | 1589 |
1 files changed, 1404 insertions, 185 deletions
diff --git a/meta/lib/oeqa/selftest/cases/devtool.py b/meta/lib/oeqa/selftest/cases/devtool.py index 58f3e58461..7d61773980 100644 --- a/meta/lib/oeqa/selftest/cases/devtool.py +++ b/meta/lib/oeqa/selftest/cases/devtool.py @@ -1,15 +1,23 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +import errno import os import re import shutil import tempfile import glob import fnmatch +import unittest +import json -import oeqa.utils.ftools as ftools from oeqa.selftest.case import OESelftestTestCase from oeqa.utils.commands import runCmd, bitbake, get_bb_var, create_temp_layer from oeqa.utils.commands import get_bb_vars, runqemu, get_test_layer -from oeqa.core.decorator.oeid import OETestID +from oeqa.core.decorator import OETestTag oldmetapath = None @@ -21,6 +29,9 @@ def setUpModule(): corecopydir = os.path.join(templayerdir, 'core-copy') bblayers_conf = os.path.join(os.environ['BUILDDIR'], 'conf', 'bblayers.conf') edited_layers = [] + # make sure user doesn't have a local workspace + result = runCmd('bitbake-layers show-layers') + assert "workspacelayer" not in result.output, "Devtool test suite cannot be run with a local workspace directory" # We need to take a copy of the meta layer so we can modify it and not # have any races against other tests that might be running in parallel @@ -35,10 +46,17 @@ def setUpModule(): canonical_layerpath = os.path.realpath(canonical_layerpath) + '/' edited_layers.append(layerpath) oldmetapath = os.path.realpath(layerpath) + + # when downloading poky from tar.gz some tests will be skipped (BUG 12389) + try: + runCmd('git rev-parse --is-inside-work-tree', cwd=canonical_layerpath) + except: + raise unittest.SkipTest("devtool tests require folder to be a git repo") + result = runCmd('git rev-parse --show-toplevel', cwd=canonical_layerpath) oldreporoot = result.output.rstrip() newmetapath = os.path.join(corecopydir, os.path.relpath(oldmetapath, oldreporoot)) - runCmd('git clone %s %s' % (oldreporoot, corecopydir), cwd=templayerdir) + runCmd('git clone file://%s %s' % (oldreporoot, corecopydir), cwd=templayerdir) # Now we need to copy any modified files # You might ask "why not just copy the entire tree instead of # cloning and doing this?" - well, the problem with that is @@ -53,7 +71,8 @@ def setUpModule(): if pth.startswith(canonical_layerpath): if relpth.endswith('/'): destdir = os.path.join(corecopydir, relpth) - shutil.copytree(pth, destdir) + # avoid race condition by not copying .pyc files YPBZ#13421,13803 + shutil.copytree(pth, destdir, ignore=shutil.ignore_patterns('*.pyc', '__pycache__')) else: destdir = os.path.join(corecopydir, os.path.dirname(relpth)) bb.utils.mkdirhier(destdir) @@ -76,32 +95,15 @@ def tearDownModule(): bb.utils.edit_bblayers_conf(bblayers_conf, None, None, bblayers_edit_cb) shutil.rmtree(templayerdir) -class DevtoolBase(OESelftestTestCase): - - @classmethod - def setUpClass(cls): - super(DevtoolBase, cls).setUpClass() - bb_vars = get_bb_vars(['TOPDIR', 'SSTATE_DIR']) - cls.original_sstate = bb_vars['SSTATE_DIR'] - cls.devtool_sstate = os.path.join(bb_vars['TOPDIR'], 'sstate_devtool') - cls.sstate_conf = 'SSTATE_DIR = "%s"\n' % cls.devtool_sstate - cls.sstate_conf += ('SSTATE_MIRRORS += "file://.* file:///%s/PATH"\n' - % cls.original_sstate) - - @classmethod - def tearDownClass(cls): - cls.logger.debug('Deleting devtool sstate cache on %s' % cls.devtool_sstate) - runCmd('rm -rf %s' % cls.devtool_sstate) - super(DevtoolBase, cls).tearDownClass() +class DevtoolTestCase(OESelftestTestCase): def setUp(self): """Test case setup function""" - super(DevtoolBase, self).setUp() + super(DevtoolTestCase, self).setUp() self.workspacedir = os.path.join(self.builddir, 'workspace') self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory ' 'under the build directory') - self.append_config(self.sstate_conf) def _check_src_repo(self, repo_dir): """Check srctree git repository""" @@ -134,6 +136,7 @@ class DevtoolBase(OESelftestTestCase): with open(recipefile, 'r') as f: invar = None invalue = None + inherits = set() for line in f: var = None if invar: @@ -155,7 +158,7 @@ class DevtoolBase(OESelftestTestCase): invar = var continue elif line.startswith('inherit '): - inherits = line.split()[1:] + inherits.update(line.split()[1:]) if var and var in checkvars: needvalue = checkvars.pop(var) @@ -230,14 +233,113 @@ class DevtoolBase(OESelftestTestCase): filelist.append(' '.join(splitline)) return filelist + def _check_diff(self, diffoutput, addlines, removelines): + """Check output from 'git diff' matches expectation""" + remaining_addlines = addlines[:] + remaining_removelines = removelines[:] + for line in diffoutput.splitlines(): + if line.startswith('+++') or line.startswith('---'): + continue + elif line.startswith('+'): + matched = False + for item in addlines: + if re.match(item, line[1:].strip()): + matched = True + remaining_addlines.remove(item) + break + self.assertTrue(matched, 'Unexpected diff add line: %s' % line) + elif line.startswith('-'): + matched = False + for item in removelines: + if re.match(item, line[1:].strip()): + matched = True + remaining_removelines.remove(item) + break + self.assertTrue(matched, 'Unexpected diff remove line: %s' % line) + if remaining_addlines: + self.fail('Expected added lines not found: %s' % remaining_addlines) + if remaining_removelines: + self.fail('Expected removed lines not found: %s' % remaining_removelines) + + def _check_runqemu_prerequisites(self): + """Check runqemu is available + + Whilst some tests would seemingly be better placed as a runtime test, + unfortunately the runtime tests run under bitbake and you can't run + devtool within bitbake (since devtool needs to run bitbake itself). + Additionally we are testing build-time functionality as well, so + really this has to be done as an oe-selftest test. + """ + machine = get_bb_var('MACHINE') + if not machine.startswith('qemu'): + self.skipTest('This test only works with qemu machines') + if not os.path.exists('/etc/runqemu-nosudo'): + self.skipTest('You must set up tap devices with scripts/runqemu-gen-tapdevs before running this test') + result = runCmd('PATH="$PATH:/sbin:/usr/sbin" ip tuntap show', ignore_status=True) + if result.status != 0: + result = runCmd('PATH="$PATH:/sbin:/usr/sbin" ifconfig -a', ignore_status=True) + if result.status != 0: + self.skipTest('Failed to determine if tap devices exist with ifconfig or ip: %s' % result.output) + for line in result.output.splitlines(): + if line.startswith('tap'): + break + else: + self.skipTest('No tap devices found - you must set up tap devices with scripts/runqemu-gen-tapdevs before running this test') + + def _test_devtool_add_git_url(self, git_url, version, pn, resulting_src_uri, srcrev=None): + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + command = 'devtool add --version %s %s %s' % (version, pn, git_url) + if srcrev : + command += ' --srcrev %s' %srcrev + result = runCmd(command) + self.assertExists(os.path.join(self.workspacedir, 'conf', 'layer.conf'), 'Workspace directory not created') + # Check the recipe name is correct + recipefile = get_bb_var('FILE', pn) + self.assertIn('%s_git.bb' % pn, recipefile, 'Recipe file incorrectly named') + self.assertIn(recipefile, result.output) + # Test devtool status + result = runCmd('devtool status') + self.assertIn(pn, result.output) + self.assertIn(recipefile, result.output) + checkvars = {} + checkvars['SRC_URI'] = resulting_src_uri + self._test_recipe_contents(recipefile, checkvars, []) + +class DevtoolBase(DevtoolTestCase): + + @classmethod + def setUpClass(cls): + super(DevtoolBase, cls).setUpClass() + bb_vars = get_bb_vars(['TOPDIR', 'SSTATE_DIR']) + cls.original_sstate = bb_vars['SSTATE_DIR'] + cls.devtool_sstate = os.path.join(bb_vars['TOPDIR'], 'sstate_devtool') + cls.sstate_conf = 'SSTATE_DIR = "%s"\n' % cls.devtool_sstate + cls.sstate_conf += ('SSTATE_MIRRORS += "file://.* file:///%s/PATH"\n' + % cls.original_sstate) + cls.sstate_conf += ('BB_HASHSERVE_UPSTREAM = "hashserv.yocto.io:8687"\n') + + @classmethod + def tearDownClass(cls): + cls.logger.debug('Deleting devtool sstate cache on %s' % cls.devtool_sstate) + runCmd('rm -rf %s' % cls.devtool_sstate) + super(DevtoolBase, cls).tearDownClass() + + def setUp(self): + """Test case setup function""" + super(DevtoolBase, self).setUp() + self.append_config(self.sstate_conf) + class DevtoolTests(DevtoolBase): - @OETestID(1158) def test_create_workspace(self): # Check preconditions result = runCmd('bitbake-layers show-layers') self.assertTrue('\nworkspace' not in result.output, 'This test cannot be run with a workspace layer in bblayers.conf') + # remove conf/devtool.conf to avoid it corrupting tests + devtoolconf = os.path.join(self.builddir, 'conf', 'devtool.conf') + self.track_for_cleanup(devtoolconf) # Try creating a workspace layer with a specific path tempdir = tempfile.mkdtemp(prefix='devtoolqa') self.track_for_cleanup(tempdir) @@ -256,14 +358,13 @@ class DevtoolTests(DevtoolBase): class DevtoolAddTests(DevtoolBase): - @OETestID(1159) def test_devtool_add(self): # Fetch source tempdir = tempfile.mkdtemp(prefix='devtoolqa') self.track_for_cleanup(tempdir) pn = 'pv' pv = '1.5.3' - url = 'http://www.ivarch.com/programs/sources/pv-1.5.3.tar.bz2' + url = 'http://downloads.yoctoproject.org/mirror/sources/pv-1.5.3.tar.bz2' result = runCmd('wget %s' % url, cwd=tempdir) result = runCmd('tar xfv %s' % os.path.basename(url), cwd=tempdir) srcdir = os.path.join(tempdir, '%s-%s' % (pn, pv)) @@ -298,7 +399,38 @@ class DevtoolAddTests(DevtoolBase): bindir = bindir[1:] self.assertTrue(os.path.isfile(os.path.join(installdir, bindir, 'pv')), 'pv binary not found in D') - @OETestID(1423) + def test_devtool_add_binary(self): + # Create a binary package containing a known test file + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + pn = 'tst-bin' + pv = '1.0' + test_file_dir = "var/lib/%s/" % pn + test_file_name = "test_file" + test_file_content = "TEST CONTENT" + test_file_package_root = os.path.join(tempdir, pn) + test_file_dir_full = os.path.join(test_file_package_root, test_file_dir) + bb.utils.mkdirhier(test_file_dir_full) + with open(os.path.join(test_file_dir_full, test_file_name), "w") as f: + f.write(test_file_content) + bin_package_path = os.path.join(tempdir, "%s.tar.gz" % pn) + runCmd("tar czf %s -C %s ." % (bin_package_path, test_file_package_root)) + + # Test devtool add -b on the binary package + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake -c cleansstate %s' % pn) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('devtool add -b %s %s' % (pn, bin_package_path)) + self.assertExists(os.path.join(self.workspacedir, 'conf', 'layer.conf'), 'Workspace directory not created') + + # Build the resulting recipe + result = runCmd('devtool build %s' % pn) + installdir = get_bb_var('D', pn) + self.assertTrue(installdir, 'Could not query installdir variable') + + # Check that a known file from the binary package has indeed been installed + self.assertTrue(os.path.isfile(os.path.join(installdir, test_file_dir, test_file_name)), '%s not found in D' % test_file_name) + def test_devtool_add_git_local(self): # We need dbus built so that DEPENDS recognition works bitbake('dbus') @@ -331,16 +463,32 @@ class DevtoolAddTests(DevtoolBase): self.assertIn(srcdir, result.output) self.assertIn(recipefile, result.output) checkvars = {} - checkvars['LICENSE'] = 'GPLv2' + checkvars['LICENSE'] = 'GPL-2.0-only' checkvars['LIC_FILES_CHKSUM'] = 'file://COPYING;md5=b234ee4d69f5fce4486a80fdaf4a4263' checkvars['S'] = '${WORKDIR}/git' - checkvars['PV'] = '0.1+git${SRCPV}' - checkvars['SRC_URI'] = 'git://git.yoctoproject.org/git/dbus-wait;protocol=https' + checkvars['PV'] = '0.1+git' + checkvars['SRC_URI'] = 'git://git.yoctoproject.org/git/dbus-wait;protocol=https;branch=master' checkvars['SRCREV'] = srcrev checkvars['DEPENDS'] = set(['dbus']) self._test_recipe_contents(recipefile, checkvars, []) - @OETestID(1162) + def test_devtool_add_git_style1(self): + version = 'v3.1.0' + pn = 'mbedtls' + # this will trigger reformat_git_uri with branch parameter in url + git_url = "'git://git@github.com/ARMmbed/mbedtls.git;branch=mbedtls-2.28;protocol=https'" + resulting_src_uri = "git://git@github.com/ARMmbed/mbedtls.git;branch=mbedtls-2.28;protocol=https" + self._test_devtool_add_git_url(git_url, version, pn, resulting_src_uri) + + def test_devtool_add_git_style2(self): + version = 'v3.1.0' + srcrev = 'v3.1.0' + pn = 'mbedtls' + # this will trigger reformat_git_uri with branch parameter in url + git_url = "'git://git@github.com/ARMmbed/mbedtls.git;protocol=https'" + resulting_src_uri = "git://git@github.com/ARMmbed/mbedtls.git;protocol=https;branch=master" + self._test_devtool_add_git_url(git_url, version, pn, resulting_src_uri, srcrev) + def test_devtool_add_library(self): # Fetch source tempdir = tempfile.mkdtemp(prefix='devtoolqa') @@ -369,7 +517,7 @@ class DevtoolAddTests(DevtoolBase): recipefile = '%s/recipes/libftdi/libftdi_%s.bb' % (self.workspacedir, version) result = runCmd('recipetool setvar %s EXTRA_OECMAKE -- \'-DPYTHON_BINDINGS=OFF -DLIBFTDI_CMAKE_CONFIG_DIR=${datadir}/cmake/Modules\'' % recipefile) with open(recipefile, 'a') as f: - f.write('\nFILES_${PN}-dev += "${datadir}/cmake/Modules"\n') + f.write('\nFILES:${PN}-dev += "${datadir}/cmake/Modules"\n') # We don't have the ability to pick up this dependency automatically yet... f.write('\nDEPENDS += "libusb1"\n') f.write('\nTESTLIBOUTPUT = "${COMPONENTS_DIR}/${TUNE_PKGARCH}/${PN}/${libdir}"\n') @@ -389,20 +537,19 @@ class DevtoolAddTests(DevtoolBase): self.assertFalse(matches, 'Stamp files exist for recipe libftdi that should have been cleaned') self.assertFalse(os.path.isfile(os.path.join(staging_libdir, 'libftdi1.so.2.1.0')), 'libftdi binary still found in STAGING_LIBDIR after cleaning') - @OETestID(1160) def test_devtool_add_fetch(self): # Fetch source tempdir = tempfile.mkdtemp(prefix='devtoolqa') self.track_for_cleanup(tempdir) testver = '0.23' - url = 'https://pypi.python.org/packages/source/M/MarkupSafe/MarkupSafe-%s.tar.gz' % testver + url = 'https://files.pythonhosted.org/packages/c0/41/bae1254e0396c0cc8cf1751cb7d9afc90a602353695af5952530482c963f/MarkupSafe-%s.tar.gz' % testver testrecipe = 'python-markupsafe' srcdir = os.path.join(tempdir, testrecipe) # Test devtool add self.track_for_cleanup(self.workspacedir) self.add_command_to_tearDown('bitbake -c cleansstate %s' % testrecipe) self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') - result = runCmd('devtool add %s %s -f %s' % (testrecipe, srcdir, url)) + result = runCmd('devtool add --no-pypi %s %s -f %s' % (testrecipe, srcdir, url)) self.assertExists(os.path.join(self.workspacedir, 'conf', 'layer.conf'), 'Workspace directory not created. %s' % result.output) self.assertTrue(os.path.isfile(os.path.join(srcdir, 'setup.py')), 'Unable to find setup.py in source directory') self.assertTrue(os.path.isdir(os.path.join(srcdir, '.git')), 'git repository for external source tree was not created') @@ -421,7 +568,7 @@ class DevtoolAddTests(DevtoolBase): result = runCmd('devtool reset -n %s' % testrecipe) shutil.rmtree(srcdir) fakever = '1.9' - result = runCmd('devtool add %s %s -f %s -V %s' % (testrecipe, srcdir, url, fakever)) + result = runCmd('devtool add --no-pypi %s %s -f %s -V %s' % (testrecipe, srcdir, url, fakever)) self.assertTrue(os.path.isfile(os.path.join(srcdir, 'setup.py')), 'Unable to find setup.py in source directory') # Test devtool status result = runCmd('devtool status') @@ -435,11 +582,11 @@ class DevtoolAddTests(DevtoolBase): checkvars['SRC_URI'] = url self._test_recipe_contents(recipefile, checkvars, []) - @OETestID(1161) def test_devtool_add_fetch_git(self): tempdir = tempfile.mkdtemp(prefix='devtoolqa') self.track_for_cleanup(tempdir) url = 'gitsm://git.yoctoproject.org/mraa' + url_branch = '%s;branch=master' % url checkrev = 'ae127b19a50aa54255e4330ccfdd9a5d058e581d' testrecipe = 'mraa' srcdir = os.path.join(tempdir, testrecipe) @@ -459,8 +606,8 @@ class DevtoolAddTests(DevtoolBase): self.assertIn('_git.bb', recipefile, 'Recipe file incorrectly named') checkvars = {} checkvars['S'] = '${WORKDIR}/git' - checkvars['PV'] = '1.0+git${SRCPV}' - checkvars['SRC_URI'] = url + checkvars['PV'] = '1.0+git' + checkvars['SRC_URI'] = url_branch checkvars['SRCREV'] = '${AUTOREV}' self._test_recipe_contents(recipefile, checkvars, []) # Try with revision and version specified @@ -478,12 +625,11 @@ class DevtoolAddTests(DevtoolBase): self.assertIn('_git.bb', recipefile, 'Recipe file incorrectly named') checkvars = {} checkvars['S'] = '${WORKDIR}/git' - checkvars['PV'] = '1.5+git${SRCPV}' - checkvars['SRC_URI'] = url + checkvars['PV'] = '1.5+git' + checkvars['SRC_URI'] = url_branch checkvars['SRCREV'] = checkrev self._test_recipe_contents(recipefile, checkvars, []) - @OETestID(1391) def test_devtool_add_fetch_simple(self): # Fetch source from a remote URL, auto-detecting name tempdir = tempfile.mkdtemp(prefix='devtoolqa') @@ -503,7 +649,7 @@ class DevtoolAddTests(DevtoolBase): result = runCmd('devtool status') self.assertIn(testrecipe, result.output) self.assertIn(srcdir, result.output) - # Check recipe + # Check recipedevtool add recipefile = get_bb_var('FILE', testrecipe) self.assertIn('%s_%s.bb' % (testrecipe, testver), recipefile, 'Recipe file incorrectly named') checkvars = {} @@ -511,17 +657,53 @@ class DevtoolAddTests(DevtoolBase): checkvars['SRC_URI'] = url.replace(testver, '${PV}') self._test_recipe_contents(recipefile, checkvars, []) + def test_devtool_add_npm(self): + collections = get_bb_var('BBFILE_COLLECTIONS').split() + if "openembedded-layer" not in collections: + self.skipTest("Test needs meta-oe for nodejs") + + pn = 'savoirfairelinux-node-server-example' + pv = '1.0.0' + url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=' + pv + # Test devtool add + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake -c cleansstate %s' % pn) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('devtool add \'%s\'' % url) + self.assertExists(os.path.join(self.workspacedir, 'conf', 'layer.conf'), 'Workspace directory not created') + self.assertExists(os.path.join(self.workspacedir, 'recipes', pn, '%s_%s.bb' % (pn, pv)), 'Recipe not created') + self.assertExists(os.path.join(self.workspacedir, 'recipes', pn, pn, 'npm-shrinkwrap.json'), 'Shrinkwrap not created') + # Test devtool status + result = runCmd('devtool status') + self.assertIn(pn, result.output) + # Clean up anything in the workdir/sysroot/sstate cache (have to do this *after* devtool add since the recipe only exists then) + bitbake('%s -c cleansstate' % pn) + # Test devtool build + result = runCmd('devtool build %s' % pn) + + def test_devtool_add_python_egg_requires(self): + # Fetch source + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + testver = '0.14.0' + url = 'https://files.pythonhosted.org/packages/e9/9e/25d59f5043cf763833b2581c8027fa92342c4cf8ee523b498ecdf460c16d/uvicorn-%s.tar.gz' % testver + testrecipe = 'python3-uvicorn' + srcdir = os.path.join(tempdir, testrecipe) + # Test devtool add + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('devtool add %s %s -f %s' % (testrecipe, srcdir, url)) + class DevtoolModifyTests(DevtoolBase): - @OETestID(1164) def test_devtool_modify(self): import oe.path tempdir = tempfile.mkdtemp(prefix='devtoolqa') self.track_for_cleanup(tempdir) self.track_for_cleanup(self.workspacedir) - self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') self.add_command_to_tearDown('bitbake -c clean mdadm') + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') result = runCmd('devtool modify mdadm -x %s' % tempdir) self.assertExists(os.path.join(tempdir, 'Makefile'), 'Extracted source could not be found') self.assertExists(os.path.join(self.workspacedir, 'conf', 'layer.conf'), 'Workspace directory not created') @@ -571,7 +753,25 @@ class DevtoolModifyTests(DevtoolBase): result = runCmd('devtool status') self.assertNotIn('mdadm', result.output) - @OETestID(1620) + def test_devtool_modify_go(self): + import oe.path + from tempfile import TemporaryDirectory + with TemporaryDirectory(prefix='devtoolqa') as tempdir: + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake -c clean go-helloworld') + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('devtool modify go-helloworld -x %s' % tempdir) + self.assertExists( + oe.path.join(tempdir, 'src', 'golang.org', 'x', 'example', 'go.mod'), + 'Extracted source could not be found' + ) + self.assertExists( + oe.path.join(self.workspacedir, 'conf', 'layer.conf'), + 'Workspace directory not created' + ) + matches = glob.glob(oe.path.join(self.workspacedir, 'appends', 'go-helloworld_*.bbappend')) + self.assertTrue(matches, 'bbappend not created %s' % result.output) + def test_devtool_buildclean(self): def assertFile(path, *paths): f = os.path.join(path, *paths) @@ -590,8 +790,8 @@ class DevtoolModifyTests(DevtoolBase): self.track_for_cleanup(tempdir_m4) self.track_for_cleanup(builddir_m4) self.track_for_cleanup(self.workspacedir) - self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') self.add_command_to_tearDown('bitbake -c clean mdadm m4') + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') self.write_recipeinc('m4', 'EXTERNALSRC_BUILD = "%s"\ndo_clean() {\n\t:\n}\n' % builddir_m4) try: runCmd('devtool modify mdadm -x %s' % tempdir_mdadm) @@ -607,6 +807,7 @@ class DevtoolModifyTests(DevtoolBase): bitbake('mdadm m4 -c buildclean') assertNoFile(tempdir_mdadm, 'mdadm') assertNoFile(builddir_m4, 'src/m4') + runCmd('echo "#Trigger rebuild" >> %s/Makefile' % tempdir_mdadm) bitbake('mdadm m4 -c compile') assertFile(tempdir_mdadm, 'mdadm') assertFile(builddir_m4, 'src/m4') @@ -618,7 +819,6 @@ class DevtoolModifyTests(DevtoolBase): finally: self.delete_recipeinc('m4') - @OETestID(1166) def test_devtool_modify_invalid(self): # Try modifying some recipes tempdir = tempfile.mkdtemp(prefix='devtoolqa') @@ -626,7 +826,7 @@ class DevtoolModifyTests(DevtoolBase): self.track_for_cleanup(self.workspacedir) self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') - testrecipes = 'perf kernel-devsrc package-index core-image-minimal meta-toolchain packagegroup-core-sdk meta-ide-support'.split() + testrecipes = 'perf kernel-devsrc package-index core-image-minimal meta-toolchain packagegroup-core-sdk'.split() # Find actual name of gcc-source since it now includes the version - crude, but good enough for this purpose result = runCmd('bitbake-layers show-recipes gcc-source*') for line in result.output.splitlines(): @@ -647,7 +847,6 @@ class DevtoolModifyTests(DevtoolBase): self.assertNotEqual(result.status, 0, 'devtool modify on %s should have failed. devtool output: %s' % (testrecipe, result.output)) self.assertIn('ERROR: ', result.output, 'devtool modify on %s should have given an ERROR' % testrecipe) - @OETestID(1365) def test_devtool_modify_native(self): # Check preconditions self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory') @@ -659,7 +858,7 @@ class DevtoolModifyTests(DevtoolBase): bbclassextended = False inheritnative = False - testrecipes = 'mtools-native apt-native desktop-file-utils-native'.split() + testrecipes = 'cdrtools-native mtools-native apt-native desktop-file-utils-native'.split() for testrecipe in testrecipes: checkextend = 'native' in (get_bb_var('BBCLASSEXTEND', testrecipe) or '').split() if not bbclassextended: @@ -676,8 +875,40 @@ class DevtoolModifyTests(DevtoolBase): self.assertTrue(bbclassextended, 'None of these recipes are BBCLASSEXTENDed to native - need to adjust testrecipes list: %s' % ', '.join(testrecipes)) self.assertTrue(inheritnative, 'None of these recipes do "inherit native" - need to adjust testrecipes list: %s' % ', '.join(testrecipes)) + def test_devtool_modify_localfiles_only(self): + # Check preconditions + testrecipe = 'base-files' + src_uri = (get_bb_var('SRC_URI', testrecipe) or '').split() + foundlocalonly = False + correct_symlink = False + for item in src_uri: + if item.startswith('file://'): + if '.patch' not in item: + foundlocalonly = True + else: + foundlocalonly = False + break + self.assertTrue(foundlocalonly, 'This test expects the %s recipe to fetch local files only and it seems that it no longer does' % testrecipe) + # Clean up anything in the workdir/sysroot/sstate cache + bitbake('%s -c cleansstate' % testrecipe) + # Try modifying a recipe + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake -c clean %s' % testrecipe) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir)) + srcfile = os.path.join(tempdir, 'share/dot.bashrc') + self.assertExists(srcfile, 'Extracted source could not be found') + matches = glob.glob(os.path.join(self.workspacedir, 'appends', '%s_*.bbappend' % testrecipe)) + self.assertTrue(matches, 'bbappend not created') + # Test devtool status + result = runCmd('devtool status') + self.assertIn(testrecipe, result.output) + self.assertIn(tempdir, result.output) + # Try building + bitbake(testrecipe) - @OETestID(1165) def test_devtool_modify_git(self): # Check preconditions testrecipe = 'psplash' @@ -689,8 +920,8 @@ class DevtoolModifyTests(DevtoolBase): tempdir = tempfile.mkdtemp(prefix='devtoolqa') self.track_for_cleanup(tempdir) self.track_for_cleanup(self.workspacedir) - self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') self.add_command_to_tearDown('bitbake -c clean %s' % testrecipe) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir)) self.assertExists(os.path.join(tempdir, 'Makefile.am'), 'Extracted source could not be found') self.assertExists(os.path.join(self.workspacedir, 'conf', 'layer.conf'), 'Workspace directory not created. devtool output: %s' % result.output) @@ -705,7 +936,122 @@ class DevtoolModifyTests(DevtoolBase): # Try building bitbake(testrecipe) - @OETestID(1167) + def test_devtool_modify_git_no_extract(self): + # Check preconditions + testrecipe = 'psplash' + src_uri = get_bb_var('SRC_URI', testrecipe) + self.assertIn('git://', src_uri, 'This test expects the %s recipe to be a git recipe' % testrecipe) + # Clean up anything in the workdir/sysroot/sstate cache + bitbake('%s -c cleansstate' % testrecipe) + # Try modifying a recipe + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake -c clean %s' % testrecipe) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('git clone https://git.yoctoproject.org/psplash %s && devtool modify -n %s %s' % (tempdir, testrecipe, tempdir)) + self.assertExists(os.path.join(self.workspacedir, 'conf', 'layer.conf'), 'Workspace directory not created. devtool output: %s' % result.output) + matches = glob.glob(os.path.join(self.workspacedir, 'appends', 'psplash_*.bbappend')) + self.assertTrue(matches, 'bbappend not created') + # Test devtool status + result = runCmd('devtool status') + self.assertIn(testrecipe, result.output) + self.assertIn(tempdir, result.output) + + def test_devtool_modify_git_crates_subpath(self): + # This tests two things in devtool context: + # - that we support local git dependencies for cargo based recipe + # - that we support patches in SRC_URI when git url contains subpath parameter + + # Check preconditions: + # recipe inherits cargo + # git:// uri with a subpath as the main package + # some crate:// in SRC_URI + # others git:// in SRC_URI + # cointains a patch + testrecipe = 'hello-rs' + bb_vars = get_bb_vars(['SRC_URI', 'FILE', 'UNPACKDIR', 'CARGO_HOME'], testrecipe) + recipefile = bb_vars['FILE'] + unpackdir = bb_vars['UNPACKDIR'] + cargo_home = bb_vars['CARGO_HOME'] + src_uri = bb_vars['SRC_URI'].split() + self.assertTrue(src_uri[0].startswith('git://'), + 'This test expects the %s recipe to have a git repo has its main uri' % testrecipe) + self.assertIn(';subpath=', src_uri[0], + 'This test expects the %s recipe to have a git uri with subpath' % testrecipe) + self.assertTrue(any([uri.startswith('crate://') for uri in src_uri]), + 'This test expects the %s recipe to have some crates in its src uris' % testrecipe) + self.assertGreaterEqual(sum(map(lambda x:x.startswith('git://'), src_uri)), 2, + 'This test expects the %s recipe to have several git:// uris' % testrecipe) + self.assertTrue(any([uri.startswith('file://') and '.patch' in uri for uri in src_uri]), + 'This test expects the %s recipe to have a patch in its src uris' % testrecipe) + + self._test_recipe_contents(recipefile, {}, ['ptest-cargo']) + + # Clean up anything in the workdir/sysroot/sstate cache + bitbake('%s -c cleansstate' % testrecipe) + # Try modifying a recipe + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake -c clean %s' % testrecipe) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir)) + self.assertExists(os.path.join(tempdir, 'Cargo.toml'), 'Extracted source could not be found') + self.assertExists(os.path.join(self.workspacedir, 'conf', 'layer.conf'), 'Workspace directory not created. devtool output: %s' % result.output) + matches = glob.glob(os.path.join(self.workspacedir, 'appends', '%s_*.bbappend' % testrecipe)) + self.assertTrue(matches, 'bbappend not created') + # Test devtool status + result = runCmd('devtool status') + self.assertIn(testrecipe, result.output) + self.assertIn(tempdir, result.output) + # Check git repo + self._check_src_repo(tempdir) + # Check that the patch is correctly applied. + # The last commit message in the tree must contain the following note: + # Notes (devtool): + # original patch: <patchname> + # .. + patchname = None + for uri in src_uri: + if uri.startswith('file://') and '.patch' in uri: + patchname = uri.replace("file://", "").partition('.patch')[0] + '.patch' + self.assertIsNotNone(patchname) + result = runCmd('git -C %s log -1' % tempdir) + self.assertIn("Notes (devtool):\n original patch: %s" % patchname, result.output) + + # Configure the recipe to check that the git dependencies are correctly patched in cargo config + bitbake('-c configure %s' % testrecipe) + + cargo_config_path = os.path.join(cargo_home, 'config') + with open(cargo_config_path, "r") as f: + cargo_config_contents = [line.strip('\n') for line in f.readlines()] + + # Get back git dependencies of the recipe (ignoring the main one) + # and check that they are all correctly patched to be fetched locally + git_deps = [uri for uri in src_uri if uri.startswith("git://")][1:] + for git_dep in git_deps: + raw_url, _, raw_parms = git_dep.partition(";") + parms = {} + for parm in raw_parms.split(";"): + name_parm, _, value_parm = parm.partition('=') + parms[name_parm]=value_parm + self.assertIn('protocol', parms, 'git dependencies uri should contain the "protocol" parameter') + self.assertIn('name', parms, 'git dependencies uri should contain the "name" parameter') + self.assertIn('destsuffix', parms, 'git dependencies uri should contain the "destsuffix" parameter') + self.assertIn('type', parms, 'git dependencies uri should contain the "type" parameter') + self.assertEqual(parms['type'], 'git-dependency', 'git dependencies uri should have "type=git-dependency"') + raw_url = raw_url.replace("git://", '%s://' % parms['protocol']) + patch_line = '[patch."%s"]' % raw_url + path_patched = os.path.join(unpackdir, parms['destsuffix']) + path_override_line = '%s = { path = "%s" }' % (parms['name'], path_patched) + # Would have been better to use tomllib to read this file :/ + self.assertIn(patch_line, cargo_config_contents) + self.assertIn(path_override_line, cargo_config_contents) + + # Try to package the recipe + bitbake('-c package_qa %s' % testrecipe) + def test_devtool_modify_localfiles(self): # Check preconditions testrecipe = 'lighttpd' @@ -722,8 +1068,8 @@ class DevtoolModifyTests(DevtoolBase): tempdir = tempfile.mkdtemp(prefix='devtoolqa') self.track_for_cleanup(tempdir) self.track_for_cleanup(self.workspacedir) - self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') self.add_command_to_tearDown('bitbake -c clean %s' % testrecipe) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir)) self.assertExists(os.path.join(tempdir, 'configure.ac'), 'Extracted source could not be found') self.assertExists(os.path.join(self.workspacedir, 'conf', 'layer.conf'), 'Workspace directory not created') @@ -736,7 +1082,6 @@ class DevtoolModifyTests(DevtoolBase): # Try building bitbake(testrecipe) - @OETestID(1378) def test_devtool_modify_virtual(self): # Try modifying a virtual recipe virtrecipe = 'virtual/make' @@ -758,9 +1103,59 @@ class DevtoolModifyTests(DevtoolBase): self._check_src_repo(tempdir) # This is probably sufficient + def test_devtool_modify_overrides(self): + # Try modifying a recipe with patches in overrides + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('devtool modify devtool-patch-overrides -x %s' % (tempdir)) + + self._check_src_repo(tempdir) + source = os.path.join(tempdir, "source") + def check(branch, expected): + runCmd('git -C %s checkout %s' % (tempdir, branch)) + with open(source, "rt") as f: + content = f.read() + self.assertEqual(content, expected) + if self.td["MACHINE"] == "qemux86": + check('devtool', 'This is a test for qemux86\n') + elif self.td["MACHINE"] == "qemuarm": + check('devtool', 'This is a test for qemuarm\n') + else: + check('devtool', 'This is a test for something\n') + check('devtool-no-overrides', 'This is a test for something\n') + check('devtool-override-qemuarm', 'This is a test for qemuarm\n') + check('devtool-override-qemux86', 'This is a test for qemux86\n') + + def test_devtool_modify_multiple_sources(self): + # This test check that recipes fetching several sources can be used with devtool modify/build + # Check preconditions + testrecipe = 'bzip2' + src_uri = get_bb_var('SRC_URI', testrecipe) + src1 = 'https://' in src_uri + src2 = 'git://' in src_uri + self.assertTrue(src1 and src2, 'This test expects the %s recipe to fetch both a git source and a tarball and it seems that it no longer does' % testrecipe) + # Clean up anything in the workdir/sysroot/sstate cache + bitbake('%s -c cleansstate' % testrecipe) + # Try modifying a recipe + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake -c clean %s' % testrecipe) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir)) + self.assertEqual(result.status, 0, "Could not modify recipe %s. Output: %s" % (testrecipe, result.output)) + # Test devtool status + result = runCmd('devtool status') + self.assertIn(testrecipe, result.output) + self.assertIn(tempdir, result.output) + # Try building + result = bitbake(testrecipe) + self.assertEqual(result.status, 0, "Bitbake failed, exit code %s, output %s" % (result.status, result.output)) + class DevtoolUpdateTests(DevtoolBase): - @OETestID(1169) def test_devtool_update_recipe(self): # Check preconditions testrecipe = 'minicom' @@ -788,15 +1183,15 @@ class DevtoolUpdateTests(DevtoolBase): result = runCmd('git commit -m "Add a new file"', cwd=tempdir) self.add_command_to_tearDown('cd %s; rm %s/*.patch; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, testrecipe, os.path.basename(recipefile))) result = runCmd('devtool update-recipe %s' % testrecipe) + result = runCmd('git add minicom', cwd=os.path.dirname(recipefile)) expected_status = [(' M', '.*/%s$' % os.path.basename(recipefile)), - ('??', '.*/0001-Change-the-README.patch$'), - ('??', '.*/0002-Add-a-new-file.patch$')] + ('A ', '.*/0001-Change-the-README.patch$'), + ('A ', '.*/0002-Add-a-new-file.patch$')] self._check_repo_status(os.path.dirname(recipefile), expected_status) - @OETestID(1172) def test_devtool_update_recipe_git(self): # Check preconditions - testrecipe = 'mtd-utils' + testrecipe = 'mtd-utils-selftest' bb_vars = get_bb_vars(['FILE', 'SRC_URI'], testrecipe) recipefile = bb_vars['FILE'] src_uri = bb_vars['SRC_URI'] @@ -830,28 +1225,12 @@ class DevtoolUpdateTests(DevtoolBase): self._check_repo_status(os.path.dirname(recipefile), expected_status) result = runCmd('git diff %s' % os.path.basename(recipefile), cwd=os.path.dirname(recipefile)) - addlines = ['SRCREV = ".*"', 'SRC_URI = "git://git.infradead.org/mtd-utils.git"'] + addlines = ['SRCREV = ".*"', 'SRC_URI = "git://git.infradead.org/mtd-utils.git;branch=master"'] srcurilines = src_uri.split() srcurilines[0] = 'SRC_URI = "' + srcurilines[0] srcurilines.append('"') removelines = ['SRCREV = ".*"'] + srcurilines - for line in result.output.splitlines(): - if line.startswith('+++') or line.startswith('---'): - continue - elif line.startswith('+'): - matched = False - for item in addlines: - if re.match(item, line[1:].strip()): - matched = True - break - self.assertTrue(matched, 'Unexpected diff add line: %s' % line) - elif line.startswith('-'): - matched = False - for item in removelines: - if re.match(item, line[1:].strip()): - matched = True - break - self.assertTrue(matched, 'Unexpected diff remove line: %s' % line) + self._check_diff(result.output, addlines, removelines) # Now try with auto mode runCmd('cd %s; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, os.path.basename(recipefile))) result = runCmd('devtool update-recipe %s' % testrecipe) @@ -863,7 +1242,6 @@ class DevtoolUpdateTests(DevtoolBase): ('??', '%s/0002-Add-a-new-file.patch' % relpatchpath)] self._check_repo_status(os.path.dirname(recipefile), expected_status) - @OETestID(1170) def test_devtool_update_recipe_append(self): # Check preconditions testrecipe = 'mdadm' @@ -902,7 +1280,7 @@ class DevtoolUpdateTests(DevtoolBase): self.assertExists(patchfile, 'Patch file not created') # Check bbappend contents - expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n', + expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', '\n', 'SRC_URI += "file://0001-Add-our-custom-version.patch"\n', '\n'] @@ -914,15 +1292,16 @@ class DevtoolUpdateTests(DevtoolBase): with open(bbappendfile, 'r') as f: self.assertEqual(expectedlines, f.readlines()) # Drop new commit and check patch gets deleted - result = runCmd('git reset HEAD^', cwd=tempsrcdir) + result = runCmd('git reset HEAD^ --hard', cwd=tempsrcdir) result = runCmd('devtool update-recipe %s -a %s' % (testrecipe, templayerdir)) self.assertNotExists(patchfile, 'Patch file not deleted') - expectedlines2 = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n', + expectedlines2 = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', '\n'] with open(bbappendfile, 'r') as f: self.assertEqual(expectedlines2, f.readlines()) # Put commit back and check we can run it if layer isn't in bblayers.conf os.remove(bbappendfile) + result = runCmd("sed 's!\\(#define VERSION\\W*\"[^\"]*\\)\"!\\1-custom\"!' -i ReadMe.c", cwd=tempsrcdir) result = runCmd('git commit -a -m "Add our custom version"', cwd=tempsrcdir) result = runCmd('bitbake-layers remove-layer %s' % templayerdir, cwd=self.builddir) result = runCmd('devtool update-recipe %s -a %s' % (testrecipe, templayerdir)) @@ -932,13 +1311,13 @@ class DevtoolUpdateTests(DevtoolBase): self.assertEqual(expectedlines, f.readlines()) # Deleting isn't expected to work under these circumstances - @OETestID(1171) def test_devtool_update_recipe_append_git(self): # Check preconditions - testrecipe = 'mtd-utils' - bb_vars = get_bb_vars(['FILE', 'SRC_URI'], testrecipe) + testrecipe = 'mtd-utils-selftest' + bb_vars = get_bb_vars(['FILE', 'SRC_URI', 'LAYERSERIES_CORENAMES'], testrecipe) recipefile = bb_vars['FILE'] src_uri = bb_vars['SRC_URI'] + corenames = bb_vars['LAYERSERIES_CORENAMES'] self.assertIn('git://', src_uri, 'This test expects the %s recipe to be a git recipe' % testrecipe) for entry in src_uri.split(): if entry.startswith('git://'): @@ -969,7 +1348,7 @@ class DevtoolUpdateTests(DevtoolBase): f.write('BBFILE_PATTERN_oeselftesttemplayer = "^${LAYERDIR}/"\n') f.write('BBFILE_PRIORITY_oeselftesttemplayer = "999"\n') f.write('BBFILE_PATTERN_IGNORE_EMPTY_oeselftesttemplayer = "1"\n') - f.write('LAYERSERIES_COMPAT_oeselftesttemplayer = "${LAYERSERIES_COMPAT_core}"\n') + f.write('LAYERSERIES_COMPAT_oeselftesttemplayer = "%s"\n' % corenames) self.add_command_to_tearDown('bitbake-layers remove-layer %s || true' % templayerdir) result = runCmd('bitbake-layers add-layer %s' % templayerdir, cwd=self.builddir) # Create the bbappend @@ -997,7 +1376,7 @@ class DevtoolUpdateTests(DevtoolBase): with open(bbappendfile, 'r') as f: self.assertEqual(expectedlines, set(f.readlines())) # Drop new commit and check SRCREV changes - result = runCmd('git reset HEAD^', cwd=tempsrcdir) + result = runCmd('git reset HEAD^ --hard', cwd=tempsrcdir) result = runCmd('devtool update-recipe -m srcrev %s -a %s' % (testrecipe, templayerdir)) self.assertNotExists(os.path.join(appenddir, testrecipe), 'Patch directory should not be created') result = runCmd('git rev-parse HEAD', cwd=tempsrcdir) @@ -1009,6 +1388,7 @@ class DevtoolUpdateTests(DevtoolBase): self.assertEqual(expectedlines, set(f.readlines())) # Put commit back and check we can run it if layer isn't in bblayers.conf os.remove(bbappendfile) + result = runCmd('echo "# Additional line" >> Makefile.am', cwd=tempsrcdir) result = runCmd('git commit -a -m "Change the Makefile"', cwd=tempsrcdir) result = runCmd('bitbake-layers remove-layer %s' % templayerdir, cwd=self.builddir) result = runCmd('devtool update-recipe -m srcrev %s -a %s' % (testrecipe, templayerdir)) @@ -1023,7 +1403,6 @@ class DevtoolUpdateTests(DevtoolBase): self.assertEqual(expectedlines, set(f.readlines())) # Deleting isn't expected to work under these circumstances - @OETestID(1370) def test_devtool_update_recipe_local_files(self): """Check that local source files are copied over instead of patched""" testrecipe = 'makedevs' @@ -1041,23 +1420,39 @@ class DevtoolUpdateTests(DevtoolBase): # Try building just to ensure we haven't broken that bitbake("%s" % testrecipe) # Edit / commit local source - runCmd('echo "/* Foobar */" >> oe-local-files/makedevs.c', cwd=tempdir) - runCmd('echo "Foo" > oe-local-files/new-local', cwd=tempdir) + runCmd('echo "/* Foobar */" >> makedevs.c', cwd=tempdir) + runCmd('echo "Foo" > new-local', cwd=tempdir) runCmd('echo "Bar" > new-file', cwd=tempdir) runCmd('git add new-file', cwd=tempdir) runCmd('git commit -m "Add new file"', cwd=tempdir) - self.add_command_to_tearDown('cd %s; git clean -fd .; git checkout .' % - os.path.dirname(recipefile)) + runCmd('git add new-local', cwd=tempdir) runCmd('devtool update-recipe %s' % testrecipe) expected_status = [(' M', '.*/%s$' % os.path.basename(recipefile)), (' M', '.*/makedevs/makedevs.c$'), ('??', '.*/makedevs/new-local$'), ('??', '.*/makedevs/0001-Add-new-file.patch$')] self._check_repo_status(os.path.dirname(recipefile), expected_status) - - @OETestID(1371) - def test_devtool_update_recipe_local_files_2(self): - """Check local source files support when oe-local-files is in Git""" + # Now try to update recipe in another layer, so first, clean it + runCmd('cd %s; git clean -fd .; git checkout .' % os.path.dirname(recipefile)) + # Create a temporary layer and add it to bblayers.conf + self._create_temp_layer(templayerdir, True, 'templayer') + # Update recipe in templayer + result = runCmd('devtool update-recipe %s -a %s' % (testrecipe, templayerdir)) + self.assertNotIn('WARNING:', result.output) + # Check recipe is still clean + self._check_repo_status(os.path.dirname(recipefile), []) + splitpath = os.path.dirname(recipefile).split(os.sep) + appenddir = os.path.join(templayerdir, splitpath[-2], splitpath[-1]) + bbappendfile = self._check_bbappend(testrecipe, recipefile, appenddir) + patchfile = os.path.join(appenddir, testrecipe, '0001-Add-new-file.patch') + new_local_file = os.path.join(appenddir, testrecipe, 'new_local') + local_file = os.path.join(appenddir, testrecipe, 'makedevs.c') + self.assertExists(patchfile, 'Patch file 0001-Add-new-file.patch not created') + self.assertExists(local_file, 'File makedevs.c not created') + self.assertExists(patchfile, 'File new_local not created') + + def _test_devtool_update_recipe_local_files_2(self): + """Check local source files support when editing local files in Git""" testrecipe = 'devtool-test-local' recipefile = get_bb_var('FILE', testrecipe) recipedir = os.path.dirname(recipefile) @@ -1072,17 +1467,13 @@ class DevtoolUpdateTests(DevtoolBase): result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir)) # Check git repo self._check_src_repo(tempdir) - # Add oe-local-files to Git - runCmd('rm oe-local-files/.gitignore', cwd=tempdir) - runCmd('git add oe-local-files', cwd=tempdir) - runCmd('git commit -m "Add local sources"', cwd=tempdir) # Edit / commit local sources - runCmd('echo "# Foobar" >> oe-local-files/file1', cwd=tempdir) + runCmd('echo "# Foobar" >> file1', cwd=tempdir) runCmd('git commit -am "Edit existing file"', cwd=tempdir) - runCmd('git rm oe-local-files/file2', cwd=tempdir) + runCmd('git rm file2', cwd=tempdir) runCmd('git commit -m"Remove file"', cwd=tempdir) - runCmd('echo "Foo" > oe-local-files/new-local', cwd=tempdir) - runCmd('git add oe-local-files/new-local', cwd=tempdir) + runCmd('echo "Foo" > new-local', cwd=tempdir) + runCmd('git add new-local', cwd=tempdir) runCmd('git commit -m "Add new local file"', cwd=tempdir) runCmd('echo "Gar" > new-file', cwd=tempdir) runCmd('git add new-file', cwd=tempdir) @@ -1091,7 +1482,7 @@ class DevtoolUpdateTests(DevtoolBase): os.path.dirname(recipefile)) # Checkout unmodified file to working copy -> devtool should still pick # the modified version from HEAD - runCmd('git checkout HEAD^ -- oe-local-files/file1', cwd=tempdir) + runCmd('git checkout HEAD^ -- file1', cwd=tempdir) runCmd('devtool update-recipe %s' % testrecipe) expected_status = [(' M', '.*/%s$' % os.path.basename(recipefile)), (' M', '.*/file1$'), @@ -1100,7 +1491,59 @@ class DevtoolUpdateTests(DevtoolBase): ('??', '.*/0001-Add-new-file.patch$')] self._check_repo_status(os.path.dirname(recipefile), expected_status) - @OETestID(1627) + def test_devtool_update_recipe_with_gitignore(self): + # First, modify the recipe + testrecipe = 'devtool-test-ignored' + bb_vars = get_bb_vars(['FILE'], testrecipe) + recipefile = bb_vars['FILE'] + patchfile = os.path.join(os.path.dirname(recipefile), testrecipe, testrecipe + '.patch') + newpatchfile = os.path.join(os.path.dirname(recipefile), testrecipe, testrecipe + '.patch.expected') + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + # (don't bother with cleaning the recipe on teardown, we won't be building it) + result = runCmd('devtool modify %s' % testrecipe) + self.add_command_to_tearDown('cd %s; rm %s/*; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, testrecipe, os.path.basename(recipefile))) + result = runCmd('devtool finish --force-patch-refresh %s meta-selftest' % testrecipe) + # Check recipe got changed as expected + with open(newpatchfile, 'r') as f: + desiredlines = f.readlines() + with open(patchfile, 'r') as f: + newlines = f.readlines() + # Ignore the initial lines, because oe-selftest creates own meta-selftest repo + # which changes the metadata subject which is added into the patch, but keep + # .patch.expected as it is in case someone runs devtool finish --force-patch-refresh + # devtool-test-ignored manually, then it should generate exactly the same .patch file + self.assertEqual(desiredlines[5:], newlines[5:]) + + def test_devtool_update_recipe_long_filename(self): + # First, modify the recipe + testrecipe = 'devtool-test-long-filename' + bb_vars = get_bb_vars(['FILE'], testrecipe) + recipefile = bb_vars['FILE'] + patchfilename = '0001-I-ll-patch-you-only-if-devtool-lets-me-to-do-it-corr.patch' + patchfile = os.path.join(os.path.dirname(recipefile), testrecipe, patchfilename) + newpatchfile = os.path.join(os.path.dirname(recipefile), testrecipe, patchfilename + '.expected') + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + # (don't bother with cleaning the recipe on teardown, we won't be building it) + result = runCmd('devtool modify %s' % testrecipe) + self.add_command_to_tearDown('cd %s; rm %s/*; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, testrecipe, os.path.basename(recipefile))) + result = runCmd('devtool finish --force-patch-refresh %s meta-selftest' % testrecipe) + # Check recipe got changed as expected + with open(newpatchfile, 'r') as f: + desiredlines = f.readlines() + with open(patchfile, 'r') as f: + newlines = f.readlines() + # Ignore the initial lines, because oe-selftest creates own meta-selftest repo + # which changes the metadata subject which is added into the patch, but keep + # .patch.expected as it is in case someone runs devtool finish --force-patch-refresh + # devtool-test-ignored manually, then it should generate exactly the same .patch file + self.assertEqual(desiredlines[5:], newlines[5:]) + def test_devtool_update_recipe_local_files_3(self): # First, modify the recipe testrecipe = 'devtool-test-localonly' @@ -1114,13 +1557,12 @@ class DevtoolUpdateTests(DevtoolBase): # (don't bother with cleaning the recipe on teardown, we won't be building it) result = runCmd('devtool modify %s' % testrecipe) # Modify one file - runCmd('echo "Another line" >> file2', cwd=os.path.join(self.workspacedir, 'sources', testrecipe, 'oe-local-files')) + runCmd('echo "Another line" >> file2', cwd=os.path.join(self.workspacedir, 'sources', testrecipe)) self.add_command_to_tearDown('cd %s; rm %s/*; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, testrecipe, os.path.basename(recipefile))) result = runCmd('devtool update-recipe %s' % testrecipe) expected_status = [(' M', '.*/%s/file2$' % testrecipe)] self._check_repo_status(os.path.dirname(recipefile), expected_status) - @OETestID(1629) def test_devtool_update_recipe_local_patch_gz(self): # First, modify the recipe testrecipe = 'devtool-test-patch-gz' @@ -1138,7 +1580,7 @@ class DevtoolUpdateTests(DevtoolBase): # Modify one file srctree = os.path.join(self.workspacedir, 'sources', testrecipe) runCmd('echo "Another line" >> README', cwd=srctree) - runCmd('git commit -a --amend --no-edit', cwd=srctree) + runCmd('git commit -a --amend --no-edit --no-verify', cwd=srctree) self.add_command_to_tearDown('cd %s; rm %s/*; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, testrecipe, os.path.basename(recipefile))) result = runCmd('devtool update-recipe %s' % testrecipe) expected_status = [(' M', '.*/%s/readme.patch.gz$' % testrecipe)] @@ -1148,7 +1590,6 @@ class DevtoolUpdateTests(DevtoolBase): if 'gzip compressed data' not in result.output: self.fail('New patch file is not gzipped - file reports:\n%s' % result.output) - @OETestID(1628) def test_devtool_update_recipe_local_files_subdir(self): # Try devtool update-recipe on a recipe that has a file with subdir= set in # SRC_URI such that it overwrites a file that was in an archive that @@ -1175,9 +1616,123 @@ class DevtoolUpdateTests(DevtoolBase): expected_status = [] self._check_repo_status(os.path.dirname(recipefile), expected_status) + def test_devtool_finish_modify_git_subdir(self): + # Check preconditions + testrecipe = 'dos2unix' + self.append_config('ERROR_QA:remove:pn-dos2unix = "patch-status"\n') + bb_vars = get_bb_vars(['SRC_URI', 'S', 'WORKDIR', 'FILE'], testrecipe) + self.assertIn('git://', bb_vars['SRC_URI'], 'This test expects the %s recipe to be a git recipe' % testrecipe) + workdir_git = '%s/git/' % bb_vars['WORKDIR'] + if not bb_vars['S'].startswith(workdir_git): + self.fail('This test expects the %s recipe to be building from a subdirectory of the git repo' % testrecipe) + subdir = bb_vars['S'].split(workdir_git, 1)[1] + # Clean up anything in the workdir/sysroot/sstate cache + bitbake('%s -c cleansstate' % testrecipe) + # Try modifying a recipe + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake -c clean %s' % testrecipe) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir)) + testsrcfile = os.path.join(tempdir, subdir, 'dos2unix.c') + self.assertExists(testsrcfile, 'Extracted source could not be found') + self.assertExists(os.path.join(self.workspacedir, 'conf', 'layer.conf'), 'Workspace directory not created. devtool output: %s' % result.output) + self.assertNotExists(os.path.join(tempdir, subdir, '.git'), 'Subdirectory has been initialised as a git repo') + # Check git repo + self._check_src_repo(tempdir) + # Modify file + runCmd("sed -i '1s:^:/* Add a comment */\\n:' %s" % testsrcfile) + result = runCmd('git commit -a -m "Add a comment"', cwd=tempdir) + # Now try updating original recipe + recipefile = bb_vars['FILE'] + recipedir = os.path.dirname(recipefile) + self.add_command_to_tearDown('cd %s; rm -f %s/*.patch; git checkout .' % (recipedir, testrecipe)) + result = runCmd('devtool update-recipe %s' % testrecipe) + expected_status = [(' M', '.*/%s$' % os.path.basename(recipefile)), + ('??', '.*/%s/%s/$' % (testrecipe, testrecipe))] + self._check_repo_status(os.path.dirname(recipefile), expected_status) + result = runCmd('git diff %s' % os.path.basename(recipefile), cwd=os.path.dirname(recipefile)) + removelines = ['SRC_URI = "git://.*"'] + addlines = [ + 'SRC_URI = "git://.* \\\\', + 'file://0001-Add-a-comment.patch;patchdir=.. \\\\', + '"' + ] + self._check_diff(result.output, addlines, removelines) + # Put things back so we can run devtool finish on a different layer + runCmd('cd %s; rm -f %s/*.patch; git checkout .' % (recipedir, testrecipe)) + # Run devtool finish + res = re.search('recipes-.*', recipedir) + self.assertTrue(res, 'Unable to find recipe subdirectory') + recipesubdir = res[0] + self.add_command_to_tearDown('rm -rf %s' % os.path.join(self.testlayer_path, recipesubdir)) + result = runCmd('devtool finish %s meta-selftest' % testrecipe) + # Check bbappend file contents + appendfn = os.path.join(self.testlayer_path, recipesubdir, '%s_%%.bbappend' % testrecipe) + with open(appendfn, 'r') as f: + appendlines = f.readlines() + expected_appendlines = [ + 'FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', + '\n', + 'SRC_URI += "file://0001-Add-a-comment.patch;patchdir=.."\n', + '\n' + ] + self.assertEqual(appendlines, expected_appendlines) + self.assertExists(os.path.join(os.path.dirname(appendfn), testrecipe, '0001-Add-a-comment.patch')) + # Try building + bitbake('%s -c patch' % testrecipe) + + def test_devtool_git_submodules(self): + # This tests if we can add a patch in a git submodule and extract it properly using devtool finish + # Check preconditions + self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory') + self.track_for_cleanup(self.workspacedir) + recipe = 'vulkan-samples' + src_uri = get_bb_var('SRC_URI', recipe) + self.assertIn('gitsm://', src_uri, 'This test expects the %s recipe to be a git recipe with submodules' % recipe) + oldrecipefile = get_bb_var('FILE', recipe) + recipedir = os.path.dirname(oldrecipefile) + result = runCmd('git status --porcelain .', cwd=recipedir) + if result.output.strip(): + self.fail('Recipe directory for %s contains uncommitted changes' % recipe) + self.assertIn('/meta/', recipedir) + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('devtool modify %s %s' % (recipe, tempdir)) + self.assertExists(os.path.join(tempdir, 'CMakeLists.txt'), 'Extracted source could not be found') + # Test devtool status + result = runCmd('devtool status') + self.assertIn(recipe, result.output) + self.assertIn(tempdir, result.output) + # Modify a source file in a submodule, (grab the first one) + result = runCmd('git submodule --quiet foreach \'echo $sm_path\'', cwd=tempdir) + submodule = result.output.splitlines()[0] + submodule_path = os.path.join(tempdir, submodule) + runCmd('echo "#This is a first comment" >> testfile', cwd=submodule_path) + result = runCmd('git status --porcelain . ', cwd=submodule_path) + self.assertIn("testfile", result.output) + runCmd('git add testfile; git commit -m "Adding a new file"', cwd=submodule_path) + + # Try finish to the original layer + self.add_command_to_tearDown('rm -rf %s ; cd %s ; git checkout %s' % (recipedir, os.path.dirname(recipedir), recipedir)) + runCmd('devtool finish -f %s meta' % recipe) + result = runCmd('devtool status') + self.assertNotIn(recipe, result.output, 'Recipe should have been reset by finish but wasn\'t') + self.assertNotExists(os.path.join(self.workspacedir, 'recipes', recipe), 'Recipe directory should not exist after finish') + expected_status = [(' M', '.*/%s$' % os.path.basename(oldrecipefile)), + ('??', '.*/.*-Adding-a-new-file.patch$')] + self._check_repo_status(recipedir, expected_status) + # Make sure the patch is added to the recipe with the correct "patchdir" option + result = runCmd('git diff .', cwd=recipedir) + addlines = [ + 'file://0001-Adding-a-new-file.patch;patchdir=%s \\\\' % submodule + ] + self._check_diff(result.output, addlines, []) + class DevtoolExtractTests(DevtoolBase): - @OETestID(1163) def test_devtool_extract(self): tempdir = tempfile.mkdtemp(prefix='devtoolqa') # Try devtool extract @@ -1188,7 +1743,6 @@ class DevtoolExtractTests(DevtoolBase): self.assertExists(os.path.join(tempdir, 'Makefile.am'), 'Extracted source could not be found') self._check_src_repo(tempdir) - @OETestID(1379) def test_devtool_extract_virtual(self): tempdir = tempfile.mkdtemp(prefix='devtoolqa') # Try devtool extract @@ -1199,7 +1753,6 @@ class DevtoolExtractTests(DevtoolBase): self.assertExists(os.path.join(tempdir, 'Makefile.am'), 'Extracted source could not be found') self._check_src_repo(tempdir) - @OETestID(1168) def test_devtool_reset_all(self): tempdir = tempfile.mkdtemp(prefix='devtoolqa') self.track_for_cleanup(tempdir) @@ -1226,34 +1779,15 @@ class DevtoolExtractTests(DevtoolBase): matches2 = glob.glob(stampprefix2 + '*') self.assertFalse(matches2, 'Stamp files exist for recipe %s that should have been cleaned' % testrecipe2) - @OETestID(1272) + @OETestTag("runqemu") def test_devtool_deploy_target(self): - # NOTE: Whilst this test would seemingly be better placed as a runtime test, - # unfortunately the runtime tests run under bitbake and you can't run - # devtool within bitbake (since devtool needs to run bitbake itself). - # Additionally we are testing build-time functionality as well, so - # really this has to be done as an oe-selftest test. - # - # Check preconditions - machine = get_bb_var('MACHINE') - if not machine.startswith('qemu'): - self.skipTest('This test only works with qemu machines') - if not os.path.exists('/etc/runqemu-nosudo'): - self.skipTest('You must set up tap devices with scripts/runqemu-gen-tapdevs before running this test') - result = runCmd('PATH="$PATH:/sbin:/usr/sbin" ip tuntap show', ignore_status=True) - if result.status != 0: - result = runCmd('PATH="$PATH:/sbin:/usr/sbin" ifconfig -a', ignore_status=True) - if result.status != 0: - self.skipTest('Failed to determine if tap devices exist with ifconfig or ip: %s' % result.output) - for line in result.output.splitlines(): - if line.startswith('tap'): - break - else: - self.skipTest('No tap devices found - you must set up tap devices with scripts/runqemu-gen-tapdevs before running this test') + self._check_runqemu_prerequisites() self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory') # Definitions testrecipe = 'mdadm' testfile = '/sbin/mdadm' + if "usrmerge" in get_bb_var('DISTRO_FEATURES'): + testfile = '/usr/sbin/mdadm' testimage = 'oe-selftest-image' testcommand = '/sbin/mdadm --help' # Build an image to run @@ -1267,8 +1801,8 @@ class DevtoolExtractTests(DevtoolBase): tempdir = tempfile.mkdtemp(prefix='devtoolqa') self.track_for_cleanup(tempdir) self.track_for_cleanup(self.workspacedir) - self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') self.add_command_to_tearDown('bitbake -c clean %s' % testrecipe) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir)) # Test that deploy-target at this point fails (properly) result = runCmd('devtool deploy-target -n %s root@localhost' % testrecipe, ignore_status=True) @@ -1312,15 +1846,14 @@ class DevtoolExtractTests(DevtoolBase): result = runCmd('ssh %s root@%s %s' % (sshargs, qemu.ip, testcommand), ignore_status=True) self.assertNotEqual(result, 0, 'undeploy-target did not remove command as it should have') - @OETestID(1366) def test_devtool_build_image(self): """Test devtool build-image plugin""" # Check preconditions self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory') image = 'core-image-minimal' self.track_for_cleanup(self.workspacedir) - self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') self.add_command_to_tearDown('bitbake -c clean %s' % image) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') bitbake('%s -c clean' % image) # Add target and native recipes to workspace recipes = ['mdadm', 'parted-native'] @@ -1348,7 +1881,14 @@ class DevtoolExtractTests(DevtoolBase): class DevtoolUpgradeTests(DevtoolBase): - @OETestID(1367) + def setUp(self): + super().setUp() + try: + runCmd("git config --global user.name") + runCmd("git config --global user.email") + except: + self.skip("Git user.name and user.email must be set") + def test_devtool_upgrade(self): # Check preconditions self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory') @@ -1393,7 +1933,6 @@ class DevtoolUpgradeTests(DevtoolBase): self.assertNotIn(recipe, result.output) self.assertNotExists(os.path.join(self.workspacedir, 'recipes', recipe), 'Recipe directory should not exist after resetting') - @OETestID(1433) def test_devtool_upgrade_git(self): # Check preconditions self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory') @@ -1430,7 +1969,100 @@ class DevtoolUpgradeTests(DevtoolBase): self.assertNotIn(recipe, result.output) self.assertNotExists(os.path.join(self.workspacedir, 'recipes', recipe), 'Recipe directory should not exist after resetting') - @OETestID(1352) + def test_devtool_upgrade_drop_md5sum(self): + # Check preconditions + self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory') + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + # For the moment, we are using a real recipe. + recipe = 'devtool-upgrade-test3' + version = '1.6.0' + oldrecipefile = get_bb_var('FILE', recipe) + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + # Check upgrade. Code does not check if new PV is older or newer that current PV, so, it may be that + # we are downgrading instead of upgrading. + result = runCmd('devtool upgrade %s %s -V %s' % (recipe, tempdir, version)) + # Check new recipe file is present + newrecipefile = os.path.join(self.workspacedir, 'recipes', recipe, '%s_%s.bb' % (recipe, version)) + self.assertExists(newrecipefile, 'Recipe file should exist after upgrade') + # Check recipe got changed as expected + with open(oldrecipefile + '.upgraded', 'r') as f: + desiredlines = f.readlines() + with open(newrecipefile, 'r') as f: + newlines = f.readlines() + self.assertEqual(desiredlines, newlines) + + def test_devtool_upgrade_all_checksums(self): + # Check preconditions + self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory') + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + # For the moment, we are using a real recipe. + recipe = 'devtool-upgrade-test4' + version = '1.6.0' + oldrecipefile = get_bb_var('FILE', recipe) + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + # Check upgrade. Code does not check if new PV is older or newer that current PV, so, it may be that + # we are downgrading instead of upgrading. + result = runCmd('devtool upgrade %s %s -V %s' % (recipe, tempdir, version)) + # Check new recipe file is present + newrecipefile = os.path.join(self.workspacedir, 'recipes', recipe, '%s_%s.bb' % (recipe, version)) + self.assertExists(newrecipefile, 'Recipe file should exist after upgrade') + # Check recipe got changed as expected + with open(oldrecipefile + '.upgraded', 'r') as f: + desiredlines = f.readlines() + with open(newrecipefile, 'r') as f: + newlines = f.readlines() + self.assertEqual(desiredlines, newlines) + + def test_devtool_upgrade_recipe_upgrade_extra_tasks(self): + # Check preconditions + self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory') + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + recipe = 'python3-guessing-game' + version = '0.2.0' + commit = '40cf004c2772ffa20ea803fa3be1528a75be3e98' + oldrecipefile = get_bb_var('FILE', recipe) + oldcratesincfile = os.path.join(os.path.dirname(oldrecipefile), os.path.basename(oldrecipefile).strip('_git.bb') + '-crates.inc') + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + # Check that recipe is not already under devtool control + result = runCmd('devtool status') + self.assertNotIn(recipe, result.output) + # Check upgrade + result = runCmd('devtool upgrade %s %s --version %s --srcrev %s' % (recipe, tempdir, version, commit)) + # Check if srctree at least is populated + self.assertTrue(len(os.listdir(tempdir)) > 0, 'srctree (%s) should be populated with new (%s) source code' % (tempdir, commit)) + # Check new recipe file and new -crates.inc files are present + newrecipefile = os.path.join(self.workspacedir, 'recipes', recipe, os.path.basename(oldrecipefile)) + newcratesincfile = os.path.join(self.workspacedir, 'recipes', recipe, os.path.basename(oldcratesincfile)) + self.assertExists(newrecipefile, 'Recipe file should exist after upgrade') + self.assertExists(newcratesincfile, 'Recipe crates.inc file should exist after upgrade') + # Check devtool status and make sure recipe is present + result = runCmd('devtool status') + self.assertIn(recipe, result.output) + self.assertIn(tempdir, result.output) + # Check recipe got changed as expected + with open(oldrecipefile + '.upgraded', 'r') as f: + desiredlines = f.readlines() + with open(newrecipefile, 'r') as f: + newlines = f.readlines() + self.assertEqual(desiredlines, newlines) + # Check crates.inc got changed as expected + with open(oldcratesincfile + '.upgraded', 'r') as f: + desiredlines = f.readlines() + with open(newcratesincfile, 'r') as f: + newlines = f.readlines() + self.assertEqual(desiredlines, newlines) + # Check devtool reset recipe + result = runCmd('devtool reset %s -n' % recipe) + result = runCmd('devtool status') + self.assertNotIn(recipe, result.output) + self.assertNotExists(os.path.join(self.workspacedir, 'recipes', recipe), 'Recipe directory should not exist after resetting') + def test_devtool_layer_plugins(self): """Test that devtool can use plugins from other layers. @@ -1449,14 +2081,29 @@ class DevtoolUpgradeTests(DevtoolBase): for p in paths: dstdir = os.path.join(dstdir, p) if not os.path.exists(dstdir): - os.makedirs(dstdir) - self.track_for_cleanup(dstdir) + try: + os.makedirs(dstdir) + except PermissionError: + return False + except OSError as e: + if e.errno == errno.EROFS: + return False + else: + raise e + if p == "lib": + # Can race with other tests + self.add_command_to_tearDown('rmdir --ignore-fail-on-non-empty %s' % dstdir) + else: + self.track_for_cleanup(dstdir) dstfile = os.path.join(dstdir, os.path.basename(srcfile)) if srcfile != dstfile: - shutil.copy(srcfile, dstfile) + try: + shutil.copy(srcfile, dstfile) + except PermissionError: + return False self.track_for_cleanup(dstfile) + return True - @OETestID(1625) def test_devtool_load_plugin(self): """Test that devtool loads only the first found plugin in BBPATH.""" @@ -1473,15 +2120,17 @@ class DevtoolUpgradeTests(DevtoolBase): plugincontent = fh.readlines() try: self.assertIn('meta-selftest', srcfile, 'wrong bbpath plugin found') - for path in searchpath: - self._copy_file_with_cleanup(srcfile, path, 'lib', 'devtool') + searchpath = [ + path for path in searchpath + if self._copy_file_with_cleanup(srcfile, path, 'lib', 'devtool') + ] result = runCmd("devtool --quiet count") self.assertEqual(result.output, '1') result = runCmd("devtool --quiet multiloaded") self.assertEqual(result.output, "no") for path in searchpath: result = runCmd("devtool --quiet bbdir") - self.assertEqual(result.output, path) + self.assertEqual(os.path.realpath(result.output), os.path.realpath(path)) os.unlink(os.path.join(result.output, 'lib', 'devtool', 'bbpath.py')) finally: with open(srcfile, 'w') as fh: @@ -1521,12 +2170,13 @@ class DevtoolUpgradeTests(DevtoolBase): recipedir = os.path.dirname(oldrecipefile) olddir = os.path.join(recipedir, recipe + '-' + oldversion) patchfn = '0001-Add-a-note-line-to-the-quick-reference.patch' + backportedpatchfn = 'backported.patch' self.assertExists(os.path.join(olddir, patchfn), 'Original patch file does not exist') - return recipe, oldrecipefile, recipedir, olddir, newversion, patchfn + self.assertExists(os.path.join(olddir, backportedpatchfn), 'Backported patch file does not exist') + return recipe, oldrecipefile, recipedir, olddir, newversion, patchfn, backportedpatchfn - @OETestID(1623) def test_devtool_finish_upgrade_origlayer(self): - recipe, oldrecipefile, recipedir, olddir, newversion, patchfn = self._setup_test_devtool_finish_upgrade() + recipe, oldrecipefile, recipedir, olddir, newversion, patchfn, backportedpatchfn = self._setup_test_devtool_finish_upgrade() # Ensure the recipe is where we think it should be (so that cleanup doesn't trash things) self.assertIn('/meta-selftest/', recipedir) # Try finish to the original layer @@ -1537,15 +2187,23 @@ class DevtoolUpgradeTests(DevtoolBase): self.assertNotExists(os.path.join(self.workspacedir, 'recipes', recipe), 'Recipe directory should not exist after finish') self.assertNotExists(oldrecipefile, 'Old recipe file should have been deleted but wasn\'t') self.assertNotExists(os.path.join(olddir, patchfn), 'Old patch file should have been deleted but wasn\'t') + self.assertNotExists(os.path.join(olddir, backportedpatchfn), 'Old backported patch file should have been deleted but wasn\'t') newrecipefile = os.path.join(recipedir, '%s_%s.bb' % (recipe, newversion)) newdir = os.path.join(recipedir, recipe + '-' + newversion) self.assertExists(newrecipefile, 'New recipe file should have been copied into existing layer but wasn\'t') self.assertExists(os.path.join(newdir, patchfn), 'Patch file should have been copied into new directory but wasn\'t') + self.assertNotExists(os.path.join(newdir, backportedpatchfn), 'Backported patch file should not have been copied into new directory but was') self.assertExists(os.path.join(newdir, '0002-Add-a-comment-to-the-code.patch'), 'New patch file should have been created but wasn\'t') + with open(newrecipefile, 'r') as f: + newcontent = f.read() + self.assertNotIn(backportedpatchfn, newcontent, "Backported patch should have been removed from the recipe but wasn't") + self.assertIn(patchfn, newcontent, "Old patch should have not been removed from the recipe but was") + self.assertIn("0002-Add-a-comment-to-the-code.patch", newcontent, "New patch should have been added to the recipe but wasn't") + self.assertIn("http://www.ivarch.com/programs/sources/pv-${PV}.tar.gz", newcontent, "New recipe no longer has upstream source in SRC_URI") + - @OETestID(1624) def test_devtool_finish_upgrade_otherlayer(self): - recipe, oldrecipefile, recipedir, olddir, newversion, patchfn = self._setup_test_devtool_finish_upgrade() + recipe, oldrecipefile, recipedir, olddir, newversion, patchfn, backportedpatchfn = self._setup_test_devtool_finish_upgrade() # Ensure the recipe is where we think it should be (so that cleanup doesn't trash things) self.assertIn('/meta-selftest/', recipedir) # Try finish to a different layer - should create a bbappend @@ -1561,10 +2219,18 @@ class DevtoolUpgradeTests(DevtoolBase): self.assertNotExists(os.path.join(self.workspacedir, 'recipes', recipe), 'Recipe directory should not exist after finish') self.assertExists(oldrecipefile, 'Old recipe file should not have been deleted') self.assertExists(os.path.join(olddir, patchfn), 'Old patch file should not have been deleted') + self.assertExists(os.path.join(olddir, backportedpatchfn), 'Old backported patch file should not have been deleted') newdir = os.path.join(newrecipedir, recipe + '-' + newversion) self.assertExists(newrecipefile, 'New recipe file should have been copied into existing layer but wasn\'t') self.assertExists(os.path.join(newdir, patchfn), 'Patch file should have been copied into new directory but wasn\'t') + self.assertNotExists(os.path.join(newdir, backportedpatchfn), 'Backported patch file should not have been copied into new directory but was') self.assertExists(os.path.join(newdir, '0002-Add-a-comment-to-the-code.patch'), 'New patch file should have been created but wasn\'t') + with open(newrecipefile, 'r') as f: + newcontent = f.read() + self.assertNotIn(backportedpatchfn, newcontent, "Backported patch should have been removed from the recipe but wasn't") + self.assertIn(patchfn, newcontent, "Old patch should have not been removed from the recipe but was") + self.assertIn("0002-Add-a-comment-to-the-code.patch", newcontent, "New patch should have been added to the recipe but wasn't") + self.assertIn("http://www.ivarch.com/programs/sources/pv-${PV}.tar.gz", newcontent, "New recipe no longer has upstream source in SRC_URI") def _setup_test_devtool_finish_modify(self): # Check preconditions @@ -1599,7 +2265,6 @@ class DevtoolUpgradeTests(DevtoolBase): self.fail('Unable to find recipe files directory for %s' % recipe) return recipe, oldrecipefile, recipedir, filesdir - @OETestID(1621) def test_devtool_finish_modify_origlayer(self): recipe, oldrecipefile, recipedir, filesdir = self._setup_test_devtool_finish_modify() # Ensure the recipe is where we think it should be (so that cleanup doesn't trash things) @@ -1614,7 +2279,6 @@ class DevtoolUpgradeTests(DevtoolBase): ('??', '.*/.*-Add-a-comment-to-the-code.patch$')] self._check_repo_status(recipedir, expected_status) - @OETestID(1622) def test_devtool_finish_modify_otherlayer(self): recipe, oldrecipefile, recipedir, filesdir = self._setup_test_devtool_finish_modify() # Ensure the recipe is where we think it should be (so that cleanup doesn't trash things) @@ -1647,7 +2311,52 @@ class DevtoolUpgradeTests(DevtoolBase): if files: self.fail('Unexpected file(s) copied next to bbappend: %s' % ', '.join(files)) - @OETestID(1626) + def test_devtool_finish_update_patch(self): + # This test uses a modified version of the sysdig recipe from meta-oe. + # - The patches have been renamed. + # - The dependencies are commented out since the recipe is not being + # built. + # + # The sysdig recipe is interesting in that it fetches two different Git + # repositories, and there are patches for both. This leads to that + # devtool will create ignore commits as it uses Git submodules to keep + # track of the second repository. + # + # This test will verify that the ignored commits actually are ignored + # when a commit in between is modified. It will also verify that the + # updated patch keeps its original name. + + # Check preconditions + self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory') + # Try modifying a recipe + self.track_for_cleanup(self.workspacedir) + recipe = 'sysdig-selftest' + recipefile = get_bb_var('FILE', recipe) + recipedir = os.path.dirname(recipefile) + result = runCmd('git status --porcelain .', cwd=recipedir) + if result.output.strip(): + self.fail('Recipe directory for %s contains uncommitted changes' % recipe) + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('devtool modify %s %s' % (recipe, tempdir)) + self.add_command_to_tearDown('cd %s; rm %s/*; git checkout %s %s' % (recipedir, recipe, recipe, os.path.basename(recipefile))) + self.assertExists(os.path.join(tempdir, 'CMakeLists.txt'), 'Extracted source could not be found') + # Make a change to one of the existing commits + result = runCmd('echo "# A comment " >> CMakeLists.txt', cwd=tempdir) + result = runCmd('git status --porcelain', cwd=tempdir) + self.assertIn('M CMakeLists.txt', result.output) + result = runCmd('git commit --fixup HEAD^ CMakeLists.txt', cwd=tempdir) + result = runCmd('git show -s --format=%s', cwd=tempdir) + self.assertIn('fixup! cmake: Pass PROBE_NAME via CFLAGS', result.output) + result = runCmd('GIT_SEQUENCE_EDITOR=true git rebase -i --autosquash devtool-base', cwd=tempdir) + result = runCmd('devtool finish %s meta-selftest' % recipe) + result = runCmd('devtool status') + self.assertNotIn(recipe, result.output, 'Recipe should have been reset by finish but wasn\'t') + self.assertNotExists(os.path.join(self.workspacedir, 'recipes', recipe), 'Recipe directory should not exist after finish') + expected_status = [(' M', '.*/0099-cmake-Pass-PROBE_NAME-via-CFLAGS.patch$')] + self._check_repo_status(recipedir, expected_status) + def test_devtool_rename(self): # Check preconditions self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory') @@ -1684,7 +2393,6 @@ class DevtoolUpgradeTests(DevtoolBase): self._test_recipe_contents(newrecipefile, checkvars, []) # Try again - change just name this time result = runCmd('devtool reset -n %s' % newrecipename) - shutil.rmtree(newsrctree) add_recipe() newrecipefile = os.path.join(self.workspacedir, 'recipes', newrecipename, '%s_%s.bb' % (newrecipename, recipever)) result = runCmd('devtool rename %s %s' % (recipename, newrecipename)) @@ -1697,7 +2405,6 @@ class DevtoolUpgradeTests(DevtoolBase): self._test_recipe_contents(newrecipefile, checkvars, []) # Try again - change just version this time result = runCmd('devtool reset -n %s' % newrecipename) - shutil.rmtree(newsrctree) add_recipe() newrecipefile = os.path.join(self.workspacedir, 'recipes', recipename, '%s_%s.bb' % (recipename, newrecipever)) result = runCmd('devtool rename %s -V %s' % (recipename, newrecipever)) @@ -1708,7 +2415,6 @@ class DevtoolUpgradeTests(DevtoolBase): checkvars['SRC_URI'] = url self._test_recipe_contents(newrecipefile, checkvars, []) - @OETestID(1577) def test_devtool_virtual_kernel_modify(self): """ Summary: The purpose of this test case is to verify that @@ -1730,17 +2436,18 @@ class DevtoolUpgradeTests(DevtoolBase): Expected: devtool modify is able to checkout the source of the kernel and modification to the source and configurations are reflected when building the kernel. - """ - kernel_provider = get_bb_var('PREFERRED_PROVIDER_virtual/kernel') - # Clean up the enviroment + """ + kernel_provider = self.td['PREFERRED_PROVIDER_virtual/kernel'] + + # Clean up the environment bitbake('%s -c clean' % kernel_provider) tempdir = tempfile.mkdtemp(prefix='devtoolqa') tempdir_cfg = tempfile.mkdtemp(prefix='config_qa') self.track_for_cleanup(tempdir) self.track_for_cleanup(tempdir_cfg) self.track_for_cleanup(self.workspacedir) - self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') self.add_command_to_tearDown('bitbake -c clean %s' % kernel_provider) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') #Step 1 #Here is just generated the config file instead of all the kernel to optimize the #time of executing this test case. @@ -1758,33 +2465,545 @@ class DevtoolUpgradeTests(DevtoolBase): self.assertExists(os.path.join(tempdir, 'Makefile'), 'Extracted source could not be found') #Step 4.2 configfile = os.path.join(tempdir,'.config') - diff = runCmd('diff %s %s' % (tmpconfig, configfile)) - self.assertEqual(0,diff.status,'Kernel .config file is not the same using bitbake and devtool') + runCmd('diff %s %s' % (tmpconfig, configfile)) + #Step 4.3 #NOTE: virtual/kernel is mapped to kernel_provider - result = runCmd('devtool build %s' % kernel_provider) - self.assertEqual(0,result.status,'Cannot build kernel using `devtool build`') + runCmd('devtool build %s' % kernel_provider) kernelfile = os.path.join(get_bb_var('KBUILD_OUTPUT', kernel_provider), 'vmlinux') self.assertExists(kernelfile, 'Kernel was not build correctly') #Modify the kernel source - modfile = os.path.join(tempdir,'arch/x86/boot/header.S') - modstring = "Use a boot loader. Devtool testing." - modapplied = runCmd("sed -i 's/Use a boot loader./%s/' %s" % (modstring, modfile)) - self.assertEqual(0,modapplied.status,'Modification to %s on kernel source failed' % modfile) + modfile = os.path.join(tempdir, 'init/version.c') + # Moved to uts.h in 6.1 onwards + modfile2 = os.path.join(tempdir, 'include/linux/uts.h') + runCmd("sed -i 's/Linux/LiNuX/g' %s %s" % (modfile, modfile2)) + #Modify the configuration - codeconfigfile = os.path.join(tempdir,'.config.new') + codeconfigfile = os.path.join(tempdir, '.config.new') modconfopt = "CONFIG_SG_POOL=n" - modconf = runCmd("sed -i 's/CONFIG_SG_POOL=y/%s/' %s" % (modconfopt, codeconfigfile)) - self.assertEqual(0,modconf.status,'Modification to %s failed' % codeconfigfile) + runCmd("sed -i 's/CONFIG_SG_POOL=y/%s/' %s" % (modconfopt, codeconfigfile)) + #Build again kernel with devtool - rebuild = runCmd('devtool build %s' % kernel_provider) - self.assertEqual(0,rebuild.status,'Fail to build kernel after modification of source and config') + runCmd('devtool build %s' % kernel_provider) + #Step 4.4 - bzimagename = 'bzImage-' + get_bb_var('KERNEL_VERSION_NAME', kernel_provider) - bzimagefile = os.path.join(get_bb_var('D', kernel_provider),'boot', bzimagename) - checkmodcode = runCmd("grep '%s' %s" % (modstring, bzimagefile)) - self.assertEqual(0,checkmodcode.status,'Modification on kernel source failed') + runCmd("grep '%s' %s" % ('LiNuX', kernelfile)) + #Step 4.5 - checkmodconfg = runCmd("grep %s %s" % (modconfopt, codeconfigfile)) - self.assertEqual(0,checkmodconfg.status,'Modification to configuration file failed') + runCmd("grep %s %s" % (modconfopt, codeconfigfile)) + + +class DevtoolIdeSdkTests(DevtoolBase): + def _write_bb_config(self, recipe_names): + """Helper to write the bitbake local.conf file""" + conf_lines = [ + 'IMAGE_CLASSES += "image-combined-dbg"', + 'IMAGE_GEN_DEBUGFS = "1"', + 'IMAGE_INSTALL:append = " gdbserver %s"' % ' '.join( + [r + '-ptest' for r in recipe_names]) + ] + self.write_config("\n".join(conf_lines)) + + def _check_workspace(self): + """Check if a workspace directory is available and setup the cleanup""" + self.assertTrue(not os.path.exists(self.workspacedir), + 'This test cannot be run with a workspace directory under the build directory') + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + + def _workspace_scripts_dir(self, recipe_name): + return os.path.realpath(os.path.join(self.builddir, 'workspace', 'ide-sdk', recipe_name, 'scripts')) + + def _sources_scripts_dir(self, src_dir): + return os.path.realpath(os.path.join(src_dir, 'oe-scripts')) + + def _workspace_gdbinit_dir(self, recipe_name): + return os.path.realpath(os.path.join(self.builddir, 'workspace', 'ide-sdk', recipe_name, 'scripts', 'gdbinit')) + + def _sources_gdbinit_dir(self, src_dir): + return os.path.realpath(os.path.join(src_dir, 'oe-gdbinit')) + + def _devtool_ide_sdk_recipe(self, recipe_name, build_file, testimage): + """Setup a recipe for working with devtool ide-sdk + + Basically devtool modify -x followed by some tests + """ + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.add_command_to_tearDown('bitbake -c clean %s' % recipe_name) + + result = runCmd('devtool modify %s -x %s' % (recipe_name, tempdir)) + self.assertExists(os.path.join(tempdir, build_file), + 'Extracted source could not be found') + self.assertExists(os.path.join(self.workspacedir, 'conf', + 'layer.conf'), 'Workspace directory not created') + matches = glob.glob(os.path.join(self.workspacedir, + 'appends', recipe_name + '.bbappend')) + self.assertTrue(matches, 'bbappend not created %s' % result.output) + + # Test devtool status + result = runCmd('devtool status') + self.assertIn(recipe_name, result.output) + self.assertIn(tempdir, result.output) + self._check_src_repo(tempdir) + + # Usually devtool ide-sdk would initiate the build of the SDK. + # But there is a circular dependency with starting Qemu and passing the IP of runqemu to devtool ide-sdk. + if testimage: + bitbake("%s qemu-native qemu-helper-native" % testimage) + deploy_dir_image = get_bb_var('DEPLOY_DIR_IMAGE') + self.add_command_to_tearDown('bitbake -c clean %s' % testimage) + self.add_command_to_tearDown( + 'rm -f %s/%s*' % (deploy_dir_image, testimage)) + + return tempdir + + def _get_recipe_ids(self, recipe_name): + """IDs needed to write recipe specific config entries into IDE config files""" + package_arch = get_bb_var('PACKAGE_ARCH', recipe_name) + recipe_id = recipe_name + "-" + package_arch + recipe_id_pretty = recipe_name + ": " + package_arch + return (recipe_id, recipe_id_pretty) + + def _verify_install_script_code(self, tempdir, recipe_name): + """Verify the scripts referred by the tasks.json file are fine. + + This function does not depend on Qemu. Therefore it verifies the scripts + exists and the delete step works as expected. But it does not try to + deploy to Qemu. + """ + recipe_id, recipe_id_pretty = self._get_recipe_ids(recipe_name) + with open(os.path.join(tempdir, '.vscode', 'tasks.json')) as tasks_j: + tasks_d = json.load(tasks_j) + tasks = tasks_d["tasks"] + task_install = next( + (task for task in tasks if task["label"] == "install && deploy-target %s" % recipe_id_pretty), None) + self.assertIsNot(task_install, None) + # execute only the bb_run_do_install script since the deploy would require e.g. Qemu running. + i_and_d_script = "install_and_deploy_" + recipe_id + i_and_d_script_path = os.path.join( + self._workspace_scripts_dir(recipe_name), i_and_d_script) + self.assertExists(i_and_d_script_path) + del_script = "delete_package_dirs_" + recipe_id + del_script_path = os.path.join( + self._workspace_scripts_dir(recipe_name), del_script) + self.assertExists(del_script_path) + runCmd(del_script_path, cwd=tempdir) + + def _devtool_ide_sdk_qemu(self, tempdir, qemu, recipe_name, example_exe): + """Verify deployment and execution in Qemu system work for one recipe. + + This function checks the entire SDK workflow: changing the code, recompiling + it and deploying it back to Qemu, and checking that the changes have been + incorporated into the provided binaries. It also runs the tests of the recipe. + """ + recipe_id, _ = self._get_recipe_ids(recipe_name) + i_and_d_script = "install_and_deploy_" + recipe_id + install_deploy_cmd = os.path.join( + self._workspace_scripts_dir(recipe_name), i_and_d_script) + self.assertExists(install_deploy_cmd, + '%s script not found' % install_deploy_cmd) + runCmd(install_deploy_cmd) + + MAGIC_STRING_ORIG = "Magic: 123456789" + MAGIC_STRING_NEW = "Magic: 987654321" + ptest_cmd = "ptest-runner " + recipe_name + + # validate that SSH is working + status, _ = qemu.run("uname") + self.assertEqual( + status, 0, msg="Failed to connect to the SSH server on Qemu") + + # Verify the unmodified example prints the magic string + status, output = qemu.run(example_exe) + self.assertEqual(status, 0, msg="%s failed: %s" % + (example_exe, output)) + self.assertIn(MAGIC_STRING_ORIG, output) + + # Verify the unmodified ptests work + status, output = qemu.run(ptest_cmd) + self.assertEqual(status, 0, msg="%s failed: %s" % (ptest_cmd, output)) + self.assertIn("PASS: cpp-example-lib", output) + + # Verify remote debugging works + self._gdb_cross_debugging( + qemu, recipe_name, example_exe, MAGIC_STRING_ORIG) + + # Replace the Magic String in the code, compile and deploy to Qemu + cpp_example_lib_hpp = os.path.join(tempdir, 'cpp-example-lib.hpp') + with open(cpp_example_lib_hpp, 'r') as file: + cpp_code = file.read() + cpp_code = cpp_code.replace(MAGIC_STRING_ORIG, MAGIC_STRING_NEW) + with open(cpp_example_lib_hpp, 'w') as file: + file.write(cpp_code) + runCmd(install_deploy_cmd, cwd=tempdir) + + # Verify the modified example prints the modified magic string + status, output = qemu.run(example_exe) + self.assertEqual(status, 0, msg="%s failed: %s" % + (example_exe, output)) + self.assertNotIn(MAGIC_STRING_ORIG, output) + self.assertIn(MAGIC_STRING_NEW, output) + + # Verify the modified example ptests work + status, output = qemu.run(ptest_cmd) + self.assertEqual(status, 0, msg="%s failed: %s" % (ptest_cmd, output)) + self.assertIn("PASS: cpp-example-lib", output) + + # Verify remote debugging works wit the modified magic string + self._gdb_cross_debugging( + qemu, recipe_name, example_exe, MAGIC_STRING_NEW) + + def _gdb_cross(self): + """Verify gdb-cross is provided by devtool ide-sdk""" + target_arch = self.td["TARGET_ARCH"] + target_sys = self.td["TARGET_SYS"] + gdb_recipe = "gdb-cross-" + target_arch + gdb_binary = target_sys + "-gdb" + + native_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", gdb_recipe) + r = runCmd("%s --version" % gdb_binary, + native_sysroot=native_sysroot, target_sys=target_sys) + self.assertEqual(r.status, 0) + self.assertIn("GNU gdb", r.output) + + def _gdb_cross_debugging(self, qemu, recipe_name, example_exe, magic_string): + """Verify gdb-cross is working + + Test remote debugging: + break main + run + continue + break CppExample::print_json() + continue + print CppExample::test_string.compare("cpp-example-lib Magic: 123456789") + $1 = 0 + print CppExample::test_string.compare("cpp-example-lib Magic: 123456789aaa") + $2 = -3 + list cpp-example-lib.hpp:13,13 + 13 inline static const std::string test_string = "cpp-example-lib Magic: 123456789"; + continue + """ + sshargs = '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' + gdbserver_script = os.path.join(self._workspace_scripts_dir( + recipe_name), 'gdbserver_1234_usr-bin-' + example_exe + '_m') + gdb_script = os.path.join(self._workspace_scripts_dir( + recipe_name), 'gdb_1234_usr-bin-' + example_exe) + + # Start a gdbserver + r = runCmd(gdbserver_script) + self.assertEqual(r.status, 0) + + # Check there is a gdbserver running + r = runCmd('ssh %s root@%s %s' % (sshargs, qemu.ip, 'ps')) + self.assertEqual(r.status, 0) + self.assertIn("gdbserver ", r.output) + + # Check the pid file is correct + test_cmd = "cat /proc/$(cat /tmp/gdbserver_1234_usr-bin-" + \ + example_exe + "/pid)/cmdline" + r = runCmd('ssh %s root@%s %s' % (sshargs, qemu.ip, test_cmd)) + self.assertEqual(r.status, 0) + self.assertIn("gdbserver", r.output) + + # Test remote debugging works + gdb_batch_cmd = " --batch -ex 'break main' -ex 'run'" + gdb_batch_cmd += " -ex 'break CppExample::print_json()' -ex 'continue'" + gdb_batch_cmd += " -ex 'print CppExample::test_string.compare(\"cpp-example-lib %s\")'" % magic_string + gdb_batch_cmd += " -ex 'print CppExample::test_string.compare(\"cpp-example-lib %saaa\")'" % magic_string + gdb_batch_cmd += " -ex 'list cpp-example-lib.hpp:13,13'" + gdb_batch_cmd += " -ex 'continue'" + r = runCmd(gdb_script + gdb_batch_cmd) + self.logger.debug("%s %s returned: %s", gdb_script, + gdb_batch_cmd, r.output) + self.assertEqual(r.status, 0) + self.assertIn("Breakpoint 1, main", r.output) + self.assertIn("$1 = 0", r.output) # test.string.compare equal + self.assertIn("$2 = -3", r.output) # test.string.compare longer + self.assertIn( + 'inline static const std::string test_string = "cpp-example-lib %s";' % magic_string, r.output) + self.assertIn("exited normally", r.output) + + # Stop the gdbserver + r = runCmd(gdbserver_script + ' stop') + self.assertEqual(r.status, 0) + + # Check there is no gdbserver running + r = runCmd('ssh %s root@%s %s' % (sshargs, qemu.ip, 'ps')) + self.assertEqual(r.status, 0) + self.assertNotIn("gdbserver ", r.output) + + def _verify_cmake_preset(self, tempdir): + """Verify the generated cmake preset works as expected + + Check if compiling works + Check if unit tests can be executed in qemu (not qemu-system) + """ + with open(os.path.join(tempdir, 'CMakeUserPresets.json')) as cmake_preset_j: + cmake_preset_d = json.load(cmake_preset_j) + config_presets = cmake_preset_d["configurePresets"] + self.assertEqual(len(config_presets), 1) + cmake_exe = config_presets[0]["cmakeExecutable"] + preset_name = config_presets[0]["name"] + + # Verify the wrapper for cmake native is available + self.assertExists(cmake_exe) + + # Verify the cmake preset generated by devtool ide-sdk is available + result = runCmd('%s --list-presets' % cmake_exe, cwd=tempdir) + self.assertIn(preset_name, result.output) + + # Verify cmake re-uses the o files compiled by bitbake + result = runCmd('%s --build --preset %s' % + (cmake_exe, preset_name), cwd=tempdir) + self.assertIn("ninja: no work to do.", result.output) + + # Verify the unit tests work (in Qemu user mode) + result = runCmd('%s --build --preset %s --target test' % + (cmake_exe, preset_name), cwd=tempdir) + self.assertIn("100% tests passed", result.output) + + # Verify re-building and testing works again + result = runCmd('%s --build --preset %s --target clean' % + (cmake_exe, preset_name), cwd=tempdir) + self.assertIn("Cleaning", result.output) + result = runCmd('%s --build --preset %s' % + (cmake_exe, preset_name), cwd=tempdir) + self.assertIn("Building", result.output) + self.assertIn("Linking", result.output) + result = runCmd('%s --build --preset %s --target test' % + (cmake_exe, preset_name), cwd=tempdir) + self.assertIn("Running tests...", result.output) + self.assertIn("100% tests passed", result.output) + + @OETestTag("runqemu") + def test_devtool_ide_sdk_none_qemu(self): + """Start qemu-system and run tests for multiple recipes. ide=none is used.""" + recipe_names = ["cmake-example", "meson-example"] + testimage = "oe-selftest-image" + + self._check_workspace() + self._write_bb_config(recipe_names) + self._check_runqemu_prerequisites() + + # Verify deployment to Qemu (system mode) works + bitbake(testimage) + with runqemu(testimage, runqemuparams="nographic") as qemu: + # cmake-example recipe + recipe_name = "cmake-example" + example_exe = "cmake-example" + build_file = "CMakeLists.txt" + tempdir = self._devtool_ide_sdk_recipe( + recipe_name, build_file, testimage) + bitbake_sdk_cmd = 'devtool ide-sdk %s %s -t root@%s -c --ide=none' % ( + recipe_name, testimage, qemu.ip) + runCmd(bitbake_sdk_cmd) + self._gdb_cross() + self._verify_cmake_preset(tempdir) + self._devtool_ide_sdk_qemu(tempdir, qemu, recipe_name, example_exe) + # Verify the oe-scripts sym-link is valid + self.assertEqual(self._workspace_scripts_dir( + recipe_name), self._sources_scripts_dir(tempdir)) + + # meson-example recipe + recipe_name = "meson-example" + example_exe = "mesonex" + build_file = "meson.build" + tempdir = self._devtool_ide_sdk_recipe( + recipe_name, build_file, testimage) + bitbake_sdk_cmd = 'devtool ide-sdk %s %s -t root@%s -c --ide=none' % ( + recipe_name, testimage, qemu.ip) + runCmd(bitbake_sdk_cmd) + self._gdb_cross() + self._devtool_ide_sdk_qemu(tempdir, qemu, recipe_name, example_exe) + # Verify the oe-scripts sym-link is valid + self.assertEqual(self._workspace_scripts_dir( + recipe_name), self._sources_scripts_dir(tempdir)) + + def test_devtool_ide_sdk_code_cmake(self): + """Verify a cmake recipe works with ide=code mode""" + recipe_name = "cmake-example" + build_file = "CMakeLists.txt" + testimage = "oe-selftest-image" + + self._check_workspace() + self._write_bb_config([recipe_name]) + tempdir = self._devtool_ide_sdk_recipe( + recipe_name, build_file, testimage) + bitbake_sdk_cmd = 'devtool ide-sdk %s %s -t root@192.168.17.17 -c --ide=code' % ( + recipe_name, testimage) + runCmd(bitbake_sdk_cmd) + self._verify_cmake_preset(tempdir) + self._verify_install_script_code(tempdir, recipe_name) + self._gdb_cross() + + def test_devtool_ide_sdk_code_meson(self): + """Verify a meson recipe works with ide=code mode""" + recipe_name = "meson-example" + build_file = "meson.build" + testimage = "oe-selftest-image" + + self._check_workspace() + self._write_bb_config([recipe_name]) + tempdir = self._devtool_ide_sdk_recipe( + recipe_name, build_file, testimage) + bitbake_sdk_cmd = 'devtool ide-sdk %s %s -t root@192.168.17.17 -c --ide=code' % ( + recipe_name, testimage) + runCmd(bitbake_sdk_cmd) + + with open(os.path.join(tempdir, '.vscode', 'settings.json')) as settings_j: + settings_d = json.load(settings_j) + meson_exe = settings_d["mesonbuild.mesonPath"] + meson_build_folder = settings_d["mesonbuild.buildFolder"] + + # Verify the wrapper for meson native is available + self.assertExists(meson_exe) + + # Verify meson re-uses the o files compiled by bitbake + result = runCmd('%s compile -C %s' % + (meson_exe, meson_build_folder), cwd=tempdir) + self.assertIn("ninja: no work to do.", result.output) + + # Verify the unit tests work (in Qemu) + runCmd('%s test -C %s' % (meson_exe, meson_build_folder), cwd=tempdir) + + # Verify re-building and testing works again + result = runCmd('%s compile -C %s --clean' % + (meson_exe, meson_build_folder), cwd=tempdir) + self.assertIn("Cleaning...", result.output) + result = runCmd('%s compile -C %s' % + (meson_exe, meson_build_folder), cwd=tempdir) + self.assertIn("Linking target", result.output) + runCmd('%s test -C %s' % (meson_exe, meson_build_folder), cwd=tempdir) + + self._verify_install_script_code(tempdir, recipe_name) + self._gdb_cross() + + def test_devtool_ide_sdk_shared_sysroots(self): + """Verify the shared sysroot SDK""" + + # Handle the workspace (which is not needed by this test case) + self._check_workspace() + + result_init = runCmd( + 'devtool ide-sdk -m shared oe-selftest-image cmake-example meson-example --ide=code') + bb_vars = get_bb_vars( + ['REAL_MULTIMACH_TARGET_SYS', 'DEPLOY_DIR_IMAGE', 'COREBASE'], "meta-ide-support") + environment_script = 'environment-setup-%s' % bb_vars['REAL_MULTIMACH_TARGET_SYS'] + deploydir = bb_vars['DEPLOY_DIR_IMAGE'] + environment_script_path = os.path.join(deploydir, environment_script) + cpp_example_src = os.path.join( + bb_vars['COREBASE'], 'meta-selftest', 'recipes-test', 'cpp', 'files') + + # Verify the cross environment script is available + self.assertExists(environment_script_path) + + def runCmdEnv(cmd, cwd): + cmd = '/bin/sh -c ". %s > /dev/null && %s"' % ( + environment_script_path, cmd) + return runCmd(cmd, cwd) + + # Verify building the C++ example works with CMake + tempdir_cmake = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir_cmake) + + result_cmake = runCmdEnv("which cmake", cwd=tempdir_cmake) + cmake_native = os.path.normpath(result_cmake.output.strip()) + self.assertExists(cmake_native) + + runCmdEnv('cmake %s' % cpp_example_src, cwd=tempdir_cmake) + runCmdEnv('cmake --build %s' % tempdir_cmake, cwd=tempdir_cmake) + + # Verify the printed note really referres to a cmake executable + cmake_native_code = "" + for line in result_init.output.splitlines(): + m = re.search(r'"cmake.cmakePath": "(.*)"', line) + if m: + cmake_native_code = m.group(1) + break + self.assertExists(cmake_native_code) + self.assertEqual(cmake_native, cmake_native_code) + + # Verify building the C++ example works with Meson + tempdir_meson = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir_meson) + + result_cmake = runCmdEnv("which meson", cwd=tempdir_meson) + meson_native = os.path.normpath(result_cmake.output.strip()) + self.assertExists(meson_native) + + runCmdEnv('meson setup %s' % tempdir_meson, cwd=cpp_example_src) + runCmdEnv('meson compile', cwd=tempdir_meson) + + def test_devtool_ide_sdk_plugins(self): + """Test that devtool ide-sdk can use plugins from other layers.""" + + # We need a workspace layer and a modified recipe (but no image) + modified_recipe_name = "meson-example" + modified_build_file = "meson.build" + testimage = "oe-selftest-image" + shared_recipe_name = "cmake-example" + + self._check_workspace() + self._write_bb_config([modified_recipe_name]) + tempdir = self._devtool_ide_sdk_recipe( + modified_recipe_name, modified_build_file, None) + + IDE_RE = re.compile(r'.*--ide \{(.*)\}.*') + + def get_ides_from_help(help_str): + m = IDE_RE.search(help_str) + return m.group(1).split(',') + + # verify the default plugins are available but the foo plugin is not + result = runCmd('devtool ide-sdk -h') + found_ides = get_ides_from_help(result.output) + self.assertIn('code', found_ides) + self.assertIn('none', found_ides) + self.assertNotIn('foo', found_ides) + + shared_config_file = os.path.join(tempdir, 'shared-config.txt') + shared_config_str = 'Dummy shared IDE config' + modified_config_file = os.path.join(tempdir, 'modified-config.txt') + modified_config_str = 'Dummy modified IDE config' + + # Generate a foo plugin in the workspace layer + plugin_dir = os.path.join( + self.workspacedir, 'lib', 'devtool', 'ide_plugins') + os.makedirs(plugin_dir) + plugin_code = 'from devtool.ide_plugins import IdeBase\n\n' + plugin_code += 'class IdeFoo(IdeBase):\n' + plugin_code += ' def setup_shared_sysroots(self, shared_env):\n' + plugin_code += ' with open("%s", "w") as config_file:\n' % shared_config_file + plugin_code += ' config_file.write("%s")\n\n' % shared_config_str + plugin_code += ' def setup_modified_recipe(self, args, image_recipe, modified_recipe):\n' + plugin_code += ' with open("%s", "w") as config_file:\n' % modified_config_file + plugin_code += ' config_file.write("%s")\n\n' % modified_config_str + plugin_code += 'def register_ide_plugin(ide_plugins):\n' + plugin_code += ' ide_plugins["foo"] = IdeFoo\n' + + plugin_py = os.path.join(plugin_dir, 'ide_foo.py') + with open(plugin_py, 'w') as plugin_file: + plugin_file.write(plugin_code) + + # Verify the foo plugin is available as well + result = runCmd('devtool ide-sdk -h') + found_ides = get_ides_from_help(result.output) + self.assertIn('code', found_ides) + self.assertIn('none', found_ides) + self.assertIn('foo', found_ides) + + # Verify the foo plugin generates a shared config + result = runCmd( + 'devtool ide-sdk -m shared --skip-bitbake --ide foo %s' % shared_recipe_name) + with open(shared_config_file) as shared_config: + shared_config_new = shared_config.read() + self.assertEqual(shared_config_str, shared_config_new) + + # Verify the foo plugin generates a modified config + result = runCmd('devtool ide-sdk --skip-bitbake --ide foo %s %s' % + (modified_recipe_name, testimage)) + with open(modified_config_file) as modified_config: + modified_config_new = modified_config.read() + self.assertEqual(modified_config_str, modified_config_new) |