aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMarkus Lehtonen <markus.lehtonen@linux.intel.com>2016-09-01 17:18:54 +0300
committerMarkus Lehtonen <markus.lehtonen@linux.intel.com>2017-05-18 11:17:51 +0300
commit20bee817e00344cb10225725eebc304e04996b29 (patch)
treed803f1751e209de5645f72cd17524c100ee228cb
parent8c2657c8418184b3af62f81f8a15dfb37d95dea5 (diff)
downloadopenembedded-core-contrib-20bee817e00344cb10225725eebc304e04996b29.tar.gz
openembedded-core-contrib-20bee817e00344cb10225725eebc304e04996b29.tar.bz2
openembedded-core-contrib-20bee817e00344cb10225725eebc304e04996b29.zip
devtools/images: add python-pgo-image
This is a special image for profiling Python 3.x in order to utilize profile-guided-optimization. Profile data can be obtained by by running bitbake python-pgo-image -c profile. Profile data will be copied into a directory pointed by PYTHON3_PROFILE_DIR on the host system. The profile task may be specified with PYTHON3_PROFILE_TASK. The default profile task is pybench, although an example for using test.regrtest is included in the recipe. [YOCTO #9338] Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
-rw-r--r--meta/recipes-devtools/images/python-pgo-image.bb85
1 files changed, 85 insertions, 0 deletions
diff --git a/meta/recipes-devtools/images/python-pgo-image.bb b/meta/recipes-devtools/images/python-pgo-image.bb
new file mode 100644
index 0000000000..76733a8c61
--- /dev/null
+++ b/meta/recipes-devtools/images/python-pgo-image.bb
@@ -0,0 +1,85 @@
+SUMMARY = "Minimal image for doing Python profiling (for PGO)"
+
+IMAGE_FEATURES += "ssh-server-dropbear"
+IMAGE_INSTALL = "packagegroup-core-boot"
+IMAGE_INSTALL += "python3-profile-opt python3-profile-opt-tests python3-profile-opt-tools"
+
+LICENSE = "MIT"
+
+inherit core-image
+
+PROFILE_DATA_WORKDIR = "${WORKDIR}/profile-data"
+
+PYTHON3_PROFILE_DIR ?= "${TMPDIR}/work-shared/${MACHINE}/python3/pgo-data"
+PYTHON3_PROFILE_TASK_DEFAULT = "-m test.regrtest --pgo -w -x test_asyncore test_gdb test_multiprocessing_fork test_multiprocessing_forkserver test_multiprocessing_main_handling test_multiprocessing_spawn test_subprocess"
+# Exclude tests that are failing on qemux86
+PYTHON3_PROFILE_TASK_DEFAULT += "test_builtin test_cmath test_concurrent_futures test_difflib test_distutils test_float test_format test_math test_optparse test_shutil test_statistics test_types test_unicode"
+# Exclude tests that are taking very long on qemux86
+PYTHON3_PROFILE_TASK_DEFAULT += "test_lib2to3 test_buffer test_pickle test_io test_threading test_asyncio test_urllib2_localnet test_itertools test_tuple test_trace test_tarfile test_unicodedata test_decimal test_long test_zipfile test_deque test_descr test_email test_venv test_bytes test_compileall test_ast test_multibytecodec"change python3 profile target to pybenchchange python3 profile target to pybench
+
+# Change default profile target to pybench. Running test.regrtest takes
+# ridiculously long, i.e. around 4 hours in qemux86 on i7-3770K desktop
+# machine. Pybench "only" takes around 55 minutes.
+PYTHON3_PROFILE_TASK_DEFAULT = "${docdir}/python3-profile-opt/Tools/pybench/pybench.py -n 2 --with-gc --with-syscheck"
+
+PYTHON3_PROFILE_TASK ?= "${PYTHON3_PROFILE_TASK_DEFAULT}"
+
+# We need these because we're utilizing the runtime test helpers from oeqa
+TEST_TARGET ?= "qemu"
+TEST_QEMUBOOT_TIMEOUT ?= "1000"
+TEST_LOG_DIR ?= "${WORKDIR}/qemulogs"
+FIND_ROOTFS = "1"
+
+def run_profile(d, profile_bin, profile_task, tgt_in_dir, host_out_dir):
+ from oeqa.targetcontrol import get_target_controller
+ from oe.utils import getstatusoutput
+
+ target = get_target_controller(d)
+ target.deploy()
+ try:
+ # Boot target
+ bootparams = None
+ if d.getVar('VIRTUAL-RUNTIME_init_manager', True) == 'systemd':
+ bootparams = 'systemd.log_level=debug systemd.log_target=console'
+ target.start(extra_bootparams=bootparams)
+
+ # Run profile task
+ ret, output = target.run(profile_bin + ' ' + profile_task, timeout=7200)
+ if ret:
+ bb.fatal("Failed to run profile task on target: %s" % output)
+ ret, output = target.run('tar czf pgo-data.tgz -C %s .' % tgt_in_dir)
+ if ret:
+ bb.fatal("Failed to archive profile data on target: %s" % output)
+
+ # Retrieve and unpack profile data
+ target.copy_from('/home/root/pgo-data.tgz', host_out_dir)
+
+ profile_tarball = os.path.join(host_out_dir, 'pgo-data.tgz')
+ ret, output = getstatusoutput('tar xf %s -C %s' % (profile_tarball, host_out_dir))
+ os.unlink(profile_tarball)
+ if ret:
+ bb.fatal("Failed to unpack python profile data: %s" % output)
+ finally:
+ target.stop()
+
+
+# Profile task for Python 3.x
+python do_profile() {
+ run_profile(d, 'python3',
+ d.getVar('PYTHON3_PROFILE_TASK', True),
+ '/home/root/python3-pgo-profiles/',
+ os.path.join(d.getVar("PROFILE_DATA_WORKDIR", True), 'python3'))
+}
+
+addtask profile after do_build
+do_profile[depends] += "qemu-native:do_populate_sysroot qemu-helper-native:do_populate_sysroot"
+do_profile[cleandirs] = "${PROFILE_DATA_WORKDIR}/python3"
+
+python do_profile_setscene () {
+ sstate_setscene(d)
+}
+
+SSTATETASKS += "do_profile"
+do_profile[sstate-inputdirs] = "${PROFILE_DATA_WORKDIR}/python3"
+do_profile[sstate-outputdirs] = "${PYTHON3_PROFILE_DIR}"
+addtask do_profile_setscene