From f124b3d63392c5d4efecd65e9330b4a1ec8d55e5 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Mon, 14 Nov 2016 17:16:42 +0200 Subject: python3: add python3-profile-opt recipe This patch adds a new recipe that builds a special version of python3 that produces profile data used for optimization. The new recipe directly includes the base python recipe so that they are build in as similar way as possible and this hopefully decreases the recipe maintenance burden, too. Also, its files clash with the "normal" python3 so they cannot be installed in an image at the same time. Normally, profile-guided-optimization in Python is done simply by doing "make profile-opt" which first builds python with profile instrumentation enabled, then runs a profile task to get the profile data, and last, re-builds python with profile data guiding the optimization. However, in our cross-build environment this gets a lot trickier. We need to split out the steps as building is done on the build host but we need to run the second step (i.e. run the profile task) on the target hardware. This patch enables the first step, i.e. building python with profile instrumentation enabled [YOCTO #9338] Signed-off-by: Markus Lehtonen --- meta/recipes-devtools/python/python3-profile-opt | 1 + .../python/python3-profile-opt_3.5.5.bb | 9 +++++++ ...efile-add-install_generate_profile-target.patch | 22 ++++++++++++++++ meta/recipes-devtools/python/python3_3.5.5.bb | 30 ++++++++++++---------- 4 files changed, 48 insertions(+), 14 deletions(-) create mode 120000 meta/recipes-devtools/python/python3-profile-opt create mode 100644 meta/recipes-devtools/python/python3-profile-opt_3.5.5.bb create mode 100644 meta/recipes-devtools/python/python3/Makefile-add-install_generate_profile-target.patch diff --git a/meta/recipes-devtools/python/python3-profile-opt b/meta/recipes-devtools/python/python3-profile-opt new file mode 120000 index 0000000000..b8a0adbbb9 --- /dev/null +++ b/meta/recipes-devtools/python/python3-profile-opt @@ -0,0 +1 @@ +python3 \ No newline at end of file diff --git a/meta/recipes-devtools/python/python3-profile-opt_3.5.5.bb b/meta/recipes-devtools/python/python3-profile-opt_3.5.5.bb new file mode 100644 index 0000000000..6940480546 --- /dev/null +++ b/meta/recipes-devtools/python/python3-profile-opt_3.5.5.bb @@ -0,0 +1,9 @@ +require python3_${PV}.bb + +SRC_URI += "file://Makefile-add-install_generate_profile-target.patch \ + " + +PYTHON3_MAKE_TARGET = "build_all_generate_profile" + +RCONFLICTS_${PN}-core = "python3-core" +RCONFLICTS_lib${BPN} = "libpython3" diff --git a/meta/recipes-devtools/python/python3/Makefile-add-install_generate_profile-target.patch b/meta/recipes-devtools/python/python3/Makefile-add-install_generate_profile-target.patch new file mode 100644 index 0000000000..3305bac47d --- /dev/null +++ b/meta/recipes-devtools/python/python3/Makefile-add-install_generate_profile-target.patch @@ -0,0 +1,22 @@ +Makefile: add install_generate_profile target + +Making it possible to install the binaries with profile generation +enabled. We need this as linking is also performed at the install stage. + +Upstream-Status: Inappropriate [oe-specific] + +Signed-off-by: Markus Lehtonen + +diff --git a/Makefile.pre.in b/Makefile.pre.in +index 245adb3..c7b8e1e 100644 +--- a/Makefile.pre.in ++++ b/Makefile.pre.in +@@ -1034,6 +1034,8 @@ QUICKTESTOPTS= $(TESTOPTS) -x test_subprocess test_io test_lib2to3 \ + quicktest: @DEF_MAKE_RULE@ platform + $(TESTRUNNER) $(QUICKTESTOPTS) + ++install_generate_profile: ++ $(MAKE) install LDFLAGS="$(LDFLAGS) $(PGO_PROF_GEN_FLAG) @LTOFLAGS@" LIBS="$(LIBS)" + + install: @FRAMEWORKINSTALLFIRST@ commoninstall bininstall maninstall @FRAMEWORKINSTALLLAST@ + if test "x$(ENSUREPIP)" != "xno" ; then \ diff --git a/meta/recipes-devtools/python/python3_3.5.5.bb b/meta/recipes-devtools/python/python3_3.5.5.bb index f893b846ad..beb9651e58 100644 --- a/meta/recipes-devtools/python/python3_3.5.5.bb +++ b/meta/recipes-devtools/python/python3_3.5.5.bb @@ -115,14 +115,10 @@ do_compile() { # then call do_install twice we get Makefile.orig == Makefile.sysroot install -m 0644 Makefile Makefile.sysroot - oe_runmake HOSTPGEN=${STAGING_BINDIR_NATIVE}/python3-native/pgen \ - HOSTPYTHON=${STAGING_BINDIR_NATIVE}/python3-native/python3 \ - STAGING_LIBDIR=${STAGING_LIBDIR} \ - STAGING_BASELIBDIR=${STAGING_BASELIBDIR} \ - STAGING_INCDIR=${STAGING_INCDIR} \ - LIB=${baselib} \ - ARCH=${TARGET_ARCH} \ - OPT="${CFLAGS}" libpython3.so + if [ "${PYTHON3_MAKE_TARGET}" = "build_all_generate_profile" ]; then + # This is only used in PGO profiling by python3-profile-opt package + export EXTRA_CFLAGS="-fprofile-dir=./python3-pgo-profiles/" + fi oe_runmake HOSTPGEN=${STAGING_BINDIR_NATIVE}/python3-native/pgen \ HOSTPYTHON=${STAGING_BINDIR_NATIVE}/python3-native/python3 \ @@ -131,7 +127,7 @@ do_compile() { STAGING_BASELIBDIR=${STAGING_BASELIBDIR} \ LIB=${baselib} \ ARCH=${TARGET_ARCH} \ - OPT="${CFLAGS}" + OPT="${CFLAGS}" ${PYTHON3_MAKE_TARGET} } do_install() { @@ -151,8 +147,14 @@ do_install() { STAGING_BASELIBDIR=${STAGING_BASELIBDIR} \ LIB=${baselib} \ ARCH=${TARGET_ARCH} \ - DESTDIR=${D} LIBDIR=${libdir} + DESTDIR=${D} LIBDIR=${libdir} ${PYTHON3_MAKE_TARGET} + if [ "${PYTHON3_MAKE_TARGET}" = "build_all_generate_profile" ]; then + # Need special make install if pgo generation is enabled + _PYTHON3_MAKE_INSTALL_TARGET="install_generate_profile" + else + _PYTHON3_MAKE_INSTALL_TARGET="install" + fi oe_runmake HOSTPGEN=${STAGING_BINDIR_NATIVE}/python3-native/pgen \ HOSTPYTHON=${STAGING_BINDIR_NATIVE}/python3-native/python3 \ STAGING_LIBDIR=${STAGING_LIBDIR} \ @@ -160,7 +162,7 @@ do_install() { STAGING_BASELIBDIR=${STAGING_BASELIBDIR} \ LIB=${baselib} \ ARCH=${TARGET_ARCH} \ - DESTDIR=${D} LIBDIR=${libdir} install + DESTDIR=${D} LIBDIR=${libdir} ${_PYTHON3_MAKE_INSTALL_TARGET} # avoid conflict with 2to3 from Python 2 rm -f ${D}/${bindir}/2to3 @@ -223,9 +225,9 @@ PACKAGES =+ "${PN}-pyvenv" FILES_${PN}-pyvenv += "${bindir}/pyvenv-${PYTHON_MAJMIN} ${bindir}/pyvenv" # package libpython3 -PACKAGES =+ "libpython3 libpython3-staticdev" -FILES_libpython3 = "${libdir}/libpython*.so.*" -FILES_libpython3-staticdev += "${libdir}/python${PYTHON_MAJMIN}/config-${PYTHON_BINABI}/libpython${PYTHON_BINABI}.a" +PACKAGES =+ "lib${BPN} lib${BPN}-staticdev" +FILES_lib${BPN} = "${libdir}/libpython*.so.*" +FILES_lib${BPN}-staticdev += "${libdir}/python${PYTHON_MAJMIN}/config-${PYTHON_BINABI}/libpython*.a" INSANE_SKIP_${PN}-dev += "dev-elf" # catch all the rest (unsorted) -- cgit 1.2.3-korg