summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2018-08-13 11:40:03 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2018-08-15 09:33:53 +0100
commit8ffd93bdb09b0a4a84b27dafcd684c6abba392ed (patch)
tree9f0282ee684abb7505a3bed988608873e445b0d2
parent0a4e6974b49bf68c4a4098d339b5d655e202a3fd (diff)
downloadopenembedded-core-contrib-8ffd93bdb09b0a4a84b27dafcd684c6abba392ed.tar.gz
openembedded-core-contrib-8ffd93bdb09b0a4a84b27dafcd684c6abba392ed.tar.bz2
openembedded-core-contrib-8ffd93bdb09b0a4a84b27dafcd684c6abba392ed.zip
image: Add locale archive optimisation
Refactor the locale archive function from the SDK to also make it work during general image creation. This reduces the size of the locales from 900MB to 220MB in core-image-lsb-sdk. The exception handling around subprocess was dropped as the standard subprocess exception printing is better handled than the catchall exception. Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--meta/classes/image.bbclass2
-rw-r--r--meta/lib/oe/package_manager.py49
-rw-r--r--meta/lib/oe/sdk.py51
3 files changed, 54 insertions, 48 deletions
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index c3e73676dc..c0a2714288 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -22,7 +22,7 @@ POPULATE_SDK_POST_TARGET_COMMAND += "rootfs_sysroot_relativelinks; "
LICENSE ?= "MIT"
PACKAGES = ""
-DEPENDS += "${@' '.join(["%s-qemuwrapper-cross" % m for m in d.getVar("MULTILIB_VARIANTS").split()])} qemuwrapper-cross depmodwrapper-cross"
+DEPENDS += "${@' '.join(["%s-qemuwrapper-cross" % m for m in d.getVar("MULTILIB_VARIANTS").split()])} qemuwrapper-cross depmodwrapper-cross cross-localedef-native"
RDEPENDS += "${PACKAGE_INSTALL} ${LINGUAS_INSTALL}"
RRECOMMENDS += "${PACKAGE_INSTALL_ATTEMPTONLY}"
PATH_prepend = "${@":".join(all_multilib_tune_values(d, 'STAGING_BINDIR_CROSS').split())}:"
diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py
index 64c8a91216..6011e87350 100644
--- a/meta/lib/oe/package_manager.py
+++ b/meta/lib/oe/package_manager.py
@@ -12,6 +12,7 @@ import oe.path
import string
from oe.gpg_sign import get_signer
import hashlib
+import fnmatch
# this can be used by all PM backends to create the index files in parallel
def create_index(arg):
@@ -89,6 +90,47 @@ def failed_postinsts_warn(pkgs, log_path):
If deferring to first boot wasn't the intent, then scriptlet failure may mean an issue in the recipe, or a regression elsewhere.
Details of the failure are in %s.""" %(pkgs, log_path))
+def generate_locale_archive(d, rootfs, target_arch, localedir):
+ # Pretty sure we don't need this for locale archive generation but
+ # keeping it to be safe...
+ locale_arch_options = { \
+ "arm": ["--uint32-align=4", "--little-endian"],
+ "armeb": ["--uint32-align=4", "--big-endian"],
+ "aarch64": ["--uint32-align=4", "--little-endian"],
+ "aarch64_be": ["--uint32-align=4", "--big-endian"],
+ "sh4": ["--uint32-align=4", "--big-endian"],
+ "powerpc": ["--uint32-align=4", "--big-endian"],
+ "powerpc64": ["--uint32-align=4", "--big-endian"],
+ "mips": ["--uint32-align=4", "--big-endian"],
+ "mipsisa32r6": ["--uint32-align=4", "--big-endian"],
+ "mips64": ["--uint32-align=4", "--big-endian"],
+ "mipsisa64r6": ["--uint32-align=4", "--big-endian"],
+ "mipsel": ["--uint32-align=4", "--little-endian"],
+ "mipsisa32r6el": ["--uint32-align=4", "--little-endian"],
+ "mips64el": ["--uint32-align=4", "--little-endian"],
+ "mipsisa64r6el": ["--uint32-align=4", "--little-endian"],
+ "i586": ["--uint32-align=4", "--little-endian"],
+ "i686": ["--uint32-align=4", "--little-endian"],
+ "x86_64": ["--uint32-align=4", "--little-endian"]
+ }
+ if target_arch in locale_arch_options:
+ arch_options = locale_arch_options[target_arch]
+ else:
+ bb.error("locale_arch_options not found for target_arch=" + target_arch)
+ bb.fatal("unknown arch:" + target_arch + " for locale_arch_options")
+
+ # Need to set this so cross-localedef knows where the archive is
+ env = dict(os.environ)
+ env["LOCALEARCHIVE"] = oe.path.join(localedir, "locale-archive")
+
+ for name in os.listdir(localedir):
+ path = os.path.join(localedir, name)
+ if os.path.isdir(path):
+ cmd = ["cross-localedef", "--verbose"]
+ cmd += arch_options
+ cmd += ["--add-to-archive", path]
+ subprocess.check_output(cmd, env=env, stderr=subprocess.STDOUT)
+
class Indexer(object, metaclass=ABCMeta):
def __init__(self, d, deploy_dir):
self.d = d
@@ -536,6 +578,13 @@ class PackageManager(object, metaclass=ABCMeta):
"'%s' returned %d:\n%s" %
(' '.join(cmd), e.returncode, e.output.decode("utf-8")))
+ target_arch = self.d.getVar('TARGET_ARCH')
+ localedir = oe.path.join(self.target_rootfs, self.d.getVar("libdir"), "locale")
+ if os.path.exists(localedir) and os.listdir(localedir):
+ generate_locale_archive(self.d, self.target_rootfs, target_arch, localedir)
+ # And now delete the binary locales
+ self.remove(fnmatch.filter(self.list_installed(), "glibc-binary-localedata-*"), False)
+
def deploy_dir_lock(self):
if self.deploy_dir is None:
raise RuntimeError("deploy_dir is not set!")
diff --git a/meta/lib/oe/sdk.py b/meta/lib/oe/sdk.py
index 0d39ea8a91..f20441ccf6 100644
--- a/meta/lib/oe/sdk.py
+++ b/meta/lib/oe/sdk.py
@@ -7,52 +7,6 @@ import shutil
import glob
import traceback
-def generate_locale_archive(d, rootfs):
- # Pretty sure we don't need this for SDK archive generation but
- # keeping it to be safe...
- target_arch = d.getVar('SDK_ARCH')
- locale_arch_options = { \
- "arm": ["--uint32-align=4", "--little-endian"],
- "armeb": ["--uint32-align=4", "--big-endian"],
- "aarch64": ["--uint32-align=4", "--little-endian"],
- "aarch64_be": ["--uint32-align=4", "--big-endian"],
- "sh4": ["--uint32-align=4", "--big-endian"],
- "powerpc": ["--uint32-align=4", "--big-endian"],
- "powerpc64": ["--uint32-align=4", "--big-endian"],
- "mips": ["--uint32-align=4", "--big-endian"],
- "mipsisa32r6": ["--uint32-align=4", "--big-endian"],
- "mips64": ["--uint32-align=4", "--big-endian"],
- "mipsisa64r6": ["--uint32-align=4", "--big-endian"],
- "mipsel": ["--uint32-align=4", "--little-endian"],
- "mipsisa32r6el": ["--uint32-align=4", "--little-endian"],
- "mips64el": ["--uint32-align=4", "--little-endian"],
- "mipsisa64r6el": ["--uint32-align=4", "--little-endian"],
- "i586": ["--uint32-align=4", "--little-endian"],
- "i686": ["--uint32-align=4", "--little-endian"],
- "x86_64": ["--uint32-align=4", "--little-endian"]
- }
- if target_arch in locale_arch_options:
- arch_options = locale_arch_options[target_arch]
- else:
- bb.error("locale_arch_options not found for target_arch=" + target_arch)
- bb.fatal("unknown arch:" + target_arch + " for locale_arch_options")
-
- localedir = oe.path.join(rootfs, d.getVar("libdir_nativesdk"), "locale")
- # Need to set this so cross-localedef knows where the archive is
- env = dict(os.environ)
- env["LOCALEARCHIVE"] = oe.path.join(localedir, "locale-archive")
-
- for name in os.listdir(localedir):
- path = os.path.join(localedir, name)
- if os.path.isdir(path):
- try:
- cmd = ["cross-localedef", "--verbose"]
- cmd += arch_options
- cmd += ["--add-to-archive", path]
- subprocess.check_output(cmd, env=env, stderr=subprocess.STDOUT)
- except Exception as e:
- bb.fatal("Cannot create locale archive: %s" % e.output)
-
class Sdk(object, metaclass=ABCMeta):
def __init__(self, d, manifest_dir):
self.d = d
@@ -144,7 +98,10 @@ class Sdk(object, metaclass=ABCMeta):
for lang in linguas.split():
pm.install("nativesdk-glibc-binary-localedata-%s.utf-8" % lang)
# Generate a locale archive of them
- generate_locale_archive(self.d, oe.path.join(self.sdk_host_sysroot, self.sdk_native_path))
+ target_arch = self.d.getVar('SDK_ARCH')
+ rootfs = oe.path.join(self.sdk_host_sysroot, self.sdk_native_path)
+ localedir = oe.path.join(rootfs, self.d.getVar("libdir_nativesdk"), "locale")
+ generate_locale_archive(self.d, rootfs, target_arch, localedir)
# And now delete the binary locales
pkgs = fnmatch.filter(pm.list_installed(), "nativesdk-glibc-binary-localedata-*.utf-8")
pm.remove(pkgs)