aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJoshua Watt <JPEWhacker@gmail.com>2021-09-01 08:44:40 -0500
committerRichard Purdie <richard.purdie@linuxfoundation.org>2021-09-03 09:03:36 +0100
commitdb9cf430e54ae68da80fbc3fba80ce88d8df164d (patch)
tree3b8b22d689a763c993ad48b5c72e5e3d714edb3f
parenta8048931701438e2f267a52a43869c27743907e3 (diff)
downloadopenembedded-core-contrib-db9cf430e54ae68da80fbc3fba80ce88d8df164d.tar.gz
openembedded-core-contrib-db9cf430e54ae68da80fbc3fba80ce88d8df164d.tar.bz2
openembedded-core-contrib-db9cf430e54ae68da80fbc3fba80ce88d8df164d.zip
classes/package: Add extended packaged data
Adds extended package data which is encoded as JSON which allows it to encode more structure than the "flat" package data files. The extended data might be much larger than the standard package data, so it is not read by default and instead requires oe.packagedata.read_subpkgdata_extended() to be called Currently, the file sizes and ELF debug sources are saved off into the extended package data Signed-off-by: Joshua Watt <JPEWhacker@gmail.com> Reviewed-by: Saul Wold <saul.wold@windriver.com> Signed-off-by: Joshua Watt <JPEWhacker@gmail.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--meta/classes/package.bbclass40
-rw-r--r--meta/lib/oe/packagedata.py12
2 files changed, 49 insertions, 3 deletions
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index c4c5515d5d..3a78e48da4 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -1224,6 +1224,14 @@ python split_and_strip_files () {
# Modified the file so clear the cache
cpath.updatecache(file)
+ def strip_pkgd_prefix(f):
+ nonlocal dvar
+
+ if f.startswith(dvar):
+ return f[len(dvar):]
+
+ return f
+
#
# First lets process debug splitting
#
@@ -1237,6 +1245,8 @@ python split_and_strip_files () {
for file in staticlibs:
results.append( (file,source_info(file, d)) )
+ d.setVar("PKGDEBUGSOURCES", {strip_pkgd_prefix(f): sorted(s) for f, s in results})
+
sources = set()
for r in results:
sources.update(r[1])
@@ -1549,6 +1559,7 @@ PKGDATA_VARS = "PN PE PV PR PKGE PKGV PKGR LICENSE DESCRIPTION SUMMARY RDEPENDS
python emit_pkgdata() {
from glob import glob
import json
+ import bb.compress.zstd
def process_postinst_on_target(pkg, mlprefix):
pkgval = d.getVar('PKG:%s' % pkg)
@@ -1621,6 +1632,8 @@ fi
with open(data_file, 'w') as fd:
fd.write("PACKAGES: %s\n" % packages)
+ pkgdebugsource = d.getVar("PKGDEBUGSOURCES") or []
+
pn = d.getVar('PN')
global_variants = (d.getVar('MULTILIB_GLOBAL_VARIANTS') or "").split()
variants = (d.getVar('MULTILIB_VARIANTS') or "").split()
@@ -1640,17 +1653,32 @@ fi
pkgval = pkg
d.setVar('PKG:%s' % pkg, pkg)
+ extended_data = {
+ "files_info": {}
+ }
+
pkgdestpkg = os.path.join(pkgdest, pkg)
files = {}
+ files_extra = {}
total_size = 0
seen = set()
for f in pkgfiles[pkg]:
- relpth = os.path.relpath(f, pkgdestpkg)
+ fpath = os.sep + os.path.relpath(f, pkgdestpkg)
+
fstat = os.lstat(f)
- files[os.sep + relpth] = fstat.st_size
+ files[fpath] = fstat.st_size
+
+ extended_data["files_info"].setdefault(fpath, {})
+ extended_data["files_info"][fpath]['size'] = fstat.st_size
+
if fstat.st_ino not in seen:
seen.add(fstat.st_ino)
total_size += fstat.st_size
+
+ if fpath in pkgdebugsource:
+ extended_data["files_info"][fpath]['debugsrc'] = pkgdebugsource[fpath]
+ del pkgdebugsource[fpath]
+
d.setVar('FILES_INFO:' + pkg , json.dumps(files, sort_keys=True))
process_postinst_on_target(pkg, d.getVar("MLPREFIX"))
@@ -1671,6 +1699,11 @@ fi
sf.write('%s:%s: %d\n' % ('PKGSIZE', pkg, total_size))
+ subdata_extended_file = pkgdatadir + "/extended/%s.json.zstd" % pkg
+ num_threads = int(d.getVar("BB_NUMBER_THREADS"))
+ with bb.compress.zstd.open(subdata_extended_file, "wt", encoding="utf-8", num_threads=num_threads) as f:
+ json.dump(extended_data, f, sort_keys=True, separators=(",", ":"))
+
# Symlinks needed for rprovides lookup
rprov = d.getVar('RPROVIDES:%s' % pkg) or d.getVar('RPROVIDES')
if rprov:
@@ -1701,7 +1734,8 @@ fi
write_extra_runtime_pkgs(global_variants, packages, pkgdatadir)
}
-emit_pkgdata[dirs] = "${PKGDESTWORK}/runtime ${PKGDESTWORK}/runtime-reverse ${PKGDESTWORK}/runtime-rprovides"
+emit_pkgdata[dirs] = "${PKGDESTWORK}/runtime ${PKGDESTWORK}/runtime-reverse ${PKGDESTWORK}/runtime-rprovides ${PKGDESTWORK}/extended"
+emit_pkgdata[vardepsexclude] = "BB_NUMBER_THREADS"
ldconfig_postinst_fragment() {
if [ x"$D" = "x" ]; then
diff --git a/meta/lib/oe/packagedata.py b/meta/lib/oe/packagedata.py
index 0b17897e40..02c81e5a52 100644
--- a/meta/lib/oe/packagedata.py
+++ b/meta/lib/oe/packagedata.py
@@ -57,6 +57,18 @@ def read_subpkgdata_dict(pkg, d):
ret[newvar] = subd[var]
return ret
+def read_subpkgdata_extended(pkg, d):
+ import json
+ import bb.compress.zstd
+
+ fn = d.expand("${PKGDATA_DIR}/extended/%s.json.zstd" % pkg)
+ try:
+ num_threads = int(d.getVar("BB_NUMBER_THREADS"))
+ with bb.compress.zstd.open(fn, "rt", encoding="utf-8", num_threads=num_threads) as f:
+ return json.load(f)
+ except FileNotFoundError:
+ return None
+
def _pkgmap(d):
"""Return a dictionary mapping package to recipe name."""