From 95de93988eb725c14102f642ebabff3920ae194f Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Wed, 30 Jan 2019 17:42:25 +0000 Subject: package: Rework debug source file handling Currently we parallel process the files we install running dwarfsrcfiles over each one in parallel threads but requiring a lock to write the results to one file. This is not ideal for performance and means we can't then use per file data for other purposes such as source code license processing. Rework the code so that the list of source files is generated per installed file and is reusable. The code still generates a null separated debugsources.list file since this is used by a shell pipeline but it no longer needs locking. Signed-off-by: Richard Purdie --- meta/classes/package.bbclass | 50 ++++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 23 deletions(-) (limited to 'meta/classes/package.bbclass') diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass index 7dd1b09a87..749c7d9ea1 100644 --- a/meta/classes/package.bbclass +++ b/meta/classes/package.bbclass @@ -344,7 +344,7 @@ def parse_debugsources_from_dwarfsrcfiles_output(dwarfsrcfiles_output): return debugfiles.keys() -def append_source_info(file, sourcefile, d, fatal=True): +def source_info(file, d, fatal=True): import subprocess cmd = ["dwarfsrcfiles", file] @@ -363,22 +363,15 @@ def append_source_info(file, sourcefile, d, fatal=True): bb.note(msg) debugsources = parse_debugsources_from_dwarfsrcfiles_output(output) - # filenames are null-separated - this is an artefact of the previous use - # of rpm's debugedit, which was writing them out that way, and the code elsewhere - # is still assuming that. - debuglistoutput = '\0'.join(debugsources) + '\0' - lf = bb.utils.lockfile(sourcefile + ".lock") - with open(sourcefile, 'a') as sf: - sf.write(debuglistoutput) - bb.utils.unlockfile(lf) + return list(debugsources) -def splitdebuginfo(file, dvar, debugdir, debuglibdir, debugappend, debugsrcdir, sourcefile, d): +def splitdebuginfo(file, dvar, debugdir, debuglibdir, debugappend, debugsrcdir, d): # Function to split a single file into two components, one is the stripped # target system binary, the other contains any debugging information. The # two files are linked to reference each other. # - # sourcefile is also generated containing a list of debugsources + # return a mapping of files:debugsources import stat import subprocess @@ -386,6 +379,7 @@ def splitdebuginfo(file, dvar, debugdir, debuglibdir, debugappend, debugsrcdir, src = file[len(dvar):] dest = debuglibdir + os.path.dirname(src) + debugdir + "/" + os.path.basename(src) + debugappend debugfile = dvar + dest + sources = [] # Split the file... bb.utils.mkdirhier(os.path.dirname(debugfile)) @@ -397,7 +391,7 @@ def splitdebuginfo(file, dvar, debugdir, debuglibdir, debugappend, debugsrcdir, # We ignore kernel modules, we don't generate debug info files. if file.find("/lib/modules/") != -1 and file.endswith(".ko"): - return 1 + return (file, sources) newmode = None if not os.access(file, os.W_OK) or os.access(file, os.R_OK): @@ -407,7 +401,7 @@ def splitdebuginfo(file, dvar, debugdir, debuglibdir, debugappend, debugsrcdir, # We need to extract the debug src information here... if debugsrcdir: - append_source_info(file, sourcefile, d) + sources = source_info(file, d) bb.utils.mkdirhier(os.path.dirname(debugfile)) @@ -419,17 +413,26 @@ def splitdebuginfo(file, dvar, debugdir, debuglibdir, debugappend, debugsrcdir, if newmode: os.chmod(file, origmode) - return 0 + return (file, sources) -def copydebugsources(debugsrcdir, d): +def copydebugsources(debugsrcdir, sources, d): # The debug src information written out to sourcefile is further processed # and copied to the destination here. import stat import subprocess - sourcefile = d.expand("${WORKDIR}/debugsources.list") - if debugsrcdir and os.path.isfile(sourcefile): + if debugsrcdir and sources: + sourcefile = d.expand("${WORKDIR}/debugsources.list") + bb.utils.remove(sourcefile) + + # filenames are null-separated - this is an artefact of the previous use + # of rpm's debugedit, which was writing them out that way, and the code elsewhere + # is still assuming that. + debuglistoutput = '\0'.join(sources) + '\0' + with open(sourcefile, 'a') as sf: + sf.write(debuglistoutput) + dvar = d.getVar('PKGD') strip = d.getVar("STRIP") objcopy = d.getVar("OBJCOPY") @@ -933,9 +936,6 @@ python split_and_strip_files () { debuglibdir = "" debugsrcdir = "/usr/src/debug" - sourcefile = d.expand("${WORKDIR}/debugsources.list") - bb.utils.remove(sourcefile) - # # First lets figure out all of the files we may have to process ... do this only once! # @@ -1040,11 +1040,15 @@ python split_and_strip_files () { # First lets process debug splitting # if (d.getVar('INHIBIT_PACKAGE_DEBUG_SPLIT') != '1'): - oe.utils.multiprocess_launch(splitdebuginfo, list(elffiles), d, extraargs=(dvar, debugdir, debuglibdir, debugappend, debugsrcdir, sourcefile, d)) + results = oe.utils.multiprocess_launch(splitdebuginfo, list(elffiles), d, extraargs=(dvar, debugdir, debuglibdir, debugappend, debugsrcdir, d)) if debugsrcdir and not targetos.startswith("mingw"): for file in staticlibs: - append_source_info(file, sourcefile, d, fatal=False) + results.extend(source_info(file, d, fatal=False)) + + sources = set() + for r in results: + sources.update(r[1]) # Hardlink our debug symbols to the other hardlink copies for ref in inodes: @@ -1092,7 +1096,7 @@ python split_and_strip_files () { # Process the debugsrcdir if requested... # This copies and places the referenced sources for later debugging... - copydebugsources(debugsrcdir, d) + copydebugsources(debugsrcdir, sources, d) # # End of debug splitting # -- cgit 1.2.3-korg