diff options
Diffstat (limited to 'meta/classes/archiver.bbclass')
| -rw-r--r-- | meta/classes/archiver.bbclass | 331 |
1 files changed, 244 insertions, 87 deletions
diff --git a/meta/classes/archiver.bbclass b/meta/classes/archiver.bbclass index ec80ad47a4..2d0bbfbd42 100644 --- a/meta/classes/archiver.bbclass +++ b/meta/classes/archiver.bbclass @@ -1,31 +1,49 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + # # This bbclass is used for creating archive for: -# 1) original (or unpacked) source: ARCHIVER_MODE[src] = "original" -# 2) patched source: ARCHIVER_MODE[src] = "patched" (default) -# 3) configured source: ARCHIVER_MODE[src] = "configured" -# 4) The patches between do_unpack and do_patch: -# ARCHIVER_MODE[diff] = "1" -# And you can set the one that you'd like to exclude from the diff: -# ARCHIVER_MODE[diff-exclude] ?= ".pc autom4te.cache patches" -# 5) The environment data, similar to 'bitbake -e recipe': -# ARCHIVER_MODE[dumpdata] = "1" -# 6) The recipe (.bb and .inc): ARCHIVER_MODE[recipe] = "1" -# 7) Whether output the .src.rpm package: -# ARCHIVER_MODE[srpm] = "1" -# 8) Filter the license, the recipe whose license in -# COPYLEFT_LICENSE_INCLUDE will be included, and in -# COPYLEFT_LICENSE_EXCLUDE will be excluded. -# COPYLEFT_LICENSE_INCLUDE = 'GPL* LGPL*' -# COPYLEFT_LICENSE_EXCLUDE = 'CLOSED Proprietary' -# 9) The recipe type that will be archived: -# COPYLEFT_RECIPE_TYPES = 'target' +# 1) original (or unpacked) source: ARCHIVER_MODE[src] = "original" +# 2) patched source: ARCHIVER_MODE[src] = "patched" (default) +# 3) configured source: ARCHIVER_MODE[src] = "configured" +# 4) source mirror: ARCHIVER_MODE[src] = "mirror" +# 5) The patches between do_unpack and do_patch: +# ARCHIVER_MODE[diff] = "1" +# And you can set the one that you'd like to exclude from the diff: +# ARCHIVER_MODE[diff-exclude] ?= ".pc autom4te.cache patches" +# 6) The environment data, similar to 'bitbake -e recipe': +# ARCHIVER_MODE[dumpdata] = "1" +# 7) The recipe (.bb and .inc): ARCHIVER_MODE[recipe] = "1" +# 8) Whether output the .src.rpm package: +# ARCHIVER_MODE[srpm] = "1" +# 9) Filter the license, the recipe whose license in +# COPYLEFT_LICENSE_INCLUDE will be included, and in +# COPYLEFT_LICENSE_EXCLUDE will be excluded. +# COPYLEFT_LICENSE_INCLUDE = 'GPL* LGPL*' +# COPYLEFT_LICENSE_EXCLUDE = 'CLOSED Proprietary' +# 10) The recipe type that will be archived: +# COPYLEFT_RECIPE_TYPES = 'target' +# 11) The source mirror mode: +# ARCHIVER_MODE[mirror] = "split" (default): Sources are split into +# per-recipe directories in a similar way to other archiver modes. +# Post-processing may be required to produce a single mirror directory. +# This does however allow inspection of duplicate sources and more +# intelligent handling. +# ARCHIVER_MODE[mirror] = "combined": All sources are placed into a single +# directory suitable for direct use as a mirror. Duplicate sources are +# ignored. +# 12) Source mirror exclusions: +# ARCHIVER_MIRROR_EXCLUDE is a list of prefixes to exclude from the mirror. +# This may be used for sources which you are already publishing yourself +# (e.g. if the URI starts with 'https://mysite.com/' and your mirror is +# going to be published to the same site). It may also be used to exclude +# local files (with the prefix 'file://') if these will be provided as part +# of an archive of the layers themselves. # -# Don't filter the license by default -COPYLEFT_LICENSE_INCLUDE ?= '' -COPYLEFT_LICENSE_EXCLUDE ?= '' # Create archive for all the recipe types COPYLEFT_RECIPE_TYPES ?= 'target native nativesdk cross crosssdk cross-canadian' inherit copyleft_filter @@ -36,49 +54,70 @@ ARCHIVER_MODE[diff] ?= "0" ARCHIVER_MODE[diff-exclude] ?= ".pc autom4te.cache patches" ARCHIVER_MODE[dumpdata] ?= "0" ARCHIVER_MODE[recipe] ?= "0" +ARCHIVER_MODE[mirror] ?= "split" +ARCHIVER_MODE[compression] ?= "xz" DEPLOY_DIR_SRC ?= "${DEPLOY_DIR}/sources" -ARCHIVER_TOPDIR ?= "${WORKDIR}/deploy-sources" -ARCHIVER_OUTDIR = "${ARCHIVER_TOPDIR}/${TARGET_SYS}/${PF}/" +ARCHIVER_TOPDIR ?= "${WORKDIR}/archiver-sources" +ARCHIVER_ARCH = "${TARGET_SYS}" +ARCHIVER_OUTDIR = "${ARCHIVER_TOPDIR}/${ARCHIVER_ARCH}/${PF}/" +ARCHIVER_RPMTOPDIR ?= "${WORKDIR}/deploy-sources-rpm" +ARCHIVER_RPMOUTDIR = "${ARCHIVER_RPMTOPDIR}/${ARCHIVER_ARCH}/${PF}/" ARCHIVER_WORKDIR = "${WORKDIR}/archiver-work/" +# When producing a combined mirror directory, allow duplicates for the case +# where multiple recipes use the same SRC_URI. +ARCHIVER_COMBINED_MIRRORDIR = "${ARCHIVER_TOPDIR}/mirror" +SSTATE_ALLOW_OVERLAP_FILES += "${DEPLOY_DIR_SRC}/mirror" + do_dumpdata[dirs] = "${ARCHIVER_OUTDIR}" do_ar_recipe[dirs] = "${ARCHIVER_OUTDIR}" do_ar_original[dirs] = "${ARCHIVER_OUTDIR} ${ARCHIVER_WORKDIR}" -do_deploy_archives[dirs] = "${WORKDIR}" -do_deploy_all_archives[dirs] = "${WORKDIR}" # This is a convenience for the shell script to use it - -python () { - pn = d.getVar('PN') - assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split() - if pn in assume_provided: - for p in d.getVar("PROVIDES").split(): - if p != pn: - pn = p - break +def include_package(d, pn): included, reason = copyleft_should_include(d) if not included: bb.debug(1, 'archiver: %s is excluded: %s' % (pn, reason)) - return + return False + else: bb.debug(1, 'archiver: %s is included: %s' % (pn, reason)) - # glibc-locale: do_fetch, do_unpack and do_patch tasks have been deleted, # so avoid archiving source here. if pn.startswith('glibc-locale'): - return + return False # We just archive gcc-source for all the gcc related recipes if d.getVar('BPN') in ['gcc', 'libgcc'] \ and not pn.startswith('gcc-source'): bb.debug(1, 'archiver: %s is excluded, covered by gcc-source' % pn) + return False + + return True + +python () { + pn = d.getVar('PN') + assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split() + if pn in assume_provided: + for p in d.getVar("PROVIDES").split(): + if p != pn: + pn = p + break + + if not include_package(d, pn): return + # TARGET_SYS in ARCHIVER_ARCH will break the stamp for gcc-source in multiconfig + if pn.startswith('gcc-source'): + d.setVar('ARCHIVER_ARCH', "allarch") + + def hasTask(task): + return bool(d.getVarFlag(task, "task", False)) and not bool(d.getVarFlag(task, "noexec", False)) + ar_src = d.getVarFlag('ARCHIVER_MODE', 'src') ar_dumpdata = d.getVarFlag('ARCHIVER_MODE', 'dumpdata') ar_recipe = d.getVarFlag('ARCHIVER_MODE', 'recipe') @@ -94,16 +133,18 @@ python () { d.appendVarFlag('do_deploy_archives', 'depends', ' %s:do_ar_patched' % pn) elif ar_src == "configured": # We can't use "addtask do_ar_configured after do_configure" since it - # will cause the deptask of do_populate_sysroot to run not matter what + # will cause the deptask of do_populate_sysroot to run no matter what # archives we need, so we add the depends here. # There is a corner case with "gcc-source-${PV}" recipes, they don't have # the "do_configure" task, so we need to use "do_preconfigure" - if pn.startswith("gcc-source-"): + if hasTask("do_preconfigure"): d.appendVarFlag('do_ar_configured', 'depends', ' %s:do_preconfigure' % pn) - else: + elif hasTask("do_configure"): d.appendVarFlag('do_ar_configured', 'depends', ' %s:do_configure' % pn) d.appendVarFlag('do_deploy_archives', 'depends', ' %s:do_ar_configured' % pn) + elif ar_src == "mirror": + d.appendVarFlag('do_deploy_archives', 'depends', '%s:do_ar_mirror' % pn) elif ar_src: bb.fatal("Invalid ARCHIVER_MODE[src]: %s" % ar_src) @@ -114,11 +155,17 @@ python () { if ar_recipe == "1": d.appendVarFlag('do_deploy_archives', 'depends', ' %s:do_ar_recipe' % pn) - # Output the srpm package - ar_srpm = d.getVarFlag('ARCHIVER_MODE', 'srpm') - if ar_srpm == "1": - if d.getVar('PACKAGES') != '' and d.getVar('IMAGE_PKGTYPE') == 'rpm': + # Output the SRPM package + if d.getVarFlag('ARCHIVER_MODE', 'srpm') == "1" and d.getVar('PACKAGES'): + if "package_rpm" not in d.getVar('PACKAGE_CLASSES'): + bb.fatal("ARCHIVER_MODE[srpm] needs package_rpm in PACKAGE_CLASSES") + + # Some recipes do not have any packaging tasks + if hasTask("do_package_write_rpm"): d.appendVarFlag('do_deploy_archives', 'depends', ' %s:do_package_write_rpm' % pn) + d.appendVarFlag('do_package_write_rpm', 'dirs', ' ${ARCHIVER_RPMTOPDIR}') + d.appendVarFlag('do_package_write_rpm', 'sstate-inputdirs', ' ${ARCHIVER_RPMTOPDIR}') + d.appendVarFlag('do_package_write_rpm', 'sstate-outputdirs', ' ${DEPLOY_DIR_SRC}') if ar_dumpdata == "1": d.appendVarFlag('do_package_write_rpm', 'depends', ' %s:do_dumpdata' % pn) if ar_recipe == "1": @@ -131,7 +178,7 @@ python () { d.appendVarFlag('do_package_write_rpm', 'depends', ' %s:do_ar_configured' % pn) } -# Take all the sources for a recipe and puts them in WORKDIR/archiver-work/. +# Take all the sources for a recipe and put them in WORKDIR/archiver-work/. # Files in SRC_URI are copied directly, anything that's a directory # (e.g. git repositories) is "unpacked" and then put into a tarball. python do_ar_original() { @@ -161,7 +208,13 @@ python do_ar_original() { del decoded[5][param] encoded = bb.fetch2.encodeurl(decoded) urls[i] = encoded - fetch = bb.fetch2.Fetch(urls, d) + + # Cleanup SRC_URI before call bb.fetch2.Fetch() since now SRC_URI is in the + # variable "urls", otherwise there might be errors like: + # The SRCREV_FORMAT variable must be set when multiple SCMs are used + ld = bb.data.createCopy(d) + ld.setVar('SRC_URI', '') + fetch = bb.fetch2.Fetch(urls, ld) tarball_suffix = {} for url in fetch.urls: local = fetch.localpath(url).rstrip("/"); @@ -213,9 +266,10 @@ python do_ar_patched() { # Get the ARCHIVER_OUTDIR before we reset the WORKDIR ar_outdir = d.getVar('ARCHIVER_OUTDIR') - ar_workdir = d.getVar('ARCHIVER_WORKDIR') + if not is_work_shared(d): + ar_workdir = d.getVar('ARCHIVER_WORKDIR') + d.setVar('WORKDIR', ar_workdir) bb.note('Archiving the patched source...') - d.setVar('WORKDIR', ar_workdir) create_tarball(d, d.getVar('S'), 'patched', ar_outdir) } @@ -242,22 +296,30 @@ python do_ar_configured() { # ${STAGING_DATADIR}/aclocal/libtool.m4, so we can't re-run the # do_configure, we archive the already configured ${S} to # instead of. - elif pn != 'libtool-native': + # The kernel class functions require it to be on work-shared, we + # don't unpack, patch, configure again, just archive the already + # configured ${S} + elif not (pn == 'libtool-native' or is_work_shared(d)): + def runTask(task): + prefuncs = d.getVarFlag(task, 'prefuncs') or '' + for func in prefuncs.split(): + if func != "sysroot_cleansstate": + bb.build.exec_func(func, d) + bb.build.exec_func(task, d) + postfuncs = d.getVarFlag(task, 'postfuncs') or '' + for func in postfuncs.split(): + if func != 'do_qa_configure': + bb.build.exec_func(func, d) + # Change the WORKDIR to make do_configure run in another dir. d.setVar('WORKDIR', d.getVar('ARCHIVER_WORKDIR')) - if bb.data.inherits_class('kernel-yocto', d): - bb.build.exec_func('do_kernel_configme', d) - if bb.data.inherits_class('cmake', d): - bb.build.exec_func('do_generate_toolchain_file', d) - prefuncs = d.getVarFlag('do_configure', 'prefuncs') - for func in (prefuncs or '').split(): - if func != "sysroot_cleansstate": - bb.build.exec_func(func, d) - bb.build.exec_func('do_configure', d) - postfuncs = d.getVarFlag('do_configure', 'postfuncs') - for func in (postfuncs or '').split(): - if func != "do_qa_configure": - bb.build.exec_func(func, d) + + preceeds = bb.build.preceedtask('do_configure', False, d) + for task in preceeds: + if task != 'do_patch' and task != 'do_prepare_recipe_sysroot': + runTask(task) + runTask('do_configure') + srcdir = d.getVar('S') builddir = d.getVar('B') if srcdir != builddir: @@ -267,27 +329,117 @@ python do_ar_configured() { create_tarball(d, srcdir, 'configured', ar_outdir) } +python do_ar_mirror() { + import subprocess + + src_uri = (d.getVar('SRC_URI') or '').split() + if len(src_uri) == 0: + return + + dl_dir = d.getVar('DL_DIR') + mirror_exclusions = (d.getVar('ARCHIVER_MIRROR_EXCLUDE') or '').split() + mirror_mode = d.getVarFlag('ARCHIVER_MODE', 'mirror') + have_mirror_tarballs = d.getVar('BB_GENERATE_MIRROR_TARBALLS') + + if mirror_mode == 'combined': + destdir = d.getVar('ARCHIVER_COMBINED_MIRRORDIR') + elif mirror_mode == 'split': + destdir = d.getVar('ARCHIVER_OUTDIR') + else: + bb.fatal('Invalid ARCHIVER_MODE[mirror]: %s' % (mirror_mode)) + + if not have_mirror_tarballs: + bb.fatal('Using `ARCHIVER_MODE[src] = "mirror"` depends on setting `BB_GENERATE_MIRROR_TARBALLS = "1"`') + + def is_excluded(url): + for prefix in mirror_exclusions: + if url.startswith(prefix): + return True + return False + + bb.note('Archiving the source as a mirror...') + + bb.utils.mkdirhier(destdir) + + fetcher = bb.fetch2.Fetch(src_uri, d) + + for ud in fetcher.expanded_urldata(): + if is_excluded(ud.url): + bb.note('Skipping excluded url: %s' % (ud.url)) + continue + + bb.note('Archiving url: %s' % (ud.url)) + ud.setup_localpath(d) + localpath = None + + # Check for mirror tarballs first. We will archive the first mirror + # tarball that we find as it's assumed that we just need one. + for mirror_fname in ud.mirrortarballs: + mirror_path = os.path.join(dl_dir, mirror_fname) + if os.path.exists(mirror_path): + bb.note('Found mirror tarball: %s' % (mirror_path)) + localpath = mirror_path + break + + if len(ud.mirrortarballs) and not localpath: + bb.warn('Mirror tarballs are listed for a source but none are present. ' \ + 'Falling back to original download.\n' \ + 'SRC_URI = %s' % (ud.url)) + + # Check original download + if not localpath: + bb.note('Using original download: %s' % (ud.localpath)) + localpath = ud.localpath + + if not localpath or not os.path.exists(localpath): + bb.fatal('Original download is missing for a source.\n' \ + 'SRC_URI = %s' % (ud.url)) + + # We now have an appropriate localpath + bb.note('Copying source mirror') + cmd = 'cp -fpPRH %s %s' % (localpath, destdir) + subprocess.check_call(cmd, shell=True) +} + def create_tarball(d, srcdir, suffix, ar_outdir): """ create the tarball from srcdir """ - import tarfile + import subprocess # Make sure we are only creating a single tarball for gcc sources if (d.getVar('SRC_URI') == ""): return + # For the kernel archive, srcdir may just be a link to the + # work-shared location. Use os.path.realpath to make sure + # that we archive the actual directory and not just the link. + srcdir = os.path.realpath(srcdir) + + compression_method = d.getVarFlag('ARCHIVER_MODE', 'compression') + if compression_method == "xz": + compression_cmd = "xz %s" % d.getVar('XZ_DEFAULTS') + # To keep compatibility with ARCHIVER_MODE[compression] + elif compression_method == "gz": + compression_cmd = "gzip" + elif compression_method == "bz2": + compression_cmd = "bzip2" + else: + bb.fatal("Unsupported compression_method: %s" % compression_method) + bb.utils.mkdirhier(ar_outdir) if suffix: - filename = '%s-%s.tar.gz' % (d.getVar('PF'), suffix) + filename = '%s-%s.tar.%s' % (d.getVar('PF'), suffix, compression_method) else: - filename = '%s.tar.gz' % d.getVar('PF') + filename = '%s.tar.%s' % (d.getVar('PF'), compression_method) tarname = os.path.join(ar_outdir, filename) bb.note('Creating %s' % tarname) - tar = tarfile.open(tarname, 'w:gz') - tar.add(srcdir, arcname=os.path.basename(srcdir)) - tar.close() + dirname = os.path.dirname(srcdir) + basename = os.path.basename(srcdir) + exclude = "--exclude=temp --exclude=patches --exclude='.pc'" + tar_cmd = "tar %s -cf - %s | %s > %s" % (exclude, basename, compression_cmd, tarname) + subprocess.check_call(tar_cmd, cwd=dirname, shell=True) # creating .diff.gz between source.orig and source def create_diff_gz(d, src_orig, src, ar_outdir): @@ -319,6 +471,10 @@ def create_diff_gz(d, src_orig, src, ar_outdir): finally: os.chdir(cwd) +def is_work_shared(d): + sharedworkdir = os.path.join(d.getVar('TMPDIR'), 'work-shared') + return d.getVar('S').startswith(sharedworkdir) + # Run do_unpack and do_patch python do_unpack_and_patch() { if d.getVarFlag('ARCHIVER_MODE', 'src') not in \ @@ -330,8 +486,8 @@ python do_unpack_and_patch() { ar_sysroot_native = d.getVar('STAGING_DIR_NATIVE') pn = d.getVar('PN') - # The kernel class functions require it to be on work-shared, so we dont change WORKDIR - if not (bb.data.inherits_class('kernel-yocto', d) or pn.startswith('gcc-source')): + # The kernel class functions require it to be on work-shared, so we don't change WORKDIR + if not is_work_shared(d): # Change the WORKDIR to make do_unpack do_patch run in another dir. d.setVar('WORKDIR', ar_workdir) # Restore the original path to recipe's native sysroot (it's relative to WORKDIR). @@ -350,8 +506,11 @@ python do_unpack_and_patch() { src_orig = '%s.orig' % src oe.path.copytree(src, src_orig) + if bb.data.inherits_class('dos2unix', d): + bb.build.exec_func('do_convert_crlf_to_lf', d) + # Make sure gcc and kernel sources are patched only once - if not (d.getVar('SRC_URI') == "" or (bb.data.inherits_class('kernel-yocto', d) or pn.startswith('gcc-source'))): + if not (d.getVar('SRC_URI') == "" or is_work_shared(d)): bb.build.exec_func('do_patch', d) # Create the patches @@ -372,7 +531,7 @@ python do_unpack_and_patch() { # of the output file ensures that we create it each time the recipe # gets rebuilt, at least as long as a PR server is used. We also rely # on that mechanism to catch changes in the file content, because the -# file content is not part of of the task signature either. +# file content is not part of the task signature either. do_ar_recipe[vardepsexclude] += "BBINCLUDED" python do_ar_recipe () { """ @@ -410,9 +569,10 @@ python do_ar_recipe () { incfile = include_re.match(line).group(1) if incfile: incfile = d.expand(incfile) + if incfile: incfile = bb.utils.which(bbpath, incfile) - if incfile: - shutil.copy(incfile, outdir) + if incfile: + shutil.copy(incfile, outdir) create_tarball(d, outdir, 'recipe', d.getVar('ARCHIVER_OUTDIR')) bb.utils.remove(outdir, recurse=True) @@ -437,7 +597,7 @@ python do_dumpdata () { SSTATETASKS += "do_deploy_archives" do_deploy_archives () { - echo "Deploying source archive files from ${ARCHIVER_TOPDIR} to ${DEPLOY_DIR_SRC}." + bbnote "Deploying source archive files from ${ARCHIVER_TOPDIR} to ${DEPLOY_DIR_SRC}." } python do_deploy_archives_setscene () { sstate_setscene(d) @@ -448,19 +608,16 @@ do_deploy_archives[sstate-outputdirs] = "${DEPLOY_DIR_SRC}" addtask do_deploy_archives_setscene addtask do_ar_original after do_unpack -addtask do_unpack_and_patch after do_patch +addtask do_unpack_and_patch after do_patch do_preconfigure addtask do_ar_patched after do_unpack_and_patch addtask do_ar_configured after do_unpack_and_patch +addtask do_ar_mirror after do_fetch addtask do_dumpdata addtask do_ar_recipe -addtask do_deploy_archives before do_build - -addtask do_deploy_all_archives after do_deploy_archives -do_deploy_all_archives[recrdeptask] = "do_deploy_archives" -do_deploy_all_archives[recideptask] = "do_${BB_DEFAULT_TASK}" -do_deploy_all_archives() { - : -} +addtask do_deploy_archives +do_build[recrdeptask] += "do_deploy_archives" +do_rootfs[recrdeptask] += "do_deploy_archives" +do_populate_sdk[recrdeptask] += "do_deploy_archives" python () { # Add tasks in the correct order, specifically for linux-yocto to avoid race condition. |