diff options
-rw-r--r-- | meta/classes/packagefeed-stability.bbclass | 270 |
1 files changed, 270 insertions, 0 deletions
diff --git a/meta/classes/packagefeed-stability.bbclass b/meta/classes/packagefeed-stability.bbclass new file mode 100644 index 0000000000..b5207d9f84 --- /dev/null +++ b/meta/classes/packagefeed-stability.bbclass @@ -0,0 +1,270 @@ +# Class to avoid copying packages into the feed if they haven't materially changed +# +# Copyright (C) 2015 Intel Corporation +# Released under the MIT license (see COPYING.MIT for details) +# +# This class effectively intercepts packages as they are written out by +# do_package_write_*, causing them to be written into a different +# directory where we can compare them to whatever older packages might +# be in the "real" package feed directory, and avoid copying the new +# package to the feed if it has not materially changed. The idea is to +# avoid unnecessary churn in the packages when dependencies trigger task +# reexecution (and thus repackaging). Enabling the class is simple: +# +# INHERIT += "packagefeed-stability" +# +# Caveats: +# 1) Latest PR values in the build system may not match those in packages +# seen on the target (naturally) +# 2) If you rebuild from sstate without the existing package feed present, +# you will lose the "state" of the package feed i.e. the preserved old +# package versions. Not the end of the world, but would negate the +# entire purpose of this class. +# +# Note that running -c cleanall on a recipe will purposely delete the old +# package files so they will definitely be copied the next time. + +python() { + # Package backend agnostic intercept + # This assumes that the package_write task is called package_write_<pkgtype> + # and that the directory in which packages should be written is + # pointed to by the variable DEPLOY_DIR_<PKGTYPE> + for pkgclass in (d.getVar('PACKAGE_CLASSES', True) or '').split(): + if pkgclass.startswith('package_'): + pkgtype = pkgclass.split('_', 1)[1] + pkgwritefunc = 'do_package_write_%s' % pkgtype + sstate_outputdirs = d.getVarFlag(pkgwritefunc, 'sstate-outputdirs', False) + deploydirvar = 'DEPLOY_DIR_%s' % pkgtype.upper() + deploydirvarref = '${' + deploydirvar + '}' + pkgcomparefunc = 'do_package_compare_%s' % pkgtype + + if bb.data.inherits_class('image', d): + d.appendVarFlag('do_rootfs', 'recrdeptask', ' ' + pkgcomparefunc) + + if bb.data.inherits_class('populate_sdk_base', d): + d.appendVarFlag('do_populate_sdk', 'recrdeptask', ' ' + pkgcomparefunc) + + if bb.data.inherits_class('populate_sdk_ext', d): + d.appendVarFlag('do_populate_sdk_ext', 'recrdeptask', ' ' + pkgcomparefunc) + + d.appendVarFlag('do_build', 'recrdeptask', ' ' + pkgcomparefunc) + + if d.getVarFlag(pkgwritefunc, 'noexec', True) or (not d.getVarFlag(pkgwritefunc, 'task', True)) or pkgwritefunc in (d.getVar('__BBDELTASKS', True) or []): + # Packaging is disabled for this recipe, we shouldn't do anything + continue + + if deploydirvarref in sstate_outputdirs: + # Set intermediate output directory + d.setVarFlag(pkgwritefunc, 'sstate-outputdirs', sstate_outputdirs.replace(deploydirvarref, deploydirvarref + '-prediff')) + + d.setVar(pkgcomparefunc, d.getVar('do_package_compare', False)) + d.setVarFlags(pkgcomparefunc, d.getVarFlags('do_package_compare', False)) + d.appendVarFlag(pkgcomparefunc, 'depends', ' build-compare-native:do_populate_sysroot') + bb.build.addtask(pkgcomparefunc, 'do_build', 'do_packagedata ' + pkgwritefunc, d) +} + +# This isn't the real task function - it's a template that we use in the +# anonymous python code above +fakeroot python do_package_compare () { + currenttask = d.getVar('BB_CURRENTTASK', True) + pkgtype = currenttask.rsplit('_', 1)[1] + package_compare_impl(pkgtype, d) +} + +def package_compare_impl(pkgtype, d): + import errno + import fnmatch + import glob + import subprocess + import oe.sstatesig + + pn = d.getVar('PN', True) + deploydir = d.getVar('DEPLOY_DIR_%s' % pkgtype.upper(), True) + prepath = deploydir + '-prediff/' + + # Find out PKGR values are + pkgdatadir = d.getVar('PKGDATA_DIR', True) + packages = [] + try: + with open(os.path.join(pkgdatadir, pn), 'r') as f: + for line in f: + if line.startswith('PACKAGES:'): + packages = line.split(':', 1)[1].split() + break + except IOError as e: + if e.errno == errno.ENOENT: + pass + + if not packages: + bb.debug(2, '%s: no packages, nothing to do' % pn) + return + + pkgrvalues = {} + rpkgnames = {} + rdepends = {} + pkgvvalues = {} + for pkg in packages: + with open(os.path.join(pkgdatadir, 'runtime', pkg), 'r') as f: + for line in f: + if line.startswith('PKGR:'): + pkgrvalues[pkg] = line.split(':', 1)[1].strip() + if line.startswith('PKGV:'): + pkgvvalues[pkg] = line.split(':', 1)[1].strip() + elif line.startswith('PKG_%s:' % pkg): + rpkgnames[pkg] = line.split(':', 1)[1].strip() + elif line.startswith('RDEPENDS_%s:' % pkg): + rdepends[pkg] = line.split(':', 1)[1].strip() + + # Prepare a list of the runtime package names for packages that were + # actually produced + rpkglist = [] + for pkg, rpkg in rpkgnames.iteritems(): + if os.path.exists(os.path.join(pkgdatadir, 'runtime', pkg + '.packaged')): + rpkglist.append((rpkg, pkg)) + rpkglist.sort(key=lambda x: len(x[0]), reverse=True) + + pvu = d.getVar('PV', False) + if '$' + '{SRCPV}' in pvu: + pvprefix = pvu.split('$' + '{SRCPV}', 1)[0] + else: + pvprefix = None + + pkgwritetask = 'package_write_%s' % pkgtype + files = [] + copypkgs = [] + manifest, _ = oe.sstatesig.sstate_get_manifest_filename(pkgwritetask, d) + with open(manifest, 'r') as f: + for line in f: + if line.startswith(prepath): + srcpath = line.rstrip() + if os.path.isfile(srcpath): + destpath = os.path.join(deploydir, os.path.relpath(srcpath, prepath)) + + # This is crude but should work assuming the output + # package file name starts with the package name + # and rpkglist is sorted by length (descending) + pkgbasename = os.path.basename(destpath) + pkgname = None + for rpkg, pkg in rpkglist: + if pkgbasename.startswith(rpkg): + pkgr = pkgrvalues[pkg] + destpathspec = destpath.replace(pkgr, '*') + if pvprefix: + pkgv = pkgvvalues[pkg] + if pkgv.startswith(pvprefix): + pkgvsuffix = pkgv[len(pvprefix):] + if '+' in pkgvsuffix: + newpkgv = pvprefix + '*+' + pkgvsuffix.split('+', 1)[1] + destpathspec = destpathspec.replace(pkgv, newpkgv) + pkgname = pkg + break + else: + bb.warn('Unable to map %s back to package' % pkgbasename) + destpathspec = destpath + + oldfiles = glob.glob(destpathspec) + oldfile = None + docopy = True + if oldfiles: + oldfile = oldfiles[-1] + result = subprocess.call(['pkg-diff.sh', oldfile, srcpath]) + if result == 0: + docopy = False + + files.append((pkgname, pkgbasename, srcpath, oldfile, destpath)) + bb.debug(2, '%s: package %s %s' % (pn, files[-1], docopy)) + if docopy: + copypkgs.append(pkgname) + + # Ensure that dependencies on specific versions (such as -dev on the + # main package) are copied in lock-step + changed = True + while changed: + rpkgdict = {x[0]: x[1] for x in rpkglist} + changed = False + for pkgname, pkgbasename, srcpath, oldfile, destpath in files: + rdeps = rdepends.get(pkgname, None) + if not rdeps: + continue + rdepvers = bb.utils.explode_dep_versions2(rdeps) + for rdep, versions in rdepvers.iteritems(): + dep = rpkgdict.get(rdep, None) + for version in versions: + if version and version.startswith('= '): + if dep in copypkgs and not pkgname in copypkgs: + bb.debug(2, '%s: copying %s because it has a fixed version dependency on %s and that package is going to be copied' % (pn, pkgname, dep)) + changed = True + copypkgs.append(pkgname) + elif pkgname in copypkgs and not dep in copypkgs: + bb.debug(2, '%s: copying %s because %s has a fixed version dependency on it and that package is going to be copied' % (pn, dep, pkgname)) + changed = True + copypkgs.append(dep) + + # Read in old manifest so we can delete any packages we aren't going to replace or preserve + pcmanifest = os.path.join(prepath, d.expand('pkg-compare-manifest-${MULTIMACH_TARGET_SYS}-${PN}')) + try: + with open(pcmanifest, 'r') as f: + knownfiles = [x[3] for x in files if x[3]] + for line in f: + fn = line.rstrip() + if fn: + if fn in knownfiles: + knownfiles.remove(fn) + else: + try: + os.remove(fn) + bb.warn('Removed old package %s' % fn) + except OSError as e: + if e.errno == errno.ENOENT: + pass + except IOError as e: + if e.errno == errno.ENOENT: + pass + + # Create new manifest + with open(pcmanifest, 'w') as f: + for pkgname, pkgbasename, srcpath, oldfile, destpath in files: + if pkgname in copypkgs: + bb.warn('Copying %s' % pkgbasename) + destdir = os.path.dirname(destpath) + bb.utils.mkdirhier(destdir) + if oldfile: + try: + os.remove(oldfile) + except OSError as e: + if e.errno == errno.ENOENT: + pass + if (os.stat(srcpath).st_dev == os.stat(destdir).st_dev): + # Use a hard link to save space + os.link(srcpath, destpath) + else: + shutil.copyfile(srcpath, destpath) + f.write('%s\n' % destpath) + else: + bb.warn('Not copying %s' % pkgbasename) + f.write('%s\n' % oldfile) + + +do_cleanall_append() { + import errno + for pkgclass in (d.getVar('PACKAGE_CLASSES', True) or '').split(): + if pkgclass.startswith('package_'): + pkgtype = pkgclass.split('_', 1)[1] + deploydir = d.getVar('DEPLOY_DIR_%s' % pkgtype.upper(), True) + prepath = deploydir + '-prediff' + pcmanifest = os.path.join(prepath, d.expand('pkg-compare-manifest-${MULTIMACH_TARGET_SYS}-${PN}')) + try: + with open(pcmanifest, 'r') as f: + for line in f: + fn = line.rstrip() + if fn: + try: + os.remove(fn) + except OSError as e: + if e.errno == errno.ENOENT: + pass + os.remove(pcmanifest) + except IOError as e: + if e.errno == errno.ENOENT: + pass +} |