# Class to avoid copying packages into the feed if they haven't materially changed # # Copyright (C) 2015 Intel Corporation # Released under the MIT license (see COPYING.MIT for details) # # This class effectively intercepts packages as they are written out by # do_package_write_*, causing them to be written into a different # directory where we can compare them to whatever older packages might # be in the "real" package feed directory, and avoid copying the new # package to the feed if it has not materially changed. The idea is to # avoid unnecessary churn in the packages when dependencies trigger task # reexecution (and thus repackaging). Enabling the class is simple: # # INHERIT += "packagefeed-stability" # # Caveats: # 1) Latest PR values in the build system may not match those in packages # seen on the target (naturally) # 2) If you rebuild from sstate without the existing package feed present, # you will lose the "state" of the package feed i.e. the preserved old # package versions. Not the end of the world, but would negate the # entire purpose of this class. # # Note that running -c cleanall on a recipe will purposely delete the old # package files so they will definitely be copied the next time. python() { # Package backend agnostic intercept # This assumes that the package_write task is called package_write_ # and that the directory in which packages should be written is # pointed to by the variable DEPLOY_DIR_ for pkgclass in (d.getVar('PACKAGE_CLASSES', True) or '').split(): if pkgclass.startswith('package_'): pkgtype = pkgclass.split('_', 1)[1] pkgwritefunc = 'do_package_write_%s' % pkgtype sstate_outputdirs = d.getVarFlag(pkgwritefunc, 'sstate-outputdirs', False) deploydirvar = 'DEPLOY_DIR_%s' % pkgtype.upper() deploydirvarref = '${' + deploydirvar + '}' pkgcomparefunc = 'do_package_compare_%s' % pkgtype if bb.data.inherits_class('image', d): d.appendVarFlag('do_rootfs', 'recrdeptask', ' ' + pkgcomparefunc) if bb.data.inherits_class('populate_sdk_base', d): d.appendVarFlag('do_populate_sdk', 'recrdeptask', ' ' + pkgcomparefunc) if bb.data.inherits_class('populate_sdk_ext', d): d.appendVarFlag('do_populate_sdk_ext', 'recrdeptask', ' ' + pkgcomparefunc) d.appendVarFlag('do_build', 'recrdeptask', ' ' + pkgcomparefunc) if d.getVarFlag(pkgwritefunc, 'noexec', True) or (not d.getVarFlag(pkgwritefunc, 'task', True)) or pkgwritefunc in (d.getVar('__BBDELTASKS', True) or []): # Packaging is disabled for this recipe, we shouldn't do anything continue if deploydirvarref in sstate_outputdirs: # Set intermediate output directory d.setVarFlag(pkgwritefunc, 'sstate-outputdirs', sstate_outputdirs.replace(deploydirvarref, deploydirvarref + '-prediff')) d.setVar(pkgcomparefunc, d.getVar('do_package_compare', False)) d.setVarFlags(pkgcomparefunc, d.getVarFlags('do_package_compare', False)) d.appendVarFlag(pkgcomparefunc, 'depends', ' build-compare-native:do_populate_sysroot') bb.build.addtask(pkgcomparefunc, 'do_build', 'do_packagedata ' + pkgwritefunc, d) } # This isn't the real task function - it's a template that we use in the # anonymous python code above fakeroot python do_package_compare () { currenttask = d.getVar('BB_CURRENTTASK', True) pkgtype = currenttask.rsplit('_', 1)[1] package_compare_impl(pkgtype, d) } def package_compare_impl(pkgtype, d): import errno import fnmatch import glob import subprocess import oe.sstatesig pn = d.getVar('PN', True) deploydir = d.getVar('DEPLOY_DIR_%s' % pkgtype.upper(), True) prepath = deploydir + '-prediff/' # Find out PKGR values are pkgdatadir = d.getVar('PKGDATA_DIR', True) packages = [] try: with open(os.path.join(pkgdatadir, pn), 'r') as f: for line in f: if line.startswith('PACKAGES:'): packages = line.split(':', 1)[1].split() break except IOError as e: if e.errno == errno.ENOENT: pass if not packages: bb.debug(2, '%s: no packages, nothing to do' % pn) return pkgrvalues = {} rpkgnames = {} rdepends = {} pkgvvalues = {} for pkg in packages: with open(os.path.join(pkgdatadir, 'runtime', pkg), 'r') as f: for line in f: if line.startswith('PKGR:'): pkgrvalues[pkg] = line.split(':', 1)[1].strip() if line.startswith('PKGV:'): pkgvvalues[pkg] = line.split(':', 1)[1].strip() elif line.startswith('PKG_%s:' % pkg): rpkgnames[pkg] = line.split(':', 1)[1].strip() elif line.startswith('RDEPENDS_%s:' % pkg): rdepends[pkg] = line.split(':', 1)[1].strip() # Prepare a list of the runtime package names for packages that were # actually produced rpkglist = [] for pkg, rpkg in rpkgnames.iteritems(): if os.path.exists(os.path.join(pkgdatadir, 'runtime', pkg + '.packaged')): rpkglist.append((rpkg, pkg)) rpkglist.sort(key=lambda x: len(x[0]), reverse=True) pvu = d.getVar('PV', False) if '$' + '{SRCPV}' in pvu: pvprefix = pvu.split('$' + '{SRCPV}', 1)[0] else: pvprefix = None pkgwritetask = 'package_write_%s' % pkgtype files = [] copypkgs = [] manifest, _ = oe.sstatesig.sstate_get_manifest_filename(pkgwritetask, d) with open(manifest, 'r') as f: for line in f: if line.startswith(prepath): srcpath = line.rstrip() if os.path.isfile(srcpath): destpath = os.path.join(deploydir, os.path.relpath(srcpath, prepath)) # This is crude but should work assuming the output # package file name starts with the package name # and rpkglist is sorted by length (descending) pkgbasename = os.path.basename(destpath) pkgname = None for rpkg, pkg in rpkglist: if pkgbasename.startswith(rpkg): pkgr = pkgrvalues[pkg] destpathspec = destpath.replace(pkgr, '*') if pvprefix: pkgv = pkgvvalues[pkg] if pkgv.startswith(pvprefix): pkgvsuffix = pkgv[len(pvprefix):] if '+' in pkgvsuffix: newpkgv = pvprefix + '*+' + pkgvsuffix.split('+', 1)[1] destpathspec = destpathspec.replace(pkgv, newpkgv) pkgname = pkg break else: bb.warn('Unable to map %s back to package' % pkgbasename) destpathspec = destpath oldfiles = glob.glob(destpathspec) oldfile = None docopy = True if oldfiles: oldfile = oldfiles[-1] result = subprocess.call(['pkg-diff.sh', oldfile, srcpath]) if result == 0: docopy = False files.append((pkgname, pkgbasename, srcpath, oldfile, destpath)) bb.debug(2, '%s: package %s %s' % (pn, files[-1], docopy)) if docopy: copypkgs.append(pkgname) # Ensure that dependencies on specific versions (such as -dev on the # main package) are copied in lock-step changed = True while changed: rpkgdict = {x[0]: x[1] for x in rpkglist} changed = False for pkgname, pkgbasename, srcpath, oldfile, destpath in files: rdeps = rdepends.get(pkgname, None) if not rdeps: continue rdepvers = bb.utils.explode_dep_versions2(rdeps) for rdep, versions in rdepvers.iteritems(): dep = rpkgdict.get(rdep, None) for version in versions: if version and version.startswith('= '): if dep in copypkgs and not pkgname in copypkgs: bb.debug(2, '%s: copying %s because it has a fixed version dependency on %s and that package is going to be copied' % (pn, pkgname, dep)) changed = True copypkgs.append(pkgname) elif pkgname in copypkgs and not dep in copypkgs: bb.debug(2, '%s: copying %s because %s has a fixed version dependency on it and that package is going to be copied' % (pn, dep, pkgname)) changed = True copypkgs.append(dep) # Read in old manifest so we can delete any packages we aren't going to replace or preserve pcmanifest = os.path.join(prepath, d.expand('pkg-compare-manifest-${MULTIMACH_TARGET_SYS}-${PN}')) try: with open(pcmanifest, 'r') as f: knownfiles = [x[3] for x in files if x[3]] for line in f: fn = line.rstrip() if fn: if fn in knownfiles: knownfiles.remove(fn) else: try: os.remove(fn) bb.warn('Removed old package %s' % fn) except OSError as e: if e.errno == errno.ENOENT: pass except IOError as e: if e.errno == errno.ENOENT: pass # Create new manifest with open(pcmanifest, 'w') as f: for pkgname, pkgbasename, srcpath, oldfile, destpath in files: if pkgname in copypkgs: bb.warn('Copying %s' % pkgbasename) destdir = os.path.dirname(destpath) bb.utils.mkdirhier(destdir) if oldfile: try: os.remove(oldfile) except OSError as e: if e.errno == errno.ENOENT: pass if (os.stat(srcpath).st_dev == os.stat(destdir).st_dev): # Use a hard link to save space os.link(srcpath, destpath) else: shutil.copyfile(srcpath, destpath) f.write('%s\n' % destpath) else: bb.warn('Not copying %s' % pkgbasename) f.write('%s\n' % oldfile) do_cleanall_append() { import errno for pkgclass in (d.getVar('PACKAGE_CLASSES', True) or '').split(): if pkgclass.startswith('package_'): pkgtype = pkgclass.split('_', 1)[1] deploydir = d.getVar('DEPLOY_DIR_%s' % pkgtype.upper(), True) prepath = deploydir + '-prediff' pcmanifest = os.path.join(prepath, d.expand('pkg-compare-manifest-${MULTIMACH_TARGET_SYS}-${PN}')) try: with open(pcmanifest, 'r') as f: for line in f: fn = line.rstrip() if fn: try: os.remove(fn) except OSError as e: if e.errno == errno.ENOENT: pass os.remove(pcmanifest) except IOError as e: if e.errno == errno.ENOENT: pass }