diff options
Diffstat (limited to 'meta/classes/staging.bbclass')
| -rw-r--r-- | meta/classes/staging.bbclass | 349 |
1 files changed, 347 insertions, 2 deletions
diff --git a/meta/classes/staging.bbclass b/meta/classes/staging.bbclass index 2512ae6f5d..1a4668e5d3 100644 --- a/meta/classes/staging.bbclass +++ b/meta/classes/staging.bbclass @@ -235,12 +235,357 @@ do_populate_sysroot[depends] += "${POPULATESYSROOTDEPS}" SSTATETASKS += "do_populate_sysroot" do_populate_sysroot[cleandirs] = "${SYSROOT_DESTDIR}" do_populate_sysroot[sstate-inputdirs] = "${SYSROOT_DESTDIR}" -do_populate_sysroot[sstate-outputdirs] = "${STAGING_DIR_HOST}/" -do_populate_sysroot[stamp-extra-info] = "${MACHINE}" +do_populate_sysroot[sstate-outputdirs] = "${STAGING_DIR}-components/${PACKAGE_ARCH}/${PN}" +do_populate_sysroot[sstate-fixmedir] = "${STAGING_DIR}-components/${PACKAGE_ARCH}/${PN}" python do_populate_sysroot_setscene () { sstate_setscene(d) } addtask do_populate_sysroot_setscene +def staging_copyfile(c, target, fixme, postinsts, stagingdir): + import errno + + if c.endswith("/fixmepath"): + fixme.append(c) + return None + if c.endswith("/fixmepath.cmd"): + return None + #bb.warn(c) + dest = c.replace(stagingdir, "") + dest = target + "/" + "/".join(dest.split("/")[3:]) + bb.utils.mkdirhier(os.path.dirname(dest)) + if "/usr/bin/postinst-" in c: + postinsts.append(dest) + if os.path.islink(c): + linkto = os.readlink(c) + if os.path.lexists(dest): + if os.readlink(dest) == linkto: + return dest + bb.fatal("Link %s already exists to a different location?" % dest) + os.symlink(linkto, dest) + #bb.warn(c) + else: + try: + os.link(c, dest) + except OSError as err: + if err.errno == errno.EXDEV: + bb.utils.copyfile(c, dest) + else: + raise + return dest + +def staging_copydir(c, target, stagingdir): + dest = c.replace(stagingdir, "") + dest = target + "/" + "/".join(dest.split("/")[3:]) + bb.utils.mkdirhier(dest) + +def staging_processfixme(fixme, target, recipesysroot, recipesysrootnative, d): + import subprocess + + if not fixme: + return + cmd = "sed -e 's:^[^/]*/:%s/:g' %s | xargs sed -i -e 's:FIXMESTAGINGDIRTARGET:%s:g; s:FIXMESTAGINGDIRHOST:%s:g'" % (target, " ".join(fixme), recipesysroot, recipesysrootnative) + for fixmevar in ['PKGDATA_DIR']: + fixme_path = d.getVar(fixmevar) + cmd += " -e 's:FIXME_%s:%s:g'" % (fixmevar, fixme_path) + bb.note(cmd) + subprocess.check_call(cmd, shell=True) + + +def staging_populate_sysroot_dir(targetsysroot, nativesysroot, native, d): + import glob + import subprocess + + fixme = [] + postinsts = [] + stagingdir = d.getVar("STAGING_DIR") + if native: + pkgarchs = ['${BUILD_ARCH}', '${BUILD_ARCH}_*'] + targetdir = nativesysroot + else: + pkgarchs = ['${MACHINE_ARCH}', '${TUNE_PKGARCH}', 'allarch'] + targetdir = targetsysroot + + bb.utils.mkdirhier(targetdir) + for pkgarch in pkgarchs: + for manifest in glob.glob(d.expand("${SSTATE_MANIFESTS}/manifest-%s-*.populate_sysroot" % pkgarch)): + if manifest.endswith("-initial.populate_sysroot"): + # skip glibc-initial and libgcc-initial due to file overlap + continue + tmanifest = targetdir + "/" + os.path.basename(manifest) + if os.path.exists(tmanifest): + continue + try: + os.link(manifest, tmanifest) + except OSError as err: + if err.errno == errno.EXDEV: + bb.utils.copyfile(manifest, tmanifest) + else: + raise + with open(manifest, "r") as f: + for l in f: + l = l.strip() + if l.endswith("/"): + staging_copydir(l, targetdir, stagingdir) + continue + staging_copyfile(l, targetdir, fixme, postinsts, stagingdir) + + staging_processfixme(fixme, targetdir, targetsysroot, nativesysroot, d) + for p in postinsts: + subprocess.check_call(p, shell=True) + +# +# Manifests here are complicated. The main sysroot area has the unpacked sstate +# which us unrelocated and tracked by the main sstate manifests. Each recipe +# specific sysroot has manifests for each dependency that is installed there. +# The task hash is used to tell whether the data needs to be reinstalled. We +# use a symlink to point to the currently installed hash. There is also a +# "complete" stamp file which is used to mark if installation completed. If +# something fails (e.g. a postinst), this won't get written and we would +# remove and reinstall the dependency. This also means partially installed +# dependencies should get cleaned up correctly. +# + +python extend_recipe_sysroot() { + import copy + import subprocess + + taskdepdata = d.getVar("BB_TASKDEPDATA", False) + mytaskname = d.getVar("BB_RUNTASK") + #bb.warn(str(taskdepdata)) + pn = d.getVar("PN") + + if mytaskname.endswith("_setscene"): + mytaskname = mytaskname.replace("_setscene", "") + + start = None + configuredeps = [] + for dep in taskdepdata: + data = taskdepdata[dep] + if data[1] == mytaskname and data[0] == pn: + start = dep + break + if start is None: + bb.fatal("Couldn't find ourself in BB_TASKDEPDATA?") + + # We need to figure out which sysroot files we need to expose to this task. + # This needs to match what would get restored from sstate, which is controlled + # ultimately by calls from bitbake to setscene_depvalid(). + # That function expects a setscene dependency tree. We build a dependency tree + # condensed to inter-sstate task dependencies, similar to that used by setscene + # tasks. We can then call into setscene_depvalid() and decide + # which dependencies we can "see" and should expose in the recipe specific sysroot. + setscenedeps = copy.deepcopy(taskdepdata) + + start = set([start]) + + sstatetasks = d.getVar("SSTATETASKS").split() + + def print_dep_tree(deptree): + data = "" + for dep in deptree: + deps = " " + "\n ".join(deptree[dep][3]) + "\n" + data = "%s:\n %s\n %s\n%s %s\n %s\n" % (deptree[dep][0], deptree[dep][1], deptree[dep][2], deps, deptree[dep][4], deptree[dep][5]) + return data + + #bb.note("Full dep tree is:\n%s" % print_dep_tree(taskdepdata)) + + #bb.note(" start2 is %s" % str(start)) + + # If start is an sstate task (like do_package) we need to add in its direct dependencies + # else the code below won't recurse into them. + for dep in set(start): + for dep2 in setscenedeps[dep][3]: + start.add(dep2) + start.remove(dep) + + #bb.note(" start3 is %s" % str(start)) + + # Create collapsed do_populate_sysroot -> do_populate_sysroot tree + for dep in taskdepdata: + data = setscenedeps[dep] + if data[1] not in sstatetasks: + for dep2 in setscenedeps: + data2 = setscenedeps[dep2] + if dep in data2[3]: + data2[3].update(setscenedeps[dep][3]) + data2[3].remove(dep) + if dep in start: + start.update(setscenedeps[dep][3]) + start.remove(dep) + del setscenedeps[dep] + + # Remove circular references + for dep in setscenedeps: + if dep in setscenedeps[dep][3]: + setscenedeps[dep][3].remove(dep) + + #bb.note("Computed dep tree is:\n%s" % print_dep_tree(setscenedeps)) + #bb.note(" start is %s" % str(start)) + + # Direct dependencies should be present and can be depended upon + for dep in set(start): + if setscenedeps[dep][1] == "do_populate_sysroot": + if dep not in configuredeps: + configuredeps.append(dep) + bb.note("Direct dependencies are %s" % str(configuredeps)) + #bb.note(" or %s" % str(start)) + + # Call into setscene_depvalid for each sub-dependency and only copy sysroot files + # for ones that would be restored from sstate. + done = list(start) + next = list(start) + while next: + new = [] + for dep in next: + data = setscenedeps[dep] + for datadep in data[3]: + if datadep in done: + continue + taskdeps = {} + taskdeps[dep] = setscenedeps[dep][:2] + taskdeps[datadep] = setscenedeps[datadep][:2] + retval = setscene_depvalid(datadep, taskdeps, [], d) + if retval: + bb.note("Skipping setscene dependency %s for installation into the sysroot" % datadep) + continue + done.append(datadep) + new.append(datadep) + if datadep not in configuredeps and setscenedeps[datadep][1] == "do_populate_sysroot": + configuredeps.append(datadep) + bb.note("Adding dependency on %s" % setscenedeps[datadep][0]) + else: + bb.note("Following dependency on %s" % setscenedeps[datadep][0]) + next = new + + stagingdir = d.getVar("STAGING_DIR") + recipesysroot = d.getVar("RECIPE_SYSROOT") + recipesysrootnative = d.getVar("RECIPE_SYSROOT_NATIVE") + current_variant = d.getVar("BBEXTENDVARIANT") + + # Detect bitbake -b usage + nodeps = d.getVar("BB_LIMITEDDEPS") or False + if nodeps: + lock = bb.utils.lockfile(recipesysroot + "/sysroot.lock") + staging_populate_sysroot_dir(recipesysroot, recipesysrootnative, True, d) + staging_populate_sysroot_dir(recipesysroot, recipesysrootnative, False, d) + bb.utils.unlockfile(lock) + + depdir = recipesysrootnative + "/installeddeps" + bb.utils.mkdirhier(depdir) + + lock = bb.utils.lockfile(recipesysroot + "/sysroot.lock") + + fixme = {} + fixme[''] = [] + fixme['native'] = [] + postinsts = [] + multilibs = {} + + for dep in configuredeps: + c = setscenedeps[dep][0] + taskhash = setscenedeps[dep][5] + taskmanifest = depdir + "/" + c + "." + taskhash + if mytaskname in ["do_sdk_depends", "do_populate_sdk_ext"] and c.endswith("-initial"): + bb.note("Skipping initial setscene dependency %s for installation into the sysroot" % c) + continue + if os.path.exists(depdir + "/" + c): + lnk = os.readlink(depdir + "/" + c) + if lnk == c + "." + taskhash and os.path.exists(depdir + "/" + c + ".complete"): + bb.note("%s exists in sysroot, skipping" % c) + continue + else: + bb.note("%s exists in sysroot, but is stale (%s vs. %s), removing." % (c, lnk, c + "." + taskhash)) + sstate_clean_manifest(depdir + "/" + lnk, d) + os.unlink(depdir + "/" + c) + elif os.path.lexists(depdir + "/" + c): + os.unlink(depdir + "/" + c) + + os.symlink(c + "." + taskhash, depdir + "/" + c) + + d2 = d + destsysroot = recipesysroot + variant = '' + if setscenedeps[dep][2].startswith("virtual:multilib"): + variant = setscenedeps[dep][2].split(":")[2] + if variant != current_variant: + if variant not in multilibs: + multilibs[variant] = get_multilib_datastore(variant, d) + d2 = multilibs[variant] + destsysroot = d2.getVar("RECIPE_SYSROOT") + + native = False + if c.endswith("-native"): + manifest = d2.expand("${SSTATE_MANIFESTS}/manifest-${BUILD_ARCH}-%s.populate_sysroot" % c) + native = True + elif c.startswith("nativesdk-"): + manifest = d2.expand("${SSTATE_MANIFESTS}/manifest-${SDK_ARCH}_${SDK_OS}-%s.populate_sysroot" % c) + elif "-cross-" in c: + manifest = d2.expand("${SSTATE_MANIFESTS}/manifest-${BUILD_ARCH}_${TARGET_ARCH}-%s.populate_sysroot" % c) + native = True + elif "-crosssdk" in c: + manifest = d2.expand("${SSTATE_MANIFESTS}/manifest-${BUILD_ARCH}_${SDK_ARCH}_${SDK_OS}-%s.populate_sysroot" % c) + native = True + else: + manifest = d2.expand("${SSTATE_MANIFESTS}/manifest-${MACHINE_ARCH}-%s.populate_sysroot" % c) + if not os.path.exists(manifest): + manifest = d2.expand("${SSTATE_MANIFESTS}/manifest-${TUNE_PKGARCH}-%s.populate_sysroot" % c) + if not os.path.exists(manifest): + manifest = d2.expand("${SSTATE_MANIFESTS}/manifest-allarch-%s.populate_sysroot" % c) + if not os.path.exists(manifest): + bb.warn("Manifest %s not found?" % manifest) + else: + with open(manifest, "r") as f, open(taskmanifest, 'w') as m: + for l in f: + l = l.strip() + if l.endswith("/"): + if native: + dest = staging_copydir(l, recipesysrootnative, stagingdir) + else: + dest = staging_copydir(l, destsysroot, stagingdir) + continue + if native: + dest = staging_copyfile(l, recipesysrootnative, fixme['native'], postinsts, stagingdir) + else: + dest = staging_copyfile(l, destsysroot, fixme[''], postinsts, stagingdir) + if dest: + m.write(dest + "\n") + + for f in fixme: + if f == '': + staging_processfixme(fixme[f], recipesysroot, recipesysroot, recipesysrootnative, d) + elif f == 'native': + staging_processfixme(fixme[f], recipesysrootnative, recipesysroot, recipesysrootnative, d) + else: + staging_processfixme(fixme[f], multilibs[f].getVar("RECIPE_SYSROOT"), recipesysroot, recipesysrootnative, d) + + for p in postinsts: + subprocess.check_call(p, shell=True) + + for dep in configuredeps: + c = setscenedeps[dep][0] + open(depdir + "/" + c + ".complete", "w").close() + + bb.utils.unlockfile(lock) +} +extend_recipe_sysroot[vardepsexclude] += "MACHINE SDK_ARCH BUILD_ARCH SDK_OS BB_TASKDEPDATA" + +python do_prepare_recipe_sysroot () { + bb.build.exec_func("extend_recipe_sysroot", d) +} +addtask do_prepare_recipe_sysroot before do_configure after do_fetch + +# Clean out the recipe specific sysroots before do_fetch +do_fetch[cleandirs] += "${RECIPE_SYSROOT} ${RECIPE_SYSROOT_NATIVE}" + +python staging_taskhandler() { + bbtasks = e.tasklist + for task in bbtasks: + deps = d.getVarFlag(task, "depends") + if deps and "populate_sysroot" in deps: + d.appendVarFlag(task, "prefuncs", " extend_recipe_sysroot") +} +staging_taskhandler[eventmask] = "bb.event.RecipeTaskPreProcess" +addhandler staging_taskhandler |