summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorChuck Wolber <chuck.wolber@boeing.com>2023-01-16 17:59:30 -0800
committerRichard Purdie <richard.purdie@linuxfoundation.org>2023-01-18 16:42:24 +0000
commitda2e01cacd98715318a5307fe0618dbca0cf1fe7 (patch)
tree12126fa6e241d24ed27b07a1b550fc871b8e49f1
parent1c51068c78d12ee02789a6dbecf5e7e91d141af5 (diff)
downloadopenembedded-core-contrib-da2e01cacd98715318a5307fe0618dbca0cf1fe7.tar.gz
scripts/oe-setup-layers: Make efficiently idempotent
The effect of subsequent setup-layers executions is now either a NOOP or the minimal set of changes required to ensure layers precisely match the JSON configuration. This change allows setup-layers to be incorporated into a team's configuration management strategy. In particular, the configuration JSON manages a "pinning policy" that documents the oversight of sources of change (a requirement for embedded development in highly regulated industries). One model for this strategy would work as follows. Team level policy is developed to regularly review upstream commits that occur between the current upstream HEAD and the previously pinned revision. The JSON configuration is periodically updated after a review, test, and approval process. In the rare instance that an upstream change is considered problematic, the bbappend mechanism can be used to make relevant changes in the team's project repository. This approach also requires that team developers regularly run the project repository copy of setup-layers. This is most easily accomplished by including setup-layers in a wrapper script that all team developers use to interact with the bitbake tool suite (e.g. "bb bitbake foo-image"). Project level policy and oversight is effectively "contained" within this wrapper script, thereby reducing a significant source of human error. Left unstated, but acknowledged here, are a number of nuances required to successfully implement the above strategy. The details are out of scope for this explanation. What should be clear is that a larger configuration management strategy can now benefit from the utility provided by setup-layers. Note: Neither the above configuration management strategy example nor the change itself is intended to alter the original intent to use "bitbake-layers create-layers-setup destdir" to keep pace with upstream activity for those who wish to use it that way. Signed-off-by: Chuck Wolber <chuck.wolber@boeing.com> Signed-off-by: Luca Ceresoli <luca.ceresoli@bootlin.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rwxr-xr-xscripts/oe-setup-layers55
1 files changed, 46 insertions, 9 deletions
diff --git a/scripts/oe-setup-layers b/scripts/oe-setup-layers
index 6ecaffed75..d0bc9f1667 100755
--- a/scripts/oe-setup-layers
+++ b/scripts/oe-setup-layers
@@ -10,12 +10,42 @@
# bitbake-layers create-layers-setup destdir
#
# It is recommended that you do not modify this file directly, but rather re-run the above command to get the freshest upstream copy.
+#
+# This script is idempotent. Subsequent runs only change what is necessary to
+# ensure your layers match your configuration.
import argparse
import json
import os
import subprocess
+def _is_layer_git_repo(layerdir):
+ git_dir = os.path.join(layerdir, ".git")
+ if not os.access(git_dir, os.R_OK):
+ return False
+ try:
+ return subprocess.check_output("git -C %s rev-parse --is-inside-git-dir" % git_dir, shell=True, stderr=subprocess.DEVNULL)
+ except subprocess.CalledProcessError:
+ return False
+
+def _is_layer_at_rev(layerdir, rev):
+ try:
+ curr_rev = subprocess.check_output("git -C %s rev-parse HEAD" % layerdir, shell=True, stderr=subprocess.DEVNULL)
+ if curr_rev.strip().decode("utf-8") == rev:
+ return True
+ except subprocess.CalledProcessError:
+ pass
+ return False
+
+def _is_layer_at_remote_uri(layerdir, remote, uri):
+ try:
+ curr_uri = subprocess.check_output("git -C %s remote get-url %s" % (layerdir, remote), shell=True, stderr=subprocess.DEVNULL)
+ if curr_uri.strip().decode("utf-8") == uri:
+ return True
+ except subprocess.CalledProcessError:
+ pass
+ return False
+
def _do_checkout(args, json):
layers = json['sources']
for l_name in layers:
@@ -36,23 +66,30 @@ def _do_checkout(args, json):
remotes = l_remote['remotes']
print('\nSetting up source {}, revision {}, branch {}'.format(l_name, desc, branch))
- cmd = 'git init -q {}'.format(layerdir)
- print("Running '{}'".format(cmd))
- subprocess.check_output(cmd, shell=True)
+ if not _is_layer_git_repo(layerdir):
+ cmd = 'git init -q {}'.format(layerdir)
+ print("Running '{}'".format(cmd))
+ subprocess.check_output(cmd, shell=True)
for remote in remotes:
- cmd = "git remote remove {} > /dev/null 2>&1; git remote add {} {}".format(remote, remote, remotes[remote]['uri'])
+ if not _is_layer_at_remote_uri(layerdir, remote, remotes[remote]['uri']):
+ cmd = "git remote remove {} > /dev/null 2>&1; git remote add {} {}".format(remote, remote, remotes[remote]['uri'])
+ print("Running '{}' in {}".format(cmd, layerdir))
+ subprocess.check_output(cmd, shell=True, cwd=layerdir)
+
+ cmd = "git fetch -q {} || true".format(remote)
+ print("Running '{}' in {}".format(cmd, layerdir))
+ subprocess.check_output(cmd, shell=True, cwd=layerdir)
+
+ if not _is_layer_at_rev(layerdir, rev):
+ cmd = "git fetch -q --all || true"
print("Running '{}' in {}".format(cmd, layerdir))
subprocess.check_output(cmd, shell=True, cwd=layerdir)
- cmd = "git fetch -q {} || true".format(remote)
+ cmd = 'git checkout -q {}'.format(rev)
print("Running '{}' in {}".format(cmd, layerdir))
subprocess.check_output(cmd, shell=True, cwd=layerdir)
- cmd = 'git checkout -q {}'.format(rev)
- print("Running '{}' in {}".format(cmd, layerdir))
- subprocess.check_output(cmd, shell=True, cwd=layerdir)
-
parser = argparse.ArgumentParser(description="A self contained python script that fetches all the needed layers and sets them to correct revisions using data in a json format from a separate file. The json data can be created from an active build directory with 'bitbake-layers create-layers-setup destdir' and there's a sample file and a schema in meta/files/")
parser.add_argument('--force-bootstraplayer-checkout', action='store_true',