1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
|
# Creates a tarball of the work directory for a recipe when one of its
# tasks fails, or any other nominated directories.
# Useful in cases where the environment in which builds are run is
# ephemeral or otherwise inaccessible for examination during
# debugging.
#
# To enable, simply add the following to your configuration:
#
# INHERIT += "retain"
#
# You can specify the recipe-specific directories to save upon failure
# or always (space-separated) e.g.:
#
# RETAIN_DIRS_FAILURE = "${WORKDIR};prefix=workdir" # default
# RETAIN_DIRS_ALWAYS = "${T}"
#
# Naturally you can use overrides to limit it to a specific recipe:
# RETAIN_DIRS_ALWAYS:pn-somerecipe = "${T}"
#
# You can also specify global (non-recipe-specific) directories to save:
#
# RETAIN_DIRS_GLOBAL_FAILURE = "${LOG_DIR}"
# RETAIN_DIRS_GLOBAL_ALWAYS = "${BUILDSTATS_BASE}"
#
# If you wish to use a different tarball name prefix than the default of
# the directory name, you can do so by specifying a ;prefix= followed by
# the desired prefix (no spaces) in any of the RETAIN_DIRS_* variables.
# e.g. to always save the log files with a "recipelogs" as the prefix for
# the tarball of ${T} you would do this:
#
# RETAIN_DIRS_ALWAYS = "${T};prefix=recipelogs"
#
# Notes:
# * For this to be useful you also need corresponding logic in your build
# orchestration tool to pick up any files written out to RETAIN_OUTDIR
# (with the other assumption being that no files are present there at
# the start of the build, since there is no logic to purge old files).
# * Work directories can be quite large, so saving them can take some time
# and of course space.
# * Tarball creation is deferred to the end of the build, thus you will
# get the state at the end, not immediately upon failure.
# * Extra directories must naturally be populated at the time the retain
# class goes to save them (build completion); to try ensure this for
# things that are also saved on build completion (e.g. buildstats), put
# the INHERIT += "retain" after the INHERIT += lines for the class that
# is writing out the data that you wish to save.
# * The tarballs have the tarball name as a top-level directory so that
# multiple tarballs can be extracted side-by-side easily.
#
# Copyright (c) 2020, 2024 Microsoft Corporation
#
# SPDX-License-Identifier: GPL-2.0-only
#
RETAIN_OUTDIR ?= "${TMPDIR}/retained"
RETAIN_DIRS_FAILURE ?= "${WORKDIR};prefix=workdir"
RETAIN_DIRS_ALWAYS ?= ""
RETAIN_DIRS_GLOBAL_FAILURE ?= ""
RETAIN_DIRS_GLOBAL_ALWAYS ?= ""
RETAIN_TARBALL_SUFFIX ?= "${DATETIME}.tar.gz"
RETAIN_ENABLED ?= "1"
def retain_retain_dir(desc, tarprefix, path, tarbasepath, d):
import datetime
outdir = d.getVar('RETAIN_OUTDIR')
bb.utils.mkdirhier(outdir)
suffix = d.getVar('RETAIN_TARBALL_SUFFIX')
tarname = '%s_%s' % (tarprefix, suffix)
tarfp = os.path.join(outdir, '%s' % tarname)
tardir = os.path.relpath(path, tarbasepath)
cmdargs = ['tar', 'cfa', tarfp]
# Prefix paths within the tarball with the tarball name so that
# multiple tarballs can be extracted side-by-side
tarname_noext = os.path.splitext(tarname)[0]
if tarname_noext.endswith('.tar'):
tarname_noext = tarname_noext[:-4]
cmdargs += ['--transform', 's:^:%s/:' % tarname_noext]
cmdargs += [tardir]
try:
bb.process.run(cmdargs, cwd=tarbasepath)
except bb.process.ExecutionError as e:
# It is possible for other tasks to be writing to the workdir
# while we are tarring it up, in which case tar will return 1,
# but we don't care in this situation (tar returns 2 for other
# errors so we we will see those)
if e.exitcode != 1:
bb.warn('retain: error saving %s: %s' % (desc, str(e)))
addhandler retain_task_handler
retain_task_handler[eventmask] = "bb.build.TaskFailed bb.build.TaskSucceeded"
addhandler retain_build_handler
retain_build_handler[eventmask] = "bb.event.BuildStarted bb.event.BuildCompleted"
python retain_task_handler() {
if d.getVar('RETAIN_ENABLED') != '1':
return
dirs = d.getVar('RETAIN_DIRS_ALWAYS')
if isinstance(e, bb.build.TaskFailed):
dirs += ' ' + d.getVar('RETAIN_DIRS_FAILURE')
dirs = dirs.strip().split()
if dirs:
outdir = d.getVar('RETAIN_OUTDIR')
bb.utils.mkdirhier(outdir)
dirlist_file = os.path.join(outdir, 'retain_dirs.list')
pn = d.getVar('PN')
taskname = d.getVar('BB_CURRENTTASK')
with open(dirlist_file, 'a') as f:
for entry in dirs:
f.write('%s %s %s\n' % (pn, taskname, entry))
}
python retain_build_handler() {
outdir = d.getVar('RETAIN_OUTDIR')
dirlist_file = os.path.join(outdir, 'retain_dirs.list')
if isinstance(e, bb.event.BuildStarted):
if os.path.exists(dirlist_file):
os.remove(dirlist_file)
return
if d.getVar('RETAIN_ENABLED') != '1':
return
savedirs = {}
try:
with open(dirlist_file, 'r') as f:
for line in f:
pn, _, path = line.rstrip().split()
if not path in savedirs:
savedirs[path] = pn
os.remove(dirlist_file)
except FileNotFoundError:
pass
if e.getFailures():
for path in (d.getVar('RETAIN_DIRS_GLOBAL_FAILURE') or '').strip().split():
savedirs[path] = ''
for path in (d.getVar('RETAIN_DIRS_GLOBAL_ALWAYS') or '').strip().split():
savedirs[path] = ''
if savedirs:
bb.plain('NOTE: retain: retaining build output...')
count = 0
for path, pn in savedirs.items():
prefix = None
if ';' in path:
pathsplit = path.split(';')
path = pathsplit[0]
for param in pathsplit[1:]:
if '=' in param:
name, value = param.split('=', 1)
if name == 'prefix':
prefix = value
else:
bb.error('retain: invalid parameter "%s" in RETAIN_* variable value' % param)
return
else:
bb.error('retain: parameter "%s" missing value in RETAIN_* variable value' % param)
return
if prefix:
itemname = prefix
else:
itemname = os.path.basename(path)
if pn:
# Always add the recipe name in front
itemname = pn + '_' + itemname
if os.path.exists(path):
retain_retain_dir(itemname, itemname, path, os.path.dirname(path), d)
count += 1
else:
bb.warn('retain: path %s does not currently exist' % path)
if count:
item = 'archive' if count == 1 else 'archives'
bb.plain('NOTE: retain: saved %d %s to %s' % (count, item, outdir))
}
|
