1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
|
#!/usr/bin/env python
# layerindex-web - bulk change implementation
#
# Copyright (C) 2013 Intel Corporation
#
# Licensed under the MIT license, see COPYING.MIT for details
import sys
import os
import os.path
import tempfile
import tarfile
import textwrap
import difflib
import recipeparse
import utils
import shutil
from django.utils.datastructures import SortedDict
logger = utils.logger_create('LayerIndexImport')
# Help us to find places to insert values
recipe_progression = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRC_URI', 'do_fetch', 'do_unpack', 'do_patch', 'EXTRA_OECONF', 'do_configure', 'EXTRA_OEMAKE', 'do_compile', 'do_install', 'do_populate_sysroot', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'do_package', 'do_deploy']
# Variables that sometimes are a bit long but shouldn't be wrapped
nowrap_vars = ['SUMMARY', 'HOMEPAGE', 'BUGTRACKER', 'LIC_FILES_CHKSUM']
meta_vars = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION']
def generate_patches(tinfoil, fetchdir, changeset, outputdir):
tmpoutdir = tempfile.mkdtemp(dir=outputdir)
last_layer = None
patchname = ''
patches = []
outfile = None
try:
for change in changeset.recipechange_set.all().order_by('recipe__layerbranch'):
fields = change.changed_fields(mapped=True)
if fields:
layerbranch = change.recipe.layerbranch
layer = layerbranch.layer
if last_layer != layer:
patchname = "%s.patch" % layer.name
patches.append(patchname)
layerfetchdir = os.path.join(fetchdir, layer.get_fetch_dir())
utils.checkout_layer_branch(layerbranch, layerfetchdir)
layerdir = os.path.join(layerfetchdir, layerbranch.vcs_subdir)
config_data_copy = utils.setup_layer(tinfoil.config_data, fetchdir, layerdir, layer, layerbranch)
if outfile:
outfile.close()
outfile = open(os.path.join(tmpoutdir, patchname), 'w')
last_layer = layer
recipefile = str(os.path.join(layerfetchdir, layerbranch.vcs_subdir, change.recipe.filepath, change.recipe.filename))
varlist = list(set(list(fields.keys()) + meta_vars))
varfiles = recipeparse.get_var_files(recipefile, varlist, config_data_copy)
filevars = localise_file_vars(recipefile, varfiles, fields.keys())
for f, fvars in filevars.items():
filefields = dict((k, fields[k]) for k in fvars)
patch = patch_recipe(f, layerfetchdir, filefields)
for line in patch:
outfile.write(line)
finally:
if outfile:
outfile.close()
# If we have more than one patch, tar it up, otherwise just return the single patch file
ret = None
if len(patches) > 1:
(tmptarfd, tmptarname) = tempfile.mkstemp('.tar.gz', 'bulkchange-', outputdir)
tmptarfile = os.fdopen(tmptarfd, "wb")
tar = tarfile.open(None, "w:gz", tmptarfile)
for patch in patches:
patchfn = os.path.join(tmpoutdir, patch)
tar.add(patchfn, arcname=patch)
tar.close()
ret = tmptarname
elif len(patches) == 1:
(tmppatchfd, tmppatchname) = tempfile.mkstemp('.patch', 'bulkchange-', outputdir)
tmppatchfile = os.fdopen(tmppatchfd, "wb")
with open(os.path.join(tmpoutdir, patches[0]), "rb") as patchfile:
shutil.copyfileobj(patchfile, tmppatchfile)
tmppatchfile.close()
ret = tmppatchname
shutil.rmtree(tmpoutdir)
return ret
def patch_recipe(fn, relpath, values):
"""Update or insert variable values into a recipe file.
Note that some manual inspection/intervention may be required
since this cannot handle all situations.
"""
remainingnames = {}
for k in values.keys():
remainingnames[k] = recipe_progression.index(k) if k in recipe_progression else -1
remainingnames = SortedDict(sorted(remainingnames.items(), key=lambda x: x[1]))
with tempfile.NamedTemporaryFile('w', delete=False) as tf:
def outputvalue(name):
if values[name]:
rawtext = '%s = "%s"\n' % (name, values[name])
if name in nowrap_vars:
tf.write(rawtext)
else:
wrapped = textwrap.wrap(rawtext)
for wrapline in wrapped[:-1]:
tf.write('%s \\\n' % wrapline)
tf.write('%s\n' % wrapped[-1])
tfn = tf.name
with open(fn, 'r') as f:
# First runthrough - find existing names (so we know not to insert based on recipe_progression)
# Second runthrough - make the changes
existingnames = []
for runthrough in [1, 2]:
currname = None
for line in f:
if not currname:
insert = False
for k in remainingnames.keys():
for p in recipe_progression:
if line.startswith(p):
if remainingnames[k] > -1 and recipe_progression.index(p) > remainingnames[k] and runthrough > 1 and not k in existingnames:
outputvalue(k)
del remainingnames[k]
break
for k in remainingnames.keys():
if line.startswith(k):
currname = k
if runthrough == 1:
existingnames.append(k)
else:
del remainingnames[k]
break
if currname and runthrough > 1:
outputvalue(currname)
if currname:
sline = line.rstrip()
if not sline.endswith('\\'):
currname = None
continue
if runthrough > 1:
tf.write(line)
f.seek(0)
if remainingnames:
tf.write('\n')
for k in remainingnames.keys():
outputvalue(k)
fromlines = open(fn, 'U').readlines()
tolines = open(tfn, 'U').readlines()
relfn = os.path.relpath(fn, relpath)
diff = difflib.unified_diff(fromlines, tolines, 'a/%s' % relfn, 'b/%s' % relfn)
os.remove(tfn)
return diff
def localise_file_vars(fn, varfiles, varlist):
from collections import defaultdict
fndir = os.path.dirname(fn) + os.sep
first_meta_file = None
for v in meta_vars:
f = varfiles.get(v, None)
if f:
actualdir = os.path.dirname(f) + os.sep
if actualdir.startswith(fndir):
first_meta_file = f
break
filevars = defaultdict(list)
for v in varlist:
f = varfiles[v]
# Only return files that are in the same directory as the recipe or in some directory below there
# (this excludes bbclass files and common inc files that wouldn't be appropriate to set the variable
# in if we were going to set a value specific to this recipe)
if f:
actualfile = f
else:
# Variable isn't in a file, if it's one of the "meta" vars, use the first file with a meta var in it
if first_meta_file:
actualfile = first_meta_file
else:
actualfile = fn
actualdir = os.path.dirname(actualfile) + os.sep
if not actualdir.startswith(fndir):
actualfile = fn
filevars[actualfile].append(v)
return filevars
def get_changeset(pk):
from layerindex.models import RecipeChangeset
res = list(RecipeChangeset.objects.filter(pk=pk)[:1])
if res:
return res[0]
return None
def usage():
print("Usage: bulkchange.py <id> <outputdir>")
def main():
if '--help' in sys.argv:
usage()
sys.exit(0)
if len(sys.argv) < 3:
usage()
sys.exit(1)
utils.setup_django()
import settings
branch = utils.get_branch('master')
fetchdir = settings.LAYER_FETCH_DIR
bitbakepath = os.path.join(fetchdir, 'bitbake')
lockfn = os.path.join(fetchdir, "layerindex.lock")
lockfile = utils.lock_file(lockfn)
if not lockfile:
sys.stderr.write("Layer index lock timeout expired\n")
sys.exit(1)
try:
(tinfoil, tempdir) = recipeparse.init_parser(settings, branch, bitbakepath, True)
changeset = get_changeset(sys.argv[1])
if not changeset:
sys.stderr.write("Unable to find changeset with id %s\n" % sys.argv[1])
sys.exit(1)
outp = generate_patches(tinfoil, fetchdir, changeset, sys.argv[2])
finally:
utils.unlock_file(lockfile)
if outp:
print(outp)
else:
sys.stderr.write("No changes to write\n")
sys.exit(1)
shutil.rmtree(tempdir)
sys.exit(0)
if __name__ == "__main__":
main()
|
