1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
|
# Internal script called by rrs_upgrade_history.py
#
# To detect package versions of the recipes the script uses the name of the recipe.
#
# Copyright (C) 2015, 2018 Intel Corporation
# Authors: Anibal Limon <anibal.limon@linux.intel.com>
# Paul Eggleton <paul.eggleton@linux.intel.com>
#
# Licensed under the MIT license, see COPYING.MIT for details
from datetime import datetime
import sys
import os
import optparse
import logging
import re
from distutils.version import LooseVersion
import shutil
sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__))))
from common import common_setup, get_pv_type, load_recipes, \
get_logger, DryRunRollbackException
common_setup()
from layerindex import utils, recipeparse
from layerindex.update_layer import split_recipe_fn
"""
Store upgrade into RecipeUpgrade model.
"""
def _save_upgrade(recipe, pv, commit, title, info, logger):
from email.utils import parsedate_tz, mktime_tz
from rrs.models import Maintainer, RecipeUpgrade
maintainer_name = info.split(';')[0]
maintainer_email = info.split(';')[1]
author_date = info.split(';')[2]
commit_date = info.split(';')[3]
maintainer = Maintainer.create_or_update(maintainer_name, maintainer_email)
upgrade = RecipeUpgrade()
upgrade.recipe = recipe
upgrade.maintainer = maintainer
upgrade.author_date = datetime.utcfromtimestamp(mktime_tz(
parsedate_tz(author_date)))
upgrade.commit_date = datetime.utcfromtimestamp(mktime_tz(
parsedate_tz(commit_date)))
upgrade.version = pv
upgrade.sha1 = commit
upgrade.title = title.strip()
upgrade.save()
"""
Create upgrade receives new recipe_data and cmp versions.
"""
def _create_upgrade(recipe_data, layerbranch, ct, title, info, logger, initial=False):
from layerindex.models import Recipe
from rrs.models import RecipeUpgrade
from bb.utils import vercmp_string
pn = recipe_data.getVar('PN', True)
pv = recipe_data.getVar('PV', True)
if '..' in pv or pv.endswith('.'):
logger.warn('Invalid version for recipe %s in commit %s, ignoring' % (recipe_data.getVar('FILE', True), ct))
return
recipes = Recipe.objects.filter(pn=pn, layerbranch=layerbranch).order_by('id')
if not recipes:
logger.warn("%s: Not found in Layer branch %s." %
(pn, str(layerbranch)))
return
recipe = recipes[0]
try:
latest_upgrade = RecipeUpgrade.objects.filter(
recipe = recipe).order_by('-commit_date')[0]
prev_pv = latest_upgrade.version
except KeyboardInterrupt:
raise
except:
prev_pv = None
if prev_pv is None:
logger.debug("%s: Initial upgrade ( -> %s)." % (recipe.pn, pv))
_save_upgrade(recipe, pv, ct, title, info, logger)
else:
from common import get_recipe_pv_without_srcpv
(ppv, _, _) = get_recipe_pv_without_srcpv(prev_pv,
get_pv_type(prev_pv))
(npv, _, _) = get_recipe_pv_without_srcpv(pv,
get_pv_type(pv))
try:
if npv == 'git':
logger.debug("%s: Avoiding upgrade to unversioned git." % \
(recipe.pn))
elif ppv == 'git' or vercmp_string(ppv, npv) == -1:
if initial is True:
logger.debug("%s: Update initial upgrade ( -> %s)." % \
(recipe.pn, pv))
latest_upgrade.version = pv
latest_upgrade.save()
else:
logger.debug("%s: detected upgrade (%s -> %s)" \
" in ct %s." % (pn, prev_pv, pv, ct))
_save_upgrade(recipe, pv, ct, title, info, logger)
except KeyboardInterrupt:
raise
except Exception as e:
logger.error("%s: fail to detect upgrade (%s -> %s)" \
" in ct %s: %s" % (pn, prev_pv, pv, ct, str(e)))
"""
Returns a list containing the fullpaths to the recipes from a commit.
"""
def _get_recipes_filenames(ct, repodir, layerdir, logger):
ct_files = []
layerdir_start = os.path.normpath(layerdir) + os.sep
files = utils.runcmd("git log --name-only --format='%n' -n 1 " + ct,
repodir, logger=logger)
for f in files.split("\n"):
if f != "":
fullpath = os.path.join(repodir, f)
# Skip deleted files in commit
if not os.path.exists(fullpath):
continue
if not fullpath.startswith(layerdir_start):
# Ignore files in repo that are outside of the layer
continue
(typename, _, filename) = recipeparse.detect_file_type(fullpath,
layerdir_start)
if typename == 'recipe':
ct_files.append(fullpath)
return ct_files
def generate_history(options, layerbranch_id, commit, logger):
from layerindex.models import LayerBranch
from rrs.models import Release
layerbranch = LayerBranch.objects.get(id=layerbranch_id)
fetchdir = settings.LAYER_FETCH_DIR
if not fetchdir:
logger.error("Please set LAYER_FETCH_DIR in settings.py")
sys.exit(1)
layer = layerbranch.layer
urldir = str(layer.get_fetch_dir())
repodir = os.path.join(fetchdir, urldir)
layerdir = os.path.join(repodir, str(layerbranch.vcs_subdir))
utils.runcmd("git checkout %s" % commit,
repodir, logger=logger)
utils.runcmd("git clean -dfx", repodir, logger=logger)
if options.initial:
fns = None
else:
fns = _get_recipes_filenames(commit, repodir, layerdir, logger)
if not fns:
return
# setup bitbake
bitbakepath = os.path.join(fetchdir, 'bitbake')
if options.bitbake_rev:
bitbake_rev = options.bitbake_rev
if not re.match('^[0-9a-f]{40}$', bitbake_rev):
# Branch name, need to check out detached
bitbake_rev = 'origin/%s' % bitbake_rev
else:
commitdate = utils.runcmd("git show -s --format=%ci", repodir, logger=logger)
bitbake_rev = '`git rev-list -1 --before="%s" origin/master`' % commitdate
utils.runcmd('git checkout %s' % bitbake_rev,
bitbakepath, logger=logger)
utils.runcmd("git clean -dfx", bitbakepath, logger=logger)
sys.path.insert(0, os.path.join(bitbakepath, 'lib'))
(tinfoil, d, recipes, tempdir) = load_recipes(layerbranch, bitbakepath,
fetchdir, settings, logger, recipe_files=fns,
nocheckout=True)
try:
if options.initial:
title = options.initial
info = 'No maintainer;;' + utils.runcmd("git log --format='%ad;%cd' --date=rfc -n 1 " \
+ commit, destdir=repodir, logger=logger)
recordcommit = ''
else:
title = utils.runcmd("git log --format='%s' -n 1 " + commit,
repodir, logger=logger)
info = utils.runcmd("git log --format='%an;%ae;%ad;%cd' --date=rfc -n 1 " \
+ commit, destdir=repodir, logger=logger)
recordcommit = commit
try:
with transaction.atomic():
for recipe_data in recipes:
_create_upgrade(recipe_data, layerbranch, recordcommit, title,
info, logger, initial=options.initial)
if options.dry_run:
raise DryRunRollbackException
except DryRunRollbackException:
pass
finally:
if tinfoil and hasattr(tinfoil, 'shutdown') and (LooseVersion(bb.__version__) > LooseVersion("1.27")):
tinfoil.shutdown()
shutil.rmtree(tempdir)
if __name__=="__main__":
logger = None
try:
utils.setup_django()
from django.db import transaction
import settings
logger = get_logger("HistoryUpgrade", settings)
parser = optparse.OptionParser(usage = """%prog [options] <layerbranchid> <commit>""")
parser.add_option("-i", "--initial",
help = "Do initial population of upgrade histories (and specify comment)",
action="store", dest="initial", default='')
parser.add_option("--bitbake-rev",
help = "Use the specified bitbake revision instead of the most recent one at the metadata commit date",
action="store", dest="bitbake_rev", default='')
parser.add_option("-d", "--debug",
help = "Enable debug output",
action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO)
parser.add_option("--dry-run",
help = "Do not write any data back to the database",
action="store_true", dest="dry_run", default=False)
options, args = parser.parse_args(sys.argv)
logger.setLevel(options.loglevel)
if len(args) < 2:
logger.error('Please specify layerbranch ID')
sys.exit(1)
if len(args) < 3:
logger.error('Please specify commit')
sys.exit(1)
generate_history(options, int(args[1]), args[2], logger)
except KeyboardInterrupt:
if logger:
logger.info('Update interrupted, exiting')
sys.exit(254)
|