diff options
Diffstat (limited to 'meta/lib/oe/buildhistory_analysis.py')
| -rw-r--r-- | meta/lib/oe/buildhistory_analysis.py | 203 |
1 files changed, 151 insertions, 52 deletions
diff --git a/meta/lib/oe/buildhistory_analysis.py b/meta/lib/oe/buildhistory_analysis.py index 3e86a46a3f..4edad01580 100644 --- a/meta/lib/oe/buildhistory_analysis.py +++ b/meta/lib/oe/buildhistory_analysis.py @@ -3,6 +3,8 @@ # Copyright (C) 2012-2013, 2016-2017 Intel Corporation # Author: Paul Eggleton <paul.eggleton@linux.intel.com> # +# SPDX-License-Identifier: GPL-2.0-only +# # Note: requires GitPython 0.3.1+ # # You can use this from the command line by running scripts/buildhistory-diff @@ -13,6 +15,7 @@ import os.path import difflib import git import re +import shlex import hashlib import collections import bb.utils @@ -31,15 +34,27 @@ ver_monitor_fields = ['PKGE', 'PKGV', 'PKGR'] monitor_numeric_threshold = 10 # Image files to monitor (note that image-info.txt is handled separately) img_monitor_files = ['installed-package-names.txt', 'files-in-image.txt'] -# Related context fields for reporting (note: PE, PV & PR are always reported for monitored package fields) -related_fields = {} -related_fields['RDEPENDS'] = ['DEPENDS'] -related_fields['RRECOMMENDS'] = ['DEPENDS'] -related_fields['FILELIST'] = ['FILES'] -related_fields['PKGSIZE'] = ['FILELIST'] -related_fields['files-in-image.txt'] = ['installed-package-names.txt', 'USER_CLASSES', 'IMAGE_CLASSES', 'ROOTFS_POSTPROCESS_COMMAND', 'IMAGE_POSTPROCESS_COMMAND'] -related_fields['installed-package-names.txt'] = ['IMAGE_FEATURES', 'IMAGE_LINGUAS', 'IMAGE_INSTALL', 'BAD_RECOMMENDATIONS', 'NO_RECOMMENDATIONS', 'PACKAGE_EXCLUDE'] +colours = { + 'colour_default': '', + 'colour_add': '', + 'colour_remove': '', +} + +def init_colours(use_colours): + global colours + if use_colours: + colours = { + 'colour_default': '\033[0m', + 'colour_add': '\033[1;32m', + 'colour_remove': '\033[1;31m', + } + else: + colours = { + 'colour_default': '', + 'colour_add': '', + 'colour_remove': '', + } class ChangeRecord: def __init__(self, path, fieldname, oldvalue, newvalue, monitored): @@ -48,7 +63,6 @@ class ChangeRecord: self.oldvalue = oldvalue self.newvalue = newvalue self.monitored = monitored - self.related = [] self.filechanges = None def __str__(self): @@ -79,7 +93,17 @@ class ChangeRecord: for name in adirs - bdirs] files_ba = [(name, sorted(os.path.basename(item) for item in bitems if os.path.dirname(item) == name)) \ for name in bdirs - adirs] - renamed_dirs = [(dir1, dir2) for dir1, files1 in files_ab for dir2, files2 in files_ba if files1 == files2] + renamed_dirs = [] + for dir1, files1 in files_ab: + rename = False + for dir2, files2 in files_ba: + if files1 == files2 and not rename: + renamed_dirs.append((dir1,dir2)) + # Make sure that we don't use this (dir, files) pair again. + files_ba.remove((dir2,files2)) + # If a dir has already been found to have a rename, stop and go no further. + rename = True + # remove files that belong to renamed dirs from aitems and bitems for dir1, dir2 in renamed_dirs: aitems = [item for item in aitems if os.path.dirname(item) not in (dir1, dir2)] @@ -88,35 +112,50 @@ class ChangeRecord: if self.fieldname in list_fields or self.fieldname in list_order_fields: renamed_dirs = [] + changed_order = False if self.fieldname in ['RPROVIDES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RREPLACES', 'RCONFLICTS']: (depvera, depverb) = compare_pkg_lists(self.oldvalue, self.newvalue) aitems = pkglist_combine(depvera) bitems = pkglist_combine(depverb) else: - aitems = self.oldvalue.split() - bitems = self.newvalue.split() if self.fieldname == 'FILELIST': + aitems = shlex.split(self.oldvalue) + bitems = shlex.split(self.newvalue) renamed_dirs, aitems, bitems = detect_renamed_dirs(aitems, bitems) + else: + aitems = self.oldvalue.split() + bitems = self.newvalue.split() removed = list(set(aitems) - set(bitems)) added = list(set(bitems) - set(aitems)) + if not removed and not added and self.fieldname in ['RPROVIDES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RREPLACES', 'RCONFLICTS']: + depvera = bb.utils.explode_dep_versions2(self.oldvalue, sort=False) + depverb = bb.utils.explode_dep_versions2(self.newvalue, sort=False) + for i, j in zip(depvera.items(), depverb.items()): + if i[0] != j[0]: + changed_order = True + break + lines = [] if renamed_dirs: for dfrom, dto in renamed_dirs: - lines.append('directory renamed %s -> %s' % (dfrom, dto)) + lines.append('directory renamed {colour_remove}{}{colour_default} -> {colour_add}{}{colour_default}'.format(dfrom, dto, **colours)) if removed or added: if removed and not bitems: - lines.append('removed all items "%s"' % ' '.join(removed)) + lines.append('removed all items "{colour_remove}{}{colour_default}"'.format(' '.join(removed), **colours)) else: if removed: - lines.append('removed "%s"' % ' '.join(removed)) + lines.append('removed "{colour_remove}{value}{colour_default}"'.format(value=' '.join(removed), **colours)) if added: - lines.append('added "%s"' % ' '.join(added)) + lines.append('added "{colour_add}{value}{colour_default}"'.format(value=' '.join(added), **colours)) else: lines.append('changed order') - out = '%s: %s' % (self.fieldname, ', '.join(lines)) + if not (removed or added or changed_order): + out = '' + else: + out = '%s: %s' % (self.fieldname, ', '.join(lines)) elif self.fieldname in numeric_fields: aval = int(self.oldvalue or 0) @@ -125,9 +164,9 @@ class ChangeRecord: percentchg = ((bval - aval) / float(aval)) * 100 else: percentchg = 100 - out = '%s changed from %s to %s (%s%d%%)' % (self.fieldname, self.oldvalue or "''", self.newvalue or "''", '+' if percentchg > 0 else '', percentchg) + out = '{} changed from {colour_remove}{}{colour_default} to {colour_add}{}{colour_default} ({}{:.0f}%)'.format(self.fieldname, self.oldvalue or "''", self.newvalue or "''", '+' if percentchg > 0 else '', percentchg, **colours) elif self.fieldname in defaultval_map: - out = '%s changed from %s to %s' % (self.fieldname, self.oldvalue, self.newvalue) + out = '{} changed from {colour_remove}{}{colour_default} to {colour_add}{}{colour_default}'.format(self.fieldname, self.oldvalue, self.newvalue, **colours) if self.fieldname == 'PKG' and '[default]' in self.newvalue: out += ' - may indicate debian renaming failure' elif self.fieldname in ['pkg_preinst', 'pkg_postinst', 'pkg_prerm', 'pkg_postrm']: @@ -142,7 +181,7 @@ class ChangeRecord: diff = difflib.unified_diff(alines, blines, self.fieldname, self.fieldname, lineterm='') out += '\n '.join(list(diff)[2:]) out += '\n --' - elif self.fieldname in img_monitor_files or '/image-files/' in self.path: + elif self.fieldname in img_monitor_files or '/image-files/' in self.path or self.fieldname == "sysroot": if self.filechanges or (self.oldvalue and self.newvalue): fieldname = self.fieldname if '/image-files/' in self.path: @@ -163,14 +202,7 @@ class ChangeRecord: else: out = '' else: - out = '%s changed from "%s" to "%s"' % (self.fieldname, self.oldvalue, self.newvalue) - - if self.related: - for chg in self.related: - if not outer and chg.fieldname in ['PE', 'PV', 'PR']: - continue - for line in chg._str_internal(False).splitlines(): - out += '\n * %s' % line + out = '{} changed from "{colour_remove}{}{colour_default}" to "{colour_add}{}{colour_default}"'.format(self.fieldname, self.oldvalue, self.newvalue, **colours) return '%s%s' % (prefix, out) if out else '' @@ -181,6 +213,7 @@ class FileChange: changetype_perms = 'P' changetype_ownergroup = 'O' changetype_link = 'L' + changetype_move = 'M' def __init__(self, path, changetype, oldvalue = None, newvalue = None): self.path = path @@ -219,10 +252,11 @@ class FileChange: return '%s changed owner/group from %s to %s' % (self.path, self.oldvalue, self.newvalue) elif self.changetype == self.changetype_link: return '%s changed symlink target from %s to %s' % (self.path, self.oldvalue, self.newvalue) + elif self.changetype == self.changetype_move: + return '%s moved to %s' % (self.path, self.oldvalue) else: return '%s changed (unknown)' % self.path - def blob_to_dict(blob): alines = [line for line in blob.data_stream.read().decode('utf-8').splitlines()] adict = {} @@ -249,11 +283,14 @@ def file_list_to_dict(lines): adict[path] = splitv[0:3] return adict +numeric_removal = str.maketrans('0123456789', 'XXXXXXXXXX') -def compare_file_lists(alines, blines): +def compare_file_lists(alines, blines, compare_ownership=True): adict = file_list_to_dict(alines) bdict = file_list_to_dict(blines) filechanges = [] + additions = [] + removals = [] for path, splitv in adict.items(): newsplitv = bdict.pop(path, None) if newsplitv: @@ -262,16 +299,20 @@ def compare_file_lists(alines, blines): newvalue = newsplitv[0][0] if oldvalue != newvalue: filechanges.append(FileChange(path, FileChange.changetype_type, oldvalue, newvalue)) + # Check permissions oldvalue = splitv[0][1:] newvalue = newsplitv[0][1:] if oldvalue != newvalue: filechanges.append(FileChange(path, FileChange.changetype_perms, oldvalue, newvalue)) - # Check owner/group - oldvalue = '%s/%s' % (splitv[1], splitv[2]) - newvalue = '%s/%s' % (newsplitv[1], newsplitv[2]) - if oldvalue != newvalue: - filechanges.append(FileChange(path, FileChange.changetype_ownergroup, oldvalue, newvalue)) + + if compare_ownership: + # Check owner/group + oldvalue = '%s/%s' % (splitv[1], splitv[2]) + newvalue = '%s/%s' % (newsplitv[1], newsplitv[2]) + if oldvalue != newvalue: + filechanges.append(FileChange(path, FileChange.changetype_ownergroup, oldvalue, newvalue)) + # Check symlink target if newsplitv[0][0] == 'l': if len(splitv) > 3: @@ -282,11 +323,67 @@ def compare_file_lists(alines, blines): if oldvalue != newvalue: filechanges.append(FileChange(path, FileChange.changetype_link, oldvalue, newvalue)) else: - filechanges.append(FileChange(path, FileChange.changetype_remove)) + removals.append(path) # Whatever is left over has been added for path in bdict: - filechanges.append(FileChange(path, FileChange.changetype_add)) + additions.append(path) + + # Rather than print additions and removals, its nicer to print file 'moves' + # where names or paths are similar. + revmap_remove = {} + for removal in removals: + translated = removal.translate(numeric_removal) + if translated not in revmap_remove: + revmap_remove[translated] = [] + revmap_remove[translated].append(removal) + + # + # We want to detect renames of large trees of files like + # /lib/modules/5.4.40-yocto-standard to /lib/modules/5.4.43-yocto-standard + # + renames = {} + for addition in additions.copy(): + if addition not in additions: + continue + translated = addition.translate(numeric_removal) + if translated in revmap_remove: + if len(revmap_remove[translated]) != 1: + continue + removal = revmap_remove[translated][0] + commondir = addition.split("/") + commondir2 = removal.split("/") + idx = None + for i in range(len(commondir)): + if commondir[i] != commondir2[i]: + idx = i + break + commondir = "/".join(commondir[:i+1]) + commondir2 = "/".join(commondir2[:i+1]) + # If the common parent is in one dict and not the other its likely a rename + # so iterate through those files and process as such + if commondir2 not in bdict and commondir not in adict: + if commondir not in renames: + renames[commondir] = commondir2 + for addition2 in additions.copy(): + if addition2.startswith(commondir): + removal2 = addition2.replace(commondir, commondir2) + if removal2 in removals: + additions.remove(addition2) + removals.remove(removal2) + continue + filechanges.append(FileChange(removal, FileChange.changetype_move, addition)) + if addition in additions: + additions.remove(addition) + if removal in removals: + removals.remove(removal) + for rename in renames: + filechanges.append(FileChange(renames[rename], FileChange.changetype_move, rename)) + + for addition in additions: + filechanges.append(FileChange(addition, FileChange.changetype_add)) + for removal in removals: + filechanges.append(FileChange(removal, FileChange.changetype_remove)) return filechanges @@ -377,15 +474,19 @@ def compare_dict_blobs(path, ablob, bblob, report_all, report_ver): if abs(percentchg) < monitor_numeric_threshold: continue elif (not report_all) and key in list_fields: - if key == "FILELIST" and path.endswith("-dbg") and bstr.strip() != '': + if key == "FILELIST" and (path.endswith("-dbg") or path.endswith("-src")) and bstr.strip() != '': continue if key in ['RPROVIDES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RREPLACES', 'RCONFLICTS']: (depvera, depverb) = compare_pkg_lists(astr, bstr) if depvera == depverb: continue - alist = astr.split() + if key == 'FILELIST': + alist = shlex.split(astr) + blist = shlex.split(bstr) + else: + alist = astr.split() + blist = bstr.split() alist.sort() - blist = bstr.split() blist.sort() # We don't care about the removal of self-dependencies if pkgname in alist and not pkgname in blist: @@ -461,7 +562,7 @@ def compare_siglists(a_blob, b_blob, taskdiff=False): elif not hash2 in hashfiles: out.append("Unable to find matching sigdata for %s with hash %s" % (desc, hash2)) else: - out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb, collapsed=True) + out2 = bb.siggen.compare_sigfiles(hashfiles[hash1]['path'], hashfiles[hash2]['path'], recursecb, collapsed=True) for line in out2: m = hashlib.sha256() m.update(line.encode('utf-8')) @@ -535,6 +636,15 @@ def process_changes(repopath, revision1, revision2='HEAD', report_all=False, rep elif filename.startswith('latest.'): chg = ChangeRecord(path, filename, d.a_blob.data_stream.read().decode('utf-8'), d.b_blob.data_stream.read().decode('utf-8'), True) changes.append(chg) + elif filename == 'sysroot': + alines = d.a_blob.data_stream.read().decode('utf-8').splitlines() + blines = d.b_blob.data_stream.read().decode('utf-8').splitlines() + filechanges = compare_file_lists(alines,blines, compare_ownership=False) + if filechanges: + chg = ChangeRecord(path, filename, None, None, True) + chg.filechanges = filechanges + changes.append(chg) + elif path.startswith('images/'): filename = os.path.basename(d.a_blob.path) if filename in img_monitor_files: @@ -594,17 +704,6 @@ def process_changes(repopath, revision1, revision2='HEAD', report_all=False, rep chg = ChangeRecord(path, filename[7:], d.a_blob.data_stream.read().decode('utf-8'), '', True) changes.append(chg) - # Link related changes - for chg in changes: - if chg.monitored: - for chg2 in changes: - # (Check dirname in the case of fields from recipe info files) - if chg.path == chg2.path or os.path.dirname(chg.path) == chg2.path: - if chg2.fieldname in related_fields.get(chg.fieldname, []): - chg.related.append(chg2) - elif chg.path == chg2.path and chg.path.startswith('packages/') and chg2.fieldname in ['PE', 'PV', 'PR']: - chg.related.append(chg2) - # filter out unwanted paths if exclude_path: for chg in changes: |