diff options
Diffstat (limited to 'scripts/gen-lockedsig-cache')
| -rwxr-xr-x | scripts/gen-lockedsig-cache | 99 |
1 files changed, 86 insertions, 13 deletions
diff --git a/scripts/gen-lockedsig-cache b/scripts/gen-lockedsig-cache index c93b2c0b99..cc674f9c1b 100755 --- a/scripts/gen-lockedsig-cache +++ b/scripts/gen-lockedsig-cache @@ -1,13 +1,13 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -# gen-lockedsig-cache <locked-sigs.inc> <input-cachedir> <output-cachedir> +# SPDX-License-Identifier: GPL-2.0-only # import os import sys -import glob import shutil import errno +import time def mkdir(d): try: @@ -16,31 +16,104 @@ def mkdir(d): if e.errno != errno.EEXIST: raise e -if len(sys.argv) < 3: +# extract the hash from past the last colon to last underscore +def extract_sha(filename): + return filename.split(':')[7].split('_')[0] + +# get all files in a directory, extract hash and make +# a map from hash to list of file with that hash +def map_sha_to_files(dir_, prefix, sha_map): + sstate_prefix_path = dir_ + '/' + prefix + '/' + if not os.path.exists(sstate_prefix_path): + return + sstate_files = os.listdir(sstate_prefix_path) + for f in sstate_files: + try: + sha = extract_sha(f) + if sha not in sha_map: + sha_map[sha] = [] + sha_map[sha].append(sstate_prefix_path + f) + except IndexError: + continue + +# given a prefix build a map of hash to list of files +def build_sha_cache(prefix): + sha_map = {} + + sstate_dir = sys.argv[2] + map_sha_to_files(sstate_dir, prefix, sha_map) + + native_sstate_dir = sys.argv[2] + '/' + sys.argv[4] + map_sha_to_files(native_sstate_dir, prefix, sha_map) + + return sha_map + +if len(sys.argv) < 5: print("Incorrect number of arguments specified") + print("syntax: gen-lockedsig-cache <locked-sigs.inc> <input-cachedir> <output-cachedir> <nativelsbstring> [filterfile]") sys.exit(1) +filterlist = [] +if len(sys.argv) > 5: + print('Reading filter file %s' % sys.argv[5]) + with open(sys.argv[5]) as f: + for l in f.readlines(): + if ":" in l: + filterlist.append(l.rstrip()) + +print('Reading %s' % sys.argv[1]) sigs = [] with open(sys.argv[1]) as f: for l in f.readlines(): if ":" in l: - sigs.append(l.split(":")[2].split()[0]) + task, sig = l.split()[0].rsplit(':', 1) + if filterlist and not task in filterlist: + print('Filtering out %s' % task) + else: + sigs.append(sig) +print('Gathering file list') +start_time = time.perf_counter() files = set() +sstate_content_cache = {} for s in sigs: - p = sys.argv[2] + "/" + s[:2] + "/*" + s + "*" - files |= set(glob.glob(p)) - p = sys.argv[2] + "/*/" + s[:2] + "/*" + s + "*" - files |= set(glob.glob(p)) + prefix = s[:2] + prefix2 = s[2:4] + if prefix not in sstate_content_cache: + sstate_content_cache[prefix] = {} + if prefix2 not in sstate_content_cache[prefix]: + sstate_content_cache[prefix][prefix2] = build_sha_cache(prefix + "/" + prefix2) + + if s in sstate_content_cache[prefix][prefix2]: + for f in sstate_content_cache[prefix][prefix2][s]: + files.add(f) +elapsed = time.perf_counter() - start_time +print("Gathering file list took %.1fs" % elapsed) + +print('Processing files') for f in files: - dst = f.replace(sys.argv[2], sys.argv[3]) + sys.stdout.write('Processing %s... ' % f) + if not f.endswith(('.tar.zst', '.siginfo', '.sig')): + # Most likely a temp file, skip it + print('skipping') + continue + dst = os.path.join(sys.argv[3], os.path.relpath(f, sys.argv[2])) destdir = os.path.dirname(dst) mkdir(destdir) + src = os.path.realpath(f) if os.path.exists(dst): os.remove(dst) - if (os.stat(f).st_dev == os.stat(destdir).st_dev): - os.link(f, dst) + if (os.stat(src).st_dev == os.stat(destdir).st_dev): + print('linking') + try: + os.link(src, dst) + except OSError as e: + print('hard linking failed, copying') + shutil.copyfile(src, dst) else: - shutil.copyfile(f, dst) + print('copying') + shutil.copyfile(src, dst) + +print('Done!') |