From 94ede642dce8cdbf09f566e3f7e9e260d33fda27 Mon Sep 17 00:00:00 2001 From: Ross Burton Date: Thu, 7 Nov 2019 23:57:33 +0000 Subject: utils: also use mmap for SHA256 and SHA1, for performance md5_file() uses a mmap() window to improve performance when hashing files, so refactor the code and do the same for SHA1 and SHA256. Signed-off-by: Ross Burton Signed-off-by: Richard Purdie Signed-off-by: Armin Kuster --- lib/bb/utils.py | 34 ++++++++++++++-------------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/lib/bb/utils.py b/lib/bb/utils.py index d035949b3..8d40bcdf8 100644 --- a/lib/bb/utils.py +++ b/lib/bb/utils.py @@ -520,22 +520,26 @@ def unlockfile(lf): fcntl.flock(lf.fileno(), fcntl.LOCK_UN) lf.close() -def md5_file(filename): - """ - Return the hex string representation of the MD5 checksum of filename. - """ - import hashlib, mmap +def _hasher(method, filename): + import mmap with open(filename, "rb") as f: - m = hashlib.md5() try: with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm: for chunk in iter(lambda: mm.read(8192), b''): - m.update(chunk) + method.update(chunk) except ValueError: # You can't mmap() an empty file so silence this exception pass - return m.hexdigest() + return method.hexdigest() + + +def md5_file(filename): + """ + Return the hex string representation of the MD5 checksum of filename. + """ + import hashlib + return _hasher(hashlib.md5(), filename) def sha256_file(filename): """ @@ -543,24 +547,14 @@ def sha256_file(filename): filename. """ import hashlib - - s = hashlib.sha256() - with open(filename, "rb") as f: - for line in f: - s.update(line) - return s.hexdigest() + return _hasher(hashlib.sha256(), filename) def sha1_file(filename): """ Return the hex string representation of the SHA1 checksum of the filename """ import hashlib - - s = hashlib.sha1() - with open(filename, "rb") as f: - for line in f: - s.update(line) - return s.hexdigest() + return _hasher(hashlib.sha1(), filename) def preserved_envvars_exported(): """Variables which are taken from the environment and placed in and exported -- cgit 1.2.3-korg