diff options
author | Ross Burton <ross.burton@intel.com> | 2019-11-07 23:57:33 +0000 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2019-11-14 14:06:37 +0000 |
commit | ecf87437ff796e17c3e4f210b5803b0136a9e8a4 (patch) | |
tree | f5195e76cf04de633cb4a84ec4fa811e11a21467 /lib/bb/utils.py | |
parent | e944d02fe678f7c6b05c62419f8bceb0709f3037 (diff) | |
download | bitbake-ecf87437ff796e17c3e4f210b5803b0136a9e8a4.tar.gz |
utils: also use mmap for SHA256 and SHA1, for performance
md5_file() uses a mmap() window to improve performance when hashing files, so
refactor the code and do the same for SHA1 and SHA256.
Signed-off-by: Ross Burton <ross.burton@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'lib/bb/utils.py')
-rw-r--r-- | lib/bb/utils.py | 34 |
1 files changed, 14 insertions, 20 deletions
diff --git a/lib/bb/utils.py b/lib/bb/utils.py index d035949b3..8d40bcdf8 100644 --- a/lib/bb/utils.py +++ b/lib/bb/utils.py @@ -520,22 +520,26 @@ def unlockfile(lf): fcntl.flock(lf.fileno(), fcntl.LOCK_UN) lf.close() -def md5_file(filename): - """ - Return the hex string representation of the MD5 checksum of filename. - """ - import hashlib, mmap +def _hasher(method, filename): + import mmap with open(filename, "rb") as f: - m = hashlib.md5() try: with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm: for chunk in iter(lambda: mm.read(8192), b''): - m.update(chunk) + method.update(chunk) except ValueError: # You can't mmap() an empty file so silence this exception pass - return m.hexdigest() + return method.hexdigest() + + +def md5_file(filename): + """ + Return the hex string representation of the MD5 checksum of filename. + """ + import hashlib + return _hasher(hashlib.md5(), filename) def sha256_file(filename): """ @@ -543,24 +547,14 @@ def sha256_file(filename): filename. """ import hashlib - - s = hashlib.sha256() - with open(filename, "rb") as f: - for line in f: - s.update(line) - return s.hexdigest() + return _hasher(hashlib.sha256(), filename) def sha1_file(filename): """ Return the hex string representation of the SHA1 checksum of the filename """ import hashlib - - s = hashlib.sha1() - with open(filename, "rb") as f: - for line in f: - s.update(line) - return s.hexdigest() + return _hasher(hashlib.sha1(), filename) def preserved_envvars_exported(): """Variables which are taken from the environment and placed in and exported |