summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2022-09-01 10:06:34 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2022-09-03 12:55:35 +0100
commit4b8de2e7d12667d69d86ffe6e9f85a7932c4c9a5 (patch)
tree821bd1af25ed7e0718be8d77882801d8cc9fcc30
parentcffcfacb747d41304c857b17bfea646e220b2389 (diff)
downloadbitbake-4b8de2e7d12667d69d86ffe6e9f85a7932c4c9a5.tar.gz
wget: Avoid bad checksum race issues
If two recipes have conflicting checksums for a file, the code will currently remove the existing file when a mismatch is downloaded, even if another task successfully fetched it. This changes the code to verify the checksum (if possible) before replacing the file. This removes a potential race window and stops builds failing everywhere from one incorrect checksum. To make this work, we need to be able to override localpath and avoid NoChecksum errors being logged. Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--lib/bb/fetch2/__init__.py11
-rw-r--r--lib/bb/fetch2/wget.py5
2 files changed, 12 insertions, 4 deletions
diff --git a/lib/bb/fetch2/__init__.py b/lib/bb/fetch2/__init__.py
index 0fb718b23..e6dd79c4a 100644
--- a/lib/bb/fetch2/__init__.py
+++ b/lib/bb/fetch2/__init__.py
@@ -545,7 +545,7 @@ def mirror_from_string(data):
bb.warn('Invalid mirror data %s, should have paired members.' % data)
return list(zip(*[iter(mirrors)]*2))
-def verify_checksum(ud, d, precomputed={}):
+def verify_checksum(ud, d, precomputed={}, localpath=None, fatal_nochecksum=True):
"""
verify the MD5 and SHA256 checksum for downloaded src
@@ -563,13 +563,16 @@ def verify_checksum(ud, d, precomputed={}):
if ud.ignore_checksums or not ud.method.supports_checksum(ud):
return {}
+ if localpath is None:
+ localpath = ud.localpath
+
def compute_checksum_info(checksum_id):
checksum_name = getattr(ud, "%s_name" % checksum_id)
if checksum_id in precomputed:
checksum_data = precomputed[checksum_id]
else:
- checksum_data = getattr(bb.utils, "%s_file" % checksum_id)(ud.localpath)
+ checksum_data = getattr(bb.utils, "%s_file" % checksum_id)(localpath)
checksum_expected = getattr(ud, "%s_expected" % checksum_id)
@@ -595,7 +598,7 @@ def verify_checksum(ud, d, precomputed={}):
checksum_lines = ["SRC_URI[%s] = \"%s\"" % (ci["name"], ci["data"])]
# If no checksum has been provided
- if ud.method.recommends_checksum(ud) and all(ci["expected"] is None for ci in checksum_infos):
+ if fatal_nochecksum and ud.method.recommends_checksum(ud) and all(ci["expected"] is None for ci in checksum_infos):
messages = []
strict = d.getVar("BB_STRICT_CHECKSUM") or "0"
@@ -627,7 +630,7 @@ def verify_checksum(ud, d, precomputed={}):
for ci in checksum_infos:
if ci["expected"] and ci["expected"] != ci["data"]:
messages.append("File: '%s' has %s checksum '%s' when '%s' was " \
- "expected" % (ud.localpath, ci["id"], ci["data"], ci["expected"]))
+ "expected" % (localpath, ci["id"], ci["data"], ci["expected"]))
bad_checksum = ci["data"]
if bad_checksum:
diff --git a/lib/bb/fetch2/wget.py b/lib/bb/fetch2/wget.py
index b2b542e1d..821afa5b5 100644
--- a/lib/bb/fetch2/wget.py
+++ b/lib/bb/fetch2/wget.py
@@ -132,6 +132,11 @@ class Wget(FetchMethod):
self._runwget(ud, d, fetchcmd, False)
+ # Try and verify any checksum now, meaning if it isn't correct, we don't remove the
+ # original file, which might be a race (imagine two recipes referencing the same
+ # source, one with an incorrect checksum)
+ bb.fetch2.verify_checksum(ud, d, localpath=localpath, fatal_nochecksum=False)
+
# Remove the ".tmp" and move the file into position atomically
# Our lock prevents multiple writers but mirroring code may grab incomplete files
os.rename(localpath, localpath[:-4])