diff options
authorRichard Purdie <richard.purdie@linuxfoundation.org>2020-08-25 14:49:12 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2020-08-25 18:14:56 +0100
commit0c9302d950c6f37bfcc4256b41001d63f668bdf7 (patch)
parentb0c807be5c2170c9481c1a04d4c11972135d7dc5 (diff)
fetch2: Drop globbing supprt in file:// SRC_URIs
Globbing in file:// urls is terminally broken. Currently when its used, the file checksum code is basically bypassed. This means changes to the source files don't change the task checksum, the task doesn't rebuild when the inputs change and things generally break. To make globbing work generically, we'd have to scan the file system for all possible matches to the glob and log whether they exist or not. We can't simply log the files which exist, we have to also know which files could later exist and influence the choice of file so we know when to reparse. For a simple file://xxx/*, this could be done but for bigger patterns, it becomes much more problemtic. We already support file://xxx/ in urls. So, lets decide we'll not support globs in file://urls. Worse case users can put files in a directory and reference that, moving files into place if needed. Remove all the glob special cases (see the comments if anyone doesn't believe this is terminally broken) and error to the user if they have such urls. Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
4 files changed, 6 insertions, 27 deletions
diff --git a/lib/bb/cache.py b/lib/bb/cache.py
index b819a0c2d..9e0c931a0 100644
--- a/lib/bb/cache.py
+++ b/lib/bb/cache.py
@@ -636,7 +636,7 @@ class Cache(NoCache):
# Have to be careful about spaces and colons in filenames
flist = self.filelist_regex.split(fl)
for f in flist:
- if not f or "*" in f:
+ if not f:
f, exist = f.split(":")
if (exist == "True" and not os.path.exists(f)) or (exist == "False" and os.path.exists(f)):
diff --git a/lib/bb/fetch2/__init__.py b/lib/bb/fetch2/__init__.py
index 756f60212..7ec1fea5d 100644
--- a/lib/bb/fetch2/__init__.py
+++ b/lib/bb/fetch2/__init__.py
@@ -1195,8 +1195,6 @@ def get_checksum_file_list(d):
paths = ud.method.localpaths(ud, d)
for f in paths:
pth = ud.decodedurl
- if '*' in pth:
- f = os.path.join(os.path.abspath(f), pth)
if f.startswith(dl_dir):
# The local fetcher's behaviour is to return a path under DL_DIR if it couldn't find the file anywhere else
if os.path.exists(f):
@@ -1365,9 +1363,6 @@ class FetchMethod(object):
# We cannot compute checksums for directories
if os.path.isdir(urldata.localpath):
return False
- if urldata.localpath.find("*") != -1:
- return False
return True
def recommends_checksum(self, urldata):
@@ -1430,11 +1425,6 @@ class FetchMethod(object):
iterate = False
file = urldata.localpath
- # Localpath can't deal with 'dir/*' entries, so it converts them to '.',
- # but it must be corrected back for local files copying
- if urldata.basename == '*' and file.endswith('/.'):
- file = '%s/%s' % (file.rstrip('/.'), urldata.path)
unpack = bb.utils.to_boolean(urldata.parm.get('unpack'), True)
except ValueError as exc:
@@ -1613,8 +1603,6 @@ class FetchMethod(object):
if os.path.exists(ud.localpath):
return True
- if ud.localpath.find("*") != -1:
- return True
return False
def implicit_urldata(self, ud, d):
diff --git a/lib/bb/fetch2/local.py b/lib/bb/fetch2/local.py
index 01d9ff9f8..25d4557db 100644
--- a/lib/bb/fetch2/local.py
+++ b/lib/bb/fetch2/local.py
@@ -17,7 +17,7 @@ import os
import urllib.request, urllib.parse, urllib.error
import bb
import bb.utils
-from bb.fetch2 import FetchMethod, FetchError
+from bb.fetch2 import FetchMethod, FetchError, ParameterError
from bb.fetch2 import logger
class Local(FetchMethod):
@@ -33,6 +33,8 @@ class Local(FetchMethod):
ud.basename = os.path.basename(ud.decodedurl)
ud.basepath = ud.decodedurl
ud.needdonestamp = False
+ if "*" in ud.decodedurl:
+ raise bb.fetch2.ParameterError("file:// urls using globbing are no longer supported. Please place the files in a directory and reference that instead.", ud.url)
def localpath(self, urldata, d):
@@ -55,12 +57,6 @@ class Local(FetchMethod):
logger.debug(2, "Searching for %s in paths:\n %s" % (path, "\n ".join(filespath.split(":"))))
newpath, hist = bb.utils.which(filespath, path, history=True)
- if (not newpath or not os.path.exists(newpath)) and path.find("*") != -1:
- # For expressions using '*', best we can do is take the first directory in FILESPATH that exists
- newpath, hist = bb.utils.which(filespath, ".", history=True)
- searched.extend(hist)
- logger.debug(2, "Searching for %s in path: %s" % (path, newpath))
- return searched
if not os.path.exists(newpath):
dldirfile = os.path.join(d.getVar("DL_DIR"), path)
logger.debug(2, "Defaulting to %s for %s" % (dldirfile, path))
@@ -70,8 +66,6 @@ class Local(FetchMethod):
return searched
def need_update(self, ud, d):
- if ud.url.find("*") != -1:
- return False
if os.path.exists(ud.localpath):
return False
return True
@@ -95,9 +89,6 @@ class Local(FetchMethod):
Check the status of the url
- if urldata.localpath.find("*") != -1:
- logger.info("URL %s looks like a glob and was therefore not checked.", urldata.url)
- return True
if os.path.exists(urldata.localpath):
return True
return False
diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py
index 61b80bede..13561cea9 100644
--- a/lib/bb/tests/fetch.py
+++ b/lib/bb/tests/fetch.py
@@ -602,8 +602,8 @@ class FetcherLocalTest(FetcherTest):
self.assertEqual(tree, ['a', 'dir/c'])
def test_local_wildcard(self):
- tree = self.fetchUnpack(['file://a', 'file://dir/*'])
- self.assertEqual(tree, ['a', 'dir/c', 'dir/d', 'dir/subdir/e'])
+ with self.assertRaises(bb.fetch2.ParameterError):
+ tree = self.fetchUnpack(['file://a', 'file://dir/*'])
def test_local_dir(self):
tree = self.fetchUnpack(['file://a', 'file://dir'])