diff options
-rw-r--r-- | SECURITY.md | 24 | ||||
-rw-r--r-- | doc/bitbake-user-manual/bitbake-user-manual-fetching.rst | 4 | ||||
-rw-r--r-- | lib/bb/__init__.py | 7 | ||||
-rw-r--r-- | lib/bb/cooker.py | 5 | ||||
-rw-r--r-- | lib/bb/data.py | 1 | ||||
-rw-r--r-- | lib/bb/fetch2/git.py | 20 | ||||
-rw-r--r-- | lib/bb/monitordisk.py | 7 | ||||
-rw-r--r-- | lib/bb/runqueue.py | 96 | ||||
-rw-r--r-- | lib/bb/siggen.py | 3 | ||||
-rw-r--r-- | lib/bb/tests/codeparser.py | 26 | ||||
-rw-r--r-- | lib/bb/tests/fetch.py | 2 | ||||
-rw-r--r-- | lib/bb/utils.py | 39 | ||||
-rw-r--r-- | lib/bblayers/layerindex.py | 1 | ||||
-rw-r--r-- | lib/toaster/toastergui/api.py | 26 |
14 files changed, 188 insertions, 73 deletions
diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000..7d2ce1f63 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,24 @@ +How to Report a Potential Vulnerability? +======================================== + +If you would like to report a public issue (for example, one with a released +CVE number), please report it using the +[https://bugzilla.yoctoproject.org/enter_bug.cgi?product=Security Security Bugzilla]. +If you have a patch ready, submit it following the same procedure as any other +patch as described in README.md. + +If you are dealing with a not-yet released or urgent issue, please send a +message to security AT yoctoproject DOT org, including as many details as +possible: the layer or software module affected, the recipe and its version, +and any example code, if available. + +Branches maintained with security fixes +--------------------------------------- + +See [https://wiki.yoctoproject.org/wiki/Stable_Release_and_LTS Stable release and LTS] +for detailed info regarding the policies and maintenance of Stable branches. + +The [https://wiki.yoctoproject.org/wiki/Releases Release page] contains a list of all +releases of the Yocto Project. Versions in grey are no longer actively maintained with +security patches, but well-tested patches may still be accepted for them for +significant issues. diff --git a/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst b/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst index 93ac18b78..75e8dd69d 100644 --- a/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst +++ b/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst @@ -405,8 +405,8 @@ This fetcher supports the following parameters: - *"nobranch":* Tells the fetcher to not check the SHA validation for the branch when set to "1". The default is "0". Set this option for - the recipe that refers to the commit that is valid for a tag instead - of the branch. + the recipe that refers to the commit that is valid for any namespace + (branch, tag, ...) instead of the branch. - *"bareclone":* Tells the fetcher to clone a bare clone into the destination directory without checking out a working tree. Only the diff --git a/lib/bb/__init__.py b/lib/bb/__init__.py index c98e23ce3..ba8039497 100644 --- a/lib/bb/__init__.py +++ b/lib/bb/__init__.py @@ -15,6 +15,13 @@ import sys if sys.version_info < (3, 5, 0): raise RuntimeError("Sorry, python 3.5.0 or later is required for this version of bitbake") +if sys.version_info < (3, 10, 0): + # With python 3.8 and 3.9, we see errors of "libgcc_s.so.1 must be installed for pthread_cancel to work" + # https://stackoverflow.com/questions/64797838/libgcc-s-so-1-must-be-installed-for-pthread-cancel-to-work + # https://bugs.ams1.psf.io/issue42888 + # so ensure libgcc_s is loaded early on + import ctypes + libgcc_s = ctypes.CDLL('libgcc_s.so.1') class BBHandledException(Exception): """ diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py index ac54d4378..6743bce58 100644 --- a/lib/bb/cooker.py +++ b/lib/bb/cooker.py @@ -13,7 +13,6 @@ import sys, os, glob, os.path, re, time import itertools import logging import multiprocessing -import sre_constants import threading from io import StringIO, UnsupportedOperation from contextlib import closing @@ -1795,7 +1794,7 @@ class CookerCollectFiles(object): try: re.compile(mask) bbmasks.append(mask) - except sre_constants.error: + except re.error: collectlog.critical("BBMASK contains an invalid regular expression, ignoring: %s" % mask) # Then validate the combined regular expressions. This should never @@ -1803,7 +1802,7 @@ class CookerCollectFiles(object): bbmask = "|".join(bbmasks) try: bbmask_compiled = re.compile(bbmask) - except sre_constants.error: + except re.error: collectlog.critical("BBMASK is not a valid regular expression, ignoring: %s" % bbmask) bbmask = None diff --git a/lib/bb/data.py b/lib/bb/data.py index b0683c518..1d21e00a1 100644 --- a/lib/bb/data.py +++ b/lib/bb/data.py @@ -301,6 +301,7 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d): value += "\n_remove of %s" % r deps |= r2.references deps = deps | (keys & r2.execs) + value = handle_contains(value, r2.contains, d) return value if "vardepvalue" in varflags: diff --git a/lib/bb/fetch2/git.py b/lib/bb/fetch2/git.py index 63a9f92b0..cad1ae820 100644 --- a/lib/bb/fetch2/git.py +++ b/lib/bb/fetch2/git.py @@ -44,7 +44,8 @@ Supported SRC_URI options are: - nobranch Don't check the SHA validation for branch. set this option for the recipe - referring to commit which is valid in tag instead of branch. + referring to commit which is valid in any namespace (branch, tag, ...) + instead of branch. The default is "0", set nobranch=1 if needed. - usehead @@ -63,6 +64,7 @@ import errno import fnmatch import os import re +import shlex import subprocess import tempfile import bb @@ -352,7 +354,7 @@ class Git(FetchMethod): # We do this since git will use a "-l" option automatically for local urls where possible if repourl.startswith("file://"): repourl = repourl[7:] - clone_cmd = "LANG=C %s clone --bare --mirror \"%s\" %s --progress" % (ud.basecmd, repourl, ud.clonedir) + clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, shlex.quote(repourl), ud.clonedir) if ud.proto.lower() != 'file': bb.fetch2.check_network_access(d, clone_cmd, ud.url) progresshandler = GitProgressHandler(d) @@ -364,8 +366,12 @@ class Git(FetchMethod): if "origin" in output: runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir) - runfetchcmd("%s remote add --mirror=fetch origin \"%s\"" % (ud.basecmd, repourl), d, workdir=ud.clonedir) - fetch_cmd = "LANG=C %s fetch -f --progress \"%s\" refs/*:refs/*" % (ud.basecmd, repourl) + runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=ud.clonedir) + + if ud.nobranch: + fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, shlex.quote(repourl)) + else: + fetch_cmd = "LANG=C %s fetch -f --progress %s refs/heads/*:refs/heads/* refs/tags/*:refs/tags/*" % (ud.basecmd, shlex.quote(repourl)) if ud.proto.lower() != 'file': bb.fetch2.check_network_access(d, fetch_cmd, ud.url) progresshandler = GitProgressHandler(d) @@ -559,7 +565,7 @@ class Git(FetchMethod): raise bb.fetch2.UnpackError("No up to date source found: " + "; ".join(source_error), ud.url) repourl = self._get_repo_url(ud) - runfetchcmd("%s remote set-url origin \"%s\"" % (ud.basecmd, repourl), d, workdir=destdir) + runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=destdir) if self._contains_lfs(ud, d, destdir): if need_lfs and not self._find_git_lfs(d): @@ -687,8 +693,8 @@ class Git(FetchMethod): d.setVar('_BB_GIT_IN_LSREMOTE', '1') try: repourl = self._get_repo_url(ud) - cmd = "%s ls-remote \"%s\" %s" % \ - (ud.basecmd, repourl, search) + cmd = "%s ls-remote %s %s" % \ + (ud.basecmd, shlex.quote(repourl), search) if ud.proto.lower() != 'file': bb.fetch2.check_network_access(d, cmd, repourl) output = runfetchcmd(cmd, d, True) diff --git a/lib/bb/monitordisk.py b/lib/bb/monitordisk.py index e7c07264a..4d243af30 100644 --- a/lib/bb/monitordisk.py +++ b/lib/bb/monitordisk.py @@ -229,9 +229,10 @@ class diskMonitor: freeInode = st.f_favail if minInode and freeInode < minInode: - # Some filesystems use dynamic inodes so can't run out - # (e.g. btrfs). This is reported by the inode count being 0. - if st.f_files == 0: + # Some filesystems use dynamic inodes so can't run out. + # This is reported by the inode count being 0 (btrfs) or the free + # inode count being -1 (cephfs). + if st.f_files == 0 or st.f_favail == -1: self.devDict[k][2] = None continue # Always show warning, the self.checked would always be False if the action is WARN diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py index 6cdc72a85..886eef1f2 100644 --- a/lib/bb/runqueue.py +++ b/lib/bb/runqueue.py @@ -1975,6 +1975,20 @@ class RunQueueExecute: self.setbuildable(revdep) logger.debug(1, "Marking task %s as buildable", revdep) + found = None + for t in sorted(self.sq_deferred.copy()): + if self.sq_deferred[t] == task: + # Allow the next deferred task to run. Any other deferred tasks should be deferred after that task. + # We shouldn't allow all to run at once as it is prone to races. + if not found: + bb.note("Deferred task %s now buildable" % t) + del self.sq_deferred[t] + update_scenequeue_data([t], self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self, summary=False) + found = t + else: + bb.note("Deferring %s after %s" % (t, found)) + self.sq_deferred[t] = found + def task_complete(self, task): self.stats.taskCompleted() bb.event.fire(runQueueTaskCompleted(task, self.stats, self.rq), self.cfgData) @@ -2084,8 +2098,6 @@ class RunQueueExecute: logger.debug(1, "%s didn't become valid, skipping setscene" % nexttask) self.sq_task_failoutright(nexttask) return True - else: - self.sqdata.outrightfail.remove(nexttask) if nexttask in self.sqdata.outrightfail: logger.debug(2, 'No package found, so skipping setscene task %s', nexttask) self.sq_task_failoutright(nexttask) @@ -2236,7 +2248,8 @@ class RunQueueExecute: if self.sq_deferred: tid = self.sq_deferred.pop(list(self.sq_deferred.keys())[0]) logger.warning("Runqeueue deadlocked on deferred tasks, forcing task %s" % tid) - self.sq_task_failoutright(tid) + if tid not in self.runq_complete: + self.sq_task_failoutright(tid) return True if len(self.failed_tids) != 0: @@ -2350,10 +2363,16 @@ class RunQueueExecute: self.updated_taskhash_queue.remove((tid, unihash)) if unihash != self.rqdata.runtaskentries[tid].unihash: - hashequiv_logger.verbose("Task %s unihash changed to %s" % (tid, unihash)) - self.rqdata.runtaskentries[tid].unihash = unihash - bb.parse.siggen.set_unihash(tid, unihash) - toprocess.add(tid) + # Make sure we rehash any other tasks with the same task hash that we're deferred against. + torehash = [tid] + for deftid in self.sq_deferred: + if self.sq_deferred[deftid] == tid: + torehash.append(deftid) + for hashtid in torehash: + hashequiv_logger.verbose("Task %s unihash changed to %s" % (hashtid, unihash)) + self.rqdata.runtaskentries[hashtid].unihash = unihash + bb.parse.siggen.set_unihash(hashtid, unihash) + toprocess.add(hashtid) # Work out all tasks which depend upon these total = set() @@ -2492,6 +2511,14 @@ class RunQueueExecute: if update_tasks: self.sqdone = False + for mc in sorted(self.sqdata.multiconfigs): + for tid in sorted([t[0] for t in update_tasks]): + if mc_from_tid(tid) != mc: + continue + h = pending_hash_index(tid, self.rqdata) + if h in self.sqdata.hashes and tid != self.sqdata.hashes[h]: + self.sq_deferred[tid] = self.sqdata.hashes[h] + bb.note("Deferring %s after %s" % (tid, self.sqdata.hashes[h])) update_scenequeue_data([t[0] for t in update_tasks], self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self, summary=False) for (tid, harddepfail, origvalid) in update_tasks: @@ -2832,6 +2859,19 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq): sqdata.stamppresent = set() sqdata.valid = set() + sqdata.hashes = {} + sqrq.sq_deferred = {} + for mc in sorted(sqdata.multiconfigs): + for tid in sorted(sqdata.sq_revdeps): + if mc_from_tid(tid) != mc: + continue + h = pending_hash_index(tid, rqdata) + if h not in sqdata.hashes: + sqdata.hashes[h] = tid + else: + sqrq.sq_deferred[tid] = sqdata.hashes[h] + bb.note("Deferring %s after %s" % (tid, sqdata.hashes[h])) + update_scenequeue_data(sqdata.sq_revdeps, sqdata, rqdata, rq, cooker, stampcache, sqrq, summary=True) def update_scenequeue_data(tids, sqdata, rqdata, rq, cooker, stampcache, sqrq, summary=True): @@ -2843,6 +2883,8 @@ def update_scenequeue_data(tids, sqdata, rqdata, rq, cooker, stampcache, sqrq, s sqdata.stamppresent.remove(tid) if tid in sqdata.valid: sqdata.valid.remove(tid) + if tid in sqdata.outrightfail: + sqdata.outrightfail.remove(tid) (mc, fn, taskname, taskfn) = split_tid_mcfn(tid) @@ -2870,32 +2912,20 @@ def update_scenequeue_data(tids, sqdata, rqdata, rq, cooker, stampcache, sqrq, s sqdata.valid |= rq.validate_hashes(tocheck, cooker.data, len(sqdata.stamppresent), False, summary=summary) - sqdata.hashes = {} - sqrq.sq_deferred = {} - for mc in sorted(sqdata.multiconfigs): - for tid in sorted(sqdata.sq_revdeps): - if mc_from_tid(tid) != mc: - continue - if tid in sqdata.stamppresent: - continue - if tid in sqdata.valid: - continue - if tid in sqdata.noexec: - continue - if tid in sqrq.scenequeue_notcovered: - continue - if tid in sqrq.scenequeue_covered: - continue - - sqdata.outrightfail.add(tid) - - h = pending_hash_index(tid, rqdata) - if h not in sqdata.hashes: - sqdata.hashes[h] = tid - else: - sqrq.sq_deferred[tid] = sqdata.hashes[h] - bb.note("Deferring %s after %s" % (tid, sqdata.hashes[h])) - + for tid in tids: + if tid in sqdata.stamppresent: + continue + if tid in sqdata.valid: + continue + if tid in sqdata.noexec: + continue + if tid in sqrq.scenequeue_covered: + continue + if tid in sqrq.scenequeue_notcovered: + continue + if tid in sqrq.sq_deferred: + continue + sqdata.outrightfail.add(tid) class TaskFailure(Exception): """ diff --git a/lib/bb/siggen.py b/lib/bb/siggen.py index 26fa7f05c..9d4f67aa9 100644 --- a/lib/bb/siggen.py +++ b/lib/bb/siggen.py @@ -318,7 +318,8 @@ class SignatureGeneratorBasic(SignatureGenerator): else: sigfile = stampbase + "." + task + ".sigbasedata" + "." + self.basehash[tid] - bb.utils.mkdirhier(os.path.dirname(sigfile)) + with bb.utils.umask(0o002): + bb.utils.mkdirhier(os.path.dirname(sigfile)) data = {} data['task'] = task diff --git a/lib/bb/tests/codeparser.py b/lib/bb/tests/codeparser.py index f48520479..f1c4f618d 100644 --- a/lib/bb/tests/codeparser.py +++ b/lib/bb/tests/codeparser.py @@ -412,6 +412,32 @@ esac # Check final value self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['anothervalue', 'yetanothervalue', 'lastone']) + def test_contains_vardeps_override_operators(self): + # Check override operators handle dependencies correctly with the contains functionality + expr_plain = 'testval' + expr_prepend = '${@bb.utils.filter("TESTVAR1", "testval1", d)} ' + expr_append = ' ${@bb.utils.filter("TESTVAR2", "testval2", d)}' + expr_remove = '${@bb.utils.contains("TESTVAR3", "no-testval", "testval", "", d)}' + # Check dependencies + self.d.setVar('ANOTHERVAR', expr_plain) + self.d.prependVar('ANOTHERVAR', expr_prepend) + self.d.appendVar('ANOTHERVAR', expr_append) + self.d.setVar('ANOTHERVAR:remove', expr_remove) + self.d.setVar('TESTVAR1', 'blah') + self.d.setVar('TESTVAR2', 'testval2') + self.d.setVar('TESTVAR3', 'no-testval') + deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), self.d) + self.assertEqual(sorted(values.splitlines()), + sorted([ + expr_prepend + expr_plain + expr_append, + '_remove of ' + expr_remove, + 'TESTVAR1{testval1} = Unset', + 'TESTVAR2{testval2} = Set', + 'TESTVAR3{no-testval} = Set', + ])) + # Check final value + self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['testval2']) + #Currently no wildcard support #def test_vardeps_wildcards(self): # self.d.setVar("oe_libinstall", "echo test") diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py index 420faa191..61dd5ccca 100644 --- a/lib/bb/tests/fetch.py +++ b/lib/bb/tests/fetch.py @@ -1338,7 +1338,7 @@ class FetchCheckStatusTest(FetcherTest): "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.2.tar.gz", "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.3.tar.gz", "https://yoctoproject.org/", - "https://yoctoproject.org/documentation", + "https://docs.yoctoproject.org/", "http://downloads.yoctoproject.org/releases/opkg/opkg-0.1.7.tar.gz", "http://downloads.yoctoproject.org/releases/opkg/opkg-0.3.0.tar.gz", "ftp://sourceware.org/pub/libffi/libffi-1.20.tar.gz", diff --git a/lib/bb/utils.py b/lib/bb/utils.py index 210e535f0..34fa0b7a6 100644 --- a/lib/bb/utils.py +++ b/lib/bb/utils.py @@ -461,13 +461,16 @@ def lockfile(name, shared=False, retry=True, block=False): consider the possibility of sending a signal to the process to break out - at which point you want block=True rather than retry=True. """ - if len(name) > 255: - root, ext = os.path.splitext(name) - name = root[:255 - len(ext)] + ext + basename = os.path.basename(name) + if len(basename) > 255: + root, ext = os.path.splitext(basename) + basename = root[:255 - len(ext)] + ext dirname = os.path.dirname(name) mkdirhier(dirname) + name = os.path.join(dirname, basename) + if not os.access(dirname, os.W_OK): logger.error("Unable to acquire lock '%s', directory is not writable", name) @@ -966,6 +969,17 @@ def which(path, item, direction = 0, history = False, executable=False): return "", hist return "" +@contextmanager +def umask(new_mask): + """ + Context manager to set the umask to a specific mask, and restore it afterwards. + """ + current_mask = os.umask(new_mask) + try: + yield + finally: + os.umask(current_mask) + def to_boolean(string, default=None): if not string: return default @@ -1567,21 +1581,22 @@ def set_process_name(name): # export common proxies variables from datastore to environment def export_proxies(d): - import os + """ export common proxies variables from datastore to environment """ variables = ['http_proxy', 'HTTP_PROXY', 'https_proxy', 'HTTPS_PROXY', 'ftp_proxy', 'FTP_PROXY', 'no_proxy', 'NO_PROXY', - 'GIT_PROXY_COMMAND'] + 'GIT_PROXY_COMMAND', 'SSL_CERT_FILE', 'SSL_CERT_DIR'] exported = False - for v in variables: - if v in os.environ.keys(): + origenv = d.getVar("BB_ORIGENV") + + for name in variables: + value = d.getVar(name) + if not value and origenv: + value = origenv.getVar(name) + if value: + os.environ[name] = value exported = True - else: - v_proxy = d.getVar(v) - if v_proxy is not None: - os.environ[v] = v_proxy - exported = True return exported diff --git a/lib/bblayers/layerindex.py b/lib/bblayers/layerindex.py index 95b67a662..f64d18e81 100644 --- a/lib/bblayers/layerindex.py +++ b/lib/bblayers/layerindex.py @@ -206,6 +206,7 @@ class LayerIndexPlugin(ActionPlugin): """ args.show_only = True args.ignore = [] + args.shallow = True self.do_layerindex_fetch(args) def register_commands(self, sp): diff --git a/lib/toaster/toastergui/api.py b/lib/toaster/toastergui/api.py index b4cdc335e..e367bd910 100644 --- a/lib/toaster/toastergui/api.py +++ b/lib/toaster/toastergui/api.py @@ -11,7 +11,7 @@ import os import re import logging import json -import subprocess +import glob from collections import Counter from orm.models import Project, ProjectTarget, Build, Layer_Version @@ -227,20 +227,18 @@ class XhrSetDefaultImageUrl(View): # same logical name # * Each project that uses a layer will have its own # LayerVersion and Project Layer for it -# * During the Paroject delete process, when the last +# * During the Project delete process, when the last # LayerVersion for a 'local_source_dir' layer is deleted # then the Layer record is deleted to remove orphans # def scan_layer_content(layer,layer_version): # if this is a local layer directory, we can immediately scan its content - if layer.local_source_dir: + if os.path.isdir(layer.local_source_dir): try: # recipes-*/*/*.bb - cmd = '%s %s' % ('ls', os.path.join(layer.local_source_dir,'recipes-*/*/*.bb')) - recipes_list = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read() - recipes_list = recipes_list.decode("utf-8").strip() - if recipes_list and 'No such' not in recipes_list: + recipes_list = glob.glob(os.path.join(layer.local_source_dir, 'recipes-*/*/*.bb')) + for recipe in recipes_list: for recipe in recipes_list.split('\n'): recipe_path = recipe[recipe.rfind('recipes-'):] recipe_name = recipe[recipe.rfind('/')+1:].replace('.bb','') @@ -260,6 +258,9 @@ def scan_layer_content(layer,layer_version): except Exception as e: logger.warning("ERROR:scan_layer_content: %s" % e) + else: + logger.warning("ERROR: wrong path given") + raise KeyError("local_source_dir") class XhrLayer(View): """ Delete, Get, Add and Update Layer information @@ -456,15 +457,18 @@ class XhrLayer(View): 'layerdetailurl': layer_dep.get_detailspage_url(project.pk)}) - # Scan the layer's content and update components - scan_layer_content(layer,layer_version) + # Only scan_layer_content if layer is local + if layer_data.get('local_source_dir', None): + # Scan the layer's content and update components + scan_layer_content(layer,layer_version) except Layer_Version.DoesNotExist: return error_response("layer-dep-not-found") except Project.DoesNotExist: return error_response("project-not-found") - except KeyError: - return error_response("incorrect-parameters") + except KeyError as e: + _log("KeyError: %s" % e) + return error_response(f"incorrect-parameters") return JsonResponse({'error': "ok", 'imported_layer': { |