summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2019-07-31 10:49:39 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2019-08-06 11:21:32 +0100
commit43d37a6eaf2224c0dda1d1436a0afc2bd34fdddf (patch)
tree273e6a02584e2222a4f2b7249018dbd487e818e4
parentd40d7e43856f176c45cf515644b5f211c708e237 (diff)
downloadopenembedded-core-contrib-43d37a6eaf2224c0dda1d1436a0afc2bd34fdddf.tar.gz
openembedded-core-contrib-43d37a6eaf2224c0dda1d1436a0afc2bd34fdddf.tar.bz2
openembedded-core-contrib-43d37a6eaf2224c0dda1d1436a0afc2bd34fdddf.zip
bitbake: hashserv: Switch from threads to multiprocessing
There were hard to debug lockups when trying to use threading to start hashserv as a thread. Switch to multiprocessing which doesn't show the same locking problems. (Bitbake rev: be23d887c8e244f1ef961298fbc9214d0fd0968a) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--bitbake/lib/bb/cooker.py20
-rw-r--r--bitbake/lib/hashserv/__init__.py14
-rw-r--r--bitbake/lib/hashserv/tests.py8
3 files changed, 27 insertions, 15 deletions
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 84f25ec94e..6e1d59bb3a 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -193,6 +193,8 @@ class BBCooker:
bb.parse.BBHandler.cached_statements = {}
self.ui_cmdline = None
+ self.hashserv = None
+ self.hashservport = None
self.initConfigurationData()
@@ -231,8 +233,6 @@ class BBCooker:
self.state = state.initial
self.parser = None
- self.hashserv = None
- self.hashservport = None
signal.signal(signal.SIGTERM, self.sigterm_exception)
# Let SIGHUP exit as SIGTERM
@@ -372,12 +372,13 @@ class BBCooker:
self.data.setVar('BB_CMDLINE', self.ui_cmdline)
if self.data.getVar("BB_HASHSERVE") == "localhost:0":
- dbfile = (self.data.getVar("PERSISTENT_DIR") or self.data.getVar("CACHE")) + "/hashserv.db"
- self.hashserv = hashserv.create_server(('localhost', 0), dbfile, '')
- self.hashservport = "localhost:" + str(self.hashserv.server_port)
- thread = threading.Thread(target=self.hashserv.serve_forever)
- thread.daemon = True
- thread.start()
+ if not self.hashserv:
+ dbfile = (self.data.getVar("PERSISTENT_DIR") or self.data.getVar("CACHE")) + "/hashserv.db"
+ self.hashserv = hashserv.create_server(('localhost', 0), dbfile, '')
+ self.hashservport = "localhost:" + str(self.hashserv.server_port)
+ self.hashserv.process = multiprocessing.Process(target=self.hashserv.serve_forever)
+ self.hashserv.process.daemon = True
+ self.hashserv.process.start()
self.data.setVar("BB_HASHSERVE", self.hashservport)
#
@@ -1658,7 +1659,8 @@ class BBCooker:
def post_serve(self):
prserv.serv.auto_shutdown()
if self.hashserv:
- self.hashserv.shutdown()
+ self.hashserv.process.terminate()
+ self.hashserv.process.join()
bb.event.fire(CookerExit(), self.data)
def shutdown(self, force = False):
diff --git a/bitbake/lib/hashserv/__init__.py b/bitbake/lib/hashserv/__init__.py
index 86aa7e9de6..1d5e08ee5a 100644
--- a/bitbake/lib/hashserv/__init__.py
+++ b/bitbake/lib/hashserv/__init__.py
@@ -13,6 +13,7 @@ import logging
import socketserver
import queue
import threading
+import signal
from datetime import datetime
logger = logging.getLogger('hashserv')
@@ -147,8 +148,14 @@ class ThreadedHTTPServer(HTTPServer):
self.handlerthread.daemon = False
self.handlerthread.start()
+
+ signal.signal(signal.SIGTERM, self.sigterm_exception)
super().serve_forever()
+ def sigterm_exception(self, signum, stackframe):
+ self.server_close()
+ os._exit(0)
+
def process_request_thread(self):
while not self.quit:
try:
@@ -163,6 +170,7 @@ class ThreadedHTTPServer(HTTPServer):
self.handle_error(request, client_address)
finally:
self.shutdown_request(request)
+ os._exit(0)
def process_request(self, request, client_address):
self.requestqueue.put((request, client_address))
@@ -208,6 +216,8 @@ def create_server(addr, dbname, prefix=''):
cursor.execute('CREATE INDEX IF NOT EXISTS taskhash_lookup ON tasks_v2 (method, taskhash)')
cursor.execute('CREATE INDEX IF NOT EXISTS outhash_lookup ON tasks_v2 (method, outhash)')
- logger.info('Starting server on %s', addr)
+ ret = ThreadedHTTPServer(addr, Handler)
+
+ logger.info('Starting server on %s\n', ret.server_port)
- return ThreadedHTTPServer(addr, Handler)
+ return ret
diff --git a/bitbake/lib/hashserv/tests.py b/bitbake/lib/hashserv/tests.py
index 15f4a52ee9..6845b53884 100644
--- a/bitbake/lib/hashserv/tests.py
+++ b/bitbake/lib/hashserv/tests.py
@@ -6,7 +6,7 @@
#
import unittest
-import threading
+import multiprocessing
import sqlite3
import hashlib
import urllib.request
@@ -21,16 +21,16 @@ class TestHashEquivalenceServer(unittest.TestCase):
self.dbfile = tempfile.NamedTemporaryFile(prefix="bb-hashserv-db-")
self.server = create_server(('localhost', 0), self.dbfile.name)
self.server_addr = 'http://localhost:%d' % self.server.socket.getsockname()[1]
- self.server_thread = threading.Thread(target=self.server.serve_forever)
+ self.server_thread = multiprocessing.Process(target=self.server.serve_forever)
+ self.server_thread.daemon = True
self.server_thread.start()
def tearDown(self):
# Shutdown server
s = getattr(self, 'server', None)
if s is not None:
- self.server.shutdown()
+ self.server_thread.terminate()
self.server_thread.join()
- self.server.server_close()
def send_get(self, path):
url = '%s/%s' % (self.server_addr, path)