aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2019-07-24 09:13:55 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2019-08-06 11:21:31 +0100
commitca04aaf7b51e3ee2bb04da970d5f20f2c9982cb8 (patch)
treeee5fe5d5e780cd965bbe7f9a0d25d91d280b1834
parentd9aafb85072bef3b5baa54408969d54a8425a111 (diff)
downloadopenembedded-core-contrib-ca04aaf7b51e3ee2bb04da970d5f20f2c9982cb8.tar.gz
openembedded-core-contrib-ca04aaf7b51e3ee2bb04da970d5f20f2c9982cb8.tar.bz2
openembedded-core-contrib-ca04aaf7b51e3ee2bb04da970d5f20f2c9982cb8.zip
bitbake: cooker/hashserv: Allow autostarting of a local hash server using BB_HASHSERVE
Its useful, particularly in the local developer model of usage, for bitbake to start and stop a hash equivalence server on local port, rather than relying on one being started by the user before the build. The new BB_HASHSERVE variable supports this. The database handling is moved internally into the hashserv code so that different threads/processes can be used for the server without errors. (Bitbake rev: a4fa8f1bd88995ae60e10430316fbed63d478587) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rwxr-xr-xbitbake/bin/bitbake-hashserv4
-rwxr-xr-xbitbake/bin/bitbake-worker1
-rw-r--r--bitbake/lib/bb/cooker.py15
-rw-r--r--bitbake/lib/bb/runqueue.py1
-rw-r--r--bitbake/lib/hashserv/__init__.py19
-rw-r--r--bitbake/lib/hashserv/tests.py7
6 files changed, 37 insertions, 10 deletions
diff --git a/bitbake/bin/bitbake-hashserv b/bitbake/bin/bitbake-hashserv
index 848a0ca39f9..6c911c098a7 100755
--- a/bitbake/bin/bitbake-hashserv
+++ b/bitbake/bin/bitbake-hashserv
@@ -41,9 +41,7 @@ def main():
console.setLevel(level)
logger.addHandler(console)
- db = sqlite3.connect(args.database)
-
- server = hashserv.create_server((args.address, args.port), db, args.prefix)
+ server = hashserv.create_server((args.address, args.port), args.database, args.prefix)
server.serve_forever()
return 0
diff --git a/bitbake/bin/bitbake-worker b/bitbake/bin/bitbake-worker
index 3e502d5ca9f..96369199f23 100755
--- a/bitbake/bin/bitbake-worker
+++ b/bitbake/bin/bitbake-worker
@@ -418,6 +418,7 @@ class BitbakeWorker(object):
bb.msg.loggerDefaultDomains = self.workerdata["logdefaultdomain"]
for mc in self.databuilder.mcdata:
self.databuilder.mcdata[mc].setVar("PRSERV_HOST", self.workerdata["prhost"])
+ self.databuilder.mcdata[mc].setVar("BB_HASHSERVE", self.workerdata["hashservport"])
def handle_newtaskhashes(self, data):
self.workerdata["newhashes"] = pickle.loads(data)
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index b4851e13ede..84f25ec94eb 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -31,6 +31,7 @@ import pyinotify
import json
import pickle
import codecs
+import hashserv
logger = logging.getLogger("BitBake")
collectlog = logging.getLogger("BitBake.Collection")
@@ -230,6 +231,8 @@ class BBCooker:
self.state = state.initial
self.parser = None
+ self.hashserv = None
+ self.hashservport = None
signal.signal(signal.SIGTERM, self.sigterm_exception)
# Let SIGHUP exit as SIGTERM
@@ -368,6 +371,15 @@ class BBCooker:
self.data.setVar('BB_CMDLINE', self.ui_cmdline)
+ if self.data.getVar("BB_HASHSERVE") == "localhost:0":
+ dbfile = (self.data.getVar("PERSISTENT_DIR") or self.data.getVar("CACHE")) + "/hashserv.db"
+ self.hashserv = hashserv.create_server(('localhost', 0), dbfile, '')
+ self.hashservport = "localhost:" + str(self.hashserv.server_port)
+ thread = threading.Thread(target=self.hashserv.serve_forever)
+ thread.daemon = True
+ thread.start()
+ self.data.setVar("BB_HASHSERVE", self.hashservport)
+
#
# Copy of the data store which has been expanded.
# Used for firing events and accessing variables where expansion needs to be accounted for
@@ -1645,9 +1657,10 @@ class BBCooker:
def post_serve(self):
prserv.serv.auto_shutdown()
+ if self.hashserv:
+ self.hashserv.shutdown()
bb.event.fire(CookerExit(), self.data)
-
def shutdown(self, force = False):
if force:
self.state = state.forceshutdown
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index 05c1200da10..82cc9af81bc 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -1254,6 +1254,7 @@ class RunQueue:
"buildname" : self.cfgData.getVar("BUILDNAME"),
"date" : self.cfgData.getVar("DATE"),
"time" : self.cfgData.getVar("TIME"),
+ "hashservport" : self.cooker.hashservport,
}
worker.stdin.write(b"<cookerconfig>" + pickle.dumps(self.cooker.configuration) + b"</cookerconfig>")
diff --git a/bitbake/lib/hashserv/__init__.py b/bitbake/lib/hashserv/__init__.py
index 768c5504cf2..7ec9b64419c 100644
--- a/bitbake/lib/hashserv/__init__.py
+++ b/bitbake/lib/hashserv/__init__.py
@@ -18,8 +18,15 @@ class HashEquivalenceServer(BaseHTTPRequestHandler):
def log_message(self, f, *args):
logger.debug(f, *args)
+ def opendb(self):
+ self.db = sqlite3.connect(self.dbname)
+ self.db.row_factory = sqlite3.Row
+
def do_GET(self):
try:
+ if not self.db:
+ self.opendb()
+
p = urllib.parse.urlparse(self.path)
if p.path != self.prefix + '/v1/equivalent':
@@ -52,6 +59,9 @@ class HashEquivalenceServer(BaseHTTPRequestHandler):
def do_POST(self):
try:
+ if not self.db:
+ self.opendb()
+
p = urllib.parse.urlparse(self.path)
if p.path != self.prefix + '/v1/equivalent':
@@ -123,14 +133,17 @@ class HashEquivalenceServer(BaseHTTPRequestHandler):
self.send_error(400, explain=traceback.format_exc())
return
-def create_server(addr, db, prefix=''):
+def create_server(addr, dbname, prefix=''):
class Handler(HashEquivalenceServer):
pass
- Handler.prefix = prefix
- Handler.db = db
+ db = sqlite3.connect(dbname)
db.row_factory = sqlite3.Row
+ Handler.prefix = prefix
+ Handler.db = None
+ Handler.dbname = dbname
+
with contextlib.closing(db.cursor()) as cursor:
cursor.execute('''
CREATE TABLE IF NOT EXISTS tasks_v2 (
diff --git a/bitbake/lib/hashserv/tests.py b/bitbake/lib/hashserv/tests.py
index 8300a255990..15f4a52ee96 100644
--- a/bitbake/lib/hashserv/tests.py
+++ b/bitbake/lib/hashserv/tests.py
@@ -11,14 +11,15 @@ import sqlite3
import hashlib
import urllib.request
import json
+import tempfile
from . import create_server
class TestHashEquivalenceServer(unittest.TestCase):
def setUp(self):
- # Start an in memory hash equivalence server in the background bound to
+ # Start a hash equivalence server in the background bound to
# an ephemeral port
- db = sqlite3.connect(':memory:', check_same_thread=False)
- self.server = create_server(('localhost', 0), db)
+ self.dbfile = tempfile.NamedTemporaryFile(prefix="bb-hashserv-db-")
+ self.server = create_server(('localhost', 0), self.dbfile.name)
self.server_addr = 'http://localhost:%d' % self.server.socket.getsockname()[1]
self.server_thread = threading.Thread(target=self.server.serve_forever)
self.server_thread.start()