bitbake: hashserve: Add support for readonly upstream

Adds support for an upstream server to be specified. The upstream server
will be queried for equivalent hashes whenever a miss is found in the
local server. If the server returns a match, it is merged into the
local database. In order to keep the get stream queries as fast as
possible since they are the critical path when bitbake is preparing the
run queue, missing tasks provided by the server are not immediately
pulled from the upstream server, but instead are put into a queue to be
backfilled by a worker task later.

Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

1 files changed, 12 insertions, 7 deletions

diff --git a/lib/hashserv/client.py b/lib/hashserv/client.py
index d0b3cf386..ae5875d1b 100644
--- a/lib/hashserv/client.py
+++ b/lib/hashserv/client.py
@@ -178,18 +178,16 @@ class AsyncClient(object):
         await self._set_mode(self.MODE_NORMAL)
         return await self.send_message({"reset-stats": None})
 
+    async def backfill_wait(self):
+        await self._set_mode(self.MODE_NORMAL)
+        return (await self.send_message({"backfill-wait": None}))["tasks"]
+
 
 class Client(object):
     def __init__(self):
         self.client = AsyncClient()
         self.loop = asyncio.new_event_loop()
 
-        def get_wrapper(self, downcall):
-            def wrapper(*args, **kwargs):
-                return self.loop.run_until_complete(downcall(*args, **kwargs))
-
-            return wrapper
-
         for call in (
             "connect_tcp",
             "connect_unix",
@@ -200,9 +198,16 @@ class Client(object):
             "get_taskhash",
             "get_stats",
             "reset_stats",
+            "backfill_wait",
         ):
             downcall = getattr(self.client, call)
-            setattr(self, call, get_wrapper(self, downcall))
+            setattr(self, call, self._get_downcall_wrapper(downcall))
+
+    def _get_downcall_wrapper(self, downcall):
+        def wrapper(*args, **kwargs):
+            return self.loop.run_until_complete(downcall(*args, **kwargs))
+
+        return wrapper
 
     @property
     def max_chunk(self):