Run the server and UI in separate processes

This uses the python multiprocessing module, both to spawn the server process
and for communication between the processes.

Signed-off-by: Bob Foerster <robert@erafx.com>
Signed-off-by: Chris Larson <chris_larson@mentor.com>

6 files changed, 171 insertions, 69 deletions

diff --git a/bin/bitbake b/bin/bitbake
index d88d6c713..346ce9bea 100755
--- a/bin/bitbake
+++ b/bin/bitbake
@@ -39,11 +39,21 @@ import bb.msg
 from bb import cooker
 from bb import ui
 from bb import server
-from bb.server import none
+from bb.server.process import ProcessServer
+
+from multiprocessing import Queue, Pipe
 
 __version__ = "1.11.0"
 logger = logging.getLogger("BitBake")
 
+class ServerCommunicator():
+    def __init__(self, connection):
+        self.connection = connection
+
+    def runCommand(self, command):
+        # @todo try/except
+        self.connection.send(command)
+        return self.connection.recv()
 
 class BBConfiguration(object):
     """
@@ -167,15 +177,6 @@ Default BBFILES are the .bb files in the current directory.""")
 
     ui_main = get_ui(configuration)
 
-    loghandler = event.LogHandler()
-    logger.addHandler(loghandler)
-
-    server = bb.server.none
-
-    # Save a logfile for cooker into the current working directory. When the
-    # server is daemonized this logfile will be truncated.
-    cooker_logfile = os.path.join(os.getcwd(), "cooker.log")
-
     bb.utils.init_logger(bb.msg, configuration.verbose, configuration.debug,
                          configuration.debug_domains)
 
@@ -184,23 +185,25 @@ Default BBFILES are the .bb files in the current directory.""")
     # of the UIs (e.g. for DISPLAY, etc.)
     bb.utils.clean_environment()
 
-    cooker = bb.cooker.BBCooker(configuration, server)
-    cooker.parseCommandLine()
+    # establish communication channels.  We use bidirectional pipes for
+    # ui <--> server command/response pairs
+    # and a queue for server -> ui event notifications
+    #
+    ui_channel, server_channel = Pipe()
+    event_queue = Queue()
 
-    serverinfo = server.BitbakeServerInfo(cooker.server)
-
-    server.BitBakeServerFork(serverinfo, cooker.serve, cooker_logfile)
-    del cooker
-
-    logger.removeHandler(loghandler)
-
-    # Setup a connection to the server (cooker)
-    server_connection = server.BitBakeServerConnection(serverinfo)
+    server = ProcessServer(server_channel, event_queue, configuration)
+    server.start()
 
     try:
         return ui_main(server_connection.connection, server_connection.events)
     finally:
-        server_connection.terminate()
+        server.stop()
+        ui_channel.close()
+        event_queue.close()
+        server.join()
+
+    return return_value
 
 if __name__ == "__main__":
     try:
@@ -210,3 +213,4 @@ if __name__ == "__main__":
         import traceback
         traceback.print_exc(5)
     sys.exit(ret)
+
diff --git a/lib/bb/command.py b/lib/bb/command.py
index b88089298..c5c8de190 100644
--- a/lib/bb/command.py
+++ b/lib/bb/command.py
@@ -82,7 +82,7 @@ class Command:
             if command not in CommandsAsync.__dict__:
                 return "No such command"
             self.currentAsyncCommand = (command, commandline)
-            self.cooker.server.register_idle_function(self.cooker.runCommands, self.cooker)
+            self.cooker.server_registration_cb(self.cooker.runCommands, self.cooker)
             return True
         except:
             import traceback
diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
index ead46d06f..13f895e25 100644
--- a/lib/bb/cooker.py
+++ b/lib/bb/cooker.py
@@ -28,7 +28,6 @@ import atexit
 import itertools
 import logging
 import multiprocessing
-import signal
 import sre_constants
 import threading
 from cStringIO import StringIO
@@ -63,16 +62,18 @@ class BBCooker:
     Manages one bitbake build run
     """
 
-    def __init__(self, configuration, server):
+    def __init__(self, configuration, server_registration_cb):
         self.status = None
         self.appendlist = {}
 
-        self.server = server.BitBakeServer(self)
+        self.server_registration_cb = server_registration_cb
 
         self.configuration = configuration
 
         self.configuration.data = bb.data.init()
 
+        self.parseCommandLine()
+
         bb.data.inheritFromOS(self.configuration.data)
 
         self.parseConfigurationFiles(self.configuration.file)
@@ -675,7 +676,7 @@ class BBCooker:
                 return True
             return 0.5
 
-        self.server.register_idle_function(buildFileIdle, rq)
+        self.server_registration_cb(buildFileIdle, rq)
 
     def buildTargets(self, targets, task):
         """
@@ -731,7 +732,7 @@ class BBCooker:
 
         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
 
-        self.server.register_idle_function(buildTargetsIdle, rq)
+        self.server_registration_cb(buildTargetsIdle, rq)
 
     def updateCache(self):
         if self.state == state.running:
@@ -882,40 +883,6 @@ class BBCooker:
             return self.appendlist[f]
         return []
 
-    def serve(self):
-
-        # Empty the environment. The environment will be populated as
-        # necessary from the data store.
-        bb.utils.empty_environment()
-
-        if self.configuration.profile:
-            try:
-                import cProfile as profile
-            except:
-                import profile
-
-            profile.runctx("self.server.serve_forever()", globals(), locals(), "profile.log")
-
-            # Redirect stdout to capture profile information
-            pout = open('profile.log.processed', 'w')
-            so = sys.stdout.fileno()
-            os.dup2(pout.fileno(), so)
-
-            import pstats
-            p = pstats.Stats('profile.log')
-            p.sort_stats('time')
-            p.print_stats()
-            p.print_callers()
-            p.sort_stats('cumulative')
-            p.print_stats()
-
-            os.dup2(so, pout.fileno())
-            pout.flush()
-            pout.close()
-        else:
-            self.server.serve_forever()
-
-        bb.event.fire(CookerExit(), self.configuration.event_data)
 
     def shutdown(self):
         self.state = state.shutdown
@@ -975,7 +942,6 @@ class CookerParser(object):
 
     def start(self):
         def init(cfg):
-            signal.signal(signal.SIGINT, signal.SIG_IGN)
             parse_file.cfg = cfg
 
         bb.event.fire(bb.event.ParseStarted(self.toparse), self.cfgdata)
diff --git a/lib/bb/event.py b/lib/bb/event.py
index b6a59fa8f..044c5572f 100644
--- a/lib/bb/event.py
+++ b/lib/bb/event.py
@@ -104,7 +104,7 @@ def fire_ui_handlers(event, d):
              # We use pickle here since it better handles object instances
              # which xmlrpc's marshaller does not. Events *must* be serializable
              # by pickle.
-            _ui_handlers[h].event.send((pickle.dumps(event)))
+            _ui_handlers[h].event.send(event)
         except:
             errors.append(h)
     for h in errors:
diff --git a/lib/bb/server/process.py b/lib/bb/server/process.py
new file mode 100644
index 000000000..541c0c4d9
--- /dev/null
+++ b/lib/bb/server/process.py
@@ -0,0 +1,135 @@
+#
+# BitBake Process based server.
+#
+# Copyright (C) 2010 Bob Foerster <robert@erafx.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+"""
+    This module implements a multiprocessing.Process based server for bitbake.
+"""
+
+import time
+import logging
+import bb
+import bb.event
+from multiprocessing import Process, Event
+from bb.cooker import BBCooker
+
+logger = logging.getLogger('BitBake')
+
+
+class BitBakeServerCommands():
+    def __init__(self, server, cooker):
+        self.cooker = cooker
+        self.server = server
+
+    def runCommand(self, command):
+        """
+        Run a cooker command on the server
+        """
+        self.server.command_channel.send(self.cooker.command.runCommand(command))
+
+    def terminateServer(self):
+        """
+        Trigger the server to quit
+        """
+        self.server.stop()
+        #print "Server (cooker) exitting"
+        return
+
+    def ping(self):
+        """
+        Dummy method which can be used to check the server is still alive
+        """
+        return True
+
+
+class EventAdapter():
+    """
+    Adapter to wrap our event queue since the caller (bb.event) expects to
+    call a send() method, but our actual queue only has put()
+    """
+    def __init__(self, queue):
+        self.queue = queue
+
+    def send(self, event):
+        try:
+            self.queue.put(event)
+        except Exception, err:
+            print("EventAdapter puked: %s" % str(err))
+
+
+class ProcessServer(Process):
+    def __init__(self, command_channel, event_queue, configuration):
+        Process.__init__(self)
+        self.command_channel = command_channel
+        self.event_queue = event_queue
+        self.event = EventAdapter(event_queue)
+        self.configuration = configuration
+        self.cooker = BBCooker(configuration, self.register_idle_function)
+        self._idlefunctions = {}
+        self.commands = BitBakeServerCommands(self, self.cooker)
+        self.event_handle = bb.event.register_UIHhandler(self)
+        self.quit = False
+
+        self.keep_running = Event()
+        self.keep_running.set()
+
+        for event in bb.event.ui_queue:
+            self.event_queue.put(event)
+
+    def register_idle_function(self, function, data):
+        """Register a function to be called while the server is idle"""
+        assert hasattr(function, '__call__')
+        self._idlefunctions[function] = data
+
+    def run(self):
+        # Ensure logging messages get sent to the UI as events
+        logger.addHandler(bb.event.LogHandler())
+
+        while self.keep_running.is_set():
+            if self.command_channel.poll():
+                command = self.command_channel.recv()
+                self.commands.runCommand(command)
+
+            self.idle_commands(.1)
+        return
+
+    def idle_commands(self, delay):
+        nextsleep = delay
+
+        for function, data in self._idlefunctions.items():
+            try:
+                retval = function(self, data, False)
+                if retval is False:
+                    del self._idlefunctions[function]
+                elif retval is True:
+                    nextsleep = None
+                elif nextsleep is None:
+                    continue
+                elif retval < nextsleep:
+                    nextsleep = retval
+            except SystemExit:
+                raise
+            except Exception:
+                logger.exception('Running idle function')
+
+        if nextsleep is not None:
+            time.sleep(nextsleep)
+
+    def stop(self):
+        self.keep_running.clear()
+        bb.event.unregister_UIHhandler(self.event_handle)
+        self.command_channel.close()
diff --git a/lib/bb/ui/knotty.py b/lib/bb/ui/knotty.py
index be0d8ce70..e5351fee7 100644
--- a/lib/bb/ui/knotty.py
+++ b/lib/bb/ui/knotty.py
@@ -22,12 +22,10 @@ from __future__ import division
 
 import os
 import sys
-import itertools
 import xmlrpclib
 import logging
 import progressbar
 import bb.msg
-from bb import ui
 from bb.ui import uihelper
 
 logger = logging.getLogger("BitBake")
@@ -99,9 +97,8 @@ def main(server, eventHandler):
     return_value = 0
     while True:
         try:
-            event = eventHandler.waitEvent(0.25)
-            if event is None:
-                continue
+            event = eventHandler.get()
+
             helper.eventHandler(event)
             if isinstance(event, bb.runqueue.runQueueExitWait):
                 if not shutdown: