diff options
author | Bob Foerster <rfoerster@layerzero.com> | 2010-11-24 12:53:12 -0500 |
---|---|---|
committer | Chris Larson <chris_larson@mentor.com> | 2010-12-16 10:39:27 -0700 |
commit | 65b615c6df4c3891e3c600947c3f96f802407fa4 (patch) | |
tree | cbb38412957c5be0d9a5f0c7c56d9ccd272e9c6e /lib | |
parent | 144887553097a288a76b8de78f71548d5ef9a350 (diff) | |
download | bitbake-65b615c6df4c3891e3c600947c3f96f802407fa4.tar.gz |
Run the server and UI in separate processes
This uses the python multiprocessing module, both to spawn the server process
and for communication between the processes.
Signed-off-by: Bob Foerster <robert@erafx.com>
Signed-off-by: Chris Larson <chris_larson@mentor.com>
Diffstat (limited to 'lib')
-rw-r--r-- | lib/bb/command.py | 2 | ||||
-rw-r--r-- | lib/bb/cooker.py | 46 | ||||
-rw-r--r-- | lib/bb/event.py | 2 | ||||
-rw-r--r-- | lib/bb/server/process.py | 135 | ||||
-rw-r--r-- | lib/bb/ui/knotty.py | 7 |
5 files changed, 145 insertions, 47 deletions
diff --git a/lib/bb/command.py b/lib/bb/command.py index b88089298..c5c8de190 100644 --- a/lib/bb/command.py +++ b/lib/bb/command.py @@ -82,7 +82,7 @@ class Command: if command not in CommandsAsync.__dict__: return "No such command" self.currentAsyncCommand = (command, commandline) - self.cooker.server.register_idle_function(self.cooker.runCommands, self.cooker) + self.cooker.server_registration_cb(self.cooker.runCommands, self.cooker) return True except: import traceback diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py index ead46d06f..13f895e25 100644 --- a/lib/bb/cooker.py +++ b/lib/bb/cooker.py @@ -28,7 +28,6 @@ import atexit import itertools import logging import multiprocessing -import signal import sre_constants import threading from cStringIO import StringIO @@ -63,16 +62,18 @@ class BBCooker: Manages one bitbake build run """ - def __init__(self, configuration, server): + def __init__(self, configuration, server_registration_cb): self.status = None self.appendlist = {} - self.server = server.BitBakeServer(self) + self.server_registration_cb = server_registration_cb self.configuration = configuration self.configuration.data = bb.data.init() + self.parseCommandLine() + bb.data.inheritFromOS(self.configuration.data) self.parseConfigurationFiles(self.configuration.file) @@ -675,7 +676,7 @@ class BBCooker: return True return 0.5 - self.server.register_idle_function(buildFileIdle, rq) + self.server_registration_cb(buildFileIdle, rq) def buildTargets(self, targets, task): """ @@ -731,7 +732,7 @@ class BBCooker: rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist) - self.server.register_idle_function(buildTargetsIdle, rq) + self.server_registration_cb(buildTargetsIdle, rq) def updateCache(self): if self.state == state.running: @@ -882,40 +883,6 @@ class BBCooker: return self.appendlist[f] return [] - def serve(self): - - # Empty the environment. The environment will be populated as - # necessary from the data store. - bb.utils.empty_environment() - - if self.configuration.profile: - try: - import cProfile as profile - except: - import profile - - profile.runctx("self.server.serve_forever()", globals(), locals(), "profile.log") - - # Redirect stdout to capture profile information - pout = open('profile.log.processed', 'w') - so = sys.stdout.fileno() - os.dup2(pout.fileno(), so) - - import pstats - p = pstats.Stats('profile.log') - p.sort_stats('time') - p.print_stats() - p.print_callers() - p.sort_stats('cumulative') - p.print_stats() - - os.dup2(so, pout.fileno()) - pout.flush() - pout.close() - else: - self.server.serve_forever() - - bb.event.fire(CookerExit(), self.configuration.event_data) def shutdown(self): self.state = state.shutdown @@ -975,7 +942,6 @@ class CookerParser(object): def start(self): def init(cfg): - signal.signal(signal.SIGINT, signal.SIG_IGN) parse_file.cfg = cfg bb.event.fire(bb.event.ParseStarted(self.toparse), self.cfgdata) diff --git a/lib/bb/event.py b/lib/bb/event.py index b6a59fa8f..044c5572f 100644 --- a/lib/bb/event.py +++ b/lib/bb/event.py @@ -104,7 +104,7 @@ def fire_ui_handlers(event, d): # We use pickle here since it better handles object instances # which xmlrpc's marshaller does not. Events *must* be serializable # by pickle. - _ui_handlers[h].event.send((pickle.dumps(event))) + _ui_handlers[h].event.send(event) except: errors.append(h) for h in errors: diff --git a/lib/bb/server/process.py b/lib/bb/server/process.py new file mode 100644 index 000000000..541c0c4d9 --- /dev/null +++ b/lib/bb/server/process.py @@ -0,0 +1,135 @@ +# +# BitBake Process based server. +# +# Copyright (C) 2010 Bob Foerster <robert@erafx.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" + This module implements a multiprocessing.Process based server for bitbake. +""" + +import time +import logging +import bb +import bb.event +from multiprocessing import Process, Event +from bb.cooker import BBCooker + +logger = logging.getLogger('BitBake') + + +class BitBakeServerCommands(): + def __init__(self, server, cooker): + self.cooker = cooker + self.server = server + + def runCommand(self, command): + """ + Run a cooker command on the server + """ + self.server.command_channel.send(self.cooker.command.runCommand(command)) + + def terminateServer(self): + """ + Trigger the server to quit + """ + self.server.stop() + #print "Server (cooker) exitting" + return + + def ping(self): + """ + Dummy method which can be used to check the server is still alive + """ + return True + + +class EventAdapter(): + """ + Adapter to wrap our event queue since the caller (bb.event) expects to + call a send() method, but our actual queue only has put() + """ + def __init__(self, queue): + self.queue = queue + + def send(self, event): + try: + self.queue.put(event) + except Exception, err: + print("EventAdapter puked: %s" % str(err)) + + +class ProcessServer(Process): + def __init__(self, command_channel, event_queue, configuration): + Process.__init__(self) + self.command_channel = command_channel + self.event_queue = event_queue + self.event = EventAdapter(event_queue) + self.configuration = configuration + self.cooker = BBCooker(configuration, self.register_idle_function) + self._idlefunctions = {} + self.commands = BitBakeServerCommands(self, self.cooker) + self.event_handle = bb.event.register_UIHhandler(self) + self.quit = False + + self.keep_running = Event() + self.keep_running.set() + + for event in bb.event.ui_queue: + self.event_queue.put(event) + + def register_idle_function(self, function, data): + """Register a function to be called while the server is idle""" + assert hasattr(function, '__call__') + self._idlefunctions[function] = data + + def run(self): + # Ensure logging messages get sent to the UI as events + logger.addHandler(bb.event.LogHandler()) + + while self.keep_running.is_set(): + if self.command_channel.poll(): + command = self.command_channel.recv() + self.commands.runCommand(command) + + self.idle_commands(.1) + return + + def idle_commands(self, delay): + nextsleep = delay + + for function, data in self._idlefunctions.items(): + try: + retval = function(self, data, False) + if retval is False: + del self._idlefunctions[function] + elif retval is True: + nextsleep = None + elif nextsleep is None: + continue + elif retval < nextsleep: + nextsleep = retval + except SystemExit: + raise + except Exception: + logger.exception('Running idle function') + + if nextsleep is not None: + time.sleep(nextsleep) + + def stop(self): + self.keep_running.clear() + bb.event.unregister_UIHhandler(self.event_handle) + self.command_channel.close() diff --git a/lib/bb/ui/knotty.py b/lib/bb/ui/knotty.py index be0d8ce70..e5351fee7 100644 --- a/lib/bb/ui/knotty.py +++ b/lib/bb/ui/knotty.py @@ -22,12 +22,10 @@ from __future__ import division import os import sys -import itertools import xmlrpclib import logging import progressbar import bb.msg -from bb import ui from bb.ui import uihelper logger = logging.getLogger("BitBake") @@ -99,9 +97,8 @@ def main(server, eventHandler): return_value = 0 while True: try: - event = eventHandler.waitEvent(0.25) - if event is None: - continue + event = eventHandler.get() + helper.eventHandler(event) if isinstance(event, bb.runqueue.runQueueExitWait): if not shutdown: |