summaryrefslogtreecommitdiffstats
path: root/lib/bb/tinfoil.py
blob: 7d53608b21460ed78189d6c35f0a469372d060ff (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
# tinfoil: a simple wrapper around cooker for bitbake-based command-line utilities
#
# Copyright (C) 2012-2017 Intel Corporation
# Copyright (C) 2011 Mentor Graphics Corporation
# Copyright (C) 2006-2012 Richard Purdie
#
# SPDX-License-Identifier: GPL-2.0-only
#

import logging
import os
import sys
import atexit
import re
from collections import OrderedDict, defaultdict
from functools import partial

import bb.cache
import bb.cooker
import bb.providers
import bb.taskdata
import bb.utils
import bb.command
import bb.remotedata
from bb.cookerdata import CookerConfiguration
from bb.main import setup_bitbake, BitBakeConfigParameters
import bb.fetch2


# We need this in order to shut down the connection to the bitbake server,
# otherwise the process will never properly exit
_server_connections = []
def _terminate_connections():
    for connection in _server_connections:
        connection.terminate()
atexit.register(_terminate_connections)

class TinfoilUIException(Exception):
    """Exception raised when the UI returns non-zero from its main function"""
    def __init__(self, returncode):
        self.returncode = returncode
    def __repr__(self):
        return 'UI module main returned %d' % self.returncode

class TinfoilCommandFailed(Exception):
    """Exception raised when run_command fails"""

class TinfoilDataStoreConnectorVarHistory:
    def __init__(self, tinfoil, dsindex):
        self.tinfoil = tinfoil
        self.dsindex = dsindex

    def remoteCommand(self, cmd, *args, **kwargs):
        return self.tinfoil.run_command('dataStoreConnectorVarHistCmd', self.dsindex, cmd, args, kwargs)

    def __getattr__(self, name):
        if not hasattr(bb.data_smart.VariableHistory, name):
            raise AttributeError("VariableHistory has no such method %s" % name)

        newfunc = partial(self.remoteCommand, name)
        setattr(self, name, newfunc)
        return newfunc

class TinfoilDataStoreConnectorIncHistory:
    def __init__(self, tinfoil, dsindex):
        self.tinfoil = tinfoil
        self.dsindex = dsindex

    def remoteCommand(self, cmd, *args, **kwargs):
        return self.tinfoil.run_command('dataStoreConnectorIncHistCmd', self.dsindex, cmd, args, kwargs)

    def __getattr__(self, name):
        if not hasattr(bb.data_smart.IncludeHistory, name):
            raise AttributeError("IncludeHistory has no such method %s" % name)

        newfunc = partial(self.remoteCommand, name)
        setattr(self, name, newfunc)
        return newfunc

class TinfoilDataStoreConnector:
    """
    Connector object used to enable access to datastore objects via tinfoil
    Method calls are transmitted to the remote datastore for processing, if a datastore is
    returned we return a connector object for the new store
    """

    def __init__(self, tinfoil, dsindex):
        self.tinfoil = tinfoil
        self.dsindex = dsindex
        self.varhistory = TinfoilDataStoreConnectorVarHistory(tinfoil, dsindex)
        self.inchistory = TinfoilDataStoreConnectorIncHistory(tinfoil, dsindex)

    def remoteCommand(self, cmd, *args, **kwargs):
        ret = self.tinfoil.run_command('dataStoreConnectorCmd', self.dsindex, cmd, args, kwargs)
        if isinstance(ret, bb.command.DataStoreConnectionHandle):
            return TinfoilDataStoreConnector(self.tinfoil, ret.dsindex)
        return ret

    def __getattr__(self, name):
        if not hasattr(bb.data._dict_type, name):
            raise AttributeError("Data store has no such method %s" % name)

        newfunc = partial(self.remoteCommand, name)
        setattr(self, name, newfunc)
        return newfunc

    def __iter__(self):
        keys = self.tinfoil.run_command('dataStoreConnectorCmd', self.dsindex, "keys", [], {})
        for k in keys:
            yield k

class TinfoilCookerAdapter:
    """
    Provide an adapter for existing code that expects to access a cooker object via Tinfoil,
    since now Tinfoil is on the client side it no longer has direct access.
    """

    class TinfoilCookerCollectionAdapter:
        """ cooker.collection adapter """
        def __init__(self, tinfoil):
            self.tinfoil = tinfoil
        def get_file_appends(self, fn):
            return self.tinfoil.get_file_appends(fn)
        def __getattr__(self, name):
            if name == 'overlayed':
                return self.tinfoil.get_overlayed_recipes()
            elif name == 'bbappends':
                return self.tinfoil.run_command('getAllAppends')
            else:
                raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))

    class TinfoilRecipeCacheAdapter:
        """ cooker.recipecache adapter """
        def __init__(self, tinfoil, mc=''):
            self.tinfoil = tinfoil
            self.mc = mc
            self._cache = {}

        def get_pkg_pn_fn(self):
            pkg_pn = defaultdict(list, self.tinfoil.run_command('getRecipes', self.mc) or [])
            pkg_fn = {}
            for pn, fnlist in pkg_pn.items():
                for fn in fnlist:
                    pkg_fn[fn] = pn
            self._cache['pkg_pn'] = pkg_pn
            self._cache['pkg_fn'] = pkg_fn

        def __getattr__(self, name):
            # Grab these only when they are requested since they aren't always used
            if name in self._cache:
                return self._cache[name]
            elif name == 'pkg_pn':
                self.get_pkg_pn_fn()
                return self._cache[name]
            elif name == 'pkg_fn':
                self.get_pkg_pn_fn()
                return self._cache[name]
            elif name == 'deps':
                attrvalue = defaultdict(list, self.tinfoil.run_command('getRecipeDepends', self.mc) or [])
            elif name == 'rundeps':
                attrvalue = defaultdict(lambda: defaultdict(list), self.tinfoil.run_command('getRuntimeDepends', self.mc) or [])
            elif name == 'runrecs':
                attrvalue = defaultdict(lambda: defaultdict(list), self.tinfoil.run_command('getRuntimeRecommends', self.mc) or [])
            elif name == 'pkg_pepvpr':
                attrvalue = self.tinfoil.run_command('getRecipeVersions', self.mc) or {}
            elif name == 'inherits':
                attrvalue = self.tinfoil.run_command('getRecipeInherits', self.mc) or {}
            elif name == 'bbfile_priority':
                attrvalue = self.tinfoil.run_command('getBbFilePriority', self.mc) or {}
            elif name == 'pkg_dp':
                attrvalue = self.tinfoil.run_command('getDefaultPreference', self.mc) or {}
            elif name == 'fn_provides':
                attrvalue = self.tinfoil.run_command('getRecipeProvides', self.mc) or {}
            elif name == 'packages':
                attrvalue = self.tinfoil.run_command('getRecipePackages', self.mc) or {}
            elif name == 'packages_dynamic':
                attrvalue = self.tinfoil.run_command('getRecipePackagesDynamic', self.mc) or {}
            elif name == 'rproviders':
                attrvalue = self.tinfoil.run_command('getRProviders', self.mc) or {}
            else:
                raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))

            self._cache[name] = attrvalue
            return attrvalue

    def __init__(self, tinfoil):
        self.tinfoil = tinfoil
        self.collection = self.TinfoilCookerCollectionAdapter(tinfoil)
        self.recipecaches = {}
        self.recipecaches[''] = self.TinfoilRecipeCacheAdapter(tinfoil)
        for mc in (tinfoil.config_data.getVar('BBMULTICONFIG') or '').split():
            self.recipecaches[mc] = self.TinfoilRecipeCacheAdapter(tinfoil, mc)
        self._cache = {}
    def __getattr__(self, name):
        # Grab these only when they are requested since they aren't always used
        if name in self._cache:
            return self._cache[name]
        elif name == 'skiplist':
            attrvalue = self.tinfoil.get_skipped_recipes()
        elif name == 'bbfile_config_priorities':
            ret = self.tinfoil.run_command('getLayerPriorities')
            bbfile_config_priorities = []
            for collection, pattern, regex, pri in ret:
                bbfile_config_priorities.append((collection, pattern, re.compile(regex), pri))

            attrvalue = bbfile_config_priorities
        else:
            raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))

        self._cache[name] = attrvalue
        return attrvalue

    def findBestProvider(self, pn):
        return self.tinfoil.find_best_provider(pn)


class TinfoilRecipeInfo:
    """
    Provides a convenient representation of the cached information for a single recipe.
    Some attributes are set on construction, others are read on-demand (which internally
    may result in a remote procedure call to the bitbake server the first time).
    Note that only information which is cached is available through this object - if
    you need other variable values you will need to parse the recipe using
    Tinfoil.parse_recipe().
    """
    def __init__(self, recipecache, d, pn, fn, fns):
        self._recipecache = recipecache
        self._d = d
        self.pn = pn
        self.fn = fn
        self.fns = fns
        self.inherit_files = recipecache.inherits[fn]
        self.depends = recipecache.deps[fn]
        (self.pe, self.pv, self.pr) = recipecache.pkg_pepvpr[fn]
        self._cached_packages = None
        self._cached_rprovides = None
        self._cached_packages_dynamic = None

    def __getattr__(self, name):
        if name == 'alternates':
            return [x for x in self.fns if x != self.fn]
        elif name == 'rdepends':
            return self._recipecache.rundeps[self.fn]
        elif name == 'rrecommends':
            return self._recipecache.runrecs[self.fn]
        elif name == 'provides':
            return self._recipecache.fn_provides[self.fn]
        elif name == 'packages':
            if self._cached_packages is None:
                self._cached_packages = []
                for pkg, fns in self._recipecache.packages.items():
                    if self.fn in fns:
                        self._cached_packages.append(pkg)
            return self._cached_packages
        elif name == 'packages_dynamic':
            if self._cached_packages_dynamic is None:
                self._cached_packages_dynamic = []
                for pkg, fns in self._recipecache.packages_dynamic.items():
                    if self.fn in fns:
                        self._cached_packages_dynamic.append(pkg)
            return self._cached_packages_dynamic
        elif name == 'rprovides':
            if self._cached_rprovides is None:
                self._cached_rprovides = []
                for pkg, fns in self._recipecache.rproviders.items():
                    if self.fn in fns:
                        self._cached_rprovides.append(pkg)
            return self._cached_rprovides
        else:
            raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
    def inherits(self, only_recipe=False):
        """
        Get the inherited classes for a recipe. Returns the class names only.
        Parameters:
            only_recipe: True to return only the classes inherited by the recipe
                         itself, False to return all classes inherited within
                         the context for the recipe (which includes globally
                         inherited classes).
        """
        if only_recipe:
            global_inherit = [x for x in (self._d.getVar('BBINCLUDED') or '').split() if x.endswith('.bbclass')]
        else:
            global_inherit = []
        for clsfile in self.inherit_files:
            if only_recipe and clsfile in global_inherit:
                continue
            clsname = os.path.splitext(os.path.basename(clsfile))[0]
            yield clsname
    def __str__(self):
        return '%s' % self.pn


class Tinfoil:
    """
    Tinfoil - an API for scripts and utilities to query
    BitBake internals and perform build operations.
    """

    def __init__(self, output=sys.stdout, tracking=False, setup_logging=True):
        """
        Create a new tinfoil object.
        Parameters:
            output: specifies where console output should be sent. Defaults
                    to sys.stdout.
            tracking: True to enable variable history tracking, False to
                    disable it (default). Enabling this has a minor
                    performance impact so typically it isn't enabled
                    unless you need to query variable history.
            setup_logging: True to setup a logger so that things like
                    bb.warn() will work immediately and timeout warnings
                    are visible; False to let BitBake do this itself.
        """
        self.logger = logging.getLogger('BitBake')
        self.config_data = None
        self.cooker = None
        self.tracking = tracking
        self.ui_module = None
        self.server_connection = None
        self.recipes_parsed = False
        self.quiet = 0
        self.oldhandlers = self.logger.handlers[:]
        if setup_logging:
            # This is the *client-side* logger, nothing to do with
            # logging messages from the server
            bb.msg.logger_create('BitBake', output)
            self.localhandlers = []
            for handler in self.logger.handlers:
                if handler not in self.oldhandlers:
                    self.localhandlers.append(handler)

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        self.shutdown()

    def prepare(self, config_only=False, config_params=None, quiet=0, extra_features=None):
        """
        Prepares the underlying BitBake system to be used via tinfoil.
        This function must be called prior to calling any of the other
        functions in the API.
        NOTE: if you call prepare() you must absolutely call shutdown()
        before your code terminates. You can use a "with" block to ensure
        this happens e.g.

            with bb.tinfoil.Tinfoil() as tinfoil:
                tinfoil.prepare()
                ...

        Parameters:
            config_only: True to read only the configuration and not load
                        the cache / parse recipes. This is useful if you just
                        want to query the value of a variable at the global
                        level or you want to do anything else that doesn't
                        involve knowing anything about the recipes in the
                        current configuration. False loads the cache / parses
                        recipes.
            config_params: optionally specify your own configuration
                        parameters. If not specified an instance of
                        TinfoilConfigParameters will be created internally.
            quiet:      quiet level controlling console output - equivalent
                        to bitbake's -q/--quiet option. Default of 0 gives
                        the same output level as normal bitbake execution.
            extra_features: extra features to be added to the feature
                        set requested from the server. See
                        CookerFeatures._feature_list for possible
                        features.
        """
        self.quiet = quiet

        if self.tracking:
            extrafeatures = [bb.cooker.CookerFeatures.BASEDATASTORE_TRACKING]
        else:
            extrafeatures = []

        if extra_features:
            extrafeatures += extra_features

        if not config_params:
            config_params = TinfoilConfigParameters(config_only=config_only, quiet=quiet)

        cookerconfig = CookerConfiguration()
        cookerconfig.setConfigParameters(config_params)

        if not config_only:
            # Disable local loggers because the UI module is going to set up its own
            for handler in self.localhandlers:
                self.logger.handlers.remove(handler)
            self.localhandlers = []

        self.server_connection, ui_module = setup_bitbake(config_params,
                            cookerconfig,
                            extrafeatures)

        self.ui_module = ui_module

        # Ensure the path to bitbake's bin directory is in PATH so that things like
        # bitbake-worker can be run (usually this is the case, but it doesn't have to be)
        path = os.getenv('PATH').split(':')
        bitbakebinpath = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', 'bin'))
        for entry in path:
            if entry.endswith(os.sep):
                entry = entry[:-1]
            if os.path.abspath(entry) == bitbakebinpath:
                break
        else:
            path.insert(0, bitbakebinpath)
            os.environ['PATH'] = ':'.join(path)

        if self.server_connection:
            _server_connections.append(self.server_connection)
            if config_only:
                config_params.updateToServer(self.server_connection.connection, os.environ.copy())
                self.run_command('parseConfiguration')
            else:
                self.run_actions(config_params)
                self.recipes_parsed = True

            self.config_data = TinfoilDataStoreConnector(self, 0)
            self.cooker = TinfoilCookerAdapter(self)
            self.cooker_data = self.cooker.recipecaches['']
        else:
            raise Exception('Failed to start bitbake server')

    def run_actions(self, config_params):
        """
        Run the actions specified in config_params through the UI.
        """
        ret = self.ui_module.main(self.server_connection.connection, self.server_connection.events, config_params)
        if ret:
            raise TinfoilUIException(ret)

    def parseRecipes(self):
        """
        Legacy function - use parse_recipes() instead.
        """
        self.parse_recipes()

    def parse_recipes(self):
        """
        Load information on all recipes. Normally you should specify
        config_only=False when calling prepare() instead of using this
        function; this function is designed for situations where you need
        to initialise Tinfoil and use it with config_only=True first and
        then conditionally call this function to parse recipes later.
        """
        config_params = TinfoilConfigParameters(config_only=False)
        self.run_actions(config_params)
        self.recipes_parsed = True

    def run_command(self, command, *params):
        """
        Run a command on the server (as implemented in bb.command).
        Note that there are two types of command - synchronous and
        asynchronous; in order to receive the results of asynchronous
        commands you will need to set an appropriate event mask
        using set_event_mask() and listen for the result using
        wait_event() - with the correct event mask you'll at least get
        bb.command.CommandCompleted and possibly other events before
        that depending on the command.
        """
        if not self.server_connection:
            raise Exception('Not connected to server (did you call .prepare()?)')

        commandline = [command]
        if params:
            commandline.extend(params)
        result = self.server_connection.connection.runCommand(commandline)
        if result[1]:
            raise TinfoilCommandFailed(result[1])
        return result[0]

    def set_event_mask(self, eventlist):
        """Set the event mask which will be applied within wait_event()"""
        if not self.server_connection:
            raise Exception('Not connected to server (did you call .prepare()?)')
        llevel, debug_domains = bb.msg.constructLogOptions()
        ret = self.run_command('setEventMask', self.server_connection.connection.getEventHandle(), llevel, debug_domains, eventlist)
        if not ret:
            raise Exception('setEventMask failed')

    def wait_event(self, timeout=0):
        """
        Wait for an event from the server for the specified time.
        A timeout of 0 means don't wait if there are no events in the queue.
        Returns the next event in the queue or None if the timeout was
        reached. Note that in order to recieve any events you will
        first need to set the internal event mask using set_event_mask()
        (otherwise whatever event mask the UI set up will be in effect).
        """
        if not self.server_connection:
            raise Exception('Not connected to server (did you call .prepare()?)')
        return self.server_connection.events.waitEvent(timeout)

    def get_overlayed_recipes(self):
        """
        Find recipes which are overlayed (i.e. where recipes exist in multiple layers)
        """
        return defaultdict(list, self.run_command('getOverlayedRecipes'))

    def get_skipped_recipes(self):
        """
        Find recipes which were skipped (i.e. SkipRecipe was raised
        during parsing).
        """
        return OrderedDict(self.run_command('getSkippedRecipes'))

    def get_all_providers(self, mc=''):
        return defaultdict(list, self.run_command('allProviders', mc))

    def find_providers(self, mc=''):
        return self.run_command('findProviders', mc)

    def find_best_provider(self, pn):
        return self.run_command('findBestProvider', pn)

    def get_runtime_providers(self, rdep):
        return self.run_command('getRuntimeProviders', rdep)

    def get_recipe_file(self, pn):
        """
        Get the file name for the specified recipe/target. Raises
        bb.providers.NoProvider if there is no match or the recipe was
        skipped.
        """
        best = self.find_best_provider(pn)
        if not best or (len(best) > 3 and not best[3]):
            skiplist = self.get_skipped_recipes()
            taskdata = bb.taskdata.TaskData(None, skiplist=skiplist)
            skipreasons = taskdata.get_reasons(pn)
            if skipreasons:
                raise bb.providers.NoProvider('%s is unavailable:\n  %s' % (pn, '  \n'.join(skipreasons)))
            else:
                raise bb.providers.NoProvider('Unable to find any recipe file matching "%s"' % pn)
        return best[3]

    def get_file_appends(self, fn):
        """
        Find the bbappends for a recipe file
        """
        return self.run_command('getFileAppends', fn)

    def all_recipes(self, mc='', sort=True):
        """
        Enable iterating over all recipes in the current configuration.
        Returns an iterator over TinfoilRecipeInfo objects created on demand.
        Parameters:
            mc: The multiconfig, default of '' uses the main configuration.
            sort: True to sort recipes alphabetically (default), False otherwise
        """
        recipecache = self.cooker.recipecaches[mc]
        if sort:
            recipes = sorted(recipecache.pkg_pn.items())
        else:
            recipes = recipecache.pkg_pn.items()
        for pn, fns in recipes:
            prov = self.find_best_provider(pn)
            recipe = TinfoilRecipeInfo(recipecache,
                                       self.config_data,
                                       pn=pn,
                                       fn=prov[3],
                                       fns=fns)
            yield recipe

    def all_recipe_files(self, mc='', variants=True, preferred_only=False):
        """
        Enable iterating over all recipe files in the current configuration.
        Returns an iterator over file paths.
        Parameters:
            mc: The multiconfig, default of '' uses the main configuration.
            variants: True to include variants of recipes created through
                      BBCLASSEXTEND (default) or False to exclude them
            preferred_only: True to include only the preferred recipe where
                      multiple exist providing the same PN, False to list
                      all recipes
        """
        recipecache = self.cooker.recipecaches[mc]
        if preferred_only:
            files = []
            for pn in recipecache.pkg_pn.keys():
                prov = self.find_best_provider(pn)
                files.append(prov[3])
        else:
            files = recipecache.pkg_fn.keys()
        for fn in sorted(files):
            if not variants and fn.startswith('virtual:'):
                continue
            yield fn


    def get_recipe_info(self, pn, mc=''):
        """
        Get information on a specific recipe in the current configuration by name (PN).
        Returns a TinfoilRecipeInfo object created on demand.
        Parameters:
            mc: The multiconfig, default of '' uses the main configuration.
        """
        recipecache = self.cooker.recipecaches[mc]
        prov = self.find_best_provider(pn)
        fn = prov[3]
        if fn:
            actual_pn = recipecache.pkg_fn[fn]
            recipe = TinfoilRecipeInfo(recipecache,
                                        self.config_data,
                                        pn=actual_pn,
                                        fn=fn,
                                        fns=recipecache.pkg_pn[actual_pn])
            return recipe
        else:
            return None

    def parse_recipe(self, pn):
        """
        Parse the specified recipe and return a datastore object
        representing the environment for the recipe.
        """
        fn = self.get_recipe_file(pn)
        return self.parse_recipe_file(fn)

    def parse_recipe_file(self, fn, appends=True, appendlist=None, config_data=None):
        """
        Parse the specified recipe file (with or without bbappends)
        and return a datastore object representing the environment
        for the recipe.
        Parameters:
            fn: recipe file to parse - can be a file path or virtual
                specification
            appends: True to apply bbappends, False otherwise
            appendlist: optional list of bbappend files to apply, if you
                        want to filter them
        """
        if self.tracking:
            # Enable history tracking just for the parse operation
            self.run_command('enableDataTracking')
        try:
            if appends and appendlist == []:
                appends = False
            if config_data:
                 dscon = self.run_command('parseRecipeFile', fn, appends, appendlist, config_data.dsindex)
            else:
                dscon = self.run_command('parseRecipeFile', fn, appends, appendlist)
            if dscon:
                return self._reconvert_type(dscon, 'DataStoreConnectionHandle')
            else:
                return None
        finally:
            if self.tracking:
                self.run_command('disableDataTracking')

    def build_file(self, buildfile, task, internal=True):
        """
        Runs the specified task for just a single recipe (i.e. no dependencies).
        This is equivalent to bitbake -b, except with the default internal=True
        no warning about dependencies will be produced, normal info messages
        from the runqueue will be silenced and BuildInit, BuildStarted and
        BuildCompleted events will not be fired.
        """
        return self.run_command('buildFile', buildfile, task, internal)

    def build_targets(self, targets, task=None, handle_events=True, extra_events=None, event_callback=None):
        """
        Builds the specified targets. This is equivalent to a normal invocation
        of bitbake. Has built-in event handling which is enabled by default and
        can be extended if needed.
        Parameters:
            targets:
                One or more targets to build. Can be a list or a
                space-separated string.
            task:
                The task to run; if None then the value of BB_DEFAULT_TASK
                will be used. Default None.
            handle_events:
                True to handle events in a similar way to normal bitbake
                invocation with knotty; False to return immediately (on the
                assumption that the caller will handle the events instead).
                Default True.
            extra_events:
                An optional list of events to add to the event mask (if
                handle_events=True). If you add events here you also need
                to specify a callback function in event_callback that will
                handle the additional events. Default None.
            event_callback:
                An optional function taking a single parameter which
                will be called first upon receiving any event (if
                handle_events=True) so that the caller can override or
                extend the event handling. Default None.
        """
        if isinstance(targets, str):
            targets = targets.split()
        if not task:
            task = self.config_data.getVar('BB_DEFAULT_TASK')

        if handle_events:
            # A reasonable set of default events matching up with those we handle below
            eventmask = [
                        'bb.event.BuildStarted',
                        'bb.event.BuildCompleted',
                        'logging.LogRecord',
                        'bb.event.NoProvider',
                        'bb.command.CommandCompleted',
                        'bb.command.CommandFailed',
                        'bb.build.TaskStarted',
                        'bb.build.TaskFailed',
                        'bb.build.TaskSucceeded',
                        'bb.build.TaskFailedSilent',
                        'bb.build.TaskProgress',
                        'bb.runqueue.runQueueTaskStarted',
                        'bb.runqueue.sceneQueueTaskStarted',
                        'bb.event.ProcessStarted',
                        'bb.event.ProcessProgress',
                        'bb.event.ProcessFinished',
                        ]
            if extra_events:
                eventmask.extend(extra_events)
            ret = self.set_event_mask(eventmask)

        includelogs = self.config_data.getVar('BBINCLUDELOGS')
        loglines = self.config_data.getVar('BBINCLUDELOGS_LINES')

        ret = self.run_command('buildTargets', targets, task)
        if handle_events:
            result = False
            # Borrowed from knotty, instead somewhat hackily we use the helper
            # as the object to store "shutdown" on
            helper = bb.ui.uihelper.BBUIHelper()
            helper.shutdown = 0
            parseprogress = None
            termfilter = bb.ui.knotty.TerminalFilter(helper, helper, self.logger.handlers, quiet=self.quiet)
            try:
                while True:
                    try:
                        event = self.wait_event(0.25)
                        if event:
                            if event_callback and event_callback(event):
                                continue
                            if helper.eventHandler(event):
                                if isinstance(event, bb.build.TaskFailedSilent):
                                    logger.warning("Logfile for failed setscene task is %s" % event.logfile)
                                elif isinstance(event, bb.build.TaskFailed):
                                    bb.ui.knotty.print_event_log(event, includelogs, loglines, termfilter)
                                continue
                            if isinstance(event, bb.event.ProcessStarted):
                                if self.quiet > 1:
                                    continue
                                parseprogress = bb.ui.knotty.new_progress(event.processname, event.total)
                                parseprogress.start(False)
                                continue
                            if isinstance(event, bb.event.ProcessProgress):
                                if self.quiet > 1:
                                    continue
                                if parseprogress:
                                    parseprogress.update(event.progress)
                                else:
                                    bb.warn("Got ProcessProgress event for someting that never started?")
                                continue
                            if isinstance(event, bb.event.ProcessFinished):
                                if self.quiet > 1:
                                    continue
                                if parseprogress:
                                    parseprogress.finish()
                                parseprogress = None
                                continue
                            if isinstance(event, bb.command.CommandCompleted):
                                result = True
                                break
                            if isinstance(event, bb.command.CommandFailed):
                                self.logger.error(str(event))
                                result = False
                                break
                            if isinstance(event, logging.LogRecord):
                                if event.taskpid == 0 or event.levelno > logging.INFO:
                                    self.logger.handle(event)
                                continue
                            if isinstance(event, bb.event.NoProvider):
                                self.logger.error(str(event))
                                result = False
                                break

                        elif helper.shutdown > 1:
                            break
                        termfilter.updateFooter()
                    except KeyboardInterrupt:
                        termfilter.clearFooter()
                        if helper.shutdown == 1:
                            print("\nSecond Keyboard Interrupt, stopping...\n")
                            ret = self.run_command("stateForceShutdown")
                            if ret and ret[2]:
                                self.logger.error("Unable to cleanly stop: %s" % ret[2])
                        elif helper.shutdown == 0:
                            print("\nKeyboard Interrupt, closing down...\n")
                            interrupted = True
                            ret = self.run_command("stateShutdown")
                            if ret and ret[2]:
                                self.logger.error("Unable to cleanly shutdown: %s" % ret[2])
                        helper.shutdown = helper.shutdown + 1
                termfilter.clearFooter()
            finally:
                termfilter.finish()
            if helper.failed_tasks:
                result = False
            return result
        else:
            return ret

    def shutdown(self):
        """
        Shut down tinfoil. Disconnects from the server and gracefully
        releases any associated resources. You must call this function if
        prepare() has been called, or use a with... block when you create
        the tinfoil object which will ensure that it gets called.
        """
        if self.server_connection:
            self.run_command('clientComplete')
            _server_connections.remove(self.server_connection)
            bb.event.ui_queue = []
            self.server_connection.terminate()
            self.server_connection = None

        # Restore logging handlers to how it looked when we started
        if self.oldhandlers:
            for handler in self.logger.handlers:
                if handler not in self.oldhandlers:
                    self.logger.handlers.remove(handler)

    def _reconvert_type(self, obj, origtypename):
        """
        Convert an object back to the right type, in the case
        that marshalling has changed it (especially with xmlrpc)
        """
        supported_types = {
            'set': set,
            'DataStoreConnectionHandle': bb.command.DataStoreConnectionHandle,
        }

        origtype = supported_types.get(origtypename, None)
        if origtype is None:
            raise Exception('Unsupported type "%s"' % origtypename)
        if type(obj) == origtype:
            newobj = obj
        elif isinstance(obj, dict):
            # New style class
            newobj = origtype()
            for k,v in obj.items():
                setattr(newobj, k, v)
        else:
            # Assume we can coerce the type
            newobj = origtype(obj)

        if isinstance(newobj, bb.command.DataStoreConnectionHandle):
            newobj = TinfoilDataStoreConnector(self, newobj.dsindex)

        return newobj


class TinfoilConfigParameters(BitBakeConfigParameters):

    def __init__(self, config_only, **options):
        self.initial_options = options
        # Apply some sane defaults
        if not 'parse_only' in options:
            self.initial_options['parse_only'] = not config_only
        #if not 'status_only' in options:
        #    self.initial_options['status_only'] = config_only
        if not 'ui' in options:
            self.initial_options['ui'] = 'knotty'
        if not 'argv' in options:
            self.initial_options['argv'] = []

        super(TinfoilConfigParameters, self).__init__()

    def parseCommandLine(self, argv=None):
        # We don't want any parameters parsed from the command line
        opts = super(TinfoilConfigParameters, self).parseCommandLine([])
        for key, val in self.initial_options.items():
            setattr(opts[0], key, val)
        return opts