aboutsummaryrefslogtreecommitdiffstats
path: root/doc/bitbake-user-manual/bitbake-user-manual-execution.xml
blob: 04c5a26b99510891c60b073d23c2602d548b1d6f (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
<!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd">

<chapter id="bitbake-user-manual-execution">
    <title>Execution</title>

    <para>
        The primary purpose for running BitBake is to produce some kind
        of output such as a single installable package, a kernel, a software
        development kit, or even a full, board-specific bootable Linux image,
        complete with bootloader, kernel, and root filesystem.
        Of course, you can execute the <filename>bitbake</filename>
        command with options that cause it to execute single tasks,
        compile single recipe files, capture or clear data, or simply
        return information about the execution environment.
    </para>

    <para>
        This chapter describes BitBake's execution process from start
        to finish when you use it to create an image.
        The execution process is launched using the following command
        form:
        <literallayout class='monospaced'>
     $ bitbake <replaceable>target</replaceable>
        </literallayout>
        For information on the BitBake command and its options,
        see
        "<link linkend='bitbake-user-manual-command'>The BitBake Command</link>"
        section.
        <note>
            <para>
                Prior to executing BitBake, you should take advantage of available
                parallel thread execution on your build host by setting the
                <link linkend='var-bb-BB_NUMBER_THREADS'><filename>BB_NUMBER_THREADS</filename></link>
                variable in your project's <filename>local.conf</filename>
                configuration file.
            </para>

            <para>
                A common method to determine this value for your build host is to run
                the following:
                <literallayout class='monospaced'>
     $ grep processor /proc/cpuinfo
                </literallayout>
                This command returns the number of processors, which takes into
                account hyper-threading.
                Thus, a quad-core build host with hyper-threading most likely
                shows eight processors, which is the value you would then assign to
                <filename>BB_NUMBER_THREADS</filename>.
            </para>

            <para>
                A possibly simpler solution is that some Linux distributions
                (e.g. Debian and Ubuntu) provide the <filename>ncpus</filename> command.
            </para>
        </note>
    </para>

    <section id='parsing-the-base-configuration-metadata'>
        <title>Parsing the Base Configuration Metadata</title>

        <para>
            The first thing BitBake does is parse base configuration
            metadata.
            Base configuration metadata consists of your project's
            <filename>bblayers.conf</filename> file to determine what
            layers BitBake needs to recognize, all necessary
            <filename>layer.conf</filename> files (one from each layer),
            and <filename>bitbake.conf</filename>.
            The data itself is of various types:
            <itemizedlist>
                <listitem><para><emphasis>Recipes:</emphasis>
                    Details about particular pieces of software.
                    </para></listitem>
                <listitem><para><emphasis>Class Data:</emphasis>
                    An abstraction of common build information
                    (e.g. how to build a Linux kernel).
                    </para></listitem>
                <listitem><para><emphasis>Configuration Data:</emphasis>
                    Machine-specific settings, policy decisions,
                    and so forth.
                    Configuration data acts as the glue to bind everything
                    together.</para></listitem>
            </itemizedlist>
        </para>

        <para>
            The <filename>layer.conf</filename> files are used to
            construct key variables such as
            <link linkend='var-bb-BBPATH'><filename>BBPATH</filename></link>
            and
            <link linkend='var-bb-BBFILES'><filename>BBFILES</filename></link>.
            <filename>BBPATH</filename> is used to search for
            configuration and class files under the
            <filename>conf</filename> and <filename>classes</filename>
            directories, respectively.
            <filename>BBFILES</filename> is used to locate both recipe
            and recipe append files
            (<filename>.bb</filename> and <filename>.bbappend</filename>).
            If there is no <filename>bblayers.conf</filename> file,
            it is assumed the user has set the <filename>BBPATH</filename>
            and <filename>BBFILES</filename> directly in the environment.
        </para>

        <para>
            Next, the <filename>bitbake.conf</filename> file is located
            using the <filename>BBPATH</filename> variable that was
            just constructed.
            The <filename>bitbake.conf</filename> file may also include other
            configuration files using the
            <filename>include</filename> or
            <filename>require</filename> directives.
        </para>

        <para>
            Prior to parsing configuration files, BitBake looks
            at certain variables, including:
            <itemizedlist>
                <listitem><para>
                    <link linkend='var-bb-BB_ENV_WHITELIST'><filename>BB_ENV_WHITELIST</filename></link>
                    </para></listitem>
                <listitem><para>
                    <link linkend='var-bb-BB_ENV_EXTRAWHITE'><filename>BB_ENV_EXTRAWHITE</filename></link>
                    </para></listitem>
                <listitem><para>
                    <link linkend='var-bb-BB_PRESERVE_ENV'><filename>BB_PRESERVE_ENV</filename></link>
                    </para></listitem>
                <listitem><para>
                    <link linkend='var-bb-BB_ORIGENV'><filename>BB_ORIGENV</filename></link>
                    </para></listitem>
                <listitem><para>
                    <link linkend='var-bb-BITBAKE_UI'><filename>BITBAKE_UI</filename></link>
                    </para></listitem>
            </itemizedlist>
            The first four variables in this list relate to how BitBake treats shell
            environment variables during task execution.
            By default, BitBake cleans the environment variables and provides tight
            control over the shell execution environment.
            However, through the use of these first four variables, you can
            apply your control regarding the
            environment variables allowed to be used by BitBake in the shell
            during execution of tasks.
            See the
            "<link linkend='passing-information-into-the-build-task-environment'>Passing Information Into the Build Task Environment</link>"
            section and the information about these variables in the
            variable glossary for more information on how they work and
            on how to use them.
        </para>

        <para>
            The base configuration metadata is global
            and therefore affects all recipes and tasks that are executed.
        </para>

        <para>
            BitBake first searches the current working directory for an
            optional <filename>conf/bblayers.conf</filename> configuration file.
            This file is expected to contain a
            <link linkend='var-bb-BBLAYERS'><filename>BBLAYERS</filename></link>
            variable that is a space-delimited list of 'layer' directories.
            Recall that if BitBake cannot find a <filename>bblayers.conf</filename>
            file, then it is assumed the user has set the <filename>BBPATH</filename>
            and <filename>BBFILES</filename> variables directly in the environment.
        </para>

        <para>
            For each directory (layer) in this list, a <filename>conf/layer.conf</filename>
            file is located and parsed with the
            <link linkend='var-bb-LAYERDIR'><filename>LAYERDIR</filename></link>
            variable being set to the directory where the layer was found.
            The idea is these files automatically set up
            <link linkend='var-bb-BBPATH'><filename>BBPATH</filename></link>
            and other variables correctly for a given build directory.
        </para>

        <para>
            BitBake then expects to find the <filename>conf/bitbake.conf</filename>
            file somewhere in the user-specified <filename>BBPATH</filename>.
            That configuration file generally has include directives to pull
            in any other metadata such as files specific to the architecture,
            the machine, the local environment, and so forth.
        </para>

        <para>
            Only variable definitions and include directives are allowed
            in BitBake <filename>.conf</filename> files.
            Some variables directly influence BitBake's behavior.
            These variables might have been set from the environment
            depending on the environment variables previously
            mentioned or set in the configuration files.
            The
            "<link linkend='ref-bb-variables-glos'>Variables Glossary</link>"
            chapter presents a full list of variables.
        </para>

        <para>
            After parsing configuration files, BitBake uses its rudimentary
            inheritance mechanism, which is through class files, to inherit
            some standard classes.
            BitBake parses a class when the inherit directive responsible
            for getting that class is encountered.
        </para>

        <para>
            The <filename>base.bbclass</filename> file is always included.
            Other classes that are specified in the configuration using the
            <link linkend='var-bb-INHERIT'><filename>INHERIT</filename></link>
            variable are also included.
            BitBake searches for class files in a
            <filename>classes</filename> subdirectory under
            the paths in <filename>BBPATH</filename> in the same way as
            configuration files.
        </para>

        <para>
            A good way to get an idea of the configuration files and
            the class files used in your execution environment is to
            run the following BitBake command:
            <literallayout class='monospaced'>
     $ bitbake -e > mybb.log
            </literallayout>
            Examining the top of the <filename>mybb.log</filename>
            shows you the many configuration files and class files
            used in your execution environment.
        </para>

        <note>
            <para>
                You need to be aware of how BitBake parses curly braces.
                If a recipe uses a closing curly brace within the function and
                the character has no leading spaces, BitBake produces a parsing
                error.
                If you use a pair of curly braces in a shell function, the
                closing curly brace must not be located at the start of the line
                without leading spaces.
            </para>

            <para>
                Here is an example that causes BitBake to produce a parsing
                error:
                <literallayout class='monospaced'>
     fakeroot create_shar() {
         cat &lt;&lt; "EOF" &gt; ${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.sh
     usage()
     {
       echo "test"
       ###### The following "}" at the start of the line causes a parsing error ######
     }
     EOF
     }
                </literallayout>
                Writing the recipe this way avoids the error:
                <literallayout class='monospaced'>
     fakeroot create_shar() {
         cat &lt;&lt; "EOF" &gt; ${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.sh
     usage()
     {
       echo "test"
       ######The following "}" with a leading space at the start of the line avoids the error ######
      }
     EOF
     }
                </literallayout>
            </para>
        </note>
    </section>

    <section id='locating-and-parsing-recipes'>
        <title>Locating and Parsing Recipes</title>

        <para>
            During the configuration phase, BitBake will have set
            <link linkend='var-bb-BBFILES'><filename>BBFILES</filename></link>.
            BitBake now uses it to construct a list of recipes to parse,
            along with any append files (<filename>.bbappend</filename>)
            to apply.
            <filename>BBFILES</filename> is a space-separated list of
            available files and supports wildcards.
            An example would be:
            <literallayout class='monospaced'>
     BBFILES = "/path/to/bbfiles/*.bb /path/to/appends/*.bbappend"
            </literallayout>
            BitBake parses each recipe and append file located
            with <filename>BBFILES</filename> and stores the values of
            various variables into the datastore.
            <note>
                Append files are applied in the order they are encountered in
                <filename>BBFILES</filename>.
            </note>
            For each file, a fresh copy of the base configuration is
            made, then the recipe is parsed line by line.
            Any inherit statements cause BitBake to find and
            then parse class files (<filename>.bbclass</filename>)
            using
            <link linkend='var-bb-BBPATH'><filename>BBPATH</filename></link>
            as the search path.
            Finally, BitBake parses in order any append files found in
            <filename>BBFILES</filename>.
        </para>

        <para>
            One common convention is to use the recipe filename to define
            pieces of metadata.
            For example, in <filename>bitbake.conf</filename> the recipe
            name and version are used to set the variables
            <link linkend='var-bb-PN'><filename>PN</filename></link> and
            <link linkend='var-bb-PV'><filename>PV</filename></link>:
            <literallayout class='monospaced'>
     PN = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}"
     PV = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[1] or '1.0'}"
            </literallayout>
            In this example, a recipe called "something_1.2.3.bb" would set
            <filename>PN</filename> to "something" and
            <filename>PV</filename> to "1.2.3".
        </para>

        <para>
            By the time parsing is complete for a recipe, BitBake
            has a list of tasks that the recipe defines and a set of
            data consisting of keys and values as well as
            dependency information about the tasks.
        </para>

        <para>
            BitBake does not need all of this information.
            It only needs a small subset of the information to make
            decisions about the recipe.
            Consequently, BitBake caches the values in which it is
            interested and does not store the rest of the information.
            Experience has shown it is faster to re-parse the metadata than to
            try and write it out to the disk and then reload it.
        </para>

        <para>
            Where possible, subsequent BitBake commands reuse this cache of
            recipe information.
            The validity of this cache is determined by first computing a
            checksum of the base configuration data (see
            <link linkend='var-bb-BB_HASHCONFIG_WHITELIST'><filename>BB_HASHCONFIG_WHITELIST</filename></link>)
            and then checking if the checksum matches.
            If that checksum matches what is in the cache and the recipe
            and class files have not changed, BitBake is able to use
            the cache.
            BitBake then reloads the cached information about the recipe
            instead of reparsing it from scratch.
        </para>

        <para>
            Recipe file collections exist to allow the user to
            have multiple repositories of
            <filename>.bb</filename> files that contain the same
            exact package.
            For example, one could easily use them to make one's
            own local copy of an upstream repository, but with
            custom modifications that one does not want upstream.
            Here is an example:
            <literallayout class='monospaced'>
    BBFILES = "/stuff/openembedded/*/*.bb /stuff/openembedded.modified/*/*.bb"
    BBFILE_COLLECTIONS = "upstream local"
    BBFILE_PATTERN_upstream = "^/stuff/openembedded/"
    BBFILE_PATTERN_local = "^/stuff/openembedded.modified/"
    BBFILE_PRIORITY_upstream = "5"
    BBFILE_PRIORITY_local = "10"
            </literallayout>
            <note>
                The layers mechanism is now the preferred method of collecting
                code.
                While the collections code remains, its main use is to set layer
                priorities and to deal with overlap (conflicts) between layers.
            </note>
        </para>
    </section>

    <section id='bb-bitbake-providers'>
        <title>Providers</title>

        <para>
            Assuming BitBake has been instructed to execute a target
            and that all the recipe files have been parsed, BitBake
            starts to figure out how to build the target.
            BitBake looks through the <filename>PROVIDES</filename> list
            for each of the recipes.
            A <filename>PROVIDES</filename> list is the list of names by which
            the recipe can be known.
            Each recipe's <filename>PROVIDES</filename> list is created
            implicitly through the recipe's
            <link linkend='var-bb-PN'><filename>PN</filename></link> variable
            and explicitly through the recipe's
            <link linkend='var-bb-PROVIDES'><filename>PROVIDES</filename></link>
            variable, which is optional.
        </para>

        <para>
            When a recipe uses <filename>PROVIDES</filename>, that recipe's
            functionality can be found under an alternative name or names other
            than the implicit <filename>PN</filename> name.
            As an example, suppose a recipe named <filename>keyboard_1.0.bb</filename>
            contained the following:
            <literallayout class='monospaced'>
     PROVIDES += "fullkeyboard"
            </literallayout>
            The <filename>PROVIDES</filename> list for this recipe becomes
            "keyboard", which is implicit, and "fullkeyboard", which is explicit.
            Consequently, the functionality found in
            <filename>keyboard_1.0.bb</filename> can be found under two
            different names.
        </para>
    </section>

    <section id='bb-bitbake-preferences'>
        <title>Preferences</title>

        <para>
            The <filename>PROVIDES</filename> list is only part of the solution
            for figuring out a target's recipes.
            Because targets might have multiple providers, BitBake needs
            to prioritize providers by determining provider preferences.
        </para>

        <para>
            A common example in which a target has multiple providers
            is "virtual/kernel", which is on the
            <filename>PROVIDES</filename> list for each kernel recipe.
            Each machine often selects the best kernel provider by using a
            line similar to the following in the machine configuration file:
            <literallayout class='monospaced'>
     PREFERRED_PROVIDER_virtual/kernel = "linux-yocto"
            </literallayout>
            The default
            <link linkend='var-bb-PREFERRED_PROVIDER'><filename>PREFERRED_PROVIDER</filename></link>
            is the provider with the same name as the target.
            BitBake iterates through each target it needs to build and
            resolves them and their dependencies using this process.
        </para>

        <para>
            Understanding how providers are chosen is made complicated by the fact
            that multiple versions might exist for a given provider.
            BitBake defaults to the highest version of a provider.
            Version comparisons are made using the same method as Debian.
            You can use the
            <link linkend='var-bb-PREFERRED_VERSION'><filename>PREFERRED_VERSION</filename></link>
            variable to specify a particular version.
            You can influence the order by using the
            <link linkend='var-bb-DEFAULT_PREFERENCE'><filename>DEFAULT_PREFERENCE</filename></link>
            variable.
        </para>

        <para>
            By default, files have a preference of "0".
            Setting <filename>DEFAULT_PREFERENCE</filename> to "-1" makes the
            recipe unlikely to be used unless it is explicitly referenced.
            Setting <filename>DEFAULT_PREFERENCE</filename> to "1" makes it
            likely the recipe is used.
            <filename>PREFERRED_VERSION</filename> overrides any
            <filename>DEFAULT_PREFERENCE</filename> setting.
            <filename>DEFAULT_PREFERENCE</filename> is often used to mark newer
            and more experimental recipe versions until they have undergone
            sufficient testing to be considered stable.
        </para>

        <para>
            When there are multiple “versions” of a given recipe,
            BitBake defaults to selecting the most recent
            version, unless otherwise specified.
            If the recipe in question has a
            <link linkend='var-bb-DEFAULT_PREFERENCE'><filename>DEFAULT_PREFERENCE</filename></link>
            set lower than the other recipes (default is 0), then
            it will not be selected.
            This allows the person or persons maintaining
            the repository of recipe files to specify
            their preference for the default selected version.
            Additionally, the user can specify their preferred version.
        </para>

        <para>
            If the first recipe is named <filename>a_1.1.bb</filename>, then the
            <link linkend='var-bb-PN'><filename>PN</filename></link> variable
            will be set to “a”, and the
            <link linkend='var-bb-PV'><filename>PV</filename></link>
            variable will be set to 1.1.
        </para>

        <para>
            Thus, if a recipe named <filename>a_1.2.bb</filename> exists, BitBake
            will choose 1.2 by default.
            However, if you define the following variable in a
            <filename>.conf</filename> file that BitBake parses, you
            can change that preference:
            <literallayout class='monospaced'>
     PREFERRED_VERSION_a = "1.1"
            </literallayout>
        </para>

        <note>
            <para>
                It is common for a recipe to provide two versions -- a stable,
                numbered (and preferred) version, and a version that is
                automatically checked out from a source code repository that
                is considered more "bleeding edge" but can be selected only
                explicitly.
            </para>

            <para>
                For example, in the OpenEmbedded codebase, there is a standard,
                versioned recipe file for BusyBox,
                <filename>busybox_1.22.1.bb</filename>,
                but there is also a Git-based version,
                <filename>busybox_git.bb</filename>, which explicitly contains the line
                <literallayout class='monospaced'>
    DEFAULT_PREFERENCE = "-1"
                </literallayout>
                to ensure that the numbered, stable version is always preferred
                unless the developer selects otherwise.
            </para>
        </note>
    </section>

    <section id='bb-bitbake-dependencies'>
        <title>Dependencies</title>

        <para>
            Each target BitBake builds consists of multiple tasks such as
            <filename>fetch</filename>, <filename>unpack</filename>,
            <filename>patch</filename>, <filename>configure</filename>,
            and <filename>compile</filename>.
            For best performance on multi-core systems, BitBake considers each
            task as an independent
            entity with its own set of dependencies.
        </para>

        <para>
            Dependencies are defined through several variables.
            You can find information about variables BitBake uses in
            the <link linkend='ref-bb-variables-glos'>Variables Glossary</link>
            near the end of this manual.
            At a basic level, it is sufficient to know that BitBake uses the
            <link linkend='var-bb-DEPENDS'><filename>DEPENDS</filename></link> and
            <link linkend='var-bb-RDEPENDS'><filename>RDEPENDS</filename></link> variables when
            calculating dependencies.
        </para>

        <para>
            For more information on how BitBake handles dependencies, see the
            "<link linkend='dependencies'>Dependencies</link>" section.
        </para>
    </section>

    <section id='ref-bitbake-tasklist'>
        <title>The Task List</title>

        <para>
            Based on the generated list of providers and the dependency information,
            BitBake can now calculate exactly what tasks it needs to run and in what
            order it needs to run them.
            The
            "<link linkend='executing-tasks'>Executing Tasks</link>" section has more
            information on how BitBake chooses which task to execute next.
        </para>

        <para>
            The build now starts with BitBake forking off threads up to the limit set in the
            <link linkend='var-bb-BB_NUMBER_THREADS'><filename>BB_NUMBER_THREADS</filename></link>
            variable.
            BitBake continues to fork threads as long as there are tasks ready to run,
            those tasks have all their dependencies met, and the thread threshold has not been
            exceeded.
        </para>

        <para>
            It is worth noting that you can greatly speed up the build time by properly setting
            the <filename>BB_NUMBER_THREADS</filename> variable.
        </para>

        <para>
            As each task completes, a timestamp is written to the directory specified by the
            <link linkend='var-bb-STAMP'><filename>STAMP</filename></link> variable.
            On subsequent runs, BitBake looks in the build directory within
            <filename>tmp/stamps</filename> and does not rerun
            tasks that are already completed unless a timestamp is found to be invalid.
            Currently, invalid timestamps are only considered on a per
            recipe file basis.
            So, for example, if the configure stamp has a timestamp greater than the
            compile timestamp for a given target, then the compile task would rerun.
            Running the compile task again, however, has no effect on other providers
            that depend on that target.
        </para>

        <para>
            The exact format of the stamps is partly configurable.
            In modern versions of BitBake, a hash is appended to the
            stamp so that if the configuration changes, the stamp becomes
            invalid and the task is automatically rerun.
            This hash, or signature used, is governed by the signature policy
            that is configured (see the
            "<link linkend='checksums'>Checksums (Signatures)</link>"
            section for information).
            It is also possible to append extra metadata to the stamp using
            the <filename>[stamp-extra-info]</filename> task flag.
            For example, OpenEmbedded uses this flag to make some tasks machine-specific.
        </para>

        <note>
            Some tasks are marked as "nostamp" tasks.
            No timestamp file is created when these tasks are run.
            Consequently, "nostamp" tasks are always rerun.
        </note>

        <para>
            For more information on tasks, see the
            "<link linkend='tasks'>Tasks</link>" section.
        </para>
    </section>

    <section id='executing-tasks'>
        <title>Executing Tasks</title>

        <para>
            Tasks can be either a shell task or a Python task.
            For shell tasks, BitBake writes a shell script to
            <filename>${</filename><link linkend='var-bb-T'><filename>T</filename></link><filename>}/run.do_taskname.<replaceable>pid</replaceable></filename>
            and then executes the script.
            The generated shell script contains all the exported variables,
            and the shell functions with all variables expanded.
            Output from the shell script goes to the file
            <filename>${T}/log.do_taskname.<replaceable>pid</replaceable></filename>.
            Looking at the expanded shell functions in the run file and
            the output in the log files is a useful debugging technique.
        </para>

        <para>
            For Python tasks, BitBake executes the task internally and logs
            information to the controlling terminal.
            Future versions of BitBake will write the functions to files
            similar to the way shell tasks are handled.
            Logging will be handled in a way similar to shell tasks as well.
        </para>

        <para>
            The order in which BitBake runs the tasks is controlled by its
            task scheduler.
            It is possible to configure the scheduler and define custom
            implementations for specific use cases.
            For more information, see these variables that control the
            behavior:
            <itemizedlist>
                <listitem><para>
                    <link linkend='var-bb-BB_SCHEDULER'><filename>BB_SCHEDULER</filename></link>
                    </para></listitem>
                <listitem><para>
                    <link linkend='var-bb-BB_SCHEDULERS'><filename>BB_SCHEDULERS</filename></link>
                    </para></listitem>
            </itemizedlist>
            It is possible to have functions run before and after a task's main
            function.
            This is done using the <filename>[prefuncs]</filename>
            and <filename>[postfuncs]</filename> flags of the task
            that lists the functions to run.
        </para>
    </section>

    <section id='checksums'>
        <title>Checksums (Signatures)</title>

        <para>
            A checksum is a unique signature of a task's inputs.
            The signature of a task can be used to determine if a task
            needs to be run.
            Because it is a change in a task's inputs that triggers running
            the task, BitBake needs to detect all the inputs to a given task.
            For shell tasks, this turns out to be fairly easy because
            BitBake generates a "run" shell script for each task and
            it is possible to create a checksum that gives you a good idea of when
            the task's data changes.
        </para>

        <para>
            To complicate the problem, some things should not be included in
            the checksum.
            First, there is the actual specific build path of a given task -
            the working directory.
            It does not matter if the working directory changes because it should not
            affect the output for target packages.
            The simplistic approach for excluding the working directory is to set
            it to some fixed value and create the checksum for the "run" script.
            BitBake goes one step better and uses the
            <link linkend='var-bb-BB_HASHBASE_WHITELIST'><filename>BB_HASHBASE_WHITELIST</filename></link>
            variable to define a list of variables that should never be included
            when generating the signatures.
        </para>

        <para>
            Another problem results from the "run" scripts containing functions that
            might or might not get called.
            The incremental build solution contains code that figures out dependencies
            between shell functions.
            This code is used to prune the "run" scripts down to the minimum set,
            thereby alleviating this problem and making the "run" scripts much more
            readable as a bonus.
        </para>

        <para>
            So far we have solutions for shell scripts.
            What about Python tasks?
            The same approach applies even though these tasks are more difficult.
            The process needs to figure out what variables a Python function accesses
            and what functions it calls.
            Again, the incremental build solution contains code that first figures out
            the variable and function dependencies, and then creates a checksum for the data
            used as the input to the task.
        </para>

        <para>
            Like the working directory case, situations exist where dependencies
            should be ignored.
            For these cases, you can instruct the build process to ignore a dependency
            by using a line like the following:
            <literallayout class='monospaced'>
     PACKAGE_ARCHS[vardepsexclude] = "MACHINE"
            </literallayout>
            This example ensures that the <filename>PACKAGE_ARCHS</filename> variable does not
            depend on the value of <filename>MACHINE</filename>, even if it does reference it.
        </para>

        <para>
            Equally, there are cases where we need to add dependencies BitBake
            is not able to find.
            You can accomplish this by using a line like the following:
            <literallayout class='monospaced'>
      PACKAGE_ARCHS[vardeps] = "MACHINE"
            </literallayout>
            This example explicitly adds the <filename>MACHINE</filename> variable as a
            dependency for <filename>PACKAGE_ARCHS</filename>.
        </para>

        <para>
            Consider a case with in-line Python, for example, where BitBake is not
            able to figure out dependencies.
            When running in debug mode (i.e. using <filename>-DDD</filename>), BitBake
            produces output when it discovers something for which it cannot figure out
            dependencies.
        </para>

        <para>
            Thus far, this section has limited discussion to the direct inputs into a task.
            Information based on direct inputs is referred to as the "basehash" in the
            code.
            However, there is still the question of a task's indirect inputs - the
            things that were already built and present in the build directory.
            The checksum (or signature) for a particular task needs to add the hashes
            of all the tasks on which the particular task depends.
            Choosing which dependencies to add is a policy decision.
            However, the effect is to generate a master checksum that combines the basehash
            and the hashes of the task's dependencies.
        </para>

        <para>
            At the code level, there are a variety of ways both the basehash and the
            dependent task hashes can be influenced.
            Within the BitBake configuration file, we can give BitBake some extra information
            to help it construct the basehash.
            The following statement effectively results in a list of global variable
            dependency excludes - variables never included in any checksum.
            This example uses variables from OpenEmbedded to help illustrate
            the concept:
            <literallayout class='monospaced'>
     BB_HASHBASE_WHITELIST ?= "TMPDIR FILE PATH PWD BB_TASKHASH BBPATH DL_DIR \
         SSTATE_DIR THISDIR FILESEXTRAPATHS FILE_DIRNAME HOME LOGNAME SHELL \
         USER FILESPATH STAGING_DIR_HOST STAGING_DIR_TARGET COREBASE PRSERV_HOST \
         PRSERV_DUMPDIR PRSERV_DUMPFILE PRSERV_LOCKDOWN PARALLEL_MAKE \
         CCACHE_DIR EXTERNAL_TOOLCHAIN CCACHE CCACHE_DISABLE LICENSE_PATH SDKPKGSUFFIX"
            </literallayout>
            The previous example excludes the work directory, which is part of
            <filename>TMPDIR</filename>.
        </para>

        <para>
            The rules for deciding which hashes of dependent tasks to include through
            dependency chains are more complex and are generally accomplished with a
            Python function.
            The code in <filename>meta/lib/oe/sstatesig.py</filename> shows two examples
            of this and also illustrates how you can insert your own policy into the system
            if so desired.
            This file defines the two basic signature generators OpenEmbedded-Core
            uses:  "OEBasic" and "OEBasicHash".
            By default, there is a dummy "noop" signature handler enabled in BitBake.
            This means that behavior is unchanged from previous versions.
            <filename>OE-Core</filename> uses the "OEBasicHash" signature handler by default
            through this setting in the <filename>bitbake.conf</filename> file:
            <literallayout class='monospaced'>
     BB_SIGNATURE_HANDLER ?= "OEBasicHash"
            </literallayout>
            The "OEBasicHash" <filename>BB_SIGNATURE_HANDLER</filename> is the same as the
            "OEBasic" version but adds the task hash to the stamp files.
            This results in any metadata change that changes the task hash, automatically
            causing the task to be run again.
            This removes the need to bump
            <link linkend='var-bb-PR'><filename>PR</filename></link>
            values, and changes to metadata automatically ripple across the build.
        </para>

        <para>
            It is also worth noting that the end result of these signature generators is to
            make some dependency and hash information available to the build.
            This information includes:
            <itemizedlist>
                <listitem><para><filename>BB_BASEHASH_task-</filename><replaceable>taskname</replaceable>:
                    The base hashes for each task in the recipe.
                    </para></listitem>
                <listitem><para><filename>BB_BASEHASH_</filename><replaceable>filename</replaceable><filename>:</filename><replaceable>taskname</replaceable>:
                    The base hashes for each dependent task.
                    </para></listitem>
                <listitem><para><filename>BBHASHDEPS_</filename><replaceable>filename</replaceable><filename>:</filename><replaceable>taskname</replaceable>:
                    The task dependencies for each task.
                    </para></listitem>
                <listitem><para><filename>BB_TASKHASH</filename>:
                    The hash of the currently running task.
                    </para></listitem>
            </itemizedlist>
        </para>

        <para>
            It is worth noting that BitBake's "-S" option lets you
            debug BitBake's processing of signatures.
            The options passed to -S allow different debugging modes
            to be used, either using BitBake's own debug functions
            or possibly those defined in the metadata/signature handler
            itself.
            The simplest parameter to pass is "none", which causes a
            set of signature information to be written out into
            <filename>STAMPS_DIR</filename>
            corresponding to the targets specified.
            The other currently available parameter is "printdiff",
            which causes BitBake to try to establish the closest
            signature match it can (e.g. in the sstate cache) and then
            run <filename>bitbake-diffsigs</filename> over the matches
            to determine the stamps and delta where these two
            stamp trees diverge.
            <note>
                It is likely that future versions of BitBake will
                provide other signature handlers triggered through
                additional "-S" parameters.
            </note>
        </para>

        <para>
            You can find more information on checksum metadata in the
            "<link linkend='task-checksums-and-setscene'>Task Checksums and Setscene</link>"
            section.
        </para>
    </section>

    <section id='setscene'>
        <title>Setscene</title>

        <para>
            The setscene process enables BitBake to handle "pre-built" artifacts.
            The ability to handle and reuse these artifacts allows BitBake
            the luxury of not having to build something from scratch every time.
            Instead, BitBake can use, when possible, existing build artifacts.
        </para>

        <para>
            BitBake needs to have reliable data indicating whether or not an
            artifact is compatible.
            Signatures, described in the previous section, provide an ideal
            way of representing whether an artifact is compatible.
            If a signature is the same, an object can be reused.
        </para>

        <para>
            If an object can be reused, the problem then becomes how to
            replace a given task or set of tasks with the pre-built artifact.
            BitBake solves the problem with the "setscene" process.
        </para>

        <para>
            When BitBake is asked to build a given target, before building anything,
            it first asks whether cached information is available for any of the
            targets it's building, or any of the intermediate targets.
            If cached information is available, BitBake uses this information instead of
            running the main tasks.
        </para>

        <para>
            BitBake first calls the function defined by the
            <link linkend='var-bb-BB_HASHCHECK_FUNCTION'><filename>BB_HASHCHECK_FUNCTION</filename></link>
            variable with a list of tasks and corresponding
            hashes it wants to build.
            This function is designed to be fast and returns a list
            of the tasks for which it believes in can obtain artifacts.
        </para>

        <para>
            Next, for each of the tasks that were returned as possibilities,
            BitBake executes a setscene version of the task that the possible
            artifact covers.
            Setscene versions of a task have the string "_setscene" appended to the
            task name.
            So, for example, the task with the name <filename>xxx</filename> has
            a setscene task named <filename>xxx_setscene</filename>.
            The setscene version of the task executes and provides the necessary
            artifacts returning either success or failure.
        </para>

        <para>
            As previously mentioned, an artifact can cover more than one task.
            For example, it is pointless to obtain a compiler if you
            already have the compiled binary.
            To handle this, BitBake calls the
            <link linkend='var-bb-BB_SETSCENE_DEPVALID'><filename>BB_SETSCENE_DEPVALID</filename></link>
            function for each successful setscene task to know whether or not it needs
            to obtain the dependencies of that task.
        </para>

        <para>
            Finally, after all the setscene tasks have executed, BitBake calls the
            function listed in
            <link linkend='var-bb-BB_SETSCENE_VERIFY_FUNCTION2'><filename>BB_SETSCENE_VERIFY_FUNCTION2</filename></link>
            with the list of tasks BitBake thinks has been "covered".
            The metadata can then ensure that this list is correct and can
            inform BitBake that it wants specific tasks to be run regardless
            of the setscene result.
        </para>

        <para>
            You can find more information on setscene metadata in the
            "<link linkend='task-checksums-and-setscene'>Task Checksums and Setscene</link>"
            section.
        </para>
    </section>

    <section id="logging">
        <title>Logging</title>
        <para>
            In addition to the standard command line option to control how
            verbose builds are when execute, bitbake also supports user defined
            configuration of the
            <ulink url='https://docs.python.org/3/library/logging.html'>Python logging</ulink>
            facilities through the
            <link linkend="var-bb-BB_LOGCONFIG"><filename>BB_LOGCONFIG</filename></link>
            variable. This variable defines a json or yaml
            <ulink url='https://docs.python.org/3/library/logging.config.html'>logging configuration</ulink>
            that will be intelligently merged into the default configuration.
            The logging configuration is merged using the following rules:
            <itemizedlist>
                <listitem><para>
                    The user defined configuration will completely replace the default
                    configuration if top level key
                    <filename>bitbake_merge</filename> is set to the value
                    <filename>False</filename>. In this case, all other rules
                    are ignored.
                </para></listitem>
                <listitem><para>
                    The user configuration must have a top level
                    <filename>version</filename> which must match the value of
                    the default configuration.
                </para></listitem>
                <listitem><para>
                    Any keys defined in the <filename>handlers</filename>,
                    <filename>formatters</filename>, or <filename>filters</filename>,
                    will be merged into the same section in the default
                    configuration, with the user specified keys taking
                    replacing a default one if there is a conflict. In
                    practice, this means that if both the default configuration
                    and user configuration specify a handler named
                    <filename>myhandler</filename>, the user defined one will
                    replace the default. To prevent the user from inadvertently
                    replacing a default handler, formatter, or filter, all of
                    the default ones are named with a prefix of
                    "<filename>BitBake.</filename>"
                </para></listitem>
                <listitem><para>
                    If a logger is defined by the user with the key
                    <filename>bitbake_merge</filename> set to
                    <filename>False</filename>, that logger will be completely
                    replaced by user configuration. In this case, no other
                    rules will apply to that logger.
                </para></listitem>
                <listitem><para>
                    All user defined <filename>filter</filename> and
                    <filename>handlers</filename> properties for a given logger
                    will be merged with corresponding properties from the
                    default logger. For example, if the user configuration adds
                    a filter called <filename>myFilter</filename> to the
                    <filename>BitBake.SigGen</filename>, and the default
                    configuration adds a filter called
                    <filename>BitBake.defaultFilter</filename>, both filters
                    will be applied to the logger
                </para></listitem>
            </itemizedlist>
        </para>

        <para>
            As an example, consider the following user logging configuration
            file which logs all Hash Equivalence related messages of VERBOSE or
            higher to a file called <filename>hashequiv.log</filename>
            <literallayout class='monospaced'>
    {
        "version": 1,
        "handlers": {
            "autobuilderlog": {
                "class": "logging.FileHandler",
                "formatter": "logfileFormatter",
                "level": "DEBUG",
                "filename": "hashequiv.log",
                "mode": "w"
            }
        },
        "formatters": {
                "logfileFormatter": {
                    "format": "%(name)s: %(levelname)s: %(message)s"
                }
        },
        "loggers": {
            "BitBake.SigGen.HashEquiv": {
                "level": "VERBOSE",
                "handlers": ["autobuilderlog"]
            },
            "BitBake.RunQueue.HashEquiv": {
                "level": "VERBOSE",
                "handlers": ["autobuilderlog"]
            }
        }
    }
            </literallayout>
        </para>
    </section>
</chapter>