ovmf: Fix other reproducibility issues

When building in longer paths, the ovmf build changes in many ways. This adds a
patch addressing various causes of problems. Full details are in the patch header.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

2 files changed, 166 insertions, 0 deletions

diff --git a/meta/recipes-core/ovmf/ovmf/reproducible.patch b/meta/recipes-core/ovmf/ovmf/reproducible.patch
new file mode 100644
index 0000000000..5d2aeaacfe
--- /dev/null
+++ b/meta/recipes-core/ovmf/ovmf/reproducible.patch
@@ -0,0 +1,165 @@
+This patch fixes various things which make the build more reproducible. Some changes
+here only change intermediate artefacts but that means when you have two build trees
+giving differing results, the differences can be isolated more easily. The issues here
+usually become apparent with longer paths.
+
+This was all debugged with:
+TMPDIR = "${TOPDIR}/tmp"
+vs.
+TMPDIR = "${TOPDIR}/tmp-inital-mylongpath-mylongpath-mylongpath-mylongpath-mylongpath-mylongpath-mylongpath-mylongpath-mylongpath"
+
+The patch specifically:
+
+ * Sorts output in GNUmakefile
+ * Always generates indirect flags files used to avoid pathlength issues else the 
+   compile commands suddenly change when using longer paths
+ * Sorts the AutoGenTimeStamp file contents
+ * Makes the TargetDescBlock objects from BuildEngine sortable to allow the makefile fix
+ * Fix ElfConvert within GenFw so that only the basename of the binary being converted 
+   is used, else the output from "GenFw XXX.bin" differs from "GenFw /long/path/XXX.bin"
+   with sufficiently long paths
+
+Upstream-Status: Pending [At least some of this might be interesting to upstream]
+Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
+
+Index: git/BaseTools/Source/Python/AutoGen/GenMake.py
+===================================================================
+--- git.orig/BaseTools/Source/Python/AutoGen/GenMake.py
++++ git/BaseTools/Source/Python/AutoGen/GenMake.py
+@@ -571,7 +571,7 @@ cleanlib:
+                 os.remove(RespFileList)

+ 

+         # convert source files and binary files to build targets

+-        self.ResultFileList = [str(T.Target) for T in MyAgo.CodaTargetList]

++        self.ResultFileList = sorted([str(T.Target) for T in MyAgo.CodaTargetList])

+         if len(self.ResultFileList) == 0 and len(MyAgo.SourceFileList) != 0:

+             EdkLogger.error("build", AUTOGEN_ERROR, "Nothing to build",

+                             ExtraData="[%s]" % str(MyAgo))

+@@ -722,7 +722,7 @@ cleanlib:
+         OutputFile = ''

+         DepsFileList = []

+ 

+-        for Cmd in self.GenFfsList:

++        for Cmd in sorted(self.GenFfsList):

+             if Cmd[2]:

+                 for CopyCmd in Cmd[2]:

+                     Src, Dst = CopyCmd

+@@ -755,7 +755,7 @@ cleanlib:
+             self.BuildTargetList.append('\t%s' % CmdString)

+ 

+             self.ParseSecCmd(DepsFileList, Cmd[1])

+-            for SecOutputFile, SecDepsFile, SecCmd in self.FfsOutputFileList :

++            for SecOutputFile, SecDepsFile, SecCmd in sorted(self.FfsOutputFileList):

+                 self.BuildTargetList.append('%s : %s' % (self.ReplaceMacro(SecOutputFile), self.ReplaceMacro(SecDepsFile)))

+                 self.BuildTargetList.append('\t%s' % self.ReplaceMacro(SecCmd))

+             self.FfsOutputFileList = []

+@@ -794,13 +794,13 @@ cleanlib:
+ 

+     def CommandExceedLimit(self):

+         FlagDict = {

+-                    'CC'    :  { 'Macro' : '$(CC_FLAGS)',    'Value' : False},

+-                    'PP'    :  { 'Macro' : '$(PP_FLAGS)',    'Value' : False},

+-                    'APP'   :  { 'Macro' : '$(APP_FLAGS)',   'Value' : False},

+-                    'ASLPP' :  { 'Macro' : '$(ASLPP_FLAGS)', 'Value' : False},

+-                    'VFRPP' :  { 'Macro' : '$(VFRPP_FLAGS)', 'Value' : False},

+-                    'ASM'   :  { 'Macro' : '$(ASM_FLAGS)',   'Value' : False},

+-                    'ASLCC' :  { 'Macro' : '$(ASLCC_FLAGS)', 'Value' : False},

++                    'CC'    :  { 'Macro' : '$(CC_FLAGS)',    'Value' : True},

++                    'PP'    :  { 'Macro' : '$(PP_FLAGS)',    'Value' : True},

++                    'APP'   :  { 'Macro' : '$(APP_FLAGS)',   'Value' : True},

++                    'ASLPP' :  { 'Macro' : '$(ASLPP_FLAGS)', 'Value' : True},

++                    'VFRPP' :  { 'Macro' : '$(VFRPP_FLAGS)', 'Value' : True},

++                    'ASM'   :  { 'Macro' : '$(ASM_FLAGS)',   'Value' : True},

++                    'ASLCC' :  { 'Macro' : '$(ASLCC_FLAGS)', 'Value' : True},

+                    }

+ 

+         RespDict = {}

+@@ -1003,9 +1003,9 @@ cleanlib:
+                 if not self.ObjTargetDict.get(T.Target.SubDir):

+                     self.ObjTargetDict[T.Target.SubDir] = set()

+                 self.ObjTargetDict[T.Target.SubDir].add(NewFile)

+-        for Type in self._AutoGenObject.Targets:

++        for Type in sorted(self._AutoGenObject.Targets):

+             resp_file_number = 0

+-            for T in self._AutoGenObject.Targets[Type]:

++            for T in sorted(self._AutoGenObject.Targets[Type]):

+                 # Generate related macros if needed

+                 if T.GenFileListMacro and T.FileListMacro not in self.FileListMacros:

+                     self.FileListMacros[T.FileListMacro] = []

+Index: git/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
+===================================================================
+--- git.orig/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
++++ git/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
+@@ -1484,6 +1484,9 @@ class ModuleAutoGen(AutoGen):
+             for File in Files:

+                 if File.lower().endswith('.pdb'):

+                     AsBuiltInfDict['binary_item'].append('DISPOSABLE|' + File)

++

++        AsBuiltInfDict['binary_item'] = sorted(AsBuiltInfDict['binary_item'])

++

+         HeaderComments = self.Module.HeaderComments

+         StartPos = 0

+         for Index in range(len(HeaderComments)):

+@@ -1759,7 +1762,7 @@ class ModuleAutoGen(AutoGen):
+             if os.path.exists (self.TimeStampPath):

+                 os.remove (self.TimeStampPath)

+ 

+-            SaveFileOnChange(self.TimeStampPath, "\n".join(FileSet), False)

++            SaveFileOnChange(self.TimeStampPath, "\n".join(sorted(FileSet)), False)

+ 

+         # Ignore generating makefile when it is a binary module

+         if self.IsBinaryModule:

+Index: git/BaseTools/Source/Python/AutoGen/BuildEngine.py
+===================================================================
+--- git.orig/BaseTools/Source/Python/AutoGen/BuildEngine.py
++++ git/BaseTools/Source/Python/AutoGen/BuildEngine.py
+@@ -70,6 +70,9 @@ class TargetDescBlock(object):
+         else:

+             return str(Other) == self.Target.Path

+ 

++    def __lt__(self, other):

++        return str(self) < str(other)

++

+     def AddInput(self, Input):

+         if Input not in self.Inputs:

+             self.Inputs.append(Input)

+Index: git/BaseTools/Source/C/GenFw/Elf64Convert.c
+===================================================================
+--- git.orig/BaseTools/Source/C/GenFw/Elf64Convert.c
++++ git/BaseTools/Source/C/GenFw/Elf64Convert.c
+@@ -14,6 +14,8 @@ SPDX-License-Identifier: BSD-2-Clause-Pa
+ #ifndef __GNUC__

+ #include <windows.h>

+ #include <io.h>

++#else

++#define _GNU_SOURCE

+ #endif

+ #include <assert.h>

+ #include <stdio.h>

+@@ -770,7 +772,7 @@ ScanSections64 (
+   }

+   mCoffOffset = mDebugOffset + sizeof(EFI_IMAGE_DEBUG_DIRECTORY_ENTRY) +

+                 sizeof(EFI_IMAGE_DEBUG_CODEVIEW_NB10_ENTRY) +

+-                strlen(mInImageName) + 1;

++                strlen(basename(mInImageName)) + 1;

+ 

+   mCoffOffset = CoffAlign(mCoffOffset);

+   if (SectionCount == 0) {

+@@ -1609,7 +1611,7 @@ WriteDebug64 (
+   EFI_IMAGE_DEBUG_DIRECTORY_ENTRY     *Dir;

+   EFI_IMAGE_DEBUG_CODEVIEW_NB10_ENTRY *Nb10;

+ 

+-  Len = strlen(mInImageName) + 1;

++  Len = strlen(basename(mInImageName)) + 1;

+ 

+   Dir = (EFI_IMAGE_DEBUG_DIRECTORY_ENTRY*)(mCoffFile + mDebugOffset);

+   Dir->Type = EFI_IMAGE_DEBUG_TYPE_CODEVIEW;

+@@ -1619,7 +1621,7 @@ WriteDebug64 (
+ 

+   Nb10 = (EFI_IMAGE_DEBUG_CODEVIEW_NB10_ENTRY*)(Dir + 1);

+   Nb10->Signature = CODEVIEW_SIGNATURE_NB10;

+-  strcpy ((char *)(Nb10 + 1), mInImageName);

++  strcpy ((char *)(Nb10 + 1), basename(mInImageName));

+ 

+ 

+   NtHdr = (EFI_IMAGE_OPTIONAL_HEADER_UNION *)(mCoffFile + mNtHdrOffset);

diff --git a/meta/recipes-core/ovmf/ovmf_git.bb b/meta/recipes-core/ovmf/ovmf_git.bb
index ef5545bf70..696c234035 100644
--- a/meta/recipes-core/ovmf/ovmf_git.bb
+++ b/meta/recipes-core/ovmf/ovmf_git.bb
@@ -18,6 +18,7 @@ SRC_URI = "gitsm://github.com/tianocore/edk2.git;branch=master;protocol=https \
            file://0004-ovmf-Update-to-latest.patch \
            file://zero.patch \
            file://debug_prefix_map.patch \
+           file://reproducible.patch \
            "
 
 PV = "edk2-stable202102"