From b2743fbf1cbf8029e4af37250f8f63c6dd252176 Mon Sep 17 00:00:00 2001
From: Michael Kubacki <michael.kubacki@microsoft.com>
Date: Fri, 28 Jun 2024 09:23:02 -0700
Subject: [PATCH] BaseTools: ProductData: Add build rule

Adds a new build rule for OEM data files which are structured compiled
data. The rule provides support for MSFT, GCC, and Clang toolchains.

Deletes the intermediate binary files in a module output directory after
the final binary is written using the files. This is important to
prevent intermediate files from potentially being used in future
incremental builds if the corresponding source file is no longer
present.
---
 .../PosixLike/ProductDataFormatter            |  14 +++
 .../WindowsLike/ProductDataFormatter.bat      |   3 +
 BaseTools/Conf/build_rule.template            |  42 ++++++-
 .../ProductDataFormatter.py                   | 119 ++++++++++++++++++
 4 files changed, 177 insertions(+), 1 deletion(-)
 create mode 100644 BaseTools/BinWrappers/PosixLike/ProductDataFormatter
 create mode 100644 BaseTools/BinWrappers/WindowsLike/ProductDataFormatter.bat
 create mode 100644 BaseTools/Source/Python/ProductDataFormatter/ProductDataFormatter.py

diff --git a/BaseTools/BinWrappers/PosixLike/ProductDataFormatter b/BaseTools/BinWrappers/PosixLike/ProductDataFormatter
new file mode 100644
index 00000000000..b088bd63e0e
--- /dev/null
+++ b/BaseTools/BinWrappers/PosixLike/ProductDataFormatter
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+#python `dirname $0`/RunToolFromSource.py `basename $0` $*
+
+# If a ${PYTHON_COMMAND} command is available, use it in preference to python
+if command -v ${PYTHON_COMMAND} >/dev/null 2>&1; then
+    python_exe=${PYTHON_COMMAND}
+fi
+
+full_cmd=${BASH_SOURCE:-$0} # see http://mywiki.wooledge.org/BashFAQ/028 for a discussion of why $0 is not a good choice here
+dir=$(dirname "$full_cmd")
+exe=$(basename "$full_cmd")
+
+export PYTHONPATH="$dir/../../Source/Python${PYTHONPATH:+:"$PYTHONPATH"}"
+exec "${python_exe:-python}" "$dir/../../Source/Python/$exe/$exe.py" "$@"
\ No newline at end of file
diff --git a/BaseTools/BinWrappers/WindowsLike/ProductDataFormatter.bat b/BaseTools/BinWrappers/WindowsLike/ProductDataFormatter.bat
new file mode 100644
index 00000000000..9616cd893be
--- /dev/null
+++ b/BaseTools/BinWrappers/WindowsLike/ProductDataFormatter.bat
@@ -0,0 +1,3 @@
+@setlocal
+@set ToolName=%~n0%
+@%PYTHON_COMMAND% %BASE_TOOLS_PATH%\Source\Python\%ToolName%\%ToolName%.py %*
diff --git a/BaseTools/Conf/build_rule.template b/BaseTools/Conf/build_rule.template
index 03ece19486a..ad3319387cc 100755
--- a/BaseTools/Conf/build_rule.template
+++ b/BaseTools/Conf/build_rule.template
@@ -112,8 +112,11 @@
 # 1.01 - Add MASM for Microsoft ARM assembly files. "MS-ARMx-Assembly-Code-File.COMMON.COMMON"
 # 2.00 - Add .slib and .efi support in the sources file
 # 2.01 - Add support for DEPS_FLAGS as well as
+# 2.23 - Add OemData Definitions
+# 2.24 - Add .arc and .ver file type to add .rsrc sections to EFI images.
+# 2.25 - Split OemData rule into intermediate binary and final binary rules.
 
-#!VERSION=2.01
+#!VERSION=2.25
 
 [C-Code-File]
     <InputFile>
@@ -546,6 +549,43 @@
         "$(MTOC)" -subsystem $(MODULE_TYPE)  $(MTOC_FLAGS) $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.dll $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.pecoff
         "$(GENFW)" -o ${dst} -c $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.pecoff $(GENFW_FLAGS)
 
+# MU_CHANGE BEGIN
+[C-Code-File-OemDataIntermediate]
+    <InputFile>
+        ?.productdatac
+
+    <OutputFile>
+        $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.productdatabin.i
+
+    <ExtraDependency>
+        $(MAKE_FILE)
+
+    <Command.MSFT>
+        "$(CC)" /Fo$(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.obj $(DEPS_FLAGS) /nologo /c /FIAutoGen.h /TC /Dmain=main $(INC) ${src}
+        "$(DLINK)" /OUT:$(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.dll /NODEFAULTLIB /ENTRY:main /SUBSYSTEM:CONSOLE $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.obj
+        "$(GENFW)" -o $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.productdatabin.i -b $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.dll $(GENFW_FLAGS)
+
+    <Command.GCC>
+        "$(CC)" $(DEPS_FLAGS) -c -o $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.obj $(CC_FLAGS) -fno-toplevel-reorder -x c $(DEPS_FLAGS) $(INC) ${src}
+        "$(DLINK)" -o $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.dll $(DLINK_COMMON) -u $(IMAGE_ENTRY_POINT) $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.obj $(CC_FLAGS)
+        "$(GENFW)" -o $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.productdatabin.i -b $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.dll $(GENFW_FLAGS)
+
+    <Command.CLANGPDB>
+        "$(CC)" $(DEPS_FLAGS) -c -o $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.obj -O0 -fkeep-static-consts --language=c $(DEPS_FLAGS) $(INC) ${src}
+        "$(DLINK)" /OUT:$(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.dll /NODEFAULTLIB /ENTRY:main /SUBSYSTEM:CONSOLE $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.obj
+        "$(GENFW)" -o $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.productdatabin.i -b $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.dll $(GENFW_FLAGS)
+
+[ProductDataIntermediate-to-ProductDataBin]
+    <InputFile>
+        ?.productdatabin.i
+
+    <OutputFile>
+        $(OUTPUT_DIR)(+)${s_dir}(+)$(MODULE_NAME).productdatabin
+
+    <Command>
+        ProductDataFormatter $(OUTPUT_DIR)(+)${s_dir} $(OUTPUT_DIR)(+)${s_dir}(+)$(MODULE_NAME).productdatabin
+
+# MU_CHANGE END
 
 [Masm16-Code-File]
     <InputFile>
diff --git a/BaseTools/Source/Python/ProductDataFormatter/ProductDataFormatter.py b/BaseTools/Source/Python/ProductDataFormatter/ProductDataFormatter.py
new file mode 100644
index 00000000000..53788c36d5d
--- /dev/null
+++ b/BaseTools/Source/Python/ProductDataFormatter/ProductDataFormatter.py
@@ -0,0 +1,119 @@
+# @file
+#
+# Formats a collection of intermediate product data binary files
+# in a given directory into a single product binary file
+#
+# Copyright (C) Microsoft Corporation. All rights reserved.
+#
+##
+
+import glob
+import io
+import os
+import struct
+from collections import namedtuple
+from pathlib import Path
+
+class ProductDataFormatter:
+    STRUCT_SIGNATURE = b"__SDSH__"
+    ITEM_SIGNATURE = b"_SDDATA_"
+    PRODUCT_ID = b"__MSFT__"
+
+    def __init__(self, directory_path):
+        if not os.path.isdir(directory_path):
+            raise NotADirectoryError(
+                f"{directory_path} is an invalid directory!")
+
+        self._directory_path = Path(directory_path)
+
+    def _get_bins(self):
+        return self._directory_path.rglob('*.productdatabin.i')
+
+    def _get_signed_item(self, file_path):
+        with open(file_path, 'rb') as bin_file:
+            bin_data = bytearray(bin_file.read())
+
+        signed_item_pos = 0
+        while True:
+            signed_item_pos = bin_data.find(
+                self.ITEM_SIGNATURE, signed_item_pos)
+
+            if signed_item_pos == -1:
+                return
+
+            ItemHeader = namedtuple('ItemHeader', 'sig header_len id len')
+
+            item_header = ItemHeader._make(struct.unpack_from(
+                '<QHHI', bin_data, signed_item_pos))
+
+            print(f"Signed data item found")
+            print(f"  Item ID: {item_header.id}")
+            print(f"  Item Length: {item_header.len} bytes")
+
+            signed_item = bin_data[signed_item_pos:signed_item_pos +
+                                   item_header.header_len + item_header.len]
+
+            yield signed_item
+            signed_item_pos += (item_header.header_len + item_header.len)
+
+    def _get_signed_structure(self):
+        signed_items = [i for b in self._get_bins() for i in
+                        self._get_signed_item(b)]
+
+        StructureHeader = namedtuple('StructureHeader',
+                                     'sig rev header_len product_id desc_count')
+
+        struct_header = StructureHeader(
+                        self.STRUCT_SIGNATURE,
+                        1,
+                        26,
+                        self.PRODUCT_ID,
+                        len(signed_items))
+
+        struct_header_data = struct.pack('<8sHI8sI', *struct_header)
+
+        with io.BytesIO() as out_buffer:
+            out_buffer.write(struct_header_data)
+            for item in signed_items:
+                # Note: Just keep things packed tight for now
+                # aligned_boundary = (((~out_buffer.tell()) + 1) &
+                #                     self.ITEM_ALIGNMENT - 1)
+                # out_buffer.seek(out_buffer.tell() + aligned_boundary)
+                out_buffer.write(item)
+            return out_buffer.getvalue()
+
+    def get_file_data(self):
+        return {f: list(self._get_signed_item(f)) for f in self._get_bins()}
+
+    def write_data(self, out_file_path):
+        with open(out_file_path, 'wb') as out_bin_file:
+            out_bin_file.write(self._get_signed_structure())
+
+    def remove_intermediate_files(self):
+        for binary in self._get_bins():
+            try:
+                os.remove(binary)
+            except OSError as e:
+                print(e)
+                print(f"File path could not be removed.")
+
+
+# Todo: Clean up arg parsing, documentation, logger, etc.
+if __name__ == '__main__':
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description=(
+            'Formats intermediate product data files into '
+            'a single consolidated binary.'))
+    parser.add_argument(
+        'input_directory',
+        help='Directory of .productdatabin.i files')
+    parser.add_argument(
+        'output_file_path',
+        help='Name of of the final .productdatabin file.')
+    args = parser.parse_args()
+
+    product_data = ProductDataFormatter(args.input_directory)
+    product_data.write_data(args.output_file_path)
+    product_data.remove_intermediate_files()