table parsers: Use closest package path relative path (#461)

INF paths in the DSC and FDF can be defined as an absolute path, relative to the workspace, or relative to any package path. To ensure consistency with paths, this change converts any provided path to be relative to the closest package path. This simplifies queries that typically use the `path` column for filtering, or `join` commands.

That is to say, if there are two package paths: `Common/`, `Common/Special/`, a path can be relative to either of these packages paths. So both `Special/Files/MyFile.c` and `Files/MyFile.c` are acceptable. However when inserting into the database, we now ensure that the closest package path relative path is used. In the above example, that means `Files/MyFile.c`.

edk2toollib/database/tables/instanced_fv_table.py

@@ -74,8 +74,15 @@ def parse(self, db_cursor: sqlite3.Cursor, pathobj: Edk2Path, env_id, env) -> No
             for inf in fdfp.FVs[fv]["Infs"]:
                 if inf.lower().startswith("ruleoverride"):
                     inf = InstancedFvTable.RULEOVERRIDE.findall(inf)[0]
-                if Path(inf).is_absolute():
-                    inf = str(Path(self.pathobj.GetEdk2RelativePathFromAbsolutePath(inf)))
+
+                # Convert to absolute, and back to relative to ensure we get the closest pp relative path
+                # i.e. if we have two package paths: ("MyPP", and "MyPP/Subfolder"), in the FDF, devs
+                # can specify INFs are either ("Subfolder/MyPkg/../MyPkg.inf" or "MyPkg/../MyPkg.inf")
+                # However in the database, we want the closest match, i.e. "MyPkg/../MyPkg.inf", even if
+                # they are providing ("Subfolder/MyPkg/../MyPkg.inf"). "GetEdk2RelativePathFromAbsolutePath"
+                # always returns the relative path from the closest package path.
+                inf = self.pathobj.GetAbsolutePathOnThisSystemFromEdk2RelativePath(inf)
+                inf = self.pathobj.GetEdk2RelativePathFromAbsolutePath(inf)
                 inf_list.append(Path(inf).as_posix())
 
             row = (self.env_id, fv, Path(self.fdf).name, self.fdf)

edk2toollib/database/tables/instanced_inf_table.py

@@ -169,6 +169,11 @@ def _build_inf_table(self, dscp: DscP):
             if arch not in self.arch:
                 continue
 
+            # Developers can set an inf path to be relative to any package path. Convert it to be the closest
+            # package path relative path to the INF, which is done by `GetEdk2RelativePathFromAbsolutePath`
+            inf = self.pathobj.GetAbsolutePathOnThisSystemFromEdk2RelativePath(inf)
+            inf = self.pathobj.GetEdk2RelativePathFromAbsolutePath(inf)
+
             logging.debug(f"Parsing Component: [{arch}][{inf}]")
             infp = InfP().SetEdk2Path(self.pathobj)
             infp.ParseFile(inf)
@@ -346,5 +351,8 @@ def _reduce_lib_instances(self, module: str, library_instance_list: list[str]) -
         for library_instance in library_instance_list:
             infp = self.inf(library_instance)
             if module.lower() in [phase.lower() for phase in infp.SupportedPhases]:
+                # Ensure the returned path is relative to the closest package path
+                library_instance = self.pathobj.GetAbsolutePathOnThisSystemFromEdk2RelativePath(library_instance)
+                library_instance = self.pathobj.GetEdk2RelativePathFromAbsolutePath(library_instance)
                 return library_instance
         return None

tests.unit/database/test_instanced_fv_table.py

@@ -6,9 +6,9 @@
 # SPDX-License-Identifier: BSD-2-Clause-Patent
 ##
 """Unittest for the InstancedFv table generator."""
+import logging
 from pathlib import Path
 
-import logging
 import pytest
 from common import Tree, empty_tree  # noqa: F401
 from edk2toollib.database import Edk2DB
@@ -30,20 +30,21 @@ def test_valid_fdf(empty_tree: Tree):  # noqa: F811
     comp1 = empty_tree.create_component("TestDriver1", "DXE_DRIVER")
     comp2 = empty_tree.create_component("TestDriver2", "DXE_DRIVER")
     comp3 = empty_tree.create_component("TestDriver3", "DXE_DRIVER")
-    comp4 = str(Path('TestPkg','Extra Drivers','TestDriver4.inf'))
+    Path(empty_tree.package / "Extra Drivers").mkdir()
+    Path(empty_tree.package / "Extra Drivers" / "TestDriver4.inf").touch()
     comp5 = empty_tree.create_component("TestDriver5", "DXE_DRIVER")
 
     dsc = empty_tree.create_dsc()
 
     # Write the FDF; includes a "infformat" FV used to test
     # All the different ways an INF can be defined in the FDF
     fdf = empty_tree.create_fdf(
-        fv_infformat = [
+        fv_testfv = [
             f"INF  {comp1}", # PP relative
             f'INF  {str(empty_tree.ws / comp2)}', # Absolute
             f'INF  RuleOverride=RESET_VECTOR {comp3}', # RuleOverride
-            f'INF  {comp4}', # Space in path
-            f'INF  ruleoverride = RESET_VECTOR {comp5}', # RuleOverride lowercase & spaces
+            'INF  TestPkg/Extra Drivers/TestDriver4.inf', # Space in path
+            f'INF  ruleoverride = RESET_VECTOR {comp5}', # RuleOverride lowercase & spaces'
         ]
     )
     env = {
@@ -54,19 +55,19 @@ def test_valid_fdf(empty_tree: Tree):  # noqa: F811
     }
     db.parse(env)
 
-    rows = db.connection.execute("SELECT key2 FROM junction where key1 == 'infformat'").fetchall()
+    rows = db.connection.execute("SELECT key2 FROM junction where key1 == 'testfv'").fetchall()
 
     assert len(rows) == 5
     assert sorted(rows) == sorted([
         (Path(comp1).as_posix(),),
         (Path(comp2).as_posix(),),
         (Path(comp3).as_posix(),),
-        (Path(comp4).as_posix(),),
+        ('TestPkg/Extra Drivers/TestDriver4.inf',),
         (Path(comp5).as_posix(),),
     ])
 
 def test_missing_dsc_and_fdf(empty_tree: Tree, caplog):
-    """Tests that the table generator is skipped if missing the necessary information"""
+    """Tests that the table generator is skipped if missing the necessary information."""
     with caplog.at_level(logging.DEBUG):
         edk2path = Edk2Path(str(empty_tree.ws), [])
         db = Edk2DB(empty_tree.ws / "db.db", pathobj=edk2path)
@@ -86,3 +87,38 @@ def test_missing_dsc_and_fdf(empty_tree: Tree, caplog):
             if record.startswith("DSC or FDF not found"):
                 count += 1
         assert count == 2
+
+def test_non_closest_inf_path(empty_tree: Tree):
+    dsc = empty_tree.create_dsc()
+    fdf = empty_tree.create_fdf(
+        fv_testfv = [
+            "INF Common/SubFolder/Drivers/TestDriver1.inf",
+        ]
+    )
+
+    # Create the Common folder, which will be a package path
+    common_folder = empty_tree.ws / "Common"
+    common_folder.mkdir()
+
+    # Create a subfolder of common folder, which is also a package path
+    sub_folder = common_folder / "SubFolder"
+    sub_folder.mkdir()
+    edk2path = Edk2Path(str(empty_tree.ws), ["Common", str(sub_folder)])
+
+    # Make the INF we want to make sure we get the closest match of
+    (sub_folder / "Drivers").mkdir()
+    (sub_folder / "Drivers" / "TestDriver1.inf").touch()
+
+    db = Edk2DB(empty_tree.ws / "db.db", pathobj=edk2path)
+    db.register(InstancedFvTable())
+    env = {
+        "ACTIVE_PLATFORM": dsc,
+        "FLASH_DEFINITION": fdf,
+        "TARGET_ARCH": "IA32 X64",
+        "TARGET": "DEBUG",
+    }
+    db.parse(env)
+
+    rows = db.connection.execute("SELECT key2 FROM junction where key1 == 'testfv'").fetchall()
+    assert len(rows) == 1
+    assert rows[0][0] == "Drivers/TestDriver1.inf"

tests.unit/database/test_instanced_inf_table.py

@@ -321,3 +321,47 @@ def test_absolute_paths_in_dsc(empty_tree: Tree):
     results = db.connection.execute("SELECT path FROM instanced_inf").fetchall()
     assert results[0] == (Path(lib1).as_posix(),)
     assert results[1] == (Path(comp1).as_posix(),)
+
+def test_closest_packagepath(empty_tree: Tree):
+    common_folder = empty_tree.ws / "Common"
+    common_folder.mkdir()
+    sub_folder = common_folder / "SubFolder"
+    sub_folder.mkdir()
+
+    (sub_folder / "Library").mkdir()
+    (sub_folder / "Drivers").mkdir()
+
+    empty_tree.create_library("TestLib", "TestCls")
+    empty_tree.create_component("TestDriver", "DXE_DRIVER", libraryclasses=["TestCls"])
+
+    # Move the files into our subfolder that has two levels of package paths
+    (empty_tree.library_folder / "TestLib.inf").rename(sub_folder / "Library" / "TestLib.inf")
+    (empty_tree.component_folder / "TestDriver.inf").rename(sub_folder / "Drivers" / "TestDriver.inf")
+
+    # Specify the file paths to be relative to the farther away package path
+    dsc = empty_tree.create_dsc(
+        libraryclasses = [
+            'TestCls|SubFolder/Library/TestLib.inf',
+        ],
+        components = [
+            'SubFolder/Drivers/TestDriver.inf',
+        ],
+    )
+
+    edk2path = Edk2Path(str(empty_tree.ws), ["Common", "Common/SubFolder"])
+    db = Edk2DB(empty_tree.ws / "db.db", pathobj=edk2path)
+    db.register(InstancedInfTable())
+
+    env = {
+        "ACTIVE_PLATFORM": dsc,
+        "TARGET_ARCH": "X64",
+        "TARGET": "DEBUG",
+    }
+
+    db.parse(env)
+
+    rows = db.connection.execute("SELECT path FROM instanced_inf").fetchall()
+
+    # Ensure that we convert the path to be relative to the clostest package path.
+    for row in rows:
+        assert row[0].startswith("Library") or row[0].startswith("Drivers")