From 931bb5b094fb6273fe4107e5e0cef432f0513850 Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Fri, 1 Sep 2023 15:38:13 -0700 Subject: [PATCH 01/33] Switch to sqlite3 Switch to using sqlite3 rather than TinyDB as this is a true database that supports SQL queries and other expected functionality that TinyDB does not. --- edk2toollib/database/__init__.py | 6 +- edk2toollib/database/edk2_db.py | 252 +++++------------- edk2toollib/database/queries/__init__.py | 14 - .../database/queries/component_query.py | 48 ---- edk2toollib/database/queries/library_query.py | 32 --- edk2toollib/database/queries/license_query.py | 46 ---- .../queries/unused_component_query.py | 94 ------- edk2toollib/database/tables/__init__.py | 2 +- edk2toollib/database/tables/base_table.py | 31 +++ .../database/tables/environment_table.py | 82 +++--- edk2toollib/database/tables/inf_table.py | 72 +++-- .../database/tables/instanced_fv_table.py | 71 ++--- .../database/tables/instanced_inf_table.py | 146 ++++++---- edk2toollib/database/tables/package_table.py | 61 +++++ edk2toollib/database/tables/source_table.py | 62 +++-- pyproject.toml | 5 +- tests.unit/database/common.py | 12 +- tests.unit/database/test_component_query.py | 77 ------ tests.unit/database/test_edk2_db.py | 63 ++--- tests.unit/database/test_environment_table.py | 83 +++--- tests.unit/database/test_inf_table.py | 32 ++- .../database/test_instanced_fv_table.py | 34 ++- .../database/test_instanced_inf_table.py | 202 +++++++------- tests.unit/database/test_library_query.py | 35 --- tests.unit/database/test_license_query.py | 75 ------ tests.unit/database/test_source_table.py | 54 ++-- .../database/test_unused_component_query.py | 192 ------------- 27 files changed, 666 insertions(+), 1217 deletions(-) delete mode 100644 edk2toollib/database/queries/__init__.py delete mode 100644 edk2toollib/database/queries/component_query.py delete mode 100644 edk2toollib/database/queries/library_query.py delete mode 100644 edk2toollib/database/queries/license_query.py delete mode 100644 edk2toollib/database/queries/unused_component_query.py create mode 100644 edk2toollib/database/tables/base_table.py create mode 100644 edk2toollib/database/tables/package_table.py delete mode 100644 tests.unit/database/test_component_query.py delete mode 100644 tests.unit/database/test_library_query.py delete mode 100644 tests.unit/database/test_license_query.py delete mode 100644 tests.unit/database/test_unused_component_query.py diff --git a/edk2toollib/database/__init__.py b/edk2toollib/database/__init__.py index 180888c3..34ebb4ae 100644 --- a/edk2toollib/database/__init__.py +++ b/edk2toollib/database/__init__.py @@ -7,8 +7,4 @@ # SPDX-License-Identifier: BSD-2-Clause-Patent ## """Core classes and methods used to interact with the database module inside edk2-pytool-library.""" - -from tinydb import Query, where # noqa: F401 -from tinyrecord import transaction # noqa: F401 - -from .edk2_db import AdvancedQuery, Edk2DB, TableGenerator # noqa: F401 +from .edk2_db import Edk2DB # noqa: F401 diff --git a/edk2toollib/database/edk2_db.py b/edk2toollib/database/edk2_db.py index fd63f245..d308fbac 100644 --- a/edk2toollib/database/edk2_db.py +++ b/edk2toollib/database/edk2_db.py @@ -7,107 +7,77 @@ ## """A class for interacting with a database implemented using json.""" import logging +import sqlite3 import time -from typing import Any, List - -from tinydb import TinyDB -from tinydb.middlewares import CachingMiddleware -from tinydb.storages import JSONStorage, MemoryStorage -from tinydb.table import Document - - -class Edk2DB(TinyDB): - """A subclass of TinyDB providing advanced queries and parser management. - - This class provides the ability to register parsers that will create / update tables in the database while also - providing the ability to run queries on the database. - - Edk2DB can be run in three modes: - - 1. File Read/Write: A database will be loaded or created at the specified path. Any changes made will be written - to the database file. This is the slowest of the three modes. Specify with Edk2DB.FILE_RW - - 2. File Read Only: A database will be loaded at the specific path. Attempting to change the database will result - in an error. This is the middle of the three in terms of performance. Specify with Edk2DB.FILE_RO - - 3. In-Memory Read/Write: A database will be created in memory. Any changes made will only exist for the lifetime - of the database object. This is the fastest of the three modes. Specify with Edk2DB.MEM_RW +import uuid +from typing import Any + +from edk2toollib.database.tables import EnvironmentTable +from edk2toollib.database.tables.base_table import TableGenerator +from edk2toollib.uefi.edk2.path_utilities import Edk2Path + +CREATE_JUNCTION_TABLE = """ +CREATE TABLE IF NOT EXISTS junction ( + env TEXT, + table1 TEXT, + key1 TEXT, + table2 TEXT, + key2 TEXT +) +""" + +class Edk2DB: + """A SQLite3 database manager for a EDKII workspace. + + This class provides the ability to register parsers that will create / update tables in the database. This will + create a SQLite dabase file that can be queried using any SQLite3 client. VSCode provides multiple extensions + for viewing and interacting with the database. Queries can also be created and run in python using the sqlite3 + module that comes with python. Edk2DB can, and should, be used as a context manager to ensure that the database is closed properly. If not using as a context manager, the `close()` method must be used to ensure that the database is closed properly and any changes are saved. - When running the parse() command, the user can specify whether or not to append the results to the database. If - not appending to the database, the entire database will be dropped before parsing. - - ```python - # Run using File storage - from edk2toollib.database.parsers import * - with Edk2DB(Edk2DB.FILE_RW, pathobj=edk2path, db_path=Path("path/to/db.db")) as db: - db.register(Parser1(), Parser2(), Parser3()) - db.parse() - - # Run using Memory storage - from edk2toollib.database.parsers import * - with Edk2DB(Edk2DB.MEM_RW, pathobj=edk2path) as db: - db.register(Parser1(), Parser2(), Parser3()) - db.parse() - - # Run some parsers in clear mode and some in append mode - from edk2toollib.database.parsers import * - with Edk2DB(Edk2DB.MEM_RW, pathobj=edk2path) as db: - db.register(Parser1()) - db.parse() - db.clear_parsers() - - db.register(Parser2(), Parser3()) - for env in env_list: - db.parse(env=env, append=True) - - # Run Queries on specific tables or on the database - from edk2toollib.database.queries import * - with Edk2DB(Edk2DB.FILE_RW, pathobj=edk2path, db_path=Path("path/to/db.db")) as db: - # Run a tinydb Query - # https://tinydb.readthedocs.io/en/latest/usage.html#queries - query_results = db.table("TABLENAME").search(Query().table_field == "value") - - # Run an advanced query - query_results = db.search(AdvancedQuerySubclass(config1 = "x", config2 = "y")) - """ - FILE_RW = 1 # Mode: File storage, Read & Write - FILE_RO = 2 # Mode: File storage, Read Only - MEM_RW = 3 # Mode: Memory storage, Read & Write + Attributes: + connection (sqlite3.Connection): The connection to the database + + !!! note + Edk2DB provides a table called `junction` that can be used to make associations between tables. It has the + following schema: `table1, key1, table2, key2`. - def __init__(self, mode: int, **kwargs: dict[str,Any]): + Example: + ```python + from edk2toollib.database.parsers import * + table = "..." + with Edk2DB(Path("path/to/db.db"), edk2path) as db: + db.register(Parser1(), Parser2(), Parser3()) + db.parse() + + db.connection.execute(f"SELECT * FROM {table}", table) + """ + def __init__(self, db_path: str, pathobj: Edk2Path, **kwargs: dict[str,Any]): """Initializes the database. Args: - mode: The mode you are opening the database with Edk2DB.FILE_RW, Edk2DB.FILE_RO, Edk2DB.MEM_RW + db_path: Path to create or load the database from + pathobj: Edk2Path object for the workspace **kwargs: see Keyword Arguments Keyword Arguments: - db_path (str): Path to create or load the database from - pathobj (Edk2Path): Edk2Path object for the workspace - - !!! note - needing db_path or pathobj depends on the mode you are opening the database with. + None """ - self.pathobj = None - self._parsers = [] - - if mode == Edk2DB.FILE_RW: - logging.debug("Database running in File Read/Write mode.") - super().__init__(kwargs.pop("db_path"), access_mode='r+', storage=CachingMiddleware(JSONStorage)) - self.pathobj = kwargs.pop("pathobj") - elif mode == Edk2DB.FILE_RO: - logging.debug("Database running in File ReadOnly mode.") - super().__init__(kwargs.pop("db_path"), access_mode='r', storage=CachingMiddleware(JSONStorage)) - elif mode == Edk2DB.MEM_RW: - logging.debug("Database running in In-Memory Read/Write mode.") - super().__init__(storage=MemoryStorage) - self.pathobj = kwargs.pop("pathobj") - else: - raise ValueError("Unknown Database mode.") + self.pathobj = pathobj + self.clear_parsers() + self.connection = sqlite3.connect(db_path) + + def __enter__(self): + """Enables the use of the `with` statement.""" + return self + + def __exit__(self, exc_type, exc_value, traceback): + """Enables the use of the `with` statement.""" + self.connection.commit() + self.connection.close() def register(self, *parsers: 'TableGenerator') -> None: """Registers a one or more table generators. @@ -120,98 +90,24 @@ def register(self, *parsers: 'TableGenerator') -> None: def clear_parsers(self) -> None: """Empties the list of registered table generators.""" - self._parsers.clear() + self._parsers = [EnvironmentTable()] - def parse(self, append: bool=False) -> None: + def parse(self, env: dict) -> None: """Runs all registered table parsers against the database. - Args: - append: Whether to append to the database or clear it first - """ - if not append: - self.drop_tables() - - for parser in self._parsers: - logging.debug(f"[{parser.__class__.__name__}] starting...") - try: - t = time.time() - parser.parse(self) - except Exception as e: - logging.error(f"[{parser.__class__.__name__}] failed.") - logging.error(str(e)) - finally: - logging.debug(f"[{parser.__class__.__name__}] finished in {time.time() - t:.2f}s") - - def search(self, advanced_query: 'AdvancedQuery') -> List[Document]: - """Runs an advanced query against the database. - - Args: - advanced_query: The query to run + !!! note + An environment table and row is always added to the database. This allows for filtering based on the environment. """ - return advanced_query.run(self) - - -class AdvancedQuery: - """An interface for an advanced query. - - One of TinyDB's limitations is that it does not support relationships between tables (i.e. Primary Key / Foreign - Key and JOINs). This means these types of queries are more complicated and require additional steps. An advanced - Query is a conceptual way to grouping these extra steps in a single place and providing a single line interface - to execute the more advanced query. - - ```python - # An example of a simple query, an interface provided by TinyDB to run a single query against a single table - db.table('table_name').search(Query().field == 'value' & Query().field2 == 'value2') - - # An example of an advanced query, which is run at the database level instead of the table level and can - # run multiple queries - db.query(MyAdvancedQuery(config1 = "a", config2 = "b")) - ``` - """ - def __init__(self, *args, **kwargs) -> None: - """Initialize the query with the specific settings.""" - - def run(self, db: Edk2DB) -> any: - """Run the query against the database.""" - raise NotImplementedError - - def columns(self, column_list: list[str], documents: list[Document], ): - """Given a list of Documents, return it with only the specified columns.""" - filtered_list = [] - for document in documents: - filtered_dict = {k: v for k, v in document.items() if k in column_list} - filtered_list.append(Document(filtered_dict, document.doc_id)) - return filtered_list - - -class TableGenerator: - """An interface for a parser that Generates an Edk2DB table. - - Allows you to parse a workspace, file, etc, and load the contents into the database as rows in a table. - - As Edk2DB is a subclass of TinyDB, it uses the same interface to interact with the database. This documentation - can be found here: https://tinydb.readthedocs.io/en/latest/usage.html#handling-data. While TinyDB provides a - default table to write to, it is suggested that a table be created for each parser using `db.table('table_name')` - - Common commands: - - `table = db.table('table_name')` Get or create a table from the database - - `table.insert(dict)` Insert a new entry into the table - - `table.insert_multiple([dict1, dict2, ...])` Insert multiple entries into the table - - !!! warning - Inserting many large entries into the database is slow! If you need to insert many entries, use tinyrecord's - transaction method which uses a record-first then execute architecture that minimizes the time we are in a - threadlock. This has been seen to cut insertion times by 90% for typical purposes. - - ```python - from tinyrecord import transaction - with transaction(table) as tr: - tr.insert_multiple - ``` - """ - def __init__(self, *args, **kwargs): - """Initialize the query with the specific settings.""" - - def parse(self, db: Edk2DB) -> None: - """Execute the parser and update the database.""" - raise NotImplementedError + self.connection.execute(CREATE_JUNCTION_TABLE) + id = str(uuid.uuid4().hex) + + # Create all tables + for table in self._parsers: + table.create_tables(self.connection.cursor()) + + # Fill all tables + for table in self._parsers: + logging.debug(f"[{table.__class__.__name__}] starting...") + time.time() + table.parse(self.connection.cursor(), self.pathobj, id, env) + self.connection.commit() diff --git a/edk2toollib/database/queries/__init__.py b/edk2toollib/database/queries/__init__.py deleted file mode 100644 index 19ce077c..00000000 --- a/edk2toollib/database/queries/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -## -# Copyright (c) Microsoft Corporation -# -# SPDX-License-Identifier: BSD-2-Clause-Patent -## -"""This file exists to satisfy pythons packaging requirements. - -Read more: https://docs.python.org/3/reference/import.html#regular-packages -""" - -from .component_query import ComponentQuery # noqa: F401 -from .library_query import LibraryQuery # noqa: F401 -from .license_query import LicenseQuery # noqa: F401 -from .unused_component_query import UnusedComponentQuery # noqa: F401 diff --git a/edk2toollib/database/queries/component_query.py b/edk2toollib/database/queries/component_query.py deleted file mode 100644 index 543e37dc..00000000 --- a/edk2toollib/database/queries/component_query.py +++ /dev/null @@ -1,48 +0,0 @@ -# @file component_query.py -# A Query that reads the database and returns information about an instanced component. -## -# Copyright (c) Microsoft Corporation -# -# SPDX-License-Identifier: BSD-2-Clause-Patent -## -"""A Query that reads the database and returns information about an instanced component.""" -from edk2toollib.database import AdvancedQuery, Edk2DB, Query - - -class ComponentQuery(AdvancedQuery): - """A query that provides information about an instanced Component.""" - def __init__(self, *args, **kwargs) -> None: - """Initializes the ComponentQuery with the specified kwargs. - - !!! note - If the database stores multiple builds of data with different environments, - that environment information should be stored in a `environment` table, and - that should be linked in the `instanced_inf` via a ENV column. - - Arguments: - args (any): non-keyword arguments - kwargs (any): keyword arguments expanded below. - - Keyword Arguments: - component (string): The component to get info on; returns all components if empty - env_idx (int): The index in the `environment` table that represents - the environment used for parsing. - - """ - self.component = kwargs.get('component', "") - self.env_id = kwargs.get('env_id', None) - - def run(self, db: Edk2DB): - """Runs the query.""" - table_name = "instanced_inf" - table = db.table(table_name) - - if self.env_id is not None: - entries = table.search((Query().PATH.search(self.component)) - & ~(Query().COMPONENT.exists()) - & (Query().ENVIRONMENT_ID == self.env_id)) - else: - entries = table.search((Query().PATH.search(self.component)) - & ~(Query().COMPONENT.exists())) - - return self.columns(["NAME", "MODULE_TYPE", "ARCH", "LIBRARIES_USED"], entries) diff --git a/edk2toollib/database/queries/library_query.py b/edk2toollib/database/queries/library_query.py deleted file mode 100644 index b0acbfb5..00000000 --- a/edk2toollib/database/queries/library_query.py +++ /dev/null @@ -1,32 +0,0 @@ -# @file library_query.py -# A Query that reads the database and returns all instances of a given LIBRARY_CLASS -## -# Copyright (c) Microsoft Corporation -# -# SPDX-License-Identifier: BSD-2-Clause-Patent -## -"""A Query that reads the database and returns all instances of a given LIBRARY_CLASS.""" -from edk2toollib.database import AdvancedQuery, Edk2DB, Query - - -class LibraryQuery(AdvancedQuery): - """A query that generates a list of library instances for a given library.""" - def __init__(self, *args, **kwargs) -> None: - """Initializes the LibraryQuery with the specified kwargs. - - Arguments: - args (any): non-keyword arguments - kwargs (any): keyword arguments expanded below. - - Keyword Arguments: - library (string): The library to get all instances of. - """ - self.library = kwargs.get('library', "") - - def run(self, db: Edk2DB): - """Runs the query.""" - table_name = "inf" - table = db.table(table_name) - - result = table.search((Query().LIBRARY_CLASS != "") & (Query().LIBRARY_CLASS.matches(self.library))) - return self.columns(["LIBRARY_CLASS", "PATH"], result) diff --git a/edk2toollib/database/queries/license_query.py b/edk2toollib/database/queries/license_query.py deleted file mode 100644 index 408da3a1..00000000 --- a/edk2toollib/database/queries/license_query.py +++ /dev/null @@ -1,46 +0,0 @@ -# @file license_query.py -# A Query that reads the database and returns files missing a license identifier. -## -# Copyright (c) Microsoft Corporation -# -# SPDX-License-Identifier: BSD-2-Clause-Patent -## -# ruff: noqa: F811 -"""A Query that reads the database and returns files missing a license identifier.""" -from edk2toollib.database import AdvancedQuery, Edk2DB, Query - - -class LicenseQuery(AdvancedQuery): - """A Query that reads the database and returns files missing a license identifier.""" - def __init__(self, *args, **kwargs) -> None: - """Initializes the LicenseQuery with the specified kwargs. - - Arguments: - args (any): non-keyword arguments - kwargs (any): keyword arguments expanded below. - - Keyword Arguments: - include (list[str]): A list of strings to search for in the file path name - exclude (list[str]): A list of strings to exclude in the file path name - """ - self.include = kwargs.get('include', None) - self.exclude = kwargs.get('exclude', None) - - if isinstance(self.include, str): - self.include = [self.include] - if isinstance(self.exclude, str): - self.exclude = [self.exclude] - - def run(self, db: Edk2DB): - """Runs the query.""" - table = db.table("source") - - regex = "^" - if self.include: - regex += f"(?=.*({'|'.join(self.include)}))" - if self.exclude: - regex += f"(?!.*({'|'.join(self.exclude)})).*$" - - result = table.search((Query().LICENSE == "") & (Query().PATH.search(regex))) - - return self.columns(['PATH', 'LICENSE'], result) diff --git a/edk2toollib/database/queries/unused_component_query.py b/edk2toollib/database/queries/unused_component_query.py deleted file mode 100644 index d39494cb..00000000 --- a/edk2toollib/database/queries/unused_component_query.py +++ /dev/null @@ -1,94 +0,0 @@ -# @file unused_component_query.py -# A Query that reads the database and returns all components and libraries defined in the DSC but unused in the FDF. -## -# Copyright (c) Microsoft Corporation -# -# SPDX-License-Identifier: BSD-2-Clause-Patent -## -"""A Query that reads the database and returns all components / libraries defined in the DSC but unused in the FDF.""" -from typing import Union - -from edk2toollib.database import AdvancedQuery, Edk2DB, Query - - -class UnusedComponentQuery(AdvancedQuery): - """A query that returns any unused components for a specific build.""" - def __init__(self, *args, **kwargs) -> None: - """Initializes the UnusedComponentQuery with the specified kwargs. - - !!! note - If the database stores multiple builds of data with different environments, - that environment information should be stored in a `environment` table, and - that should be linked in the `instanced_inf` via a ENV column. - - Arguments: - args (any): non-keyword arguments - kwargs (any): keyword arguments expanded below. - - Keyword Arguments: - ignore_app (bool): Whether to ingore UEFI_APPLICATIONs or not - env_idx (int): The index in the `environment` table that represents - the environment to use for parsing. - """ - self.ignore_app = kwargs.get('ignore_app', False) - self.env_id = kwargs.get('env_id', None) - - def run(self, db: Edk2DB) -> Union[str, str]: - """Returns (unused_components, unused_libraries).""" - dsc_infs = db.table("instanced_inf") - fdf_fvs = db.table("instanced_fv") - - dsc_components = [] - fdf_components = [] - - if self.env_id is not None: - dsc_rows = dsc_infs.search((~Query().COMPONENT.exists()) & (Query().ENVIRONMENT_ID == self.env_id)) - fv_rows = fdf_fvs.search(Query().ENVIRONMENT_ID == self.env_id) - else: - dsc_rows = dsc_infs.search(~Query().COMPONENT.exists()) - fv_rows = fdf_fvs.all() - - # Grab all components in the DSC - for entry in dsc_rows: - if self.ignore_app and entry["MODULE_TYPE"] == "UEFI_APPLICATION": - continue - dsc_components.append(entry["PATH"]) - - # Grab all components from the fdf - for fv in fv_rows: - fdf_components.extend(fv["INF_LIST"]) - - unused_components = set(dsc_components) - set(fdf_components) - used_components = set(fdf_components) - - unused_library_list = [] - used_library_list = [] - - # Grab all libraries used by unused_components - for component in unused_components: - self._recurse_inf(component, dsc_infs, unused_library_list) - - # Grab all libraries used by used_components - for component in used_components: - self._recurse_inf(component, dsc_infs, used_library_list) - - unused_libraries = set(unused_library_list) - set(used_library_list) - - return (list(unused_components), list(unused_libraries)) - - def _recurse_inf(self, inf, table, library_list): - if inf in library_list: - return - - if self.env_id is not None: - search_results = table.search((Query().PATH == inf) & (Query().ENVIRONMENT_ID == self.env_id)) - else: - search_results = table.search(Query().PATH == inf) - - for result in search_results: - # Only mark a inf as visited if it is a library - if "COMPONENT" in result: - library_list.append(inf) - - for inf in result["LIBRARIES_USED"]: - self._recurse_inf(inf[1], table, library_list) diff --git a/edk2toollib/database/tables/__init__.py b/edk2toollib/database/tables/__init__.py index 5984a955..b4dd2df3 100644 --- a/edk2toollib/database/tables/__init__.py +++ b/edk2toollib/database/tables/__init__.py @@ -4,9 +4,9 @@ # SPDX-License-Identifier: BSD-2-Clause-Patent ## """A collection of table generators that run against the workspace.""" - from .environment_table import EnvironmentTable # noqa: F401 from .inf_table import InfTable # noqa: F401 from .instanced_fv_table import InstancedFvTable # noqa: F401 from .instanced_inf_table import InstancedInfTable # noqa: F401 +from .package_table import PackageTable # noqa: F401 from .source_table import SourceTable # noqa: F401 diff --git a/edk2toollib/database/tables/base_table.py b/edk2toollib/database/tables/base_table.py new file mode 100644 index 00000000..43fc231a --- /dev/null +++ b/edk2toollib/database/tables/base_table.py @@ -0,0 +1,31 @@ +# @file base_table.py +# An interface for a parser that generates a sqlite3 table maintained by Edk2DB. +## +# Copyright (c) Microsoft Corporation +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +"""An interface for a parser that generates a sqlite3 table maintained by Edk2DB.""" +import sqlite3 + +from edk2toollib.uefi.edk2.path_utilities import Edk2Path + + +class TableGenerator: + """An interface for a parser that generates a sqlite3 table maintained by Edk2DB. + + Allows you to parse a workspace, file, etc, and load the contents into the database as rows in a table. + + Edk2Db provides a connection to a sqlite3 database and will commit any changes made during `parse` once + the parser has finished executing and has returned. Review sqlite3 documentation for more information on + how to interact with the database. + """ + def __init__(self, *args, **kwargs): + """Initialize the query with the specific settings.""" + + def create_tables(self, db_cursor: sqlite3.Cursor) -> None: + """Create the tables necessary for this parser.""" + raise NotImplementedError + + def parse(self, db_cursor: sqlite3.Cursor, pathobj: Edk2Path) -> None: + """Execute the parser and update the database.""" + raise NotImplementedError diff --git a/edk2toollib/database/tables/environment_table.py b/edk2toollib/database/tables/environment_table.py index 4eea69d6..9d2c3a7c 100644 --- a/edk2toollib/database/tables/environment_table.py +++ b/edk2toollib/database/tables/environment_table.py @@ -6,46 +6,54 @@ # SPDX-License-Identifier: BSD-2-Clause-Patent ## """A module to run a table generator that creates or appends to a table with environment information.""" -from datetime import date - -from tinyrecord import transaction - -from edk2toollib.database import Edk2DB, TableGenerator - +import datetime +import sqlite3 + +import git + +from edk2toollib.database.tables.base_table import TableGenerator +from edk2toollib.uefi.edk2.path_utilities import Edk2Path + +CREATE_ENV_TABLE_COMMAND = ''' +CREATE TABLE IF NOT EXISTS environment ( + id TEXT PRIMARY KEY, + date TEXT, + version TEXT +); +''' + +CREATE_ENV_VALUES_TABLE_COMMAND = ''' +CREATE TABLE IF NOT EXISTS environment_values ( + id TEXT, + key TEXT, + value TEXT, + FOREIGN KEY (id) REFERENCES environment(id) +); +''' class EnvironmentTable(TableGenerator): - """A Workspace parser that records import environment information for a given parsing execution. - - Generates a table with the following schema: - - - ``` py - table_name = "environment" - |--------------------------------------| - | DATE | VERSION | ENV | PACKAGES_PATH | - |--------------------------------------| - ``` - """ # noqa: E501 + """A Workspace parser that records import environment information for a given parsing execution.""" # noqa: E501 def __init__(self, *args, **kwargs): """Initialize the query with the specific settings.""" - self.env = kwargs.pop("env") - def parse(self, db: Edk2DB) -> None: + def create_tables(self, db_cursor: sqlite3.Cursor) -> None: + """Create the tables necessary for this parser.""" + db_cursor.execute(CREATE_ENV_VALUES_TABLE_COMMAND) + db_cursor.execute(CREATE_ENV_TABLE_COMMAND) + + def parse(self, db_cursor: sqlite3.Cursor, pathobj: Edk2Path, id, env) -> None: """Parses the environment and adds the data to the table.""" - table_name = 'environment' - table = db.table(table_name, cache_size=None) - today = date.today() - - # Pull out commonly used environment variables as their own entry rather than in the dict. - version = self.env.pop('VERSION', "UNKNOWN") - pp = self.env.pop('PACKAGES_PATH', []) - - entry = { - "DATE": str(today), - "VERSION": version, - "ENV": self.env, - "PACKAGES_PATH": pp, - } - - with transaction(table) as tr: - tr.insert(entry) + dtime = datetime.datetime.now() + + try: + version = git.Repo(pathobj.WorkspacePath).head.commit.hexsha + except git.InvalidGitRepositoryError: + version = "UNKNOWN" + + # Insert into environment table + entry = (id,str(dtime),version,) + db_cursor.execute("INSERT INTO environment (id, date,version) VALUES (?, ?, ?)", entry) + + # Insert into environment_values table + data = [(db_cursor.lastrowid, key, value) for key, value in env.items()] + db_cursor.executemany("INSERT INTO environment_values VALUES (?, ?, ?)", data) diff --git a/edk2toollib/database/tables/inf_table.py b/edk2toollib/database/tables/inf_table.py index a7935328..8de6a219 100644 --- a/edk2toollib/database/tables/inf_table.py +++ b/edk2toollib/database/tables/inf_table.py @@ -10,26 +10,42 @@ import logging import time from pathlib import Path +from sqlite3 import Cursor from joblib import Parallel, delayed -from tinyrecord import transaction -from edk2toollib.database import Edk2DB, TableGenerator +from edk2toollib.database.tables.base_table import TableGenerator from edk2toollib.uefi.edk2.parsers.inf_parser import InfParser as InfP +from edk2toollib.uefi.edk2.path_utilities import Edk2Path +CREATE_INF_TABLE = ''' +CREATE TABLE IF NOT EXISTS inf ( + path TEXT PRIMARY KEY, + guid TEXT, + library_class TEXT, + package TEXT +); +''' -class InfTable(TableGenerator): - """A Table Generator that parses all INF files in the workspace and generates a table. +CREATE_LIBRARY_CLASS_TABLE = ''' +CREATE TABLE IF NOT EXISTS library_class ( + class TEXT +) +''' + +INSERT_JUNCTION_ROW = ''' +INSERT INTO junction (env, table1, key1, table2, key2) +VALUES (?, ?, ?, ?, ?) +''' - Generates a table with the following schema: +INSERT_INF_ROW = ''' +INSERT OR REPLACE INTO inf (path, guid, library_class, package) +VALUES (?, ?, ?, ?) +''' - ``` py - table_name = "inf" - |----------------------------------------------------------------------------------------------------------------------------| - | GUID | LIBRARY_CLASS | PATH | PHASES | SOURCES_USED | LIBRARIES_USED | PROTOCOLS_USED | GUIDS_USED | PPIS_USED | PCDS_USED | - |----------------------------------------------------------------------------------------------------------------------------| - ``` - """ # noqa: E501 +class InfTable(TableGenerator): + """A Table Generator that parses all INF files in the workspace and generates a table.""" + # TODO: Add phase, protocol, guid, ppi, pcd tables and associations once necessary def __init__(self, *args, **kwargs): """Initializes the INF Table Parser. @@ -42,24 +58,41 @@ def __init__(self, *args, **kwargs): """ self.n_jobs = kwargs.get("n_jobs", -1) - def parse(self, db: Edk2DB) -> None: + def create_tables(self, db_cursor: Cursor) -> None: + """Create the tables necessary for this parser.""" + db_cursor.execute(CREATE_INF_TABLE) + db_cursor.execute(CREATE_LIBRARY_CLASS_TABLE) + + def parse(self, db_cursor: Cursor, pathobj: Edk2Path, env_id: str, env: dict) -> None: """Parse the workspace and update the database.""" - ws = Path(db.pathobj.WorkspacePath) - inf_table = db.table("inf", cache_size=None) + ws = Path(pathobj.WorkspacePath) inf_entries = [] start = time.time() files = list(ws.glob("**/*.inf")) files = [file for file in files if not file.is_relative_to(ws / "Build")] - inf_entries = Parallel(n_jobs=self.n_jobs)(delayed(self._parse_file)(ws, fname, db.pathobj) for fname in files) + inf_entries = Parallel(n_jobs=self.n_jobs)(delayed(self._parse_file)(fname, pathobj) for fname in files) logging.debug( f"{self.__class__.__name__}: Parsed {len(inf_entries)} .inf files took; " f"{round(time.time() - start, 2)} seconds.") - with transaction(inf_table) as tr: - tr.insert_multiple(inf_entries) + # Insert the data into the database + for inf in inf_entries: + row = (inf["PATH"], inf["GUID"], inf["LIBRARY_CLASS"], inf["PACKAGE"]) + db_cursor.execute(INSERT_INF_ROW, row) + + for library in inf["LIBRARIES_USED"]: + row = (env_id, "inf", inf["PATH"], "library_class", library) + db_cursor.execute(INSERT_JUNCTION_ROW, row) + + for source in inf["SOURCES_USED"]: + source_path = pathobj.GetEdk2RelativePathFromAbsolutePath( + (Path(inf["PATH"]).parent / source).resolve().as_posix() + ) + row = (env_id, "inf", inf["PATH"], "source", source_path) + db_cursor.execute(INSERT_JUNCTION_ROW, row) - def _parse_file(self, ws, filename, pathobj) -> dict: + def _parse_file(self, filename, pathobj) -> dict: inf_parser = InfP().SetEdk2Path(pathobj) inf_parser.ParseFile(filename) @@ -79,5 +112,6 @@ def _parse_file(self, ws, filename, pathobj) -> dict: data["GUIDS_USED"] = inf_parser.GuidsUsed data["PPIS_USED"] = inf_parser.PpisUsed data["PCDS_USED"] = inf_parser.PcdsUsed + data["PACKAGE"] = pkg return data diff --git a/edk2toollib/database/tables/instanced_fv_table.py b/edk2toollib/database/tables/instanced_fv_table.py index 4f253489..3174e6c7 100644 --- a/edk2toollib/database/tables/instanced_fv_table.py +++ b/edk2toollib/database/tables/instanced_fv_table.py @@ -8,51 +8,61 @@ ## """A module to generate a table containing fv information.""" import re +import sqlite3 from pathlib import Path -from tinyrecord import transaction - -from edk2toollib.database import Edk2DB, TableGenerator +from edk2toollib.database.tables.base_table import TableGenerator from edk2toollib.uefi.edk2.parsers.fdf_parser import FdfParser as FdfP +from edk2toollib.uefi.edk2.path_utilities import Edk2Path +CREATE_INSTANCED_FV_TABLE = """ +CREATE TABLE IF NOT EXISTS instanced_fv ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + env INTEGER, + fv_name TEXT, + fdf TEXT, + path TEXT +) +""" -class InstancedFvTable(TableGenerator): - """A Table Generator that parses a single FDF file and generates a table containing FV information. +INSERT_INSTANCED_FV_ROW = """ +INSERT INTO instanced_fv (env, fv_name, fdf, path) +VALUES (?, ?, ?, ?) +""" - Generates a table with the following schema: +INSERT_JUNCTION_ROW = ''' +INSERT INTO junction (env, table1, key1, table2, key2) +VALUES (?, ?, ?, ?, ?) +''' - ``` py - table_name = "instanced_fv" - |------------------------------------------------------| - | FV_NAME | FDF | PATH | TARGET | INF_LIST | FILE_LIST | - |------------------------------------------------------| - ``` - """ # noqa: E501 +class InstancedFvTable(TableGenerator): + """A Table Generator that parses a single FDF file and generates a table containing FV information.""" # noqa: E501 RULEOVERRIDE = re.compile(r'RuleOverride\s*=.+\s+(.+\.inf)', re.IGNORECASE) def __init__(self, *args, **kwargs): """Initialize the query with the specific settings.""" - self.env = kwargs.pop("env") + + + def create_tables(self, db_cursor: sqlite3.Cursor) -> None: + """Create the tables necessary for this parser.""" + db_cursor.execute(CREATE_INSTANCED_FV_TABLE) + + def parse(self, db_cursor: sqlite3.Cursor, pathobj: Edk2Path, id, env) -> None: + """Parse the workspace and update the database.""" + self.pathobj = pathobj + self.ws = Path(self.pathobj.WorkspacePath) + self.env = env self.dsc = self.env["ACTIVE_PLATFORM"] self.fdf = self.env["FLASH_DEFINITION"] self.arch = self.env["TARGET_ARCH"].split(" ") self.target = self.env["TARGET"] - def parse(self, db: Edk2DB) -> None: - """Parse the workspace and update the database.""" - self.pathobj = db.pathobj - self.ws = Path(self.pathobj.WorkspacePath) - # Our DscParser subclass can now parse components, their scope, and their overrides fdfp = FdfP().SetEdk2Path(self.pathobj) fdfp.SetInputVars(self.env) fdfp.ParseFile(self.fdf) - table_name = 'instanced_fv' - table = db.table(table_name, cache_size=None) - - entry_list = [] for fv in fdfp.FVs: inf_list = [] # Some INF's start with RuleOverride. We only need the INF @@ -63,13 +73,10 @@ def parse(self, db: Edk2DB) -> None: inf = str(Path(self.pathobj.GetEdk2RelativePathFromAbsolutePath(inf))) inf_list.append(Path(inf).as_posix()) - entry_list.append({ - "FV_NAME": fv, - "FDF": Path(self.fdf).name, - "PATH": self.fdf, - "INF_LIST": inf_list, - "FILE_LIST": fdfp.FVs[fv]["Files"] - }) + row = (id, fv, Path(self.fdf).name, self.fdf) + db_cursor.execute(INSERT_INSTANCED_FV_ROW, row) + fv_id = db_cursor.lastrowid - with transaction(table) as tr: - tr.insert_multiple(entry_list) + for inf in inf_list: + row = (id, "instanced_fv", fv_id, "inf", inf) + db_cursor.execute(INSERT_JUNCTION_ROW, row) diff --git a/edk2toollib/database/tables/instanced_inf_table.py b/edk2toollib/database/tables/instanced_inf_table.py index 2d3a9eae..052e5e85 100644 --- a/edk2toollib/database/tables/instanced_inf_table.py +++ b/edk2toollib/database/tables/instanced_inf_table.py @@ -10,26 +10,45 @@ import logging import re from pathlib import Path +from sqlite3 import Cursor -from tinyrecord import transaction - -from edk2toollib.database.edk2_db import Edk2DB, TableGenerator +from edk2toollib.database.tables.base_table import TableGenerator from edk2toollib.uefi.edk2.parsers.dsc_parser import DscParser as DscP from edk2toollib.uefi.edk2.parsers.inf_parser import InfParser as InfP - +from edk2toollib.uefi.edk2.path_utilities import Edk2Path + +CREATE_INSTANCED_INF_TABLE = ''' +CREATE TABLE IF NOT EXISTS instanced_inf ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + env INTEGER, + path TEXT, + class TEXT, + name TEXT, + arch TEXT, + dsc TEXT, + component TEXT, + FOREIGN KEY(env) REFERENCES environment(env) +) +''' + +INSERT_INSTANCED_INF_ROW = ''' +INSERT INTO instanced_inf (env, path, class, name, arch, dsc, component) +VALUES (?, ?, ?, ?, ?, ?, ?) +''' + +INSERT_JUNCTION_ROW = ''' +INSERT INTO junction (env, table1, key1, table2, key2) +VALUES (?, ?, ?, ?, ?) +''' + +GET_ROW_ID = ''' +SELECT id FROM instanced_inf +WHERE env = ? and path = ? and dsc = ? and (class = ? OR class IS NULL) +LIMIT 1 +''' class InstancedInfTable(TableGenerator): - """A Table Generator that parses a single DSC file and generates a table. - - Generates a table with the following schema: - - ``` py - table_name = "instanced_inf" - |----------------------------------------------------------------------------------------------------------------------------------------------| - | DSC | PATH | NAME | LIBRARY_CLASS | COMPONENT | MODULE_TYPE | ARCH | SOURCES_USED | LIBRARIES_USED | PROTOCOLS_USED | GUIDS_USED | PCDS_USED | - |----------------------------------------------------------------------------------------------------------------------------------------------| - ``` - """ # noqa: E501 + """A Table Generator that parses a single DSC file and generates a table.""" SECTION_LIBRARY = "LibraryClasses" SECTION_COMPONENT = "Components" SECTION_REGEX = re.compile(r"\[(.*)\]") @@ -37,11 +56,10 @@ class InstancedInfTable(TableGenerator): def __init__(self, *args, **kwargs): """Initialize the query with the specific settings.""" - self.env = kwargs.pop("env") - self.dsc = self.env["ACTIVE_PLATFORM"] # REQUIRED - self.fdf = self.env.get("FLASH_DEFINITION", "") # OPTIONAL - self.arch = self.env["TARGET_ARCH"].split(" ") # REQUIRED - self.target = self.env["TARGET"] # REQUIRED + + def create_tables(self, db_cursor: Cursor) -> None: + """Create the tables necessary for this parser.""" + db_cursor.execute(CREATE_INSTANCED_INF_TABLE) # Prevent parsing the same INF multiple times self._parsed_infs = {} @@ -56,15 +74,22 @@ def inf(self, inf: str) -> InfP: self._parsed_infs[inf] = infp return infp - def parse(self, db: Edk2DB) -> None: + def parse(self, db_cursor: Cursor, pathobj: Edk2Path, id: str, env: dict) -> None: """Parse the workspace and update the database.""" - self.pathobj = db.pathobj + self.pathobj = pathobj self.ws = Path(self.pathobj.WorkspacePath) + self.id = id + self.env = env + self.dsc = self.env["ACTIVE_PLATFORM"] # REQUIRED + self.fdf = self.env.get("FLASH_DEFINITION", "") # OPTIONAL + self.arch = self.env["TARGET_ARCH"].split(" ") # REQUIRED + self.target = self.env["TARGET"] # REQUIRED - # Our DscParser subclass can now parse components, their scope, and their overrides dscp = DscP().SetEdk2Path(self.pathobj) dscp.SetInputVars(self.env) dscp.ParseFile(self.dsc) + + # General Debugging logging.debug(f"All DSCs included in {self.dsc}:") for dsc in dscp.GetAllDscPaths(): logging.debug(f" {dsc}") @@ -74,19 +99,37 @@ def parse(self, db: Edk2DB) -> None: logging.debug(f" {line}") logging.debug("End of DSC") - # Create the instanced inf entries, including components and libraries. multiple entries + # Create the instanced inf entries, including components and libraries. Multiple entries # of the same library will exist if multiple components use it. # # This is where we merge DSC parser information with INF parser information. inf_entries = self._build_inf_table(dscp) - for entry in inf_entries: - if Path(entry["PATH"]).is_absolute(): - entry["PATH"] = self.pathobj.GetEdk2RelativePathFromAbsolutePath(entry["PATH"]) + for e in inf_entries: + if Path(e["PATH"]).is_absolute(): + e["PATH"] = self.pathobj.GetEdk2RelativePathFromAbsolutePath(e["PATH"]) - table_name = 'instanced_inf' - table = db.table(table_name, cache_size=None) - with transaction(table) as tr: - tr.insert_multiple(inf_entries) + # add instanced_inf entries + for e in inf_entries: + row = (id, e["PATH"], e["LIBRARY_CLASS"], e["NAME"], e["ARCH"], e["DSC"], e["COMPONENT"]) + db_cursor.execute(INSERT_INSTANCED_INF_ROW, row) + + for e in inf_entries: + inf_id = db_cursor.execute(GET_ROW_ID, (id, e["PATH"], e["DSC"], e["LIBRARY_CLASS"])).fetchone()[0] + + # Add junction entries to link source the source files used by an INF + for source in e["SOURCES_USED"]: + row = (id, "instanced_inf", inf_id, "source", source) + db_cursor.execute(INSERT_JUNCTION_ROW, row) + + # Add junction entires to link libraries / components to the libraries they consume. + for cls, instance in e["LIBRARIES_USED"]: + if instance is None: + used_inf_id = None # no library instance found for this library class + else: + used_inf_id = db_cursor.execute(GET_ROW_ID, (id, instance, e["DSC"], cls)).fetchone()[0] + + row = (id, "instanced_inf", inf_id, "instanced_inf", used_inf_id) + db_cursor.execute(INSERT_JUNCTION_ROW, row) def _build_inf_table(self, dscp: DscP): @@ -104,17 +147,20 @@ def _build_inf_table(self, dscp: DscP): if "MODULE_TYPE" in infp.Dict: scope += f".{infp.Dict['MODULE_TYPE']}".lower() - inf_entries += self._parse_inf_recursively(inf, inf, dscp.ScopedLibraryDict, overrides, scope, []) - - # Move entries to correct table - for entry in inf_entries: - if entry["PATH"] == entry["COMPONENT"]: - del entry["COMPONENT"] + inf_entries += self._parse_inf_recursively(inf, None, inf, dscp.ScopedLibraryDict, overrides, scope, []) return inf_entries def _parse_inf_recursively( - self, inf: str, component: str, library_dict: dict, override_dict: dict, scope: str, visited): + self, + inf: str, + lib_cls: str, + component: str, + library_dict: dict, + override_dict: dict, + scope: str, + visited: list[str] + ): """Recurses down all libraries starting from a single INF. Will immediately return if the INF has already been visited. @@ -142,19 +188,24 @@ def _parse_inf_recursively( lib = lib.split(" ")[0] library_instance_list.append(self._lib_to_instance(lib.lower(), scope, library_dict, override_dict)) library_class_list.append(lib) - # Append all NULL library instances + + # + # 2. Append all NULL library instances + # for null_lib in override_dict["NULL"]: library_instance_list.append(null_lib) library_class_list.append("NULL") - to_parse = list(filter(lambda lib: lib is not None, library_instance_list)) - - # Time to visit in libraries that we have not visited yet. + # + # 3. Recursively parse used libraries + # to_return = [] - for library in filter(lambda lib: lib not in visited, to_parse): - to_return += self._parse_inf_recursively(library, component, - library_dict, override_dict, scope, visited) - + for cls, instance in zip(library_class_list, library_instance_list): + if instance is None or instance in visited: + continue + to_return += self._parse_inf_recursively( + instance, cls, component, library_dict, override_dict, scope, visited + ) # Transform path to edk2 relative form (POSIX) def to_posix(path): if path is None: @@ -166,8 +217,9 @@ def to_posix(path): to_return.append({ "DSC": Path(self.dsc).name, "PATH": Path(inf).as_posix(), + "GUID": infp.Dict.get("FILE_GUID", ""), "NAME": infp.Dict["BASE_NAME"], - "LIBRARY_CLASS": infp.LibraryClass, + "LIBRARY_CLASS": lib_cls, "COMPONENT": Path(component).as_posix(), "MODULE_TYPE": infp.Dict["MODULE_TYPE"], "ARCH": scope.split(".")[0].upper(), diff --git a/edk2toollib/database/tables/package_table.py b/edk2toollib/database/tables/package_table.py new file mode 100644 index 00000000..ccc5e5c7 --- /dev/null +++ b/edk2toollib/database/tables/package_table.py @@ -0,0 +1,61 @@ +# @file repository_table.py +# A module to associate the packages in a workspace with the repositories they come from. +## +# Copyright (c) Microsoft Corporation +# +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +"""A module to generate a table containing information about a package.""" +from pathlib import Path +from sqlite3 import Cursor + +import git + +from edk2toollib.database.tables.base_table import TableGenerator +from edk2toollib.uefi.edk2.path_utilities import Edk2Path + +CREATE_PACKAGE_TABLE = """ +CREATE TABLE IF NOT EXISTS package ( + name TEXT PRIMARY KEY, + repository TEXT +) +""" + +INSERT_PACKAGE_ROW = """ +INSERT OR REPLACE INTO package (name, repository) +VALUES (?, ?) +""" +class PackageTable(TableGenerator): + """A Table Generator that associates packages with their repositories.""" + def __init__(self, *args, **kwargs): + """Initializes the Repository Table Parser. + + Args: + args (any): non-keyword arguments + kwargs (any): keyword arguments described below + + Keyword Arguments: + None + """ + + def create_tables(self, db_cursor: Cursor) -> None: + """Create the table necessary for this parser.""" + db_cursor.execute(CREATE_PACKAGE_TABLE) + + def parse(self, db_cursor: Cursor, pathobj: Edk2Path, id: str, env: dict) -> None: + """Glob for packages and insert them into the table.""" + try: + repo = git.Repo(pathobj.WorkspacePath) + except git.InvalidGitRepositoryError: + return + + for file in Path(pathobj.WorkspacePath).rglob("*.dec"): + pkg = pathobj.GetContainingPackage(str(file)) + containing_repo = "BASE" + if repo: + for submodule in repo.submodules: + if submodule.abspath in str(file): + containing_repo = submodule.name + break + row = (pkg, containing_repo) + db_cursor.execute(INSERT_PACKAGE_ROW, row) diff --git a/edk2toollib/database/tables/source_table.py b/edk2toollib/database/tables/source_table.py index 691f9b00..87b37667 100644 --- a/edk2toollib/database/tables/source_table.py +++ b/edk2toollib/database/tables/source_table.py @@ -10,27 +10,35 @@ import re import time from pathlib import Path +from sqlite3 import Cursor from joblib import Parallel, delayed -from tinyrecord import transaction -from edk2toollib.database import Edk2DB, TableGenerator +from edk2toollib.database.tables.base_table import TableGenerator +from edk2toollib.uefi.edk2.path_utilities import Edk2Path SOURCE_FILES = ["*.c", "*.h", "*.cpp", "*.asm", "*.s", "*.nasm", "*.masm", "*.rs"] +CREATE_SOURCE_TABLE = ''' +CREATE TABLE IF NOT EXISTS source ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + path TEXT UNIQUE, + license TEXT, + total_lines INTEGER, + code_lines INTEGER, + comment_lines INTEGER, + blank_lines INTEGER +) +''' -class SourceTable(TableGenerator): - """A Table Generator that parses all c and h files in the workspace. +INSERT_SOURCE_ROW = ''' +INSERT OR REPLACE INTO source (path, license, total_lines, code_lines, comment_lines, blank_lines) +VALUES (?, ?, ?, ?, ?, ?) +''' - Generates a table with the following schema: - ``` py - table_name = "source" - |-------------------------------------------------------------------------| - | PATH | LICENSE | TOTAL_LINES | CODE_LINES | COMMENT_LINES | BLANK_LINES | - |-------------------------------------------------------------------------| - ``` - """ # noqa: E501 +class SourceTable(TableGenerator): + """A Table Generator that parses all c and h files in the workspace.""" def __init__(self, *args, **kwargs): """Initializes the Source Table Parser. @@ -43,10 +51,14 @@ def __init__(self, *args, **kwargs): """ self.n_jobs = kwargs.get("n_jobs", -1) - def parse(self, db: Edk2DB) -> None: + def create_tables(self, db_cursor: Cursor) -> None: + """Create the tables necessary for this parser.""" + db_cursor.execute(CREATE_SOURCE_TABLE) + + def parse(self, db_cursor: Cursor, pathobj: Edk2Path, id: str, env: dict) -> None: """Parse the workspace and update the database.""" - ws = Path(db.pathobj.WorkspacePath) - src_table = db.table("source", cache_size=None) + ws = Path(pathobj.WorkspacePath) + self.pathobj = pathobj start = time.time() files = [] @@ -58,8 +70,7 @@ def parse(self, db: Edk2DB) -> None: f"{self.__class__.__name__}: Parsed {len(src_entries)} files; " f"took {round(time.time() - start, 2)} seconds.") - with transaction(src_table) as tr: - tr.insert_multiple(src_entries) + db_cursor.executemany(INSERT_SOURCE_ROW, src_entries) def _parse_file(self, ws, filename: Path) -> dict: """Parse a C file and return the results.""" @@ -69,12 +80,11 @@ def _parse_file(self, ws, filename: Path) -> dict: match = re.search(r"SPDX-License-Identifier:\s*(.*)$", line) # TODO: This is not a standard format. if match: license = match.group(1) - - return { - "PATH": filename.relative_to(ws).as_posix(), - "LICENSE": license, - "TOTAL_LINES": 0, - "CODE_LINES": 0, - "COMMENT_LINES": 0, - "BLANK_LINES": 0, - } + return ( + self.pathobj.GetEdk2RelativePathFromAbsolutePath(filename.as_posix()), # path + license or "Unknown", # license + 0, # total_lines + 0, # code_lines + 0, # comment_lines + 0, # blank_lines + ) diff --git a/pyproject.toml b/pyproject.toml index 035fad82..931201e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,9 +13,8 @@ dependencies = [ "pyasn1 >= 0.4.8", "pyasn1-modules >= 0.2.8", "cryptography >= 39.0.1", - "tinydb == 4.8.0", - "joblib == 1.3.2", - "tinyrecord == 0.2.0", + "joblib >= 1.3.2", + "GitPython >= 3.1.30" ] classifiers=[ "Programming Language :: Python :: 3", diff --git a/tests.unit/database/common.py b/tests.unit/database/common.py index 7b32c57a..5a26ac08 100644 --- a/tests.unit/database/common.py +++ b/tests.unit/database/common.py @@ -10,19 +10,9 @@ from pathlib import Path import pytest -from edk2toollib.database import Edk2DB, Query, transaction from edk2toollib.uefi.edk2.path_utilities import Edk2Path -def correlate_env(db: Edk2DB): - """Correlates the environment table with the other tables.""" - idx = len(db.table("environment")) - 1 - for table in filter(lambda table: table != "environment", db.tables()): - table = db.table(table) - - with transaction(table) as tr: - tr.update({'ENVIRONMENT_ID': idx}, ~Query().ENVIRONMENT_ID.exists()) - def write_file(file, contents): """Writes contents to a file.""" file.write_text(contents) @@ -44,7 +34,7 @@ def make_edk2_cfg_file(*args, **kwargs)->str: out += "[Defines]\n" for key, value in kwargs["defines"].items(): # Must exist - out += f' {key} = {value}\n' + out += f' {key.split(" ")[0]} = {value}\n' for key, values in kwargs.items(): if key == "defines": diff --git a/tests.unit/database/test_component_query.py b/tests.unit/database/test_component_query.py deleted file mode 100644 index d761fcc9..00000000 --- a/tests.unit/database/test_component_query.py +++ /dev/null @@ -1,77 +0,0 @@ -## -# unittest for the ComponentQuery query -# -# Copyright (c) Microsoft Corporation -# -# Spdx-License-Identifier: BSD-2-Clause-Patent -## -# ruff: noqa: F811 -"""Unittest for the ComponentQuery query.""" -from common import Tree, correlate_env, empty_tree # noqa: F401 -from edk2toollib.database import Edk2DB -from edk2toollib.database.queries import ComponentQuery -from edk2toollib.database.tables import EnvironmentTable, InstancedInfTable -from edk2toollib.uefi.edk2.path_utilities import Edk2Path - - -def test_simple_component(empty_tree: Tree): - """Tests that components are detected.""" - lib1 = empty_tree.create_library("TestLib1", "TestCls") - lib2 = empty_tree.create_library("TestLib2", "TestCls") - lib3 = empty_tree.create_library("TestLib3", "TestNullCls") - - comp1 = empty_tree.create_component( - "TestDriver1", "DXE_DRIVER", - libraryclasses = ["TestCls"] - ) - comp2 = empty_tree.create_component( - "TestDriver2", "DXE_DRIVER", - libraryclasses = ["TestCls"] - ) - - dsc = empty_tree.create_dsc( - libraryclasses = [ - f'TestCls|{lib1}' - ], - components = [ - f'{comp2}', - f'{comp1} {{', - '', - '!if $(TARGET) == "DEBUG"', - f'TestCls|{lib2}', - f'NULL|{lib3}', - '!endif', - '}', - ] - ) - - edk2path = Edk2Path(str(empty_tree.ws), []) - db = Edk2DB(Edk2DB.MEM_RW, pathobj = edk2path) - env = { - "ACTIVE_PLATFORM": dsc, - "TARGET_ARCH": "IA32", - "TARGET": "DEBUG", - } - db.register(InstancedInfTable(env = env), EnvironmentTable(env = env)) - db.parse() - correlate_env(db) - - # Ensure that a component query with an invalid env id returns nothing and does not crash - result = db.search(ComponentQuery(env_id = 1)) - assert len(result) == 0 - - # ensure that a component query with a valid env id returns the correct result - result = db.search(ComponentQuery(env_id = 0)) - assert len(result) == 2 - - # ensure that a component query without an env id returns the correct result - result = db.search(ComponentQuery()) - assert len(result) == 2 - - result = db.search(ComponentQuery(component = "TestDriver1")) - assert len(result) == 1 - - assert sorted(result[0]['LIBRARIES_USED']) == sorted([('TestCls','TestPkg/Library/TestLib2.inf'), ('NULL','TestPkg/Library/TestLib3.inf')]) - - result = db.search(ComponentQuery(component = "NonExistantDriver")) - assert len(result) == 0 diff --git a/tests.unit/database/test_edk2_db.py b/tests.unit/database/test_edk2_db.py index 1dcd65a7..5d34bde3 100644 --- a/tests.unit/database/test_edk2_db.py +++ b/tests.unit/database/test_edk2_db.py @@ -7,33 +7,15 @@ ## # ruff: noqa: F811 """Unittest for the Edk2DB class.""" -import logging import pytest -from common import Tree, correlate_env, empty_tree # noqa: F401 -from edk2toollib.database import AdvancedQuery, Edk2DB, TableGenerator -from edk2toollib.database.queries import LibraryQuery +from common import Tree, empty_tree # noqa: F401 +from edk2toollib.database import Edk2DB from edk2toollib.database.tables import InfTable +from edk2toollib.database.tables.base_table import TableGenerator from edk2toollib.uefi.edk2.path_utilities import Edk2Path -def test_load_each_db_mode(empty_tree: Tree): - edk2path = Edk2Path(str(empty_tree.ws), []) - db_path = empty_tree.ws / "test.db" - - with Edk2DB(Edk2DB.FILE_RW, pathobj=edk2path, db_path=db_path): - pass - - with Edk2DB(Edk2DB.FILE_RO, db_path=db_path): - pass - - with Edk2DB(Edk2DB.MEM_RW, pathobj=edk2path): - pass - - with pytest.raises(ValueError, match = "Unknown Database mode."): - with Edk2DB(5): - pass - def test_load_existing_db(empty_tree: Tree): """Test that we can create a json database and load it later.""" empty_tree.create_library("TestLib1", "TestCls") @@ -42,44 +24,37 @@ def test_load_existing_db(empty_tree: Tree): db_path = empty_tree.ws / "test.db" assert db_path.exists() is False - with Edk2DB(Edk2DB.FILE_RW, pathobj=edk2path, db_path=db_path) as db: + with Edk2DB(db_path, pathobj=edk2path) as db: db.register(InfTable()) - db.parse(edk2path) - assert len(db.search(LibraryQuery())) == 1 + db.parse({}) + result = db.connection.execute("SELECT name FROM sqlite_master WHERE type='table' AND name=?", ("inf",)).fetchone() + assert result is not None assert db_path.exists() # Ensure we can load an existing database - with Edk2DB(Edk2DB.FILE_RW, pathobj=edk2path, db_path=db_path) as db: - assert len(db.search(LibraryQuery())) == 1 + with Edk2DB(db_path, pathobj=edk2path) as db: + result = db.connection.execute("SELECT name FROM sqlite_master WHERE type='table' AND name=?", ("inf",)).fetchone() + assert result is not None -def test_catch_bad_parser_and_query(empty_tree: Tree, caplog): +def test_catch_bad_parser_and_query(empty_tree: Tree): """Test that a bad parser will be caught and logged.""" - caplog.set_level(logging.ERROR) edk2path = Edk2Path(str(empty_tree.ws), []) db_path = empty_tree.ws / "test.db" assert db_path.exists() is False - with Edk2DB(Edk2DB.FILE_RW, pathobj=edk2path, db_path=db_path) as db: - db.register(TableGenerator()) # Not implemented, will throw an error. Caught and logged. - db.parse(edk2path) - - with pytest.raises(NotImplementedError): - db.search(AdvancedQuery()) # Not implemented, will throw an error - - for message in [r.message for r in caplog.records]: - if "failed." in message: - break - else: - pytest.fail("No error message was logged for a failed parser.") + with pytest.raises(NotImplementedError): + with Edk2DB(db_path, pathobj=edk2path) as db: + db.register(TableGenerator()) + db.parse({}) def test_clear_parsers(empty_tree: Tree): - """Test that we can clear all parsers.""" + """Test that we can clear all parsers. EnvironmentTable should always persist.""" edk2path = Edk2Path(str(empty_tree.ws), []) - with Edk2DB(Edk2DB.MEM_RW, pathobj=edk2path) as db: + with Edk2DB(empty_tree.ws / "test.db", pathobj=edk2path) as db: db.register(TableGenerator()) - assert len(db._parsers) == 1 + assert len(db._parsers) == 2 db.clear_parsers() - assert len(db._parsers) == 0 + assert len(db._parsers) == 1 diff --git a/tests.unit/database/test_environment_table.py b/tests.unit/database/test_environment_table.py index c46d5c1b..468769e5 100644 --- a/tests.unit/database/test_environment_table.py +++ b/tests.unit/database/test_environment_table.py @@ -7,62 +7,75 @@ ## # ruff: noqa: F811 """Tests for build an inf file table.""" -import os from datetime import date -from edk2toollib.database.tables import EnvironmentTable + from edk2toollib.database import Edk2DB from edk2toollib.uefi.edk2.path_utilities import Edk2Path -def test_environment_no_version(): + +def test_environment_no_version(tmp_path): """Test that version is set if not found in the environment variables.""" - edk2path = Edk2Path(os.getcwd(), []) - db = Edk2DB(Edk2DB.MEM_RW, pathobj=edk2path) - env_table = EnvironmentTable(env={}) + edk2path = Edk2Path(str(tmp_path), []) + db = Edk2DB(tmp_path / "db.db", pathobj=edk2path) + + db.parse({}) - env_table.parse(db) - table = db.table("environment") + rows = list(db.connection.cursor().execute("SELECT * FROM environment")) - assert len(table) == 1 - row = table.all()[0] + assert len(rows) == 1 + _, actual_date, actual_version = rows[0] - assert row['VERSION'] == 'UNKNOWN' - assert row["DATE"] == str(date.today()) - assert row["ENV"] == {} + assert actual_version == 'UNKNOWN' + assert actual_date.split(" ")[0] == str(date.today()) -def test_environment_version(): + rows = list(db.connection.cursor().execute("SELECT key, value FROM environment_values")) + assert len(rows) == 0 + +def test_environment_version(tmp_path): """Test that version is detected out of environment variables.""" - edk2path = Edk2Path(os.getcwd(), []) - db = Edk2DB(Edk2DB.MEM_RW, pathobj=edk2path) - env = {"VERSION": "abcdef1"} - env_table = EnvironmentTable(env=env) + edk2path = Edk2Path(str(tmp_path), []) + db = Edk2DB(tmp_path / "db.db", pathobj=edk2path) + + db.parse({}) + + rows = list(db.connection.cursor().execute("SELECT * FROM environment")) - env_table.parse(db) - table = db.table("environment") + assert len(rows) == 1 + _, actual_date, actual_version = rows[0] - assert len(table) == 1 - row = table.all()[0] + assert actual_date.split(" ")[0] == str(date.today()) + assert actual_version == 'UNKNOWN' - assert row['VERSION'] == 'abcdef1' - assert row["DATE"] == str(date.today()) - assert row["ENV"] == {} + rows = list(db.connection.cursor().execute("SELECT key, value FROM environment_values")) + assert len(rows) == 0 -def test_environment_with_vars(): +def test_environment_with_vars(tmp_path): """Tests that environment variables are recorded.""" - edk2path = Edk2Path(os.getcwd(), []) - db = Edk2DB(Edk2DB.MEM_RW, pathobj=edk2path) env = { "ACTIVE_PLATFORM": "TestPkg/TestPkg.dsc", "TARGET_ARCH": "X64", "TOOL_CHAIN_TAG": "VS2019", "FLASH_DEFINITION": "TestPkg/TestPkg.fdf", } - env_table = EnvironmentTable(env = env) - env_table.parse(db) + edk2path = Edk2Path(str(tmp_path), []) + db = Edk2DB(tmp_path / "db.db", pathobj=edk2path) + + db.parse(env) + + rows = list(db.connection.cursor().execute("SELECT * FROM environment")) + + assert len(rows) == 1 + _, actual_date, actual_version = rows[0] + + assert actual_date.split(" ")[0] == str(date.today()) + assert actual_version == 'UNKNOWN' + + rows = list(db.connection.cursor().execute("SELECT key, value FROM environment_values WHERE id = 1")) + assert len(rows) == 4 + + db.parse(env) - assert len(db.table("environment")) == 1 - row = db.table("environment").all()[0] + rows = list(db.connection.cursor().execute("SELECT * FROM environment")) - assert row['VERSION'] == 'UNKNOWN' - assert row["DATE"] == str(date.today()) - assert row["ENV"] == env + assert len(rows) == 2 diff --git a/tests.unit/database/test_inf_table.py b/tests.unit/database/test_inf_table.py index 9072aee7..cc88cb93 100644 --- a/tests.unit/database/test_inf_table.py +++ b/tests.unit/database/test_inf_table.py @@ -16,8 +16,8 @@ def test_valid_inf(empty_tree: Tree): """Tests that a valid Inf with typical settings is properly parsed.""" edk2path = Edk2Path(str(empty_tree.ws), []) - db = Edk2DB(Edk2DB.MEM_RW, pathobj=edk2path) - inf_table = InfTable(n_jobs = 1) + db = Edk2DB(empty_tree.ws / "db.db", pathobj=edk2path) + db.register(InfTable(n_jobs = 1)) # Configure inf libs = ["TestLib2", "TestLib3"] @@ -36,14 +36,24 @@ def test_valid_inf(empty_tree: Tree): sources_ia32 = sources_ia32, sources_x64 = sources_x64, ) - inf_table.parse(db) - table = db.table("inf") + lib2 = empty_tree.create_library( + "TestLib2", "TestCls", + libraryclasses = libs, + protocols = protocols, + guids = guids, + sources = sources, + sources_ia32 = sources_ia32, + sources_x64 = sources_x64, + ) + db.parse({}) + + rows = list(db.connection.cursor().execute("SELECT path, library_class FROM inf")) + assert len(rows) == 2 - assert len(table) == 1 - row = table.all()[0] + for path, library_class in rows: + assert path in [lib1, lib2] + assert library_class == "TestCls" - assert row['PATH'] in (empty_tree.ws / lib1).as_posix() - assert row['LIBRARIES_USED'] == libs - assert row['PROTOCOLS_USED'] == protocols - assert row['GUIDS_USED'] == guids - assert sorted(row['SOURCES_USED']) == sorted(sources + sources_ia32 + sources_x64) + for inf in [lib1, lib2]: + rows = db.connection.execute("SELECT * FROM junction WHERE key1 = ? AND table2 = 'source'", (inf,)).fetchall() + assert len(rows) == 3 diff --git a/tests.unit/database/test_instanced_fv_table.py b/tests.unit/database/test_instanced_fv_table.py index fbb13506..470bdcf8 100644 --- a/tests.unit/database/test_instanced_fv_table.py +++ b/tests.unit/database/test_instanced_fv_table.py @@ -14,16 +14,22 @@ from edk2toollib.database.tables import InstancedFvTable from edk2toollib.uefi.edk2.path_utilities import Edk2Path +GET_INF_LIST_QUERY = """ +SELECT i.path +FROM inf AS i +JOIN junction AS j ON ? = j.key1 and j.table2 = "inf" +""" def test_valid_fdf(empty_tree: Tree): # noqa: F811 """Tests that a typical fdf can be properly parsed.""" edk2path = Edk2Path(str(empty_tree.ws), []) - db = Edk2DB(Edk2DB.MEM_RW, pathobj=edk2path) + db = Edk2DB(empty_tree.ws / "db.db", pathobj=edk2path) + db.register(InstancedFvTable()) # raise exception if the Table generator is missing required information to # Generate the table. with pytest.raises(KeyError): - fv_table = InstancedFvTable(env = {}) + db.parse({}) comp1 = empty_tree.create_component("TestDriver1", "DXE_DRIVER") comp2 = empty_tree.create_component("TestDriver2", "DXE_DRIVER") @@ -44,26 +50,16 @@ def test_valid_fdf(empty_tree: Tree): # noqa: F811 f'INF ruleoverride = RESET_VECTOR {comp5}', # RuleOverride lowercase & spaces ] ) - - fv_table = InstancedFvTable(env = { + env = { "ACTIVE_PLATFORM": dsc, "FLASH_DEFINITION": fdf, "TARGET_ARCH": "IA32 X64", "TARGET": "DEBUG", - }) - # Parse the FDF - fv_table.parse(db) + } + db.parse(env) - # Ensure tests pass for expected output - for fv in db.table("instanced_fv").all(): + fv_id = db.connection.execute("SELECT id FROM instanced_fv WHERE fv_name = 'infformat'").fetchone()[0] + rows = db.connection.execute("SELECT key2 FROM junction where key1 == ?", (fv_id,)).fetchall() - # Test INF's were parsed correctly. Paths should be posix as - # That is the EDK2 standard - if fv['FV_NAME'] == "infformat": - assert sorted(fv['INF_LIST']) == sorted([ - Path(comp1).as_posix(), - Path(comp2).as_posix(), - Path(comp3).as_posix(), - Path(comp5).as_posix(), - Path(comp4).as_posix(), - ]) + assert len(rows) == 5 + assert sorted(rows) == sorted([(comp1,), (comp2,), (comp3,), (comp4,), (comp5,)]) diff --git a/tests.unit/database/test_instanced_inf_table.py b/tests.unit/database/test_instanced_inf_table.py index 4469fd31..1b988484 100644 --- a/tests.unit/database/test_instanced_inf_table.py +++ b/tests.unit/database/test_instanced_inf_table.py @@ -11,16 +11,28 @@ from pathlib import Path import pytest -from common import Tree, create_inf_file, empty_tree # noqa: F401 -from edk2toollib.database import Edk2DB, Query +from common import Tree, empty_tree # noqa: F401 +from edk2toollib.database import Edk2DB from edk2toollib.database.tables import InstancedInfTable from edk2toollib.uefi.edk2.path_utilities import Edk2Path +GET_USED_LIBRARIES_QUERY = """ +SELECT i.path +FROM instanced_inf AS i +JOIN junction AS j ON i.id = j.key2 and j.table2 = "instanced_inf" +WHERE j.key1 = ( + SELECT id + FROM instanced_inf + WHERE name = ? AND arch = ? + LIMIT 1 +); +""" def test_valid_dsc(empty_tree: Tree): """Tests that a typical dsc can be correctly parsed.""" edk2path = Edk2Path(str(empty_tree.ws), []) - db = Edk2DB(Edk2DB.MEM_RW, pathobj=edk2path) + db = Edk2DB(empty_tree.ws / "db.db", pathobj=edk2path) + db.register(InstancedInfTable()) comp1 = empty_tree.create_component("TestComponent1", "DXE_DRIVER") lib1 = empty_tree.create_library("TestLib1", "TestCls") @@ -29,37 +41,37 @@ def test_valid_dsc(empty_tree: Tree): components = [str(empty_tree.ws / comp1), lib1] # absolute comp path ) - inf_table = InstancedInfTable(env = { + env = { "ACTIVE_PLATFORM": dsc, "TARGET_ARCH": "IA32", "TARGET": "DEBUG", - }) - inf_table.parse(db) + } + db.parse(env) - # Check that only 1 component is picked up, as libraries in the component section are ignored - assert len(db.table("instanced_inf")) == 1 - entry = db.table("instanced_inf").all()[0] - assert entry["NAME"] == Path(comp1).stem + rows = db.connection.cursor().execute("SELECT * FROM instanced_inf").fetchall() + assert len(rows) == 1 + assert rows[0][4] == Path(comp1).stem def test_no_active_platform(empty_tree: Tree, caplog): """Tests that the dsc table returns immediately when no ACTIVE_PLATFORM is defined.""" caplog.set_level(logging.DEBUG) edk2path = Edk2Path(str(empty_tree.ws), []) - Edk2DB(Edk2DB.MEM_RW, pathobj=edk2path) + db = Edk2DB(empty_tree.ws / "db.db", pathobj=edk2path) + db.register(InstancedInfTable()) # Test 1: raise error for missing ACTIVE_PLATFORM with pytest.raises(KeyError, match = "ACTIVE_PLATFORM"): - InstancedInfTable(env = {}) + db.parse({}) # Test 2: raise error for missing TARGET_ARCH with pytest.raises(KeyError, match = "TARGET_ARCH"): - InstancedInfTable(env = { + db.parse({ "ACTIVE_PLATFORM": "Test.dsc" }) # Test 3: raise error for missing TARGET with pytest.raises(KeyError, match = "TARGET"): - InstancedInfTable(env = { + db.parse({ "ACTIVE_PLATFORM": "Test.dsc", "TARGET_ARCH": "IA32", }) @@ -67,7 +79,8 @@ def test_no_active_platform(empty_tree: Tree, caplog): def test_dsc_with_conditional(empty_tree: Tree): """Tests that conditionals inside a DSC works as expected.""" edk2path = Edk2Path(str(empty_tree.ws), []) - db = Edk2DB(Edk2DB.MEM_RW, pathobj=edk2path) + db = Edk2DB(empty_tree.ws / "db.db", pathobj=edk2path) + db.register(InstancedInfTable()) empty_tree.create_library("TestLib", "SortLib") comp1 = empty_tree.create_component('TestComponent1', 'DXE_DRIVER') @@ -79,20 +92,20 @@ def test_dsc_with_conditional(empty_tree: Tree): "!endif" ]) - inf_table = InstancedInfTable(env = { + env = { "ACTIVE_PLATFORM": dsc, "TARGET_ARCH": "IA32 X64", "TARGET": "DEBUG", - }) + } + db.parse(env) - inf_table.parse(db) - - assert len(db.table("instanced_inf")) == 0 + assert db.connection.cursor().execute("SELECT * FROM instanced_inf").fetchall() == [] def test_library_override(empty_tree: Tree): """Tests that overrides and null library overrides can be parsed as expected.""" edk2path = Edk2Path(str(empty_tree.ws), []) - db = Edk2DB(Edk2DB.MEM_RW, pathobj=edk2path) + db = Edk2DB(empty_tree.ws / "db.db", pathobj=edk2path) + db.register(InstancedInfTable()) lib1 = empty_tree.create_library("TestLib1", "TestCls") lib2 = empty_tree.create_library("TestLib2", "TestCls") @@ -118,21 +131,17 @@ def test_library_override(empty_tree: Tree): ] ) - inf_table = InstancedInfTable(env = { + env = { "ACTIVE_PLATFORM": dsc, "TARGET_ARCH": "IA32 X64", "TARGET": "DEBUG", - }) - inf_table.parse(db) + } + db.parse(env) + db.connection.execute("SELECT * FROM junction").fetchall() + library_list = db.connection.cursor().execute(GET_USED_LIBRARIES_QUERY, ("TestDriver1", "IA32")) - # Ensure the Test Driver is using TestLib2 from the override and the NULL library was added - for row in db.table("instanced_inf").all(): - if (row["NAME"] == Path(comp1).stem - and ("TestCls", Path(lib2).as_posix()) in row["LIBRARIES_USED"] - and ("NULL", Path(lib3).as_posix()) in row["LIBRARIES_USED"]): - break - else: - assert False + for path, in library_list: + assert path in [lib2, lib3] def test_scoped_libraries1(empty_tree: Tree): """Ensure that the correct libraries in regards to scoping. @@ -143,7 +152,8 @@ def test_scoped_libraries1(empty_tree: Tree): 2. $(ARCH) """ edk2path = Edk2Path(str(empty_tree.ws), []) - db = Edk2DB(Edk2DB.MEM_RW, pathobj=edk2path) + db = Edk2DB(empty_tree.ws / "db.db", pathobj=edk2path) + db.register(InstancedInfTable()) lib1 = empty_tree.create_library("TestLib1", "TestCls") lib2 = empty_tree.create_library("TestLib2", "TestCls") @@ -162,17 +172,17 @@ def test_scoped_libraries1(empty_tree: Tree): components_ia32 = [comp2, comp3] ) - inf_table = InstancedInfTable(env = { + env = { "ACTIVE_PLATFORM": dsc, "TARGET_ARCH": "IA32 X64", "TARGET": "DEBUG", - }) - inf_table.parse(db) + } + db.parse(env) - # For each driver, verify that the the driver number (1, 2, 3) uses the corresponding lib number (1, 2, 3) - for row in db.table("instanced_inf").all(): - if "COMPONENT" not in row: # Only care about looking at drivers, which do not have a component - assert row["NAME"].replace("Driver", "Lib") in row["LIBRARIES_USED"][0][1] + for arch in ["IA32", "X64"]: + for component, in db.connection.execute("SELECT name FROM instanced_inf WHERE component IS NULL and arch is ?;", (arch,)): + component_lib = db.connection.execute(GET_USED_LIBRARIES_QUERY, (component, arch)).fetchone()[0] + assert component.replace("Driver", "Lib") in component_lib def test_scoped_libraries2(empty_tree: Tree): """Ensure that the correct libraries in regards to scoping. @@ -183,7 +193,8 @@ def test_scoped_libraries2(empty_tree: Tree): 2. common """ edk2path = Edk2Path(str(empty_tree.ws), []) - db = Edk2DB(Edk2DB.MEM_RW, pathobj=edk2path) + db = Edk2DB(empty_tree.ws / "db.db", pathobj=edk2path) + db.register(InstancedInfTable()) lib1 = empty_tree.create_library("TestLib1", "TestCls") lib2 = empty_tree.create_library("TestLib2", "TestCls") @@ -198,21 +209,23 @@ def test_scoped_libraries2(empty_tree: Tree): components_x64 = [comp1, comp2], ) - inf_table = InstancedInfTable(env = { + env = { "ACTIVE_PLATFORM": dsc, "TARGET_ARCH": "IA32 X64", "TARGET": "DEBUG", - }) - inf_table.parse(db) + } + db.parse(env) - for row in db.table("instanced_inf").all(): - if "COMPONENT" not in row: - assert row["NAME"].replace("Driver", "Lib") in row["LIBRARIES_USED"][0][1] + for arch in ["IA32", "X64"]: + for component, in db.connection.execute("SELECT name FROM instanced_inf WHERE component IS NULL and arch is ?;", (arch,)): + component_lib = db.connection.execute(GET_USED_LIBRARIES_QUERY, (component, arch)).fetchone()[0] + assert component.replace("Driver", "Lib") in component_lib -def test_missing_library(empty_tree: Tree, caplog): +def test_missing_library(empty_tree: Tree): """Test when a library is missing.""" edk2path = Edk2Path(str(empty_tree.ws), []) - db = Edk2DB(Edk2DB.MEM_RW, pathobj=edk2path) + db = Edk2DB(empty_tree.ws / "db.db", pathobj=edk2path) + db.register(InstancedInfTable()) comp1 = empty_tree.create_component("TestDriver1", "PEIM", libraryclasses = ["TestCls"]) @@ -222,76 +235,55 @@ def test_missing_library(empty_tree: Tree, caplog): components_x64 = [comp1], ) - inf_table = InstancedInfTable(env = { + env = { "ACTIVE_PLATFORM": dsc, "TARGET_ARCH": "IA32 X64", "TARGET": "DEBUG", - }) + } + db.parse(env) + key2 = db.connection.execute("SELECT key2 FROM junction").fetchone()[0] + assert key2 is None # This library class does not have an instance available, so key2 should be None - with caplog.at_level(logging.WARNING): - inf_table.parse(db) - - assert len(caplog.records) == 1 - assert 'testcls' in caplog.records[0].message - -def test_skip_library_with_unsupported_module_type(empty_tree: Tree): - """Library class INFs can specify what module types they support. - - In situations where a library class is in the [LibraryClasses] section, it may not necessarily - support all module types as the LIBRARY_CLASS section of the INF may limit its supported - module types. This test ensures that a library instance is ignored if the library instance - itself states it does not support a module type. - - i.e. LIBRARY_CLASS = TestCls| PEIM only supports PEIM's, even if it is in the [LibraryClasses] section. - """ +def test_multiple_library_class(empty_tree: Tree): + """Test that a library INF that has multiple library class definitions is handled correctly.""" edk2path = Edk2Path(str(empty_tree.ws), []) - db = Edk2DB(Edk2DB.MEM_RW, pathobj=edk2path) - - testlib1 = empty_tree.create_library("TestLib1", "TestCls", - defines = { - "LIBRARY_CLASS": "TestCls| PEIM" - } - ) - testlib2 = empty_tree.create_library("TestLib2", "TestCls") - comp1 = empty_tree.create_component("TestDriver1", "DXE_DRIVER", libraryclasses = ["TestCls"]) + db = Edk2DB(empty_tree.ws / "db.db", pathobj=edk2path) + db.register(InstancedInfTable()) + + lib1 = empty_tree.create_library("TestLib", "TestCls", default = { + "MODULE_TYPE": "BASE", + "BASE_NAME": "TestLib1", + "LIBRARY_CLASS 1": "TestCls1", + "LIBRARY_CLASS 2": "TestCls2", + }) - # Generate the DSC with testlib1 first - dsc1 = empty_tree.create_dsc( - libraryclasses = [ - f'TestCls|{testlib1}', - f'TestCls|{testlib2}', + comp1 = empty_tree.create_component("TestDriver1", "DXE_RUNTIME_DRIVER", libraryclasses = ["TestCls1"]) + comp2 = empty_tree.create_component("TestDriver2", "DXE_DRIVER", libraryclasses = ["TestCls2"]) - ], - components = [comp1], - ) - - # Generate the DSC with testlib2 first - dsc2 = empty_tree.create_dsc( + dsc = empty_tree.create_dsc( libraryclasses = [ - f'TestCls|{testlib2}', - f'TestCls|{testlib1}', - + f'TestCls1|{lib1}', + f'TestCls2|{lib1}' ], - components = [comp1], + components = [comp1, comp2], ) - inf_table = InstancedInfTable(env = { - "ACTIVE_PLATFORM": dsc1, + env = { + "ACTIVE_PLATFORM": dsc, "TARGET_ARCH": "X64", "TARGET": "DEBUG", - }) + } - inf_table.parse(db) + db.parse(env) - inf_table = InstancedInfTable(env = { - "ACTIVE_PLATFORM": dsc2, - "TARGET_ARCH": "X64", - "TARGET": "DEBUG", - }) + results = db.connection.execute("SELECT key1, key2 FROM junction").fetchall() - inf_table.parse(db) + # Verify that TestDrver1 uses TestLib acting as TestCls1 + assert results[0] == ('2','1') # idx 2 is TestDriver1, idx1 is TestLib1 acting as TestCsl1 + assert ("TestLib", "TestCls1") == db.connection.execute("SELECT name, class FROM instanced_inf where id = 1").fetchone() + assert ("TestDriver1",) == db.connection.execute("SELECT name FROM instanced_inf where id = 2").fetchone() - component_list = db.table("instanced_inf").search(~Query().COMPONENT.exists()) - assert len(component_list) == 2 - for component in component_list: - assert component["LIBRARIES_USED"][0] == ("TestCls", Path(testlib2).as_posix()) + # Verify that TestDriver2 uses TestLib acting as TestCls2 + assert results[1] == ('4', '3') # idx 4 is TestDriver2, idx 3 is TestLib1 acting as TestCls2 + assert ("TestLib", "TestCls2") == db.connection.execute("SELECT name, class FROM instanced_inf where id = 3").fetchone() + assert ("TestDriver2",) == db.connection.execute("SELECT name FROM instanced_inf where id = 4").fetchone() diff --git a/tests.unit/database/test_library_query.py b/tests.unit/database/test_library_query.py deleted file mode 100644 index b77811cf..00000000 --- a/tests.unit/database/test_library_query.py +++ /dev/null @@ -1,35 +0,0 @@ -## -# unittest for the LibraryQuery query -# -# Copyright (c) Microsoft Corporation -# -# Spdx-License-Identifier: BSD-2-Clause-Patent -## -# ruff: noqa: F811 -"""Unittest for the LibraryQuery query.""" -from common import Tree, empty_tree # noqa: F401 -from edk2toollib.database import Edk2DB -from edk2toollib.database.queries import LibraryQuery -from edk2toollib.database.tables import InfTable -from edk2toollib.uefi.edk2.path_utilities import Edk2Path - - -def test_simple_library_query(empty_tree: Tree): - """Tests that libraries are detected.""" - empty_tree.create_library("TestLib1", "TestCls") - empty_tree.create_library("TestLib2", "TestCls") - empty_tree.create_library("TestLib3", "TestOtherCls") - - edk2path = Edk2Path(str(empty_tree.ws), []) - db = Edk2DB(Edk2DB.MEM_RW, pathobj=edk2path) - db.register(InfTable()) - db.parse() - - result = db.search(LibraryQuery(library = "TestCls")) - assert len(result) == 2 - - result = db.search(LibraryQuery(library = "TestOtherCls")) - assert len(result) == 1 - - result = db.search(LibraryQuery()) - assert len(result) == 3 diff --git a/tests.unit/database/test_license_query.py b/tests.unit/database/test_license_query.py deleted file mode 100644 index d5bc34cd..00000000 --- a/tests.unit/database/test_license_query.py +++ /dev/null @@ -1,75 +0,0 @@ -## -# unittest for the LicenseQuery query -# -# Copyright (c) Microsoft Corporation -# -# Spdx-License-Identifier: BSD-2-Clause-Patent -## -"""Unittest for the LicenseQuery query.""" -from common import Tree, empty_tree # noqa: F401 -from edk2toollib.database import Edk2DB -from edk2toollib.database.queries import LicenseQuery -from edk2toollib.database.tables import SourceTable -from edk2toollib.uefi.edk2.path_utilities import Edk2Path - - -def test_simple_license(empty_tree: Tree): - """Tests that missing licenses are detected.""" - f1 = empty_tree.library_folder / "File.c" - f1.touch() - - with open(f1, 'a+') as f: - f.writelines([ - '/**' - ' Nothing to see here!' - '**/' - ]) - - f2 = empty_tree.library_folder / "File2.c" - f2.touch() - - with open(f2, 'a+') as f: - f.writelines([ - '/**' - ' SPDX-License-Identifier: Fake-License' - '**/' - ]) - - f3 = empty_tree.component_folder / "File3.c" - f3.touch() - - with open(f3, 'a+') as f: - f.writelines([ - '/**' - ' SPDX-License-Identifier: BSD-2-Clause-Patent' - '**/' - ]) - - f4 = empty_tree.component_folder / "File4.c" - f4.touch() - - with open(f4, 'a+') as f: - f.writelines([ - '/**' - ' Nothing to see here!' - '**/' - ]) - - edk2path = Edk2Path(str(empty_tree.ws), []) - db = Edk2DB(Edk2DB.MEM_RW, pathobj=edk2path) - db.register(SourceTable()) - db.parse() - - # Test with no filters - result = db.search(LicenseQuery()) - assert len(result) == 2 - - # Test with include filter - result = db.search(LicenseQuery(include = "Library")) - assert len(result) == 1 - assert "Library" in result[0]["PATH"] - - # Test with exclude filter - result = db.search(LicenseQuery(exclude = "Library")) - assert len(result) == 1 - assert "Driver" in result[0]["PATH"] diff --git a/tests.unit/database/test_source_table.py b/tests.unit/database/test_source_table.py index f31cff23..b803bcd3 100644 --- a/tests.unit/database/test_source_table.py +++ b/tests.unit/database/test_source_table.py @@ -7,7 +7,7 @@ ## """Tests for building a source file table.""" from common import write_file -from edk2toollib.database import Edk2DB +from edk2toollib.database.edk2_db import Edk2DB from edk2toollib.database.tables import SourceTable from edk2toollib.uefi.edk2.path_utilities import Edk2Path @@ -32,54 +32,46 @@ def test_source_with_license(tmp_path): """Tests that a source with a license is detected and the license is set.""" edk2path = Edk2Path(str(tmp_path), []) - db = Edk2DB(Edk2DB.MEM_RW, pathobj=edk2path) - source_table = SourceTable(n_jobs = 1) + db = Edk2DB(tmp_path / "db.db", pathobj=edk2path) + db.register(SourceTable(n_jobs = 1)) # Verify we detect c and h files for file in ["file.c", "file.h", "file.asm", "file.cpp"]: write_file(tmp_path / file, SOURCE_LICENSE) - source_table.parse(db) - table = db.table("source") - assert len(table) == 1 - row = table.all()[0] - assert row["PATH"] == (tmp_path / file).relative_to(tmp_path).as_posix() - assert row["LICENSE"] == "BSD-2-Clause-Patent" + db.parse({}) - db.drop_table("source") - (tmp_path / file).unlink() + rows = list(db.connection.cursor().execute("SELECT license FROM source")) + assert len(rows) == 4 + for license, in rows: + assert license == "BSD-2-Clause-Patent" - # Ensure we don't catch a file that isnt a c / h file. - write_file(tmp_path / "file1.py", SOURCE_LICENSE) - source_table.parse(db) - table = db.table("source") - assert len(table) == 0 - def test_source_without_license(tmp_path): """Tests that a source without a license is detected.""" edk2path = Edk2Path(str(tmp_path), []) - db = Edk2DB(Edk2DB.MEM_RW, pathobj=edk2path) - source_table = SourceTable(n_jobs = 1) - + db = Edk2DB(tmp_path / "db.db", pathobj=edk2path) + db.register(SourceTable(n_jobs = 1)) # Verify we detect c and h files for file in ["file.c", "file.h"]: write_file(tmp_path / file, SOURCE_NO_LICENSE) - source_table.parse(db) - table = db.table("source") - assert len(table) == 1 - row = table.all()[0] - assert row["PATH"] == (tmp_path / file).relative_to(tmp_path).as_posix() - assert row["LICENSE"] == "" + db.parse({}) - db.drop_table("source") - (tmp_path / file).unlink() + rows = list(db.connection.cursor().execute("SELECT license FROM source")) + assert len(rows) == 2 + for license, in rows: + assert license == "Unknown" +def test_invalid_filetype(tmp_path): + """Tests that a source file that is not of the valid type is skipped.""" + edk2path = Edk2Path(str(tmp_path), []) + db = Edk2DB(tmp_path / "db.db", pathobj=edk2path) + db.register(SourceTable(n_jobs = 1)) # Ensure we don't catch a file that isnt a c / h file. write_file(tmp_path / "file1.py", SOURCE_LICENSE) - source_table.parse(db) - table = db.table("source") - assert len(table) == 0 + db.parse({}) + rows = list(db.connection.cursor().execute("SELECT license FROM source")) + assert len(rows) == 0 diff --git a/tests.unit/database/test_unused_component_query.py b/tests.unit/database/test_unused_component_query.py deleted file mode 100644 index 19434e62..00000000 --- a/tests.unit/database/test_unused_component_query.py +++ /dev/null @@ -1,192 +0,0 @@ -## -# unittest for the UnusedComponentQuery query -# -# Copyright (c) Microsoft Corporation -# -# Spdx-License-Identifier: BSD-2-Clause-Patent -## -# ruff: noqa: F811 -"""Unittest for the ComponentQuery query.""" -from pathlib import Path - -from common import Tree, correlate_env, empty_tree # noqa: F401 -from edk2toollib.database import Edk2DB -from edk2toollib.database.queries import UnusedComponentQuery -from edk2toollib.database.tables import EnvironmentTable, InstancedFvTable, InstancedInfTable -from edk2toollib.uefi.edk2.path_utilities import Edk2Path - - -def test_simple_unused_component(empty_tree: Tree): - """Tests that unused components are detected.""" - lib1 = empty_tree.create_library("TestLib1", "TestCls1") - lib2 = empty_tree.create_library("TestLib2", "TestCls2") - empty_tree.create_library("TestLib3", "TestCls3") - - comp1 = empty_tree.create_component( - "TestDriver1", "DXE_DRIVER", - libraryclasses = ["TestCls1"] - ) - comp2 = empty_tree.create_component( - "TestDriver2", "DXE_DRIVER", - libraryclasses = ["TestCls2"] - ) - - dsc = empty_tree.create_dsc( - libraryclasses = [ - f'TestCls1|{lib1}', - f'TestCls2|{lib2}', - ], - components = [ - comp1, - comp2, - ] - ) - - fdf = empty_tree.create_fdf( - fv_testfv = [ - f"INF {comp1}" - ] - ) - - edk2path = Edk2Path(str(empty_tree.ws), []) - db = Edk2DB(Edk2DB.MEM_RW, pathobj=edk2path) - env = { - "ACTIVE_PLATFORM": dsc, - "FLASH_DEFINITION": fdf, - "TARGET_ARCH": "IA32 X64", - "TARGET": "DEBUG", - } - db.register(InstancedFvTable(env=env), InstancedInfTable(env=env)) - db.parse() - comps, libs = db.search(UnusedComponentQuery()) - - assert len(comps) == 1 and comps[0] == Path(comp2).as_posix() - assert len(libs) == 1 and libs[0] == Path(lib2).as_posix() - -def test_env_unused_component(empty_tree: Tree): - """Tests that unused components are detected for different runs.""" - lib1 = empty_tree.create_library("TestLib1", "TestCls") - lib2 = empty_tree.create_library("TestLib2", "TestCls") - lib3 = empty_tree.create_library("TestLib3", "TestCls2") - lib4 = empty_tree.create_library("TestLib4", "TestCls2") - - comp1 = empty_tree.create_component( - "TestDriver1", "DXE_DRIVER", - libraryclasses = ["TestCls"] - ) - comp2 = empty_tree.create_component( - "TestDriver2", "DXE_DRIVER", - libraryclasses = ["TestCls2"] - ) - - dsc = empty_tree.create_dsc( - libraryclasses_ia32 = [ - f'TestCls|{lib1}', - f'TestCls2|{lib3}', - ], - libraryclasses_x64 = [ - f'TestCls|{lib2}', - f'TestCls2|{lib4}', - ], - components = [ - comp1, - comp2, - ] - ) - - fdf = empty_tree.create_fdf( - fv_testfv = [ - f"INF {comp1}" - ] - ) - - edk2path = Edk2Path(str(empty_tree.ws), []) - db = Edk2DB(Edk2DB.MEM_RW, pathobj=edk2path) - - env = { - "ACTIVE_PLATFORM": dsc, - "FLASH_DEFINITION": fdf, - "TARGET_ARCH": "IA32", - "TARGET": "DEBUG", - } - db.register( - InstancedFvTable(env=env), - InstancedInfTable(env=env), - EnvironmentTable(env=env), - ) - db.parse() - correlate_env(db) - - db.clear_parsers() - env = { - "ACTIVE_PLATFORM": dsc, - "FLASH_DEFINITION": fdf, - "TARGET_ARCH": "X64", - "TARGET": "DEBUG", - } - db.register( - InstancedFvTable(env=env), - InstancedInfTable(env=env), - EnvironmentTable(env=env), - ) - db.parse(append=True) - correlate_env(db) - - # Driver 2 goes unused as it's not in the FDF - # Driver 2 is built with Lib3 as our first scan is for ARCH IA32 - comps, libs = db.search(UnusedComponentQuery(env_id = 0)) - assert len(comps) == 1 and comps[0] == Path(comp2).as_posix() - assert len(libs) == 1 and libs[0] == Path(lib3).as_posix() - - # Driver 2 goes unused as it's not in the FDF - # Driver 2 is built with Lib2 as our second scan is for ARCH X64 - comps, libs = db.search(UnusedComponentQuery(env_id = 1)) - assert len(comps) == 1 and comps[0] == Path(comp2).as_posix() - assert len(libs) == 1 and libs[0] == Path(lib4).as_posix() - - # Driver 2 goes unused twice (It is built for IA32 and X64) as it's not in the FDF - # Driver 2 uses Lib3 for once instance, and Lib4 for the other, so both are considered unused - comps, libs = db.search(UnusedComponentQuery()) - assert len(comps) == 1 and comps[0] == Path(comp2).as_posix() - assert len(libs) == 2 and sorted(libs) == sorted([Path(lib3).as_posix(), Path(lib4).as_posix()]) - -def test_ignore_uefi_application(empty_tree: Tree): - """Tests that UEFI_APPLICATION components are ignored.""" - lib1 = empty_tree.create_library("TestLib1", "TestCls1") - - comp1 = empty_tree.create_component( - "TestDriver1", "UEFI_APPLICATION", - libraryclasses = ["TestCls1"] - ) - - dsc = empty_tree.create_dsc( - libraryclasses = [ - f'TestCls1|{lib1}', - ], - components = [ - comp1, - ] - ) - - fdf = empty_tree.create_fdf( - fv_testfv = [] - ) - - edk2path = Edk2Path(str(empty_tree.ws), []) - db = Edk2DB(Edk2DB.MEM_RW, pathobj=edk2path) - env = { - "ACTIVE_PLATFORM": dsc, - "FLASH_DEFINITION": fdf, - "TARGET_ARCH": "IA32 X64", - "TARGET": "DEBUG", - } - db.register(InstancedFvTable(env=env), InstancedInfTable(env=env)) - db.parse(env) - comps, libs = db.search(UnusedComponentQuery()) - - assert len(comps) == 1 and comps[0] == Path(comp1).as_posix() - assert len(libs) == 1 and libs[0] == Path(lib1).as_posix() - - comps, libs = db.search(UnusedComponentQuery(ignore_app = True)) - assert len(comps) == 0 - assert len(libs) == 0 From 15f73467ed37cabe884f706a53ea4d9cc059822c Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Wed, 6 Sep 2023 15:59:51 -0700 Subject: [PATCH 02/33] mend --- edk2toollib/database/edk2_db.py | 10 ++++------ edk2toollib/database/tables/base_table.py | 2 +- edk2toollib/database/tables/package_table.py | 4 +--- tests.unit/database/test_instanced_inf_table.py | 2 +- 4 files changed, 7 insertions(+), 11 deletions(-) diff --git a/edk2toollib/database/edk2_db.py b/edk2toollib/database/edk2_db.py index d308fbac..b49b174f 100644 --- a/edk2toollib/database/edk2_db.py +++ b/edk2toollib/database/edk2_db.py @@ -30,7 +30,7 @@ class Edk2DB: """A SQLite3 database manager for a EDKII workspace. This class provides the ability to register parsers that will create / update tables in the database. This will - create a SQLite dabase file that can be queried using any SQLite3 client. VSCode provides multiple extensions + create a SQLite datbase file that can be queried using any SQLite3 client. VSCode provides multiple extensions for viewing and interacting with the database. Queries can also be created and run in python using the sqlite3 module that comes with python. @@ -61,10 +61,7 @@ def __init__(self, db_path: str, pathobj: Edk2Path, **kwargs: dict[str,Any]): Args: db_path: Path to create or load the database from pathobj: Edk2Path object for the workspace - **kwargs: see Keyword Arguments - - Keyword Arguments: - None + **kwargs: None """ self.pathobj = pathobj self.clear_parsers() @@ -96,7 +93,8 @@ def parse(self, env: dict) -> None: """Runs all registered table parsers against the database. !!! note - An environment table and row is always added to the database. This allows for filtering based on the environment. + An environment table and row is always added to the database. This allows for filtering based on the + environment. """ self.connection.execute(CREATE_JUNCTION_TABLE) id = str(uuid.uuid4().hex) diff --git a/edk2toollib/database/tables/base_table.py b/edk2toollib/database/tables/base_table.py index 43fc231a..6c8448d7 100644 --- a/edk2toollib/database/tables/base_table.py +++ b/edk2toollib/database/tables/base_table.py @@ -26,6 +26,6 @@ def create_tables(self, db_cursor: sqlite3.Cursor) -> None: """Create the tables necessary for this parser.""" raise NotImplementedError - def parse(self, db_cursor: sqlite3.Cursor, pathobj: Edk2Path) -> None: + def parse(self, db_cursor: sqlite3.Cursor, pathobj: Edk2Path, id: str, env: dict) -> None: """Execute the parser and update the database.""" raise NotImplementedError diff --git a/edk2toollib/database/tables/package_table.py b/edk2toollib/database/tables/package_table.py index ccc5e5c7..8d2224ba 100644 --- a/edk2toollib/database/tables/package_table.py +++ b/edk2toollib/database/tables/package_table.py @@ -32,10 +32,8 @@ def __init__(self, *args, **kwargs): Args: args (any): non-keyword arguments - kwargs (any): keyword arguments described below + kwargs (any): None - Keyword Arguments: - None """ def create_tables(self, db_cursor: Cursor) -> None: diff --git a/tests.unit/database/test_instanced_inf_table.py b/tests.unit/database/test_instanced_inf_table.py index 1b988484..04e80acd 100644 --- a/tests.unit/database/test_instanced_inf_table.py +++ b/tests.unit/database/test_instanced_inf_table.py @@ -278,7 +278,7 @@ def test_multiple_library_class(empty_tree: Tree): results = db.connection.execute("SELECT key1, key2 FROM junction").fetchall() - # Verify that TestDrver1 uses TestLib acting as TestCls1 + # Verify that TestDriver1 uses TestLib acting as TestCls1 assert results[0] == ('2','1') # idx 2 is TestDriver1, idx1 is TestLib1 acting as TestCsl1 assert ("TestLib", "TestCls1") == db.connection.execute("SELECT name, class FROM instanced_inf where id = 1").fetchone() assert ("TestDriver1",) == db.connection.execute("SELECT name FROM instanced_inf where id = 2").fetchone() From f8f2f4505407a62a60812e5f040300c69c62bfe7 Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Wed, 6 Sep 2023 16:15:18 -0700 Subject: [PATCH 03/33] mend --- tests.unit/database/test_instanced_fv_table.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests.unit/database/test_instanced_fv_table.py b/tests.unit/database/test_instanced_fv_table.py index 470bdcf8..4d0f5b27 100644 --- a/tests.unit/database/test_instanced_fv_table.py +++ b/tests.unit/database/test_instanced_fv_table.py @@ -62,4 +62,10 @@ def test_valid_fdf(empty_tree: Tree): # noqa: F811 rows = db.connection.execute("SELECT key2 FROM junction where key1 == ?", (fv_id,)).fetchall() assert len(rows) == 5 - assert sorted(rows) == sorted([(comp1,), (comp2,), (comp3,), (comp4,), (comp5,)]) + assert sorted(rows) == sorted([ + (Path(comp1).as_posix(),), + (Path(comp2).as_posix(),), + (Path(comp3).as_posix(),), + (Path(comp4).as_posix(),), + (Path(comp5).as_posix(),), + ]) From 63315d8b171d756e3febce825dac6cfca48fa8e0 Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Thu, 7 Sep 2023 08:31:59 -0700 Subject: [PATCH 04/33] mend --- edk2toollib/database/tables/instanced_inf_table.py | 2 +- tests.unit/database/test_inf_table.py | 4 +++- tests.unit/database/test_instanced_inf_table.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/edk2toollib/database/tables/instanced_inf_table.py b/edk2toollib/database/tables/instanced_inf_table.py index 052e5e85..48264ed4 100644 --- a/edk2toollib/database/tables/instanced_inf_table.py +++ b/edk2toollib/database/tables/instanced_inf_table.py @@ -285,7 +285,7 @@ def _lib_to_instance(self, library_class_name, scope, library_dict, override_dic logging.debug(f'scoped library contents: {library_dict}') logging.debug(f'override dictionary: {override_dict}') e = f'Cannot find library class [{library_class_name}] for scope [{scope}] when evaluating {self.dsc}' - logging.warn(e) + logging.warning(e) return None def _reduce_lib_instances(self, module: str, library_instance_list: list[str]) -> str: diff --git a/tests.unit/database/test_inf_table.py b/tests.unit/database/test_inf_table.py index cc88cb93..a0bc9bb6 100644 --- a/tests.unit/database/test_inf_table.py +++ b/tests.unit/database/test_inf_table.py @@ -7,6 +7,8 @@ ## # ruff: noqa: F811 """Tests for build an inf file table.""" +from pathlib import Path + from common import Tree, empty_tree # noqa: F401 from edk2toollib.database import Edk2DB from edk2toollib.database.tables import InfTable @@ -51,7 +53,7 @@ def test_valid_inf(empty_tree: Tree): assert len(rows) == 2 for path, library_class in rows: - assert path in [lib1, lib2] + assert path in [Path(lib1).as_posix(), Path(lib2).as_posix()] assert library_class == "TestCls" for inf in [lib1, lib2]: diff --git a/tests.unit/database/test_instanced_inf_table.py b/tests.unit/database/test_instanced_inf_table.py index 04e80acd..d04dfc3c 100644 --- a/tests.unit/database/test_instanced_inf_table.py +++ b/tests.unit/database/test_instanced_inf_table.py @@ -141,7 +141,7 @@ def test_library_override(empty_tree: Tree): library_list = db.connection.cursor().execute(GET_USED_LIBRARIES_QUERY, ("TestDriver1", "IA32")) for path, in library_list: - assert path in [lib2, lib3] + assert path in [Path(lib2).as_posix(), Path(lib3).as_posix()] def test_scoped_libraries1(empty_tree: Tree): """Ensure that the correct libraries in regards to scoping. From a1a325bb9d8ba7ee1837e4cb0ba18d0b76cc056d Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Thu, 7 Sep 2023 09:08:55 -0700 Subject: [PATCH 05/33] mend --- docs/user/features/edk2_db.md | 130 ++++++++++++-------------------- edk2toollib/database/edk2_db.py | 2 +- 2 files changed, 49 insertions(+), 83 deletions(-) diff --git a/docs/user/features/edk2_db.md b/docs/user/features/edk2_db.md index c955fbc3..97af6fb3 100644 --- a/docs/user/features/edk2_db.md +++ b/docs/user/features/edk2_db.md @@ -1,120 +1,97 @@ # Edk2 Database -The general purpose of Edk2DB is to allow EDKII repos to query specific information about their environment. `Edk2Db` is -a subclass of [TinyDB](https://tinydb.readthedocs.io/en/latest/), with functionality expanded to meet the purpose, but -also more narrowly scoped in it's expected usage. The most notable is that `Edk2DB` narrows the types of ways it can be -instantiated to the following three scenarios: +Edk2DB enables EDKII repository developers or maintainers to query specific information about their workspace. `Edk2Db` +utilizes the sqlite3 python module to create and manipulate a sqlite database. Multiple Table generators are provided +with edk2-pytool-library that developers can register and use, however a [Table Generator](#table-generators) interface +is also provided to allow the creation of additional parsers that create tables and insert rows into them. -1. File Storage Read & Write `FILE_RW`: This mode is intended for generating a database file that can be stored and used - multiple times. This mode is slow as all database changes are written to file -2. File Storage Read Only `FILE_RO`: This mode is intended for consuming a database file generated in the above scenario - , and running queries against it -3. In-Memory Storage Read & Write `MEM_RW`: This mode is intended for running quick queries with no persistent data. +Edk2DB automatically registers an environment table which records the current environment at the time of parsing, and +provides a unique key (a uuid) for that parse to all table generators. This unique key can optionally be used as a +column in the table to distinguish common values between parsing (Such as having a database that contains parsed +information about a platform as if it was built in DEBUG mode and as if it was built in RELEASE mode. Another example +is database that contains parsed information for multiple platforms or packages.) -`Edk2DB` also adds the concepts of Managing and running [Table Generators](#table-generators) / -[Advanced Queries](#advanced-queries), which will be discussed in more detail, in their own sections. +Edk2DB automatically registers a junction table, `junction`, that acts as a lookup table between unique keys in two +tables to link them together, primarily for a one-to-many relation. One example used in the codebase is to associate +an INF file with the many source files it uses. + +The database generated in an actual sqlite database and any tools that work on a sqlite database will work on this +database. VSCode provides multiple extensions for viewing and running queries on a standalone database, along with +other downloadable tools. ## General Flow The expected usage of Edk2DB is fairly simple: -1. Instantiate the DB in the necessary mode +1. Instantiate the DB 2. Register and run the necessary table generators -3. (optional) run advanced queries to generate wanted data +3. (optional) run queries on the database through python's sqlite3 module 4. Release the database +5. (optional) run queries on the database through external tools ### Instantiate Edk2DB -As mentioned above, there are three ways to instantiate the database, depending on your needs. Edk2DB requires that you -define the mode you are attempting to insatiate in, then provide the additional required arguments as kwargs. +Edk2DB supports normal instantiation and instantiation through a context manager. It is suggested to open the database +through a context manager, but if using it through normal instantion, remember to do a a final `db.connection.commit()` +and `db.connection.close()` to cleanly close the database. ``` python -# File Storage, Read and Write -db = Edk2DB(Edk2DB.FILE_RW, db_path=db_path, pathobj=pathobj) +db = Edk2DB(db_path, pathobj=pathobj) -# File Storage, Read Only -db = Edk2DB(Edk2DB.FILE_RO, db_path=db_path) +with Edk2DB(db_path, pathobj=pathobj) as db: + ... -# In-Memory Storage, Read Only -db = Edk2DB(Edk2DB.MEM_RW, pathobj=pathobj) -``` - -Additionally, you can **and should** instantiate the database using a context manager, to ensure the database is -properly released when finished: - -``` python -with Edk2DB(mode, **kwargs) as db: - ... ``` ### Register and run table generators A [Table Generator](#table-generators) is a type of parser that creates a table in the database and fills it with rows -of data. While each table generator can generate one or more tables, they should never rely on, or expect, other tables -existing to generate it's own table. Advanced Queries should be used to make those associations. - -It's simple to register a table generator! simply call the `register()` with one or more of the instantiated parsers: +of data. A Table Generator should never expect specific data in a table to exist. It's simple to register a table +generator! simply call the `register()` with one or more of the instantiated parsers: ``` python db.register(Parser1()) db.register(Parser2(), Parser3()) ``` -If your parser needs some type of metadata (As an example, a few of the provided parsers need environment information), -then it can be set using the initializer of the Parser (`__init__(*args, **kwargs)`). +If your parser needs some type of metadata, then that metadata can be set in the initialization of the Parser +(`__init__(*args, **kwargs)`). -You can also clear your registered parsers, which may be necessary in some situations, such as re-running the same -parser with different environment information: +``` python +db.register(Parser1(x = 5, y = 7)) +``` + +A method is provided to clear any registered parsers: ``` python db.clear_parsers() ``` -Lastly is running all registered parsers. The `parse()` command will iterate through all registered parsers and run -them. If you need to run the same Parser with different sets of metadata, you have two options: +Lastly is running all registered parsers. The `parse(env: dict)` method expects to be provided a dictionary of +environment variables used when building a platform. Depending on the parser, the dictionary can be empty. + +The `parse(env: dict)` command will perform two loops across the parsers.The first loop will create all tables for all +table parsers. This ensures that any dependencies on tables existing between parsers is handled. The second loop +performs the parsing and row insertion. The order in which parsers execute is the same as the order that they are +registered. ```python # Option 1: parse one at a time -db.register(Parser(env=env1)) +db.register(Parser(key=value2)) db.parse() db.clear_parsers() -db.register(Parser(env=env2)) -db.parse(append=True) +db.register(Parser(key=value2)) +db.parse(env) # Option 2: parse together -db.register(Parser(env=env1), Parser(env=env2)) -db.parse() +db.register(Parser(key=value1), Parser(key=value2)) +db.parse(env) ``` -If for some reason, it is necessary to keep Table generators and the database separate, you can reverse the function call: - -```python -# Before -db.register(Parser()) -db.parse() - -# Reversed -Parser().parse(db) -``` - -### Run Advanced Queries - -Running advanced queries is simple! Similar to Table Generators, you will pass the instantiated Query, passing any -necessary metadata to the query to work properly: - -```python -db.search(AdvancedQuery(cfg1=cfg1, cfg2=cfg2)) -``` - -TinyDB does not support relationships betweed tables (i.e. Primary / Foreign Keys and JOINs). Due to this, the intent of -the Advanced Query is to compartmentalize the multiple query calls that may be necessary to mock that functionality. -Therefore, the expected return should continue to be a list of rows (json objects), similar to any other database query. -What the caller wishes to do with that data is up to them. - ### Release the Database -If you are using a context manager, then this is handled automatically for you. Otherwise, you need to call `.close()` -on the database. +If you are using a context manager, then this is handled automatically for you. Otherwise, you need to call +`db.connection.commit()` and `db.connection.close()` on the database (or `__exit__()`) ## Table Generators @@ -122,14 +99,3 @@ Table generators are just that, classes that subclass the [TableGenerator](/api/ , parse some type of information (typically the workspace) and insert the data into one of the tables managed by Edk2DB. Multiple table generators are provided by edk2toollib, and can be seen at [edk2toollib/database/tables](https://github.com/tianocore/edk2-pytool-library/tree/master/edk2toollib/database/tables). Edk2DB can use any class that implements the `TableGenerator` interface. - -## Advanced Queries - -Edk2DB supports running simple queries as defined by [TinyDb Query](https://tinydb.readthedocs.io/en/latest/usage.html#queries), -however TinyDB does not support relationships between tables (i.e. Primary Key / Foreign keys and JOINs). Due to this -limitation, the concept of `Advanced Queries` was created to compartmentalize the extra steps necessary to emulate the -functionality above. The [AdvancedQuery](/api/database/edk2_db/#edk2toollib.database.edk2_db.AdvancedQuery) class is the -interface that should be subclassed when creating a more complex query than filtering on a single database table. As -with the `TableGenerator`, multiple Advanced are provided by edk2toollib, and can be seen at -[edk2toollib/database/queries](https://github.com/tianocore/edk2-pytool-library/tree/master/edk2toollib/database/queries). -Edb2DB can use any class that implements the `AdvancedQuery` interface. diff --git a/edk2toollib/database/edk2_db.py b/edk2toollib/database/edk2_db.py index b49b174f..d96be69f 100644 --- a/edk2toollib/database/edk2_db.py +++ b/edk2toollib/database/edk2_db.py @@ -55,7 +55,7 @@ class Edk2DB: db.connection.execute(f"SELECT * FROM {table}", table) """ - def __init__(self, db_path: str, pathobj: Edk2Path, **kwargs: dict[str,Any]): + def __init__(self, db_path: str, pathobj: Edk2Path = None, **kwargs: dict[str,Any]): """Initializes the database. Args: From 7d310e47a70fd97d7a506892cf72f3ca26cd83ee Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Fri, 8 Sep 2023 12:05:21 -0700 Subject: [PATCH 06/33] Update --- edk2toollib/database/tables/environment_table.py | 2 +- edk2toollib/database/tables/inf_table.py | 2 +- edk2toollib/database/tables/instanced_inf_table.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/edk2toollib/database/tables/environment_table.py b/edk2toollib/database/tables/environment_table.py index 9d2c3a7c..801e7cb4 100644 --- a/edk2toollib/database/tables/environment_table.py +++ b/edk2toollib/database/tables/environment_table.py @@ -55,5 +55,5 @@ def parse(self, db_cursor: sqlite3.Cursor, pathobj: Edk2Path, id, env) -> None: db_cursor.execute("INSERT INTO environment (id, date,version) VALUES (?, ?, ?)", entry) # Insert into environment_values table - data = [(db_cursor.lastrowid, key, value) for key, value in env.items()] + data = [(id, key, value) for key, value in env.items()] db_cursor.executemany("INSERT INTO environment_values VALUES (?, ?, ?)", data) diff --git a/edk2toollib/database/tables/inf_table.py b/edk2toollib/database/tables/inf_table.py index 8de6a219..db7523b1 100644 --- a/edk2toollib/database/tables/inf_table.py +++ b/edk2toollib/database/tables/inf_table.py @@ -102,7 +102,7 @@ def _parse_file(self, filename, pathobj) -> dict: path = path[path.find(pkg):] data = {} data["GUID"] = inf_parser.Dict.get("FILE_GUID", "") - data["LIBRARY_CLASS"] = inf_parser.LibraryClass + data["LIBRARY_CLASS"] = inf_parser.LibraryClass or None data["PATH"] = Path(path).as_posix() data["PHASES"] = inf_parser.SupportedPhases data["SOURCES_USED"] = inf_parser.Sources diff --git a/edk2toollib/database/tables/instanced_inf_table.py b/edk2toollib/database/tables/instanced_inf_table.py index 48264ed4..2025452e 100644 --- a/edk2toollib/database/tables/instanced_inf_table.py +++ b/edk2toollib/database/tables/instanced_inf_table.py @@ -110,7 +110,7 @@ def parse(self, db_cursor: Cursor, pathobj: Edk2Path, id: str, env: dict) -> Non # add instanced_inf entries for e in inf_entries: - row = (id, e["PATH"], e["LIBRARY_CLASS"], e["NAME"], e["ARCH"], e["DSC"], e["COMPONENT"]) + row = (id, e["PATH"], e["LIBRARY_CLASS"] or None, e["NAME"], e["ARCH"], e["DSC"], e["COMPONENT"]) db_cursor.execute(INSERT_INSTANCED_INF_ROW, row) for e in inf_entries: From e6087c49e10274df2f8f6781c7d18697aac520e0 Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Fri, 8 Sep 2023 12:31:29 -0700 Subject: [PATCH 07/33] tmp update --- edk2toollib/database/tables/instanced_inf_table.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/edk2toollib/database/tables/instanced_inf_table.py b/edk2toollib/database/tables/instanced_inf_table.py index 2025452e..f7f0b6f6 100644 --- a/edk2toollib/database/tables/instanced_inf_table.py +++ b/edk2toollib/database/tables/instanced_inf_table.py @@ -126,7 +126,11 @@ def parse(self, db_cursor: Cursor, pathobj: Edk2Path, id: str, env: dict) -> Non if instance is None: used_inf_id = None # no library instance found for this library class else: - used_inf_id = db_cursor.execute(GET_ROW_ID, (id, instance, e["DSC"], cls)).fetchone()[0] + try: + used_inf_id = db_cursor.execute(GET_ROW_ID, (id, instance, e["DSC"], cls)).fetchone()[0] + except Exception: + logging.warning(f"Failed to get row id for {id},{instance},{e['DSC']}{cls}") + used_inf_id = None row = (id, "instanced_inf", inf_id, "instanced_inf", used_inf_id) db_cursor.execute(INSERT_JUNCTION_ROW, row) From 750579597dcc71d3d31ea059e57851e9cd141891 Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Fri, 8 Sep 2023 13:29:49 -0700 Subject: [PATCH 08/33] update --- edk2toollib/database/tables/instanced_inf_table.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/edk2toollib/database/tables/instanced_inf_table.py b/edk2toollib/database/tables/instanced_inf_table.py index f7f0b6f6..87c3dd89 100644 --- a/edk2toollib/database/tables/instanced_inf_table.py +++ b/edk2toollib/database/tables/instanced_inf_table.py @@ -43,7 +43,7 @@ class TEXT, GET_ROW_ID = ''' SELECT id FROM instanced_inf -WHERE env = ? and path = ? and dsc = ? and (class = ? OR class IS NULL) +WHERE env = ? and path = ? and component = ? LIMIT 1 ''' @@ -114,7 +114,7 @@ def parse(self, db_cursor: Cursor, pathobj: Edk2Path, id: str, env: dict) -> Non db_cursor.execute(INSERT_INSTANCED_INF_ROW, row) for e in inf_entries: - inf_id = db_cursor.execute(GET_ROW_ID, (id, e["PATH"], e["DSC"], e["LIBRARY_CLASS"])).fetchone()[0] + inf_id = db_cursor.execute(GET_ROW_ID, (id, e["PATH"], e["COMPONENT"])).fetchone()[0] # Add junction entries to link source the source files used by an INF for source in e["SOURCES_USED"]: @@ -126,11 +126,7 @@ def parse(self, db_cursor: Cursor, pathobj: Edk2Path, id: str, env: dict) -> Non if instance is None: used_inf_id = None # no library instance found for this library class else: - try: - used_inf_id = db_cursor.execute(GET_ROW_ID, (id, instance, e["DSC"], cls)).fetchone()[0] - except Exception: - logging.warning(f"Failed to get row id for {id},{instance},{e['DSC']}{cls}") - used_inf_id = None + used_inf_id = db_cursor.execute(GET_ROW_ID, (id, instance, e["COMPONENT"])).fetchone()[0] row = (id, "instanced_inf", inf_id, "instanced_inf", used_inf_id) db_cursor.execute(INSERT_JUNCTION_ROW, row) From 6d961e2cfa64b9551bf7898d0a25c3f3519d40b3 Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Fri, 8 Sep 2023 15:50:22 -0700 Subject: [PATCH 09/33] update --- edk2toollib/uefi/edk2/parsers/dsc_parser.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/edk2toollib/uefi/edk2/parsers/dsc_parser.py b/edk2toollib/uefi/edk2/parsers/dsc_parser.py index 463bf888..4f994a2d 100644 --- a/edk2toollib/uefi/edk2/parsers/dsc_parser.py +++ b/edk2toollib/uefi/edk2/parsers/dsc_parser.py @@ -366,6 +366,8 @@ def _parse_libraries(self): for scope in current_scope: key = f"{scope.strip()}.{lib.strip()}".lower() value = instance.strip() + if os.path.isabs(value): + value = self._Edk2PathUtil.GetEdk2RelativePathFromAbsolutePath(value) if key in self.ScopedLibraryDict and value not in self.ScopedLibraryDict[key]: self.ScopedLibraryDict[key].insert(0, value) else: @@ -400,6 +402,8 @@ def _parse_components(self): for scope in current_scope: # Components without a specific scope (common or empty) are added to all current scopes + if os.path.isabs(line.strip(" {")): + line = self._Edk2PathUtil.GetEdk2RelativePathFromAbsolutePath(line.strip(" {")) if "common" in current_scope[0]: for arch in self.InputVars.get("TARGET_ARCH", "").split(" "): scope = current_scope[0].replace("common", arch).lower() @@ -459,8 +463,11 @@ def _build_library_override_dictionary(self, lines): if section == self.SECTION_LIBRARY: logging.debug(f" Library Section Override: {line}") lib, instance = map(str.strip, line.split("|")) - lib = lib.lower() + if os.path.isabs(instance): + instance = self._Edk2PathUtil.GetEdk2RelativePathFromAbsolutePath(instance) + + lib = lib.lower() if lib == "null": library_override_dictionary["NULL"].append(instance) else: From 53362884690c7dc25b76547774e662001ad99496 Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Mon, 11 Sep 2023 07:34:25 -0700 Subject: [PATCH 10/33] Update Tests --- tests.unit/database/test_environment_table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests.unit/database/test_environment_table.py b/tests.unit/database/test_environment_table.py index 468769e5..01fbacdc 100644 --- a/tests.unit/database/test_environment_table.py +++ b/tests.unit/database/test_environment_table.py @@ -71,7 +71,7 @@ def test_environment_with_vars(tmp_path): assert actual_date.split(" ")[0] == str(date.today()) assert actual_version == 'UNKNOWN' - rows = list(db.connection.cursor().execute("SELECT key, value FROM environment_values WHERE id = 1")) + rows = list(db.connection.cursor().execute("SELECT * FROM environment_values")) assert len(rows) == 4 db.parse(env) From fd7152959975acb0e9b897244ef1178f05753b58 Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Mon, 11 Sep 2023 13:10:22 -0700 Subject: [PATCH 11/33] update --- tests.unit/database/test_edk2_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests.unit/database/test_edk2_db.py b/tests.unit/database/test_edk2_db.py index 5d34bde3..403ee85a 100644 --- a/tests.unit/database/test_edk2_db.py +++ b/tests.unit/database/test_edk2_db.py @@ -25,7 +25,7 @@ def test_load_existing_db(empty_tree: Tree): assert db_path.exists() is False with Edk2DB(db_path, pathobj=edk2path) as db: - db.register(InfTable()) + db.register(InfTable(n_jobs = 1)) db.parse({}) result = db.connection.execute("SELECT name FROM sqlite_master WHERE type='table' AND name=?", ("inf",)).fetchone() assert result is not None From b99b3b8ac373dccce8f16e1d7bfaee613ca51a5e Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Tue, 12 Sep 2023 08:08:59 -0700 Subject: [PATCH 12/33] Skip instanced_fv_table if the FDF is not set --- edk2toollib/database/tables/instanced_fv_table.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/edk2toollib/database/tables/instanced_fv_table.py b/edk2toollib/database/tables/instanced_fv_table.py index 3174e6c7..8d9e5edf 100644 --- a/edk2toollib/database/tables/instanced_fv_table.py +++ b/edk2toollib/database/tables/instanced_fv_table.py @@ -7,6 +7,7 @@ # SPDX-License-Identifier: BSD-2-Clause-Patent ## """A module to generate a table containing fv information.""" +import logging import re import sqlite3 from pathlib import Path @@ -53,11 +54,15 @@ def parse(self, db_cursor: sqlite3.Cursor, pathobj: Edk2Path, id, env) -> None: self.pathobj = pathobj self.ws = Path(self.pathobj.WorkspacePath) self.env = env - self.dsc = self.env["ACTIVE_PLATFORM"] - self.fdf = self.env["FLASH_DEFINITION"] + self.dsc = self.env.get("ACTIVE_PLATFORM", None) + self.fdf = self.env.get("FLASH_DEFINITION", None) self.arch = self.env["TARGET_ARCH"].split(" ") self.target = self.env["TARGET"] + if self.dsc is None or self.fdf is None: + logging.debug("DSC or FDF not found in environment. Skipping InstancedFvTable") + return + # Our DscParser subclass can now parse components, their scope, and their overrides fdfp = FdfP().SetEdk2Path(self.pathobj) fdfp.SetInputVars(self.env) From 4353ab7a1f59c2e261b25fa5955307c419bf2848 Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Fri, 15 Sep 2023 11:35:48 -0700 Subject: [PATCH 13/33] Update --- tests.unit/database/test_inf_table.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests.unit/database/test_inf_table.py b/tests.unit/database/test_inf_table.py index a0bc9bb6..996622b1 100644 --- a/tests.unit/database/test_inf_table.py +++ b/tests.unit/database/test_inf_table.py @@ -9,7 +9,7 @@ """Tests for build an inf file table.""" from pathlib import Path -from common import Tree, empty_tree # noqa: F401 +from common import Tree, empty_tree, write_file # noqa: F401 from edk2toollib.database import Edk2DB from edk2toollib.database.tables import InfTable from edk2toollib.uefi.edk2.path_utilities import Edk2Path @@ -47,6 +47,12 @@ def test_valid_inf(empty_tree: Tree): sources_ia32 = sources_ia32, sources_x64 = sources_x64, ) + + (empty_tree.library_folder / "IA32").mkdir() + (empty_tree.library_folder / "X64").mkdir() + for file in sources + sources_ia32 + sources_x64: + write_file((empty_tree.library_folder / file).resolve(), "FILLER") + db.parse({}) rows = list(db.connection.cursor().execute("SELECT path, library_class FROM inf")) From d2c70df4cfe97526333d3c257d95a0e951ad3592 Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Fri, 15 Sep 2023 12:13:13 -0700 Subject: [PATCH 14/33] Update --- edk2toollib/database/tables/inf_table.py | 4 +--- tests.unit/database/test_inf_table.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/edk2toollib/database/tables/inf_table.py b/edk2toollib/database/tables/inf_table.py index db7523b1..df709625 100644 --- a/edk2toollib/database/tables/inf_table.py +++ b/edk2toollib/database/tables/inf_table.py @@ -86,9 +86,7 @@ def parse(self, db_cursor: Cursor, pathobj: Edk2Path, env_id: str, env: dict) -> db_cursor.execute(INSERT_JUNCTION_ROW, row) for source in inf["SOURCES_USED"]: - source_path = pathobj.GetEdk2RelativePathFromAbsolutePath( - (Path(inf["PATH"]).parent / source).resolve().as_posix() - ) + source_path = (Path(inf["PATH"]).parent / source).as_posix() row = (env_id, "inf", inf["PATH"], "source", source_path) db_cursor.execute(INSERT_JUNCTION_ROW, row) diff --git a/tests.unit/database/test_inf_table.py b/tests.unit/database/test_inf_table.py index 996622b1..ff73832e 100644 --- a/tests.unit/database/test_inf_table.py +++ b/tests.unit/database/test_inf_table.py @@ -62,6 +62,6 @@ def test_valid_inf(empty_tree: Tree): assert path in [Path(lib1).as_posix(), Path(lib2).as_posix()] assert library_class == "TestCls" - for inf in [lib1, lib2]: + for inf in [Path(lib1).as_posix(), Path(lib2).as_posix()]: rows = db.connection.execute("SELECT * FROM junction WHERE key1 = ? AND table2 = 'source'", (inf,)).fetchall() assert len(rows) == 3 From d1e4b5e0df6cbfa4edbf3829d3d6243896ba9f0d Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Mon, 18 Sep 2023 09:40:38 -0700 Subject: [PATCH 15/33] Update --- docs/user/features/edk2_db.md | 2 ++ edk2toollib/database/edk2_db.py | 8 ++--- .../database/tables/instanced_inf_table.py | 6 ++-- tests.unit/database/test_edk2_db.py | 12 +++++-- .../database/test_instanced_fv_table.py | 28 +++++++++++++--- .../database/test_instanced_inf_table.py | 32 +++++++++++++++++++ 6 files changed, 73 insertions(+), 15 deletions(-) diff --git a/docs/user/features/edk2_db.md b/docs/user/features/edk2_db.md index 97af6fb3..5e66c088 100644 --- a/docs/user/features/edk2_db.md +++ b/docs/user/features/edk2_db.md @@ -37,6 +37,8 @@ and `db.connection.close()` to cleanly close the database. ``` python db = Edk2DB(db_path, pathobj=pathobj) +db.commit() +db.close() with Edk2DB(db_path, pathobj=pathobj) as db: ... diff --git a/edk2toollib/database/edk2_db.py b/edk2toollib/database/edk2_db.py index d96be69f..ac5186e8 100644 --- a/edk2toollib/database/edk2_db.py +++ b/edk2toollib/database/edk2_db.py @@ -35,8 +35,8 @@ class Edk2DB: module that comes with python. Edk2DB can, and should, be used as a context manager to ensure that the database is closed properly. If - not using as a context manager, the `close()` method must be used to ensure that the database is closed properly - and any changes are saved. + not using as a context manager, the `db.connection.commit()` and `db.connection.close()` must be used to cleanly + close the database. Attributes: connection (sqlite3.Connection): The connection to the database @@ -93,8 +93,8 @@ def parse(self, env: dict) -> None: """Runs all registered table parsers against the database. !!! note - An environment table and row is always added to the database. This allows for filtering based on the - environment. + To enable queries to differentiate between two parses, an environment table is always created if it does + not exist, and a row is added for each call of this command. """ self.connection.execute(CREATE_JUNCTION_TABLE) id = str(uuid.uuid4().hex) diff --git a/edk2toollib/database/tables/instanced_inf_table.py b/edk2toollib/database/tables/instanced_inf_table.py index 87c3dd89..c32f7e50 100644 --- a/edk2toollib/database/tables/instanced_inf_table.py +++ b/edk2toollib/database/tables/instanced_inf_table.py @@ -104,9 +104,9 @@ def parse(self, db_cursor: Cursor, pathobj: Edk2Path, id: str, env: dict) -> Non # # This is where we merge DSC parser information with INF parser information. inf_entries = self._build_inf_table(dscp) - for e in inf_entries: - if Path(e["PATH"]).is_absolute(): - e["PATH"] = self.pathobj.GetEdk2RelativePathFromAbsolutePath(e["PATH"]) + # for e in inf_entries: + # if Path(e["PATH"]).is_absolute(): + # e["PATH"] = self.pathobj.GetEdk2RelativePathFromAbsolutePath(e["PATH"]) # add instanced_inf entries for e in inf_entries: diff --git a/tests.unit/database/test_edk2_db.py b/tests.unit/database/test_edk2_db.py index 403ee85a..5e0014f1 100644 --- a/tests.unit/database/test_edk2_db.py +++ b/tests.unit/database/test_edk2_db.py @@ -44,11 +44,17 @@ def test_catch_bad_parser_and_query(empty_tree: Tree): db_path = empty_tree.ws / "test.db" assert db_path.exists() is False - with pytest.raises(NotImplementedError): - with Edk2DB(db_path, pathobj=edk2path) as db: - db.register(TableGenerator()) + + with Edk2DB(db_path, pathobj=edk2path) as db: + parser = TableGenerator() + db.register(parser) + + with pytest.raises(NotImplementedError): db.parse({}) + with pytest.raises(NotImplementedError): + parser.parse(db.connection.cursor(), db.pathobj, 0, {}) + def test_clear_parsers(empty_tree: Tree): """Test that we can clear all parsers. EnvironmentTable should always persist.""" edk2path = Edk2Path(str(empty_tree.ws), []) diff --git a/tests.unit/database/test_instanced_fv_table.py b/tests.unit/database/test_instanced_fv_table.py index 4d0f5b27..919cb602 100644 --- a/tests.unit/database/test_instanced_fv_table.py +++ b/tests.unit/database/test_instanced_fv_table.py @@ -8,6 +8,7 @@ """Unittest for the InstancedFv table generator.""" from pathlib import Path +import logging import pytest from common import Tree, empty_tree # noqa: F401 from edk2toollib.database import Edk2DB @@ -26,11 +27,6 @@ def test_valid_fdf(empty_tree: Tree): # noqa: F811 db = Edk2DB(empty_tree.ws / "db.db", pathobj=edk2path) db.register(InstancedFvTable()) - # raise exception if the Table generator is missing required information to - # Generate the table. - with pytest.raises(KeyError): - db.parse({}) - comp1 = empty_tree.create_component("TestDriver1", "DXE_DRIVER") comp2 = empty_tree.create_component("TestDriver2", "DXE_DRIVER") comp3 = empty_tree.create_component("TestDriver3", "DXE_DRIVER") @@ -69,3 +65,25 @@ def test_valid_fdf(empty_tree: Tree): # noqa: F811 (Path(comp4).as_posix(),), (Path(comp5).as_posix(),), ]) + +def test_missing_dsc_and_fdf(empty_tree: Tree, caplog): + """Tests that the table generator is skipped if missing the necessary information""" + with caplog.at_level(logging.DEBUG): + edk2path = Edk2Path(str(empty_tree.ws), []) + db = Edk2DB(empty_tree.ws / "db.db", pathobj=edk2path) + db.register(InstancedFvTable()) + + # raise exception if the Table generator is missing required information to Generate the table. + with pytest.raises(KeyError): + db.parse({}) + + db.parse({"TARGET_ARCH": "", "TARGET": "DEBUG"}) + db.parse({"TARGET_ARCH": "", "TARGET": "DEBUG", "ACTIVE_PLATFORM": "Pkg.dsc"}) + + # check that we skipped (instead of asserting) twice, once for missing ACTIVE_PLATFORM and once for the + # missing FLASH_DEFINITION + count = 0 + for _, _, record in caplog.record_tuples: + if record.startswith("DSC or FDF not found"): + count += 1 + assert count == 2 diff --git a/tests.unit/database/test_instanced_inf_table.py b/tests.unit/database/test_instanced_inf_table.py index d04dfc3c..fcdd79e5 100644 --- a/tests.unit/database/test_instanced_inf_table.py +++ b/tests.unit/database/test_instanced_inf_table.py @@ -287,3 +287,35 @@ def test_multiple_library_class(empty_tree: Tree): assert results[1] == ('4', '3') # idx 4 is TestDriver2, idx 3 is TestLib1 acting as TestCls2 assert ("TestLib", "TestCls2") == db.connection.execute("SELECT name, class FROM instanced_inf where id = 3").fetchone() assert ("TestDriver2",) == db.connection.execute("SELECT name FROM instanced_inf where id = 4").fetchone() + +def test_absolute_path_for_library(empty_tree: Tree): + """Test that a library INF in the dsc works if the INF path is aboslute.""" + edk2path = Edk2Path(str(empty_tree.ws), []) + db = Edk2DB(empty_tree.ws / "db.db", pathobj=edk2path) + db.register(InstancedInfTable()) + + lib1 = empty_tree.create_library("TestLib", "TestCls") + lib2 = empty_tree.create_library("NullLib", "NullCls") + + comp1 = empty_tree.create_component("TestDriver1", "DXE_RUNTIME_DRIVER", libraryclasses = ["TestCls"]) + x = str(empty_tree.ws / comp1) + dsc = empty_tree.create_dsc( + libraryclasses = [ + f'TestCls|{str(empty_tree.ws / lib1)}' + ], + components = [ + f'{str(empty_tree.ws / comp1)} {{', + '', + f'NULL|{str(empty_tree.ws / lib2)}', + '}', + ] + ) + + env = { + "ACTIVE_PLATFORM": dsc, + "TARGET_ARCH": "X64", + "TARGET": "DEBUG", + } + + db.parse(env) + print("HELLO") From 1ff96d8517470b6ff8d1d8a17cab32e3706b90f1 Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Mon, 18 Sep 2023 09:45:48 -0700 Subject: [PATCH 16/33] update --- .../database/test_instanced_inf_table.py | 32 ------------------- 1 file changed, 32 deletions(-) diff --git a/tests.unit/database/test_instanced_inf_table.py b/tests.unit/database/test_instanced_inf_table.py index fcdd79e5..d04dfc3c 100644 --- a/tests.unit/database/test_instanced_inf_table.py +++ b/tests.unit/database/test_instanced_inf_table.py @@ -287,35 +287,3 @@ def test_multiple_library_class(empty_tree: Tree): assert results[1] == ('4', '3') # idx 4 is TestDriver2, idx 3 is TestLib1 acting as TestCls2 assert ("TestLib", "TestCls2") == db.connection.execute("SELECT name, class FROM instanced_inf where id = 3").fetchone() assert ("TestDriver2",) == db.connection.execute("SELECT name FROM instanced_inf where id = 4").fetchone() - -def test_absolute_path_for_library(empty_tree: Tree): - """Test that a library INF in the dsc works if the INF path is aboslute.""" - edk2path = Edk2Path(str(empty_tree.ws), []) - db = Edk2DB(empty_tree.ws / "db.db", pathobj=edk2path) - db.register(InstancedInfTable()) - - lib1 = empty_tree.create_library("TestLib", "TestCls") - lib2 = empty_tree.create_library("NullLib", "NullCls") - - comp1 = empty_tree.create_component("TestDriver1", "DXE_RUNTIME_DRIVER", libraryclasses = ["TestCls"]) - x = str(empty_tree.ws / comp1) - dsc = empty_tree.create_dsc( - libraryclasses = [ - f'TestCls|{str(empty_tree.ws / lib1)}' - ], - components = [ - f'{str(empty_tree.ws / comp1)} {{', - '', - f'NULL|{str(empty_tree.ws / lib2)}', - '}', - ] - ) - - env = { - "ACTIVE_PLATFORM": dsc, - "TARGET_ARCH": "X64", - "TARGET": "DEBUG", - } - - db.parse(env) - print("HELLO") From e57b6bdc34d17a66ed0794495c09ec0d33f18745 Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Mon, 18 Sep 2023 12:45:29 -0700 Subject: [PATCH 17/33] update --- edk2toollib/database/tables/instanced_inf_table.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/edk2toollib/database/tables/instanced_inf_table.py b/edk2toollib/database/tables/instanced_inf_table.py index c32f7e50..726c9ca2 100644 --- a/edk2toollib/database/tables/instanced_inf_table.py +++ b/edk2toollib/database/tables/instanced_inf_table.py @@ -104,9 +104,6 @@ def parse(self, db_cursor: Cursor, pathobj: Edk2Path, id: str, env: dict) -> Non # # This is where we merge DSC parser information with INF parser information. inf_entries = self._build_inf_table(dscp) - # for e in inf_entries: - # if Path(e["PATH"]).is_absolute(): - # e["PATH"] = self.pathobj.GetEdk2RelativePathFromAbsolutePath(e["PATH"]) # add instanced_inf entries for e in inf_entries: From 56f8df316b242cfe6b9c6130d90e2a9598186b34 Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Mon, 18 Sep 2023 13:14:45 -0700 Subject: [PATCH 18/33] update --- .../database/test_instanced_inf_table.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests.unit/database/test_instanced_inf_table.py b/tests.unit/database/test_instanced_inf_table.py index d04dfc3c..7b1fd66c 100644 --- a/tests.unit/database/test_instanced_inf_table.py +++ b/tests.unit/database/test_instanced_inf_table.py @@ -287,3 +287,32 @@ def test_multiple_library_class(empty_tree: Tree): assert results[1] == ('4', '3') # idx 4 is TestDriver2, idx 3 is TestLib1 acting as TestCls2 assert ("TestLib", "TestCls2") == db.connection.execute("SELECT name, class FROM instanced_inf where id = 3").fetchone() assert ("TestDriver2",) == db.connection.execute("SELECT name FROM instanced_inf where id = 4").fetchone() + +def test_absolute_paths_in_dsc(empty_tree: Tree): + edk2path = Edk2Path(str(empty_tree.ws), []) + db = Edk2DB(empty_tree.ws / "db.db", pathobj=edk2path) + db.register(InstancedInfTable()) + + lib1 = empty_tree.create_library("TestLib", "TestCls") + comp1 = empty_tree.create_component("TestDriver", "DXE_DRIVER", libraryclasses=["TestCls"]) + + dsc = empty_tree.create_dsc( + libraryclasses = [ + f'TestCls| {str(empty_tree.ws / lib1)}', + ], + components = [ + str(empty_tree.ws / comp1), + ], + ) + + env = { + "ACTIVE_PLATFORM": dsc, + "TARGET_ARCH": "X64", + "TARGET": "DEBUG", + } + + db.parse(env) + + results = db.connection.execute("SELECT path FROM instanced_inf").fetchall() + assert results[0] == (Path(lib1).as_posix(),) + assert results[1] == (Path(comp1).as_posix(),) From c21338b8d228a510439a6422462b6772e8babb7d Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Mon, 18 Sep 2023 13:24:20 -0700 Subject: [PATCH 19/33] Update --- tests.unit/database/test_instanced_inf_table.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests.unit/database/test_instanced_inf_table.py b/tests.unit/database/test_instanced_inf_table.py index 7b1fd66c..00e4194b 100644 --- a/tests.unit/database/test_instanced_inf_table.py +++ b/tests.unit/database/test_instanced_inf_table.py @@ -155,9 +155,9 @@ def test_scoped_libraries1(empty_tree: Tree): db = Edk2DB(empty_tree.ws / "db.db", pathobj=edk2path) db.register(InstancedInfTable()) - lib1 = empty_tree.create_library("TestLib1", "TestCls") - lib2 = empty_tree.create_library("TestLib2", "TestCls") - lib3 = empty_tree.create_library("TestLib3", "TestCls") + lib1 = empty_tree.create_library("TestLib1", "TestCls", sources = ["File1.c"]) + lib2 = empty_tree.create_library("TestLib2", "TestCls", sources = ["File2.c"]) + lib3 = empty_tree.create_library("TestLib3", "TestCls", sources = ["File3.c"]) comp1 = empty_tree.create_component("TestDriver1", "PEIM", libraryclasses = ["TestCls"]) comp2 = empty_tree.create_component("TestDriver2", "SEC", libraryclasses = ["TestCls"]) @@ -184,6 +184,11 @@ def test_scoped_libraries1(empty_tree: Tree): component_lib = db.connection.execute(GET_USED_LIBRARIES_QUERY, (component, arch)).fetchone()[0] assert component.replace("Driver", "Lib") in component_lib + results = db.connection.execute('SELECT key2 FROM junction WHERE table1 = "instanced_inf" AND table2 = "source"').fetchall() + assert len(results) == 3 + for source, in results: + assert source in ["File1.c", "File2.c", "File3.c"] + def test_scoped_libraries2(empty_tree: Tree): """Ensure that the correct libraries in regards to scoping. From b1809fc33cda80406a19f0cf77c6e712d630cebb Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Mon, 18 Sep 2023 15:39:24 -0700 Subject: [PATCH 20/33] Add test_package_table --- tests.unit/database/test_package_table.py | 42 +++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 tests.unit/database/test_package_table.py diff --git a/tests.unit/database/test_package_table.py b/tests.unit/database/test_package_table.py new file mode 100644 index 00000000..e6279dcb --- /dev/null +++ b/tests.unit/database/test_package_table.py @@ -0,0 +1,42 @@ +## +# unittest for the PackageTable generator +# +# Copyright (c) Microsoft Corporation +# +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +"""Tests for building a package table.""" +import git +from common import Tree, empty_tree # noqa: F401 +from edk2toollib.database import Edk2DB +from edk2toollib.database.tables import PackageTable +from edk2toollib.uefi.edk2.path_utilities import Edk2Path + + +def test_basic_parse(tmp_path): + """Tests basic PackageTable functionality.""" + # Clone the repo and init a single submodule. + repo_path = tmp_path / "mu_tiano_platforms" + repo = git.Repo.clone_from("https://github.com/microsoft/mu_tiano_platforms", repo_path) + if repo is None: + raise Exception("Failed to clone mu_tiano_platforms") + repo.git.submodule("update", "--init", "Features/CONFIG") + + edk2path = Edk2Path(str(repo_path), ["Platforms", "Features/CONFIG"]) + db = Edk2DB(tmp_path / "db.db", pathobj=edk2path) + db.register(PackageTable()) + db.parse({}) + + results = db.connection.cursor().execute("SELECT * FROM package").fetchall() + + to_pass = { + ("QemuPkg", "BASE"): False, + ("QemuSbsaPkg", "BASE"): False, + ("QemuQ35Pkg", "BASE"): False, + ("SetupDataPkg", "Features/CONFIG"): False, + } + for result in results: + to_pass[result] = True + + # Assert that all expected items in to_pass were found and set to True + assert all(to_pass.values()) From 24570c647b3e79b004008f8c5820d3056790cbb1 Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Mon, 18 Sep 2023 16:03:44 -0700 Subject: [PATCH 21/33] update --- tests.unit/database/test_package_table.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests.unit/database/test_package_table.py b/tests.unit/database/test_package_table.py index e6279dcb..05ebb0f1 100644 --- a/tests.unit/database/test_package_table.py +++ b/tests.unit/database/test_package_table.py @@ -6,13 +6,16 @@ # SPDX-License-Identifier: BSD-2-Clause-Patent ## """Tests for building a package table.""" +import sys + import git -from common import Tree, empty_tree # noqa: F401 +import pytest from edk2toollib.database import Edk2DB from edk2toollib.database.tables import PackageTable from edk2toollib.uefi.edk2.path_utilities import Edk2Path +@pytest.mark.skipif(sys.platform.startswith("win"), reason="Linux only") def test_basic_parse(tmp_path): """Tests basic PackageTable functionality.""" # Clone the repo and init a single submodule. From 7bb22dcb75caa0de551a235e136712a9e00ad6ef Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Mon, 18 Sep 2023 16:08:54 -0700 Subject: [PATCH 22/33] update --- tests.unit/database/test_package_table.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tests.unit/database/test_package_table.py b/tests.unit/database/test_package_table.py index 05ebb0f1..5a808022 100644 --- a/tests.unit/database/test_package_table.py +++ b/tests.unit/database/test_package_table.py @@ -6,24 +6,22 @@ # SPDX-License-Identifier: BSD-2-Clause-Patent ## """Tests for building a package table.""" -import sys import git -import pytest from edk2toollib.database import Edk2DB from edk2toollib.database.tables import PackageTable from edk2toollib.uefi.edk2.path_utilities import Edk2Path -@pytest.mark.skipif(sys.platform.startswith("win"), reason="Linux only") +#@pytest.mark.skipif(sys.platform.startswith("win"), reason="Linux only") def test_basic_parse(tmp_path): """Tests basic PackageTable functionality.""" # Clone the repo and init a single submodule. repo_path = tmp_path / "mu_tiano_platforms" - repo = git.Repo.clone_from("https://github.com/microsoft/mu_tiano_platforms", repo_path) - if repo is None: - raise Exception("Failed to clone mu_tiano_platforms") - repo.git.submodule("update", "--init", "Features/CONFIG") + with git.Repo.clone_from("https://github.com/microsoft/mu_tiano_platforms", repo_path) as repo: + if repo is None: + raise Exception("Failed to clone mu_tiano_platforms") + repo.git.submodule("update", "--init", "Features/CONFIG") edk2path = Edk2Path(str(repo_path), ["Platforms", "Features/CONFIG"]) db = Edk2DB(tmp_path / "db.db", pathobj=edk2path) From 3a9622a4b06a862ad63bb58c19588c8c09ebf7e7 Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Mon, 18 Sep 2023 16:17:14 -0700 Subject: [PATCH 23/33] update --- tests.unit/database/test_package_table.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests.unit/database/test_package_table.py b/tests.unit/database/test_package_table.py index 5a808022..6de09d14 100644 --- a/tests.unit/database/test_package_table.py +++ b/tests.unit/database/test_package_table.py @@ -18,6 +18,7 @@ def test_basic_parse(tmp_path): """Tests basic PackageTable functionality.""" # Clone the repo and init a single submodule. repo_path = tmp_path / "mu_tiano_platforms" + repo_path.mkdir() with git.Repo.clone_from("https://github.com/microsoft/mu_tiano_platforms", repo_path) as repo: if repo is None: raise Exception("Failed to clone mu_tiano_platforms") From 7c3ead8971a0aec3a4a61e172567a77eba299bfe Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Mon, 18 Sep 2023 16:26:43 -0700 Subject: [PATCH 24/33] update --- tests.unit/database/test_package_table.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests.unit/database/test_package_table.py b/tests.unit/database/test_package_table.py index 6de09d14..a22a37cb 100644 --- a/tests.unit/database/test_package_table.py +++ b/tests.unit/database/test_package_table.py @@ -7,13 +7,16 @@ ## """Tests for building a package table.""" +import sys + import git +import pytest from edk2toollib.database import Edk2DB from edk2toollib.database.tables import PackageTable from edk2toollib.uefi.edk2.path_utilities import Edk2Path -#@pytest.mark.skipif(sys.platform.startswith("win"), reason="Linux only") +@pytest.mark.skipif(sys.platform.startswith("win"), reason="Linux only") def test_basic_parse(tmp_path): """Tests basic PackageTable functionality.""" # Clone the repo and init a single submodule. From 6f293007782907b573f78450f26f7a167aa49d2d Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Mon, 18 Sep 2023 17:42:26 -0700 Subject: [PATCH 25/33] update --- edk2toollib/database/edk2_db.py | 3 ++- edk2toollib/database/tables/instanced_inf_table.py | 13 ++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/edk2toollib/database/edk2_db.py b/edk2toollib/database/edk2_db.py index ac5186e8..f4e5b2fa 100644 --- a/edk2toollib/database/edk2_db.py +++ b/edk2toollib/database/edk2_db.py @@ -106,6 +106,7 @@ def parse(self, env: dict) -> None: # Fill all tables for table in self._parsers: logging.debug(f"[{table.__class__.__name__}] starting...") - time.time() + t = time.time() table.parse(self.connection.cursor(), self.pathobj, id, env) self.connection.commit() + logging.debug(f"Finished in {round(time.time() - t, 2)}") diff --git a/edk2toollib/database/tables/instanced_inf_table.py b/edk2toollib/database/tables/instanced_inf_table.py index 726c9ca2..1f630b2e 100644 --- a/edk2toollib/database/tables/instanced_inf_table.py +++ b/edk2toollib/database/tables/instanced_inf_table.py @@ -74,11 +74,10 @@ def inf(self, inf: str) -> InfP: self._parsed_infs[inf] = infp return infp - def parse(self, db_cursor: Cursor, pathobj: Edk2Path, id: str, env: dict) -> None: + def parse(self, db_cursor: Cursor, pathobj: Edk2Path, env_id: str, env: dict) -> None: """Parse the workspace and update the database.""" self.pathobj = pathobj self.ws = Path(self.pathobj.WorkspacePath) - self.id = id self.env = env self.dsc = self.env["ACTIVE_PLATFORM"] # REQUIRED self.fdf = self.env.get("FLASH_DEFINITION", "") # OPTIONAL @@ -107,15 +106,15 @@ def parse(self, db_cursor: Cursor, pathobj: Edk2Path, id: str, env: dict) -> Non # add instanced_inf entries for e in inf_entries: - row = (id, e["PATH"], e["LIBRARY_CLASS"] or None, e["NAME"], e["ARCH"], e["DSC"], e["COMPONENT"]) + row = (env_id, e["PATH"], e["LIBRARY_CLASS"] or None, e["NAME"], e["ARCH"], e["DSC"], e["COMPONENT"]) db_cursor.execute(INSERT_INSTANCED_INF_ROW, row) for e in inf_entries: - inf_id = db_cursor.execute(GET_ROW_ID, (id, e["PATH"], e["COMPONENT"])).fetchone()[0] + inf_id = db_cursor.execute(GET_ROW_ID, (env_id, e["PATH"], e["COMPONENT"])).fetchone()[0] # Add junction entries to link source the source files used by an INF for source in e["SOURCES_USED"]: - row = (id, "instanced_inf", inf_id, "source", source) + row = (env_id, "instanced_inf", inf_id, "source", source) db_cursor.execute(INSERT_JUNCTION_ROW, row) # Add junction entires to link libraries / components to the libraries they consume. @@ -123,9 +122,9 @@ def parse(self, db_cursor: Cursor, pathobj: Edk2Path, id: str, env: dict) -> Non if instance is None: used_inf_id = None # no library instance found for this library class else: - used_inf_id = db_cursor.execute(GET_ROW_ID, (id, instance, e["COMPONENT"])).fetchone()[0] + used_inf_id = db_cursor.execute(GET_ROW_ID, (env_id, instance, e["COMPONENT"])).fetchone()[0] - row = (id, "instanced_inf", inf_id, "instanced_inf", used_inf_id) + row = (env_id, "instanced_inf", inf_id, "instanced_inf", used_inf_id) db_cursor.execute(INSERT_JUNCTION_ROW, row) def _build_inf_table(self, dscp: DscP): From fb2fcd82d4bb099d69ffecf95b94a97588e1b8be Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Mon, 18 Sep 2023 18:41:05 -0700 Subject: [PATCH 26/33] Improve performance of instanced_inf_table --- .../database/tables/instanced_inf_table.py | 57 ++++++++++++------- 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/edk2toollib/database/tables/instanced_inf_table.py b/edk2toollib/database/tables/instanced_inf_table.py index 1f630b2e..1b833987 100644 --- a/edk2toollib/database/tables/instanced_inf_table.py +++ b/edk2toollib/database/tables/instanced_inf_table.py @@ -98,37 +98,50 @@ def parse(self, db_cursor: Cursor, pathobj: Edk2Path, env_id: str, env: dict) -> logging.debug(f" {line}") logging.debug("End of DSC") - # Create the instanced inf entries, including components and libraries. Multiple entries - # of the same library will exist if multiple components use it. - # - # This is where we merge DSC parser information with INF parser information. inf_entries = self._build_inf_table(dscp) + return self._insert_db_rows(db_cursor, env_id, inf_entries) + + def _insert_db_rows(self, db_cursor, env_id, inf_entries) -> int: + """Inserts data into the database. - # add instanced_inf entries - for e in inf_entries: - row = (env_id, e["PATH"], e["LIBRARY_CLASS"] or None, e["NAME"], e["ARCH"], e["DSC"], e["COMPONENT"]) + Inserts all inf's into the instanced_inf table and links source files and used libraries via the junction + table. + """ + # Must use "execute" so that lastrowid is updated. + rows = [ + (env_id, e["PATH"], e["LIBRARY_CLASS"] or None, e["NAME"], e["ARCH"], e["DSC"], e["COMPONENT"]) + for e in inf_entries + ] + for row in rows: db_cursor.execute(INSERT_INSTANCED_INF_ROW, row) + last_inserted_id = db_cursor.lastrowid - for e in inf_entries: - inf_id = db_cursor.execute(GET_ROW_ID, (env_id, e["PATH"], e["COMPONENT"])).fetchone()[0] + # Mapping used for linking libraries together in the junction table so that the value does not need to be + # queried, which reduces performance greatly. + id_mapping = {} + for idx, e in enumerate(inf_entries): + id_mapping[e["PATH"], e["COMPONENT"]] = last_inserted_id - len(inf_entries) + 1 + idx - # Add junction entries to link source the source files used by an INF - for source in e["SOURCES_USED"]: - row = (env_id, "instanced_inf", inf_id, "source", source) - db_cursor.execute(INSERT_JUNCTION_ROW, row) + for idx, e in enumerate(inf_entries): + inf_id = last_inserted_id - len(inf_entries) + 1 + idx - # Add junction entires to link libraries / components to the libraries they consume. - for cls, instance in e["LIBRARIES_USED"]: - if instance is None: - used_inf_id = None # no library instance found for this library class - else: - used_inf_id = db_cursor.execute(GET_ROW_ID, (env_id, instance, e["COMPONENT"])).fetchone()[0] + # Link all source files to this instanced_inf + rows = [(env_id, "instanced_inf", inf_id, "source", source) for source in e["SOURCES_USED"]] + db_cursor.executemany(INSERT_JUNCTION_ROW, rows) - row = (env_id, "instanced_inf", inf_id, "instanced_inf", used_inf_id) - db_cursor.execute(INSERT_JUNCTION_ROW, row) + # link other libraries used by this instanced_inf + rows = [ + (env_id, "instanced_inf", inf_id, "instanced_inf", id_mapping.get((library, e["COMPONENT"]), None)) + for library in e["LIBRARIES_USED"] + ] + db_cursor.executemany(INSERT_JUNCTION_ROW, rows) def _build_inf_table(self, dscp: DscP): + """Create the instanced inf entries, including components and libraries. + Multiple entries of the same library will exist if multiple components use it. + This is where we merge DSC parser information with INF parser information. + """ inf_entries = [] for (inf, scope, overrides) in dscp.Components: logging.debug(f"Parsing Component: [{inf}]") @@ -220,7 +233,7 @@ def to_posix(path): "MODULE_TYPE": infp.Dict["MODULE_TYPE"], "ARCH": scope.split(".")[0].upper(), "SOURCES_USED": list(map(lambda p: Path(p).as_posix(), infp.Sources)), - "LIBRARIES_USED": list(zip(library_class_list, library_instance_list)), + "LIBRARIES_USED": list(library_instance_list), "PROTOCOLS_USED": [], # TODO "GUIDS_USED": [], # TODO "PPIS_USED": [], # TODO From 4aff8b27f76e961a952db2f552d9ad92c7148494 Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Mon, 18 Sep 2023 18:51:37 -0700 Subject: [PATCH 27/33] Update --- .../database/tables/instanced_inf_table.py | 31 ++++++++----------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/edk2toollib/database/tables/instanced_inf_table.py b/edk2toollib/database/tables/instanced_inf_table.py index 1b833987..e708147b 100644 --- a/edk2toollib/database/tables/instanced_inf_table.py +++ b/edk2toollib/database/tables/instanced_inf_table.py @@ -41,12 +41,6 @@ class TEXT, VALUES (?, ?, ?, ?, ?) ''' -GET_ROW_ID = ''' -SELECT id FROM instanced_inf -WHERE env = ? and path = ? and component = ? -LIMIT 1 -''' - class InstancedInfTable(TableGenerator): """A Table Generator that parses a single DSC file and generates a table.""" SECTION_LIBRARY = "LibraryClasses" @@ -56,16 +50,17 @@ class InstancedInfTable(TableGenerator): def __init__(self, *args, **kwargs): """Initialize the query with the specific settings.""" + self._parsed_infs = {} def create_tables(self, db_cursor: Cursor) -> None: """Create the tables necessary for this parser.""" db_cursor.execute(CREATE_INSTANCED_INF_TABLE) - # Prevent parsing the same INF multiple times - self._parsed_infs = {} - def inf(self, inf: str) -> InfP: - """Returns a parsed INF object.""" + """Returns a parsed INF object. + + Caches the parsed inf information to reduce multiple re-parses. + """ if inf in self._parsed_infs: infp = self._parsed_infs[inf] else: @@ -79,10 +74,9 @@ def parse(self, db_cursor: Cursor, pathobj: Edk2Path, env_id: str, env: dict) -> self.pathobj = pathobj self.ws = Path(self.pathobj.WorkspacePath) self.env = env - self.dsc = self.env["ACTIVE_PLATFORM"] # REQUIRED - self.fdf = self.env.get("FLASH_DEFINITION", "") # OPTIONAL - self.arch = self.env["TARGET_ARCH"].split(" ") # REQUIRED - self.target = self.env["TARGET"] # REQUIRED + self.dsc = self.env["ACTIVE_PLATFORM"] + self.arch = self.env["TARGET_ARCH"].split(" ") + self.target = self.env["TARGET"] dscp = DscP().SetEdk2Path(self.pathobj) dscp.SetInputVars(self.env) @@ -98,6 +92,7 @@ def parse(self, db_cursor: Cursor, pathobj: Edk2Path, env_id: str, env: dict) -> logging.debug(f" {line}") logging.debug("End of DSC") + # Parse and insert inf_entries = self._build_inf_table(dscp) return self._insert_db_rows(db_cursor, env_id, inf_entries) @@ -107,7 +102,7 @@ def _insert_db_rows(self, db_cursor, env_id, inf_entries) -> int: Inserts all inf's into the instanced_inf table and links source files and used libraries via the junction table. """ - # Must use "execute" so that lastrowid is updated. + # Must use "execute" so that db_cursor.lastrowid is updated. rows = [ (env_id, e["PATH"], e["LIBRARY_CLASS"] or None, e["NAME"], e["ARCH"], e["DSC"], e["COMPONENT"]) for e in inf_entries @@ -122,6 +117,7 @@ def _insert_db_rows(self, db_cursor, env_id, inf_entries) -> int: for idx, e in enumerate(inf_entries): id_mapping[e["PATH"], e["COMPONENT"]] = last_inserted_id - len(inf_entries) + 1 + idx + # Insert rows into the tables for idx, e in enumerate(inf_entries): inf_id = last_inserted_id - len(inf_entries) + 1 + idx @@ -163,7 +159,7 @@ def _build_inf_table(self, dscp: DscP): def _parse_inf_recursively( self, inf: str, - lib_cls: str, + library_class: str, component: str, library_dict: dict, override_dict: dict, @@ -182,7 +178,6 @@ def _parse_inf_recursively( library_instance_list = [] library_class_list = [] - # # 0. Use the existing parser to parse the INF file. This parser parses an INF as an independent file # and does not take into account the context of a DSC. @@ -228,7 +223,7 @@ def to_posix(path): "PATH": Path(inf).as_posix(), "GUID": infp.Dict.get("FILE_GUID", ""), "NAME": infp.Dict["BASE_NAME"], - "LIBRARY_CLASS": lib_cls, + "LIBRARY_CLASS": library_class, "COMPONENT": Path(component).as_posix(), "MODULE_TYPE": infp.Dict["MODULE_TYPE"], "ARCH": scope.split(".")[0].upper(), From e785c2e14f74c1f848df3e8539d9c042b5765902 Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Mon, 18 Sep 2023 19:09:27 -0700 Subject: [PATCH 28/33] Update --- edk2toollib/database/tables/instanced_inf_table.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/edk2toollib/database/tables/instanced_inf_table.py b/edk2toollib/database/tables/instanced_inf_table.py index e708147b..a3f76ae8 100644 --- a/edk2toollib/database/tables/instanced_inf_table.py +++ b/edk2toollib/database/tables/instanced_inf_table.py @@ -104,7 +104,7 @@ def _insert_db_rows(self, db_cursor, env_id, inf_entries) -> int: """ # Must use "execute" so that db_cursor.lastrowid is updated. rows = [ - (env_id, e["PATH"], e["LIBRARY_CLASS"] or None, e["NAME"], e["ARCH"], e["DSC"], e["COMPONENT"]) + (env_id, e["PATH"], e.get("LIBRARY_CLASS"), e["NAME"], e["ARCH"], e["DSC"], e["COMPONENT"]) for e in inf_entries ] for row in rows: @@ -118,19 +118,19 @@ def _insert_db_rows(self, db_cursor, env_id, inf_entries) -> int: id_mapping[e["PATH"], e["COMPONENT"]] = last_inserted_id - len(inf_entries) + 1 + idx # Insert rows into the tables + rows = [] for idx, e in enumerate(inf_entries): inf_id = last_inserted_id - len(inf_entries) + 1 + idx # Link all source files to this instanced_inf - rows = [(env_id, "instanced_inf", inf_id, "source", source) for source in e["SOURCES_USED"]] - db_cursor.executemany(INSERT_JUNCTION_ROW, rows) + rows += [(env_id, "instanced_inf", inf_id, "source", source) for source in e["SOURCES_USED"]] # link other libraries used by this instanced_inf - rows = [ + rows += [ (env_id, "instanced_inf", inf_id, "instanced_inf", id_mapping.get((library, e["COMPONENT"]), None)) for library in e["LIBRARIES_USED"] ] - db_cursor.executemany(INSERT_JUNCTION_ROW, rows) + db_cursor.executemany(INSERT_JUNCTION_ROW, rows) def _build_inf_table(self, dscp: DscP): """Create the instanced inf entries, including components and libraries. From 0d1960be681596aa61ca5a8664daff46758d7912 Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Tue, 19 Sep 2023 10:37:18 -0700 Subject: [PATCH 29/33] Update edk2toollib/database/edk2_db.py --- edk2toollib/database/edk2_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edk2toollib/database/edk2_db.py b/edk2toollib/database/edk2_db.py index f4e5b2fa..437efe29 100644 --- a/edk2toollib/database/edk2_db.py +++ b/edk2toollib/database/edk2_db.py @@ -43,7 +43,7 @@ class Edk2DB: !!! note Edk2DB provides a table called `junction` that can be used to make associations between tables. It has the - following schema: `table1, key1, table2, key2`. + following schema: `env_id, table1, key1, table2, key2`. Example: ```python From 3da9deb7c923f6c97533a7cbbf2d397f942128b5 Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Mon, 18 Sep 2023 20:01:32 -0700 Subject: [PATCH 30/33] remove unecessary table --- edk2toollib/database/tables/inf_table.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/edk2toollib/database/tables/inf_table.py b/edk2toollib/database/tables/inf_table.py index df709625..941edf83 100644 --- a/edk2toollib/database/tables/inf_table.py +++ b/edk2toollib/database/tables/inf_table.py @@ -27,12 +27,6 @@ ); ''' -CREATE_LIBRARY_CLASS_TABLE = ''' -CREATE TABLE IF NOT EXISTS library_class ( - class TEXT -) -''' - INSERT_JUNCTION_ROW = ''' INSERT INTO junction (env, table1, key1, table2, key2) VALUES (?, ?, ?, ?, ?) @@ -61,7 +55,6 @@ def __init__(self, *args, **kwargs): def create_tables(self, db_cursor: Cursor) -> None: """Create the tables necessary for this parser.""" db_cursor.execute(CREATE_INF_TABLE) - db_cursor.execute(CREATE_LIBRARY_CLASS_TABLE) def parse(self, db_cursor: Cursor, pathobj: Edk2Path, env_id: str, env: dict) -> None: """Parse the workspace and update the database.""" From e43f72c50e6c81e9a78bc35f4dfa10a4ac100242 Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Tue, 19 Sep 2023 10:43:08 -0700 Subject: [PATCH 31/33] Update edk2toollib/database/tables/environment_table.py --- edk2toollib/database/tables/environment_table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edk2toollib/database/tables/environment_table.py b/edk2toollib/database/tables/environment_table.py index 801e7cb4..c7854074 100644 --- a/edk2toollib/database/tables/environment_table.py +++ b/edk2toollib/database/tables/environment_table.py @@ -52,7 +52,7 @@ def parse(self, db_cursor: sqlite3.Cursor, pathobj: Edk2Path, id, env) -> None: # Insert into environment table entry = (id,str(dtime),version,) - db_cursor.execute("INSERT INTO environment (id, date,version) VALUES (?, ?, ?)", entry) + db_cursor.execute("INSERT INTO environment (id, date, version) VALUES (?, ?, ?)", entry) # Insert into environment_values table data = [(id, key, value) for key, value in env.items()] From 4c7e84105496ffa3fb8f93b47b89cff5e5cf19ad Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Tue, 19 Sep 2023 10:44:15 -0700 Subject: [PATCH 32/33] Update edk2toollib/database/tables/package_table.py Co-authored-by: Michael Kubacki --- edk2toollib/database/tables/package_table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edk2toollib/database/tables/package_table.py b/edk2toollib/database/tables/package_table.py index 8d2224ba..9e24f1cb 100644 --- a/edk2toollib/database/tables/package_table.py +++ b/edk2toollib/database/tables/package_table.py @@ -1,4 +1,4 @@ -# @file repository_table.py +# @file package_table.py # A module to associate the packages in a workspace with the repositories they come from. ## # Copyright (c) Microsoft Corporation From c5211a58ff679eebe9358b2fc5767b8c418714e1 Mon Sep 17 00:00:00 2001 From: Joey Vagedes Date: Mon, 18 Sep 2023 20:09:47 -0700 Subject: [PATCH 33/33] Update --- edk2toollib/database/edk2_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edk2toollib/database/edk2_db.py b/edk2toollib/database/edk2_db.py index 437efe29..f789cf77 100644 --- a/edk2toollib/database/edk2_db.py +++ b/edk2toollib/database/edk2_db.py @@ -53,7 +53,7 @@ class Edk2DB: db.register(Parser1(), Parser2(), Parser3()) db.parse() - db.connection.execute(f"SELECT * FROM {table}", table) + db.connection.execute("SELECT * FROM ?", table) """ def __init__(self, db_path: str, pathobj: Edk2Path = None, **kwargs: dict[str,Any]): """Initializes the database.