From d83403dbda9f2fdc7ae621d2d12a33fb78037ea1 Mon Sep 17 00:00:00 2001 From: Jacob Urbanczyk Date: Sat, 9 Mar 2024 21:56:51 +0100 Subject: [PATCH] Clean up pytest --- scripts/run_tests.py | 5 +---- test/conftest.py | 14 ++++++++++++++ test/regression/conftest.py | 21 ++++++++------------- test/regression/test_regression.py | 24 +++++++++++++++++------- transactron/testing/infrastructure.py | 2 +- 5 files changed, 41 insertions(+), 25 deletions(-) diff --git a/scripts/run_tests.py b/scripts/run_tests.py index d75cb1e53..2223b8cf1 100755 --- a/scripts/run_tests.py +++ b/scripts/run_tests.py @@ -33,12 +33,9 @@ def main(): pytest_arguments = ["--max-worker-restart=1"] if args.trace: - os.environ["__COREBLOCKS_DUMP_TRACES"] = "1" pytest_arguments.append("--coreblocks-traces") - if args.profile: - os.environ["__TRANSACTRON_PROFILE"] = "1" - + pytest_arguments.append("--coreblocks-profile") if args.test_name: pytest_arguments += [f"--coreblocks-test-name={args.test_name}"] if args.count: diff --git a/test/conftest.py b/test/conftest.py index 1095fba03..d0a77f9ec 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,4 +1,5 @@ import re +import os from typing import Optional import pytest @@ -13,6 +14,7 @@ def pytest_addoption(parser: pytest.Parser): help="Simulation backend for regression tests", ) group.addoption("--coreblocks-traces", action="store_true", help="Generate traces from regression tests") + group.addoption("--coreblocks-profile", action="store_true", help="Write execution profiles") group.addoption("--coreblocks-list", action="store_true", help="List all tests in flatten format.") group.addoption( "--coreblocks-test-name", @@ -90,3 +92,15 @@ def deselect_based_on_count(items: list[pytest.Item], config: pytest.Config) -> def pytest_collection_modifyitems(items: list[pytest.Item], config: pytest.Config) -> None: deselect_based_on_flatten_name(items, config) deselect_based_on_count(items, config) + + +def pytest_runtest_setup(item: pytest.Item): + """ + This function is called to perform the setup phase for every test, so + it is a perfect moment to set environment variables. + """ + if item.config.getoption("--coreblocks-traces", False): # type: ignore + os.environ["__TRANSACTRON_DUMP_TRACES"] = "1" + + if item.config.getoption("--coreblocks-profile", False): # type: ignore + os.environ["__TRANSACTRON_PROFILE"] = "1" diff --git a/test/regression/conftest.py b/test/regression/conftest.py index 144fe5c93..bf0f1cc96 100644 --- a/test/regression/conftest.py +++ b/test/regression/conftest.py @@ -2,7 +2,6 @@ from pathlib import Path import pytest import subprocess -import sys test_dir = Path(__file__).parent.parent riscv_tests_dir = test_dir.joinpath("external/riscv-tests") @@ -19,7 +18,6 @@ def load_regression_tests() -> list[str]: res = subprocess.run(["make", "-C", "test/external/riscv-tests"]) if res.returncode != 0: print("Couldn't build regression tests") - sys.exit(1) all_tests = set(get_all_test_names()) exclude = {"rv32ui-ma_data", "rv32ui-fence_i"} @@ -28,19 +26,16 @@ def load_regression_tests() -> list[str]: def pytest_generate_tests(metafunc: pytest.Metafunc): - if not metafunc.config.getoption("coreblocks_regression"): - # Add regression to skiped tests - metafunc.parametrize(["test_name", "backend", "traces", "verbose"], []) - return - all_tests = ( load_regression_tests() ) # The list has to be always in the same order (e.g. sorted) to allow for parallel testing - traces = metafunc.config.getoption("coreblocks_traces") - backend = metafunc.config.getoption("coreblocks_backend") - verbose = bool(metafunc.config.getoption("verbose")) - if {"test_name", "backend", "traces", "verbose"}.issubset(metafunc.fixturenames): + if "test_name" in metafunc.fixturenames: metafunc.parametrize( - ["test_name", "backend", "traces", "verbose"], - [(test_name, backend, traces, verbose) for test_name in all_tests], + "test_name", + [test_name for test_name in all_tests], ) + + +def pytest_runtest_setup(item: pytest.Item): + if not item.config.getoption("--coreblocks-regression", default=False): # type: ignore + pytest.skip("need --coreblocks-regression option to run this test") diff --git a/test/regression/test_regression.py b/test/regression/test_regression.py index a85c9c7f4..53d1d2e95 100644 --- a/test/regression/test_regression.py +++ b/test/regression/test_regression.py @@ -81,11 +81,11 @@ def regression_body_with_cocotb(test_name: str, traces: bool): assert len(list(tree.iter("failure"))) == 0 -def regression_body_with_pysim(test_name: str, traces: bool, verbose: bool): +def regression_body_with_pysim(test_name: str, traces: bool): traces_file = None if traces: traces_file = REGRESSION_TESTS_PREFIX + test_name - asyncio.run(run_test(PySimulation(verbose, traces_file=traces_file), test_name)) + asyncio.run(run_test(PySimulation(verbose=False, traces_file=traces_file), test_name)) @pytest.fixture(scope="session") @@ -126,8 +126,18 @@ def verilate_model(worker_id, request: pytest.FixtureRequest): os.remove(counter_path) -def test_entrypoint(test_name: str, backend: Literal["pysim", "cocotb"], traces: bool, verbose: bool, verilate_model): - if backend == "cocotb": - regression_body_with_cocotb(test_name, traces) - elif backend == "pysim": - regression_body_with_pysim(test_name, traces, verbose) +@pytest.fixture +def sim_backend(request: pytest.FixtureRequest): + return request.config.getoption("coreblocks_backend") + + +@pytest.fixture +def traces_enabled(request: pytest.FixtureRequest): + return request.config.getoption("coreblocks_traces") + + +def test_entrypoint(test_name: str, sim_backend: Literal["pysim", "cocotb"], traces_enabled: bool, verilate_model): + if sim_backend == "cocotb": + regression_body_with_cocotb(test_name, traces_enabled) + elif sim_backend == "pysim": + regression_body_with_pysim(test_name, traces_enabled) diff --git a/transactron/testing/infrastructure.py b/transactron/testing/infrastructure.py index c01763ad9..dc4a5404e 100644 --- a/transactron/testing/infrastructure.py +++ b/transactron/testing/infrastructure.py @@ -241,7 +241,7 @@ def add_all_mocks(self, sim: PysimSimulator, frame_locals: dict) -> None: @contextmanager def run_simulation(self, module: HasElaborate, max_cycles: float = 10e4, add_transaction_module=True): traces_file = None - if "__COREBLOCKS_DUMP_TRACES" in os.environ: + if "__TRANSACTRON_DUMP_TRACES" in os.environ: traces_file = unittest.TestCase.id(self) clk_period = 1e-6