From dbdec1429ce63eeafa32841c212386f72eb50509 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Urba=C5=84czyk?= Date: Sun, 10 Mar 2024 15:12:27 +0000 Subject: [PATCH 1/5] Clean up pytest (#611) --- scripts/run_tests.py | 5 +---- test/conftest.py | 14 ++++++++++++++ test/regression/conftest.py | 21 ++++++++------------- test/regression/test_regression.py | 24 +++++++++++++++++------- transactron/testing/infrastructure.py | 2 +- 5 files changed, 41 insertions(+), 25 deletions(-) diff --git a/scripts/run_tests.py b/scripts/run_tests.py index d75cb1e53..2223b8cf1 100755 --- a/scripts/run_tests.py +++ b/scripts/run_tests.py @@ -33,12 +33,9 @@ def main(): pytest_arguments = ["--max-worker-restart=1"] if args.trace: - os.environ["__COREBLOCKS_DUMP_TRACES"] = "1" pytest_arguments.append("--coreblocks-traces") - if args.profile: - os.environ["__TRANSACTRON_PROFILE"] = "1" - + pytest_arguments.append("--coreblocks-profile") if args.test_name: pytest_arguments += [f"--coreblocks-test-name={args.test_name}"] if args.count: diff --git a/test/conftest.py b/test/conftest.py index 1095fba03..d0a77f9ec 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,4 +1,5 @@ import re +import os from typing import Optional import pytest @@ -13,6 +14,7 @@ def pytest_addoption(parser: pytest.Parser): help="Simulation backend for regression tests", ) group.addoption("--coreblocks-traces", action="store_true", help="Generate traces from regression tests") + group.addoption("--coreblocks-profile", action="store_true", help="Write execution profiles") group.addoption("--coreblocks-list", action="store_true", help="List all tests in flatten format.") group.addoption( "--coreblocks-test-name", @@ -90,3 +92,15 @@ def deselect_based_on_count(items: list[pytest.Item], config: pytest.Config) -> def pytest_collection_modifyitems(items: list[pytest.Item], config: pytest.Config) -> None: deselect_based_on_flatten_name(items, config) deselect_based_on_count(items, config) + + +def pytest_runtest_setup(item: pytest.Item): + """ + This function is called to perform the setup phase for every test, so + it is a perfect moment to set environment variables. + """ + if item.config.getoption("--coreblocks-traces", False): # type: ignore + os.environ["__TRANSACTRON_DUMP_TRACES"] = "1" + + if item.config.getoption("--coreblocks-profile", False): # type: ignore + os.environ["__TRANSACTRON_PROFILE"] = "1" diff --git a/test/regression/conftest.py b/test/regression/conftest.py index 144fe5c93..bf0f1cc96 100644 --- a/test/regression/conftest.py +++ b/test/regression/conftest.py @@ -2,7 +2,6 @@ from pathlib import Path import pytest import subprocess -import sys test_dir = Path(__file__).parent.parent riscv_tests_dir = test_dir.joinpath("external/riscv-tests") @@ -19,7 +18,6 @@ def load_regression_tests() -> list[str]: res = subprocess.run(["make", "-C", "test/external/riscv-tests"]) if res.returncode != 0: print("Couldn't build regression tests") - sys.exit(1) all_tests = set(get_all_test_names()) exclude = {"rv32ui-ma_data", "rv32ui-fence_i"} @@ -28,19 +26,16 @@ def load_regression_tests() -> list[str]: def pytest_generate_tests(metafunc: pytest.Metafunc): - if not metafunc.config.getoption("coreblocks_regression"): - # Add regression to skiped tests - metafunc.parametrize(["test_name", "backend", "traces", "verbose"], []) - return - all_tests = ( load_regression_tests() ) # The list has to be always in the same order (e.g. sorted) to allow for parallel testing - traces = metafunc.config.getoption("coreblocks_traces") - backend = metafunc.config.getoption("coreblocks_backend") - verbose = bool(metafunc.config.getoption("verbose")) - if {"test_name", "backend", "traces", "verbose"}.issubset(metafunc.fixturenames): + if "test_name" in metafunc.fixturenames: metafunc.parametrize( - ["test_name", "backend", "traces", "verbose"], - [(test_name, backend, traces, verbose) for test_name in all_tests], + "test_name", + [test_name for test_name in all_tests], ) + + +def pytest_runtest_setup(item: pytest.Item): + if not item.config.getoption("--coreblocks-regression", default=False): # type: ignore + pytest.skip("need --coreblocks-regression option to run this test") diff --git a/test/regression/test_regression.py b/test/regression/test_regression.py index a85c9c7f4..53d1d2e95 100644 --- a/test/regression/test_regression.py +++ b/test/regression/test_regression.py @@ -81,11 +81,11 @@ def regression_body_with_cocotb(test_name: str, traces: bool): assert len(list(tree.iter("failure"))) == 0 -def regression_body_with_pysim(test_name: str, traces: bool, verbose: bool): +def regression_body_with_pysim(test_name: str, traces: bool): traces_file = None if traces: traces_file = REGRESSION_TESTS_PREFIX + test_name - asyncio.run(run_test(PySimulation(verbose, traces_file=traces_file), test_name)) + asyncio.run(run_test(PySimulation(verbose=False, traces_file=traces_file), test_name)) @pytest.fixture(scope="session") @@ -126,8 +126,18 @@ def verilate_model(worker_id, request: pytest.FixtureRequest): os.remove(counter_path) -def test_entrypoint(test_name: str, backend: Literal["pysim", "cocotb"], traces: bool, verbose: bool, verilate_model): - if backend == "cocotb": - regression_body_with_cocotb(test_name, traces) - elif backend == "pysim": - regression_body_with_pysim(test_name, traces, verbose) +@pytest.fixture +def sim_backend(request: pytest.FixtureRequest): + return request.config.getoption("coreblocks_backend") + + +@pytest.fixture +def traces_enabled(request: pytest.FixtureRequest): + return request.config.getoption("coreblocks_traces") + + +def test_entrypoint(test_name: str, sim_backend: Literal["pysim", "cocotb"], traces_enabled: bool, verilate_model): + if sim_backend == "cocotb": + regression_body_with_cocotb(test_name, traces_enabled) + elif sim_backend == "pysim": + regression_body_with_pysim(test_name, traces_enabled) diff --git a/transactron/testing/infrastructure.py b/transactron/testing/infrastructure.py index c01763ad9..dc4a5404e 100644 --- a/transactron/testing/infrastructure.py +++ b/transactron/testing/infrastructure.py @@ -241,7 +241,7 @@ def add_all_mocks(self, sim: PysimSimulator, frame_locals: dict) -> None: @contextmanager def run_simulation(self, module: HasElaborate, max_cycles: float = 10e4, add_transaction_module=True): traces_file = None - if "__COREBLOCKS_DUMP_TRACES" in os.environ: + if "__TRANSACTRON_DUMP_TRACES" in os.environ: traces_file = unittest.TestCase.id(self) clk_period = 1e-6 From 09e3e7340a724e6d512731c95a1f65debf0ca33e Mon Sep 17 00:00:00 2001 From: piotro888 Date: Sun, 10 Mar 2024 16:56:04 +0100 Subject: [PATCH 2/5] Cache riscv-arch-test build (#606) --- .github/workflows/main.yml | 65 +++++++++++++++++++++++----- .gitmodules | 3 ++ test/external/riscof/riscv-arch-test | 1 + 3 files changed, 58 insertions(+), 11 deletions(-) create mode 160000 test/external/riscof/riscv-arch-test diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 831efa175..90b928b38 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -57,35 +57,63 @@ jobs: defaults: run: working-directory: test/external/riscof/ + steps: - name: Checkout uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Cache compiled and reference riscv-arch-test + id: cache-riscv-arch-test + uses: actions/cache@v3 + env: + cache-name: cache-riscv-arch-test + with: + path: | + test/external/riscof/riscv-arch-test/**/*.elf + test/external/riscof/riscof_work/**/*.signature + test/external/riscof/**/*Makefile* - - name: Setup PATH + key: ${{ env.cache-name }}-${{ runner.os }}-${{ hashFiles( + '**/test/external/riscof/coreblocks/**', + '**/test/external/riscof/spike_simple/**', + '**/test/external/riscof/config.ini', + '**/.git/modules/test/external/riscof/riscv-arch-test/HEAD', + '**/docker/riscv-toolchain.Dockerfile' + ) }} + restore-keys: | + ${{ env.cache-name }}-${{ runner.os }}- + + - if: ${{ steps.cache-riscv-arch-test.outputs.cache-hit != 'true' }} + name: Setup PATH run: echo "/.pyenv/bin" >> $GITHUB_PATH - - name: Setup pyenv python + - if: ${{ steps.cache-riscv-arch-test.outputs.cache-hit != 'true' }} + name: Setup pyenv python run: | eval "$(pyenv init --path)" pyenv global 3.6.15 . /venv3.6/bin/activate - - name: Setup arch test suite + - if: ${{ steps.cache-riscv-arch-test.outputs.cache-hit != 'true' }} + name: Setup arch test suite run: | . /venv3.6/bin/activate - riscof --verbose info arch-test --clone riscof testlist --config=config.ini --suite=riscv-arch-test/riscv-test-suite/ --env=riscv-arch-test/riscv-test-suite/env - - name: Build and run tests on reference and generate Makefiles + - if: ${{ steps.cache-riscv-arch-test.outputs.cache-hit != 'true' }} + name: Build and run tests on reference and generate Makefiles run: | . /venv3.6/bin/activate riscof run --config=config.ini --suite=riscv-arch-test/riscv-test-suite/ --env=riscv-arch-test/riscv-test-suite/env - - name: Build tests for Coreblocks + - if: ${{ steps.cache-riscv-arch-test.outputs.cache-hit != 'true' }} + name: Build tests for Coreblocks run: | MAKEFILE_PATH=riscof_work/Makefile.build-DUT-coreblocks ../../../ci/riscof_run_makefile.sh - - - uses: actions/upload-artifact@v3 + - if: ${{ steps.cache-riscv-arch-test.outputs.cache-hit != 'true' }} + uses: actions/upload-artifact@v3 with: name: "riscof-tests" path: | @@ -102,6 +130,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 + with: + submodules: recursive - name: Set up Python uses: actions/setup-python@v4 @@ -120,10 +150,23 @@ jobs: name: "verilog-full-core" path: . - - uses: actions/download-artifact@v3 + - uses: actions/cache@v3 + env: + cache-name: cache-riscv-arch-test with: - name: "riscof-tests" - path: test/external/riscof/ + path: | + test/external/riscof/riscv-arch-test/**/*.elf + test/external/riscof/riscof_work/**/*.signature + test/external/riscof/**/*Makefile* + + key: ${{ env.cache-name }}-${{ runner.os }}-${{ hashFiles( + '**/test/external/riscof/coreblocks/**', + '**/test/external/riscof/spike_simple/**', + '**/test/external/riscof/config.ini', + '**/.git/modules/test/external/riscof/riscv-arch-test/HEAD', + '**/docker/riscv-toolchain.Dockerfile' + ) }} + fail-on-cache-miss: true - name: Run tests on Coreblocks run: | diff --git a/.gitmodules b/.gitmodules index 6ecd7035d..8dea05eb8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,3 +5,6 @@ path = test/external/embench/embench-iot url = https://github.com/embench/embench-iot.git ignore = dirty +[submodule "test/external/riscof/riscv-arch-test"] + path = test/external/riscof/riscv-arch-test + url = https://github.com/riscv-non-isa/riscv-arch-test.git diff --git a/test/external/riscof/riscv-arch-test b/test/external/riscof/riscv-arch-test new file mode 160000 index 000000000..8a52b016d --- /dev/null +++ b/test/external/riscof/riscv-arch-test @@ -0,0 +1 @@ +Subproject commit 8a52b016dbe1e2733cc168b9d6e5c93e39059d4d From 04871936e2c20c9a2429a8543f72114ff36d62e9 Mon Sep 17 00:00:00 2001 From: lekcyjna123 <34948061+lekcyjna123@users.noreply.github.com> Date: Mon, 11 Mar 2024 17:05:12 +0100 Subject: [PATCH 3/5] Add CI runs for #604 (#608) --- .github/workflows/main.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 90b928b38..5ee446967 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -263,6 +263,12 @@ jobs: . venv/bin/activate scripts/run_tests.py -a regression + - name: Check regression with pysim + run: | + . venv/bin/activate + ./scripts/run_tests.py -c 1 -a -b pysim regression + + unit-test: name: Run unit tests runs-on: ubuntu-latest @@ -292,6 +298,9 @@ jobs: - name: Check traces and profiles run: ./scripts/run_tests.py -t -p -c 1 TestCore + - name: Check listing tests + run: ./scripts/run_tests.py -l + lint: name: Check code formatting and typing runs-on: ubuntu-latest From 6d8bba8dad67f1cb9a884762ef495bcbd13850da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Urba=C5=84czyk?= Date: Tue, 12 Mar 2024 16:49:18 +0000 Subject: [PATCH 4/5] Add B extension support to the toolchain Dockerfile (#615) --- docker/riscv-toolchain.Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/riscv-toolchain.Dockerfile b/docker/riscv-toolchain.Dockerfile index 957141eb0..a998e79e3 100644 --- a/docker/riscv-toolchain.Dockerfile +++ b/docker/riscv-toolchain.Dockerfile @@ -12,8 +12,8 @@ RUN apt-get update && \ RUN git clone --shallow-since=2023.05.01 https://github.com/riscv/riscv-gnu-toolchain && \ cd riscv-gnu-toolchain && \ - git checkout 2023.05.14 && \ - ./configure --with-multilib-generator="rv32i-ilp32--a*zifence*zicsr;rv32im-ilp32--a*zifence*zicsr;rv32ic-ilp32--a*zifence*zicsr;rv32imc-ilp32--a*zifence*zicsr;rv32imfc-ilp32f--a*zifence;rv32i_zmmul-ilp32--a*zifence*zicsr;rv32ic_zmmul-ilp32--a*zifence*zicsr" && \ + git checkout 2023.12.10 && \ + ./configure --with-multilib-generator="rv32i-ilp32--a*zifence*zicsr;rv32im-ilp32--a*zifence*zicsr;rv32ic-ilp32--a*zifence*zicsr;rv32imc-ilp32--a*zifence*zicsr;rv32imfc-ilp32f--a*zifence;rv32imc_zba_zbb_zbc_zbs-ilp32--a*zifence*zicsr" && \ make -j$(nproc) && \ cd / && rm -rf riscv-gnu-toolchain From 58249da625a3b981af7d4bafab97b47b6e73f811 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Urba=C5=84czyk?= Date: Tue, 12 Mar 2024 17:21:11 +0000 Subject: [PATCH 5/5] Enable division and bit manipulation extensions in benchmarks (#616) --- .github/workflows/benchmark.yml | 2 +- .github/workflows/main.yml | 4 ++-- test/external/embench/board_config/coreblocks-sim/board.cfg | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 45429f68d..1fc7fac1d 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -63,7 +63,7 @@ jobs: build-perf-benchmarks: name: Build performance benchmarks runs-on: ubuntu-latest - container: ghcr.io/kuznia-rdzeni/riscv-toolchain:2023.11.19_v + container: ghcr.io/kuznia-rdzeni/riscv-toolchain:2024.03.12 steps: - name: Checkout uses: actions/checkout@v3 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5ee446967..23c1aa56c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -48,7 +48,7 @@ jobs: build-riscof-tests: name: Build regression tests (riscv-arch-test) runs-on: ubuntu-latest - container: ghcr.io/kuznia-rdzeni/riscv-toolchain:2023.11.19_v + container: ghcr.io/kuznia-rdzeni/riscv-toolchain:2024.03.12 timeout-minutes: 10 env: PYENV_ROOT: "/root/.pyenv" @@ -180,7 +180,7 @@ jobs: build-regression-tests: name: Build regression tests (riscv-tests) runs-on: ubuntu-latest - container: ghcr.io/kuznia-rdzeni/riscv-toolchain:2023.11.19_v + container: ghcr.io/kuznia-rdzeni/riscv-toolchain:2024.03.12 timeout-minutes: 10 outputs: cache_hit: ${{ steps.cache-regression.outputs.cache-hit }} diff --git a/test/external/embench/board_config/coreblocks-sim/board.cfg b/test/external/embench/board_config/coreblocks-sim/board.cfg index 96eaae307..b1a885340 100644 --- a/test/external/embench/board_config/coreblocks-sim/board.cfg +++ b/test/external/embench/board_config/coreblocks-sim/board.cfg @@ -1,5 +1,5 @@ cc = 'riscv64-unknown-elf-gcc' -cflags = (['-c', '-fdata-sections', '-march=rv32ic_zmmul_zicsr', '-mabi=ilp32']) -ldflags = (['-Wl,-gc-sections', '-march=rv32ic_zmmul_zicsr', '-mabi=ilp32', '-nostartfiles', '-T../../../common/link.ld']) +cflags = (['-c', '-fdata-sections', '-march=rv32imc_zba_zbb_zbc_zbs_zicsr', '-mabi=ilp32']) +ldflags = (['-Wl,-gc-sections', '-march=rv32imc_zba_zbb_zbc_zbs_zicsr', '-mabi=ilp32', '-nostartfiles', '-T../../../common/link.ld']) user_libs = (['-lm']) cpu_mhz = 0.01