Merge branch 'master' into lekcyjna/fix-profiles-06032024

.github/workflows/benchmark.yml

@@ -63,7 +63,7 @@ jobs:
   build-perf-benchmarks:
     name: Build performance benchmarks
     runs-on: ubuntu-latest
-    container: ghcr.io/kuznia-rdzeni/riscv-toolchain:2023.11.19_v
+    container: ghcr.io/kuznia-rdzeni/riscv-toolchain:2024.03.12
     steps:
       - name: Checkout
         uses: actions/checkout@v3

.github/workflows/main.yml

@@ -48,7 +48,7 @@ jobs:
   build-riscof-tests:
     name: Build regression tests (riscv-arch-test)
     runs-on: ubuntu-latest
-    container: ghcr.io/kuznia-rdzeni/riscv-toolchain:2023.11.19_v
+    container: ghcr.io/kuznia-rdzeni/riscv-toolchain:2024.03.12
     timeout-minutes: 10
     env:
       PYENV_ROOT: "/root/.pyenv"
@@ -57,35 +57,63 @@ jobs:
     defaults:
       run:
         working-directory: test/external/riscof/
+
     steps:
       - name: Checkout
         uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - name: Cache compiled and reference riscv-arch-test
+        id: cache-riscv-arch-test
+        uses: actions/cache@v3
+        env:
+          cache-name: cache-riscv-arch-test
+        with:
+          path: |
+            test/external/riscof/riscv-arch-test/**/*.elf
+            test/external/riscof/riscof_work/**/*.signature
+            test/external/riscof/**/*Makefile*
+
+          key: ${{ env.cache-name }}-${{ runner.os }}-${{ hashFiles(
+              '**/test/external/riscof/coreblocks/**',
+              '**/test/external/riscof/spike_simple/**',
+              '**/test/external/riscof/config.ini',
+              '**/.git/modules/test/external/riscof/riscv-arch-test/HEAD',
+              '**/docker/riscv-toolchain.Dockerfile'
+            ) }}
+          restore-keys: |
+            ${{ env.cache-name }}-${{ runner.os }}-
 
-      - name: Setup PATH
+      - if: ${{ steps.cache-riscv-arch-test.outputs.cache-hit != 'true' }}
+        name: Setup PATH
         run: echo "/.pyenv/bin" >> $GITHUB_PATH
 
-      - name: Setup pyenv python
+      - if: ${{ steps.cache-riscv-arch-test.outputs.cache-hit != 'true' }}
+        name: Setup pyenv python
         run: |
           eval "$(pyenv init --path)"
           pyenv global 3.6.15
           . /venv3.6/bin/activate
 
-      - name: Setup arch test suite
+      - if: ${{ steps.cache-riscv-arch-test.outputs.cache-hit != 'true' }}
+        name: Setup arch test suite
         run: |
           . /venv3.6/bin/activate
-          riscof --verbose info arch-test --clone
           riscof testlist --config=config.ini --suite=riscv-arch-test/riscv-test-suite/ --env=riscv-arch-test/riscv-test-suite/env
 
-      - name: Build and run tests on reference and generate Makefiles
+      - if: ${{ steps.cache-riscv-arch-test.outputs.cache-hit != 'true' }}
+        name: Build and run tests on reference and generate Makefiles
         run: |
           . /venv3.6/bin/activate
           riscof run --config=config.ini --suite=riscv-arch-test/riscv-test-suite/ --env=riscv-arch-test/riscv-test-suite/env
 
-      - name: Build tests for Coreblocks
+      - if: ${{ steps.cache-riscv-arch-test.outputs.cache-hit != 'true' }}
+        name: Build tests for Coreblocks
         run: |
           MAKEFILE_PATH=riscof_work/Makefile.build-DUT-coreblocks ../../../ci/riscof_run_makefile.sh
-
-      - uses: actions/upload-artifact@v3
+      - if: ${{ steps.cache-riscv-arch-test.outputs.cache-hit != 'true' }}
+        uses: actions/upload-artifact@v3
         with:
           name: "riscof-tests"
           path: |
@@ -102,6 +130,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
+        with:
+          submodules: recursive
 
       - name: Set up Python
         uses: actions/setup-python@v4
@@ -120,10 +150,23 @@ jobs:
           name: "verilog-full-core"
           path: .
 
-      - uses: actions/download-artifact@v3
+      - uses: actions/cache@v3
+        env:
+          cache-name: cache-riscv-arch-test
         with:
-          name: "riscof-tests"
-          path: test/external/riscof/
+          path: |
+            test/external/riscof/riscv-arch-test/**/*.elf
+            test/external/riscof/riscof_work/**/*.signature
+            test/external/riscof/**/*Makefile*
+
+          key: ${{ env.cache-name }}-${{ runner.os }}-${{ hashFiles(
+              '**/test/external/riscof/coreblocks/**',
+              '**/test/external/riscof/spike_simple/**',
+              '**/test/external/riscof/config.ini',
+              '**/.git/modules/test/external/riscof/riscv-arch-test/HEAD',
+              '**/docker/riscv-toolchain.Dockerfile'
+            ) }}
+          fail-on-cache-miss: true
 
       - name: Run tests on Coreblocks
         run: |
@@ -137,7 +180,7 @@ jobs:
   build-regression-tests:
     name: Build regression tests (riscv-tests)
     runs-on: ubuntu-latest
-    container: ghcr.io/kuznia-rdzeni/riscv-toolchain:2023.11.19_v
+    container: ghcr.io/kuznia-rdzeni/riscv-toolchain:2024.03.12
     timeout-minutes: 10
     outputs:
         cache_hit: ${{ steps.cache-regression.outputs.cache-hit }}
@@ -220,6 +263,12 @@ jobs:
           . venv/bin/activate
           scripts/run_tests.py -a regression
 
+      - name: Check regression with pysim
+        run: |
+          . venv/bin/activate
+          ./scripts/run_tests.py -c 1 -a -b pysim regression
+
+
   unit-test:
     name: Run unit tests
     runs-on: ubuntu-latest
@@ -249,6 +298,9 @@ jobs:
       - name: Check traces and profiles
         run: ./scripts/run_tests.py -t -p -c 1 TestCore
 
+      - name: Check listing tests
+        run: ./scripts/run_tests.py -l
+
   lint:
     name: Check code formatting and typing
     runs-on: ubuntu-latest

.gitmodules

@@ -5,3 +5,6 @@
 	path = test/external/embench/embench-iot
 	url = https://github.com/embench/embench-iot.git
 	ignore = dirty
+[submodule "test/external/riscof/riscv-arch-test"]
+	path = test/external/riscof/riscv-arch-test
+	url = https://github.com/riscv-non-isa/riscv-arch-test.git

docker/riscv-toolchain.Dockerfile

@@ -12,8 +12,8 @@ RUN apt-get update && \
 
 RUN git clone --shallow-since=2023.05.01 https://github.com/riscv/riscv-gnu-toolchain && \
     cd riscv-gnu-toolchain && \
-    git checkout 2023.05.14 && \
-    ./configure --with-multilib-generator="rv32i-ilp32--a*zifence*zicsr;rv32im-ilp32--a*zifence*zicsr;rv32ic-ilp32--a*zifence*zicsr;rv32imc-ilp32--a*zifence*zicsr;rv32imfc-ilp32f--a*zifence;rv32i_zmmul-ilp32--a*zifence*zicsr;rv32ic_zmmul-ilp32--a*zifence*zicsr" && \
+    git checkout 2023.12.10 && \
+    ./configure --with-multilib-generator="rv32i-ilp32--a*zifence*zicsr;rv32im-ilp32--a*zifence*zicsr;rv32ic-ilp32--a*zifence*zicsr;rv32imc-ilp32--a*zifence*zicsr;rv32imfc-ilp32f--a*zifence;rv32imc_zba_zbb_zbc_zbs-ilp32--a*zifence*zicsr" && \
     make -j$(nproc) && \
     cd / && rm -rf riscv-gnu-toolchain
 

scripts/run_tests.py

@@ -33,12 +33,9 @@ def main():
     pytest_arguments = ["--max-worker-restart=1"]
 
     if args.trace:
-        os.environ["__COREBLOCKS_DUMP_TRACES"] = "1"
         pytest_arguments.append("--coreblocks-traces")
-
     if args.profile:
-        os.environ["__TRANSACTRON_PROFILE"] = "1"
-
+        pytest_arguments.append("--coreblocks-profile")
     if args.test_name:
         pytest_arguments += [f"--coreblocks-test-name={args.test_name}"]
     if args.count:

test/conftest.py

@@ -1,4 +1,5 @@
 import re
+import os
 from typing import Optional
 import pytest
 
@@ -13,6 +14,7 @@ def pytest_addoption(parser: pytest.Parser):
         help="Simulation backend for regression tests",
     )
     group.addoption("--coreblocks-traces", action="store_true", help="Generate traces from regression tests")
+    group.addoption("--coreblocks-profile", action="store_true", help="Write execution profiles")
     group.addoption("--coreblocks-list", action="store_true", help="List all tests in flatten format.")
     group.addoption(
         "--coreblocks-test-name",
@@ -90,3 +92,15 @@ def deselect_based_on_count(items: list[pytest.Item], config: pytest.Config) ->
 def pytest_collection_modifyitems(items: list[pytest.Item], config: pytest.Config) -> None:
     deselect_based_on_flatten_name(items, config)
     deselect_based_on_count(items, config)
+
+
+def pytest_runtest_setup(item: pytest.Item):
+    """
+    This function is called to perform the setup phase for every test, so
+    it is a perfect moment to set environment variables.
+    """
+    if item.config.getoption("--coreblocks-traces", False):  # type: ignore
+        os.environ["__TRANSACTRON_DUMP_TRACES"] = "1"
+
+    if item.config.getoption("--coreblocks-profile", False):  # type: ignore
+        os.environ["__TRANSACTRON_PROFILE"] = "1"

test/external/embench/board_config/coreblocks-sim/board.cfg

@@ -1,5 +1,5 @@
 cc = 'riscv64-unknown-elf-gcc'
-cflags = (['-c', '-fdata-sections', '-march=rv32ic_zmmul_zicsr', '-mabi=ilp32'])
-ldflags = (['-Wl,-gc-sections', '-march=rv32ic_zmmul_zicsr', '-mabi=ilp32', '-nostartfiles', '-T../../../common/link.ld'])
+cflags = (['-c', '-fdata-sections', '-march=rv32imc_zba_zbb_zbc_zbs_zicsr', '-mabi=ilp32'])
+ldflags = (['-Wl,-gc-sections', '-march=rv32imc_zba_zbb_zbc_zbs_zicsr', '-mabi=ilp32', '-nostartfiles', '-T../../../common/link.ld'])
 user_libs = (['-lm'])
 cpu_mhz = 0.01

test/regression/conftest.py

@@ -2,7 +2,6 @@
 from pathlib import Path
 import pytest
 import subprocess
-import sys
 
 test_dir = Path(__file__).parent.parent
 riscv_tests_dir = test_dir.joinpath("external/riscv-tests")
@@ -19,7 +18,6 @@ def load_regression_tests() -> list[str]:
         res = subprocess.run(["make", "-C", "test/external/riscv-tests"])
         if res.returncode != 0:
             print("Couldn't build regression tests")
-            sys.exit(1)
         all_tests = set(get_all_test_names())
 
     exclude = {"rv32ui-ma_data", "rv32ui-fence_i"}
@@ -28,19 +26,16 @@ def load_regression_tests() -> list[str]:
 
 
 def pytest_generate_tests(metafunc: pytest.Metafunc):
-    if not metafunc.config.getoption("coreblocks_regression"):
-        # Add regression to skiped tests
-        metafunc.parametrize(["test_name", "backend", "traces", "verbose"], [])
-        return
-
     all_tests = (
         load_regression_tests()
     )  # The list has to be always in the same order (e.g. sorted) to allow for parallel testing
-    traces = metafunc.config.getoption("coreblocks_traces")
-    backend = metafunc.config.getoption("coreblocks_backend")
-    verbose = bool(metafunc.config.getoption("verbose"))
-    if {"test_name", "backend", "traces", "verbose"}.issubset(metafunc.fixturenames):
+    if "test_name" in metafunc.fixturenames:
         metafunc.parametrize(
-            ["test_name", "backend", "traces", "verbose"],
-            [(test_name, backend, traces, verbose) for test_name in all_tests],
+            "test_name",
+            [test_name for test_name in all_tests],
         )
+
+
+def pytest_runtest_setup(item: pytest.Item):
+    if not item.config.getoption("--coreblocks-regression", default=False):  # type: ignore
+        pytest.skip("need --coreblocks-regression option to run this test")

test/regression/test_regression.py

@@ -81,11 +81,11 @@ def regression_body_with_cocotb(test_name: str, traces: bool):
     assert len(list(tree.iter("failure"))) == 0
 
 
-def regression_body_with_pysim(test_name: str, traces: bool, verbose: bool):
+def regression_body_with_pysim(test_name: str, traces: bool):
     traces_file = None
     if traces:
         traces_file = REGRESSION_TESTS_PREFIX + test_name
-    asyncio.run(run_test(PySimulation(verbose, traces_file=traces_file), test_name))
+    asyncio.run(run_test(PySimulation(verbose=False, traces_file=traces_file), test_name))
 
 
 @pytest.fixture(scope="session")
@@ -126,8 +126,18 @@ def verilate_model(worker_id, request: pytest.FixtureRequest):
         os.remove(counter_path)
 
 
-def test_entrypoint(test_name: str, backend: Literal["pysim", "cocotb"], traces: bool, verbose: bool, verilate_model):
-    if backend == "cocotb":
-        regression_body_with_cocotb(test_name, traces)
-    elif backend == "pysim":
-        regression_body_with_pysim(test_name, traces, verbose)
+@pytest.fixture
+def sim_backend(request: pytest.FixtureRequest):
+    return request.config.getoption("coreblocks_backend")
+
+
+@pytest.fixture
+def traces_enabled(request: pytest.FixtureRequest):
+    return request.config.getoption("coreblocks_traces")
+
+
+def test_entrypoint(test_name: str, sim_backend: Literal["pysim", "cocotb"], traces_enabled: bool, verilate_model):
+    if sim_backend == "cocotb":
+        regression_body_with_cocotb(test_name, traces_enabled)
+    elif sim_backend == "pysim":
+        regression_body_with_pysim(test_name, traces_enabled)

transactron/testing/infrastructure.py

@@ -241,7 +241,7 @@ def add_all_mocks(self, sim: PysimSimulator, frame_locals: dict) -> None:
     @contextmanager
     def run_simulation(self, module: HasElaborate, max_cycles: float = 10e4, add_transaction_module=True):
         traces_file = None
-        if "__COREBLOCKS_DUMP_TRACES" in os.environ:
+        if "__TRANSACTRON_DUMP_TRACES" in os.environ:
             traces_file = unittest.TestCase.id(self)
 
         clk_period = 1e-6