Profiles for regression tests (#596)

scripts/run_benchmarks.py

@@ -144,6 +144,7 @@ def main():
     parser = argparse.ArgumentParser()
     parser.add_argument("-l", "--list", action="store_true", help="List all benchmarks")
     parser.add_argument("-t", "--trace", action="store_true", help="Dump waveforms")
+    parser.add_argument("-p", "--profile", action="store_true", help="Write execution profiles")
     parser.add_argument("-v", "--verbose", action="store_true", help="Verbose output")
     parser.add_argument("-b", "--backend", default="cocotb", choices=["cocotb", "pysim"], help="Simulation backend")
     parser.add_argument(
@@ -171,6 +172,9 @@ def main():
             print(f"Could not find benchmark '{args.benchmark_name}'")
             sys.exit(1)
 
+    if args.profile:
+        os.environ["__TRANSACTRON_PROFILE"] = "1"
+
     success = run_benchmarks(benchmarks, args.backend, args.trace, args.verbose)
     if not success:
         print("Benchmark execution failed")

test/regression/benchmark.py

@@ -9,6 +9,7 @@
 test_dir = Path(__file__).parent.parent
 embench_dir = test_dir.joinpath("external/embench/build/src")
 results_dir = test_dir.joinpath("regression/benchmark_results")
+profile_dir = test_dir.joinpath("__profiles__")
 
 
 @dataclass_json
@@ -77,6 +78,10 @@ async def run_benchmark(sim_backend: SimulationBackend, benchmark_name: str):
 
     result = await sim_backend.run(mem_model, timeout_cycles=2000000)
 
+    if result.profile is not None:
+        os.makedirs(profile_dir, exist_ok=True)
+        result.profile.encode(f"{profile_dir}/benchmark.{benchmark_name}.json")
+
     if not result.success:
         raise RuntimeError("Simulation timed out")
 

test/regression/cocotb.py

@@ -8,13 +8,14 @@
 import cocotb
 from cocotb.clock import Clock, Timer
 from cocotb.handle import ModifiableObject
-from cocotb.triggers import FallingEdge, Event, with_timeout
+from cocotb.triggers import FallingEdge, Event, RisingEdge, with_timeout
 from cocotb_bus.bus import Bus
 from cocotb.result import SimTimeoutError
 
 from .memory import *
 from .common import SimulationBackend, SimulationExecutionResult
 
+from transactron.profiler import CycleProfile, MethodSamples, Profile, ProfileSamples, TransactionSamples
 from transactron.utils.gen import GenerationInfo
 
 
@@ -151,12 +152,43 @@ def get_cocotb_handle(self, path_components: list[str]) -> ModifiableObject:
         obj = self.dut
         # Skip the first component, as it is already referenced in "self.dut"
         for component in path_components[1:]:
-            # As the component may start with '_' character, we need to use '_id'
-            # function instead of 'getattr' - this is required by cocotb.
-            obj = obj._id(component, extended=False)
+            try:
+                # As the component may start with '_' character, we need to use '_id'
+                # function instead of 'getattr' - this is required by cocotb.
+                obj = obj._id(component, extended=False)
+            except AttributeError:
+                if component[0] == "\\" and component[-1] == " ":
+                    # workaround for cocotb/verilator weirdness
+                    # for some escaped names lookup fails, but works when unescaped
+                    obj = obj._id(component[1:-1], extended=False)
+                else:
+                    raise
 
         return obj
 
+    async def profile_handler(self, clock, profile: Profile):
+        clock_edge_event = RisingEdge(clock)
+
+        while True:
+            samples = ProfileSamples()
+
+            for transaction_id, location in self.gen_info.transaction_signals_location.items():
+                request_val = self.get_cocotb_handle(location.request)
+                runnable_val = self.get_cocotb_handle(location.runnable)
+                grant_val = self.get_cocotb_handle(location.grant)
+                samples.transactions[transaction_id] = TransactionSamples(
+                    bool(request_val.value), bool(runnable_val.value), bool(grant_val.value)
+                )
+
+            for method_id, location in self.gen_info.method_signals_location.items():
+                run_val = self.get_cocotb_handle(location.run)
+                samples.methods[method_id] = MethodSamples(bool(run_val.value))
+
+            cprof = CycleProfile.make(samples, self.gen_info.profile_data)
+            profile.cycles.append(cprof)
+
+            await clock_edge_event  # type: ignore
+
     async def assert_handler(self, clock):
         clock_edge_event = FallingEdge(clock)
 
@@ -182,6 +214,12 @@ async def run(self, mem_model: CoreMemoryModel, timeout_cycles: int = 5000) -> S
         data_wb = WishboneSlave(self.dut, "wb_data", self.dut.clk, mem_model, is_instr_bus=False)
         cocotb.start_soon(data_wb.start())
 
+        profile = None
+        if "__TRANSACTRON_PROFILE" in os.environ:
+            profile = Profile()
+            profile.transactions_and_methods = self.gen_info.profile_data.transactions_and_methods
+            cocotb.start_soon(self.profile_handler(self.dut.clk, profile))
+
         cocotb.start_soon(self.assert_handler(self.dut.clk))
 
         success = True
@@ -192,6 +230,8 @@ async def run(self, mem_model: CoreMemoryModel, timeout_cycles: int = 5000) -> S
 
         result = SimulationExecutionResult(success)
 
+        result.profile = profile
+
         for metric_name, metric_loc in self.gen_info.metrics_location.items():
             result.metric_values[metric_name] = {}
             for reg_name, reg_loc in metric_loc.regs.items():

test/regression/common.py

@@ -1,6 +1,8 @@
 from abc import ABC, abstractmethod
 from dataclasses import dataclass, field
+from typing import Optional
 from .memory import CoreMemoryModel
+from transactron.profiler import Profile
 
 
 @dataclass
@@ -18,6 +20,7 @@ class SimulationExecutionResult:
 
     success: bool
     metric_values: dict[str, dict[str, int]] = field(default_factory=dict)
+    profile: Optional[Profile] = None
 
 
 class SimulationBackend(ABC):

test/regression/conftest.py

@@ -6,6 +6,7 @@
 
 test_dir = Path(__file__).parent.parent
 riscv_tests_dir = test_dir.joinpath("external/riscv-tests")
+profile_dir = test_dir.joinpath("__profiles__")
 
 
 def get_all_test_names():

test/regression/pysim.py

@@ -1,13 +1,16 @@
 import re
+import os
 
 from amaranth.sim import Passive, Settle
 from amaranth.utils import exact_log2
 from amaranth import *
 
+from transactron.core import TransactionManagerKey
+
 from .memory import *
 from .common import SimulationBackend, SimulationExecutionResult
 
-from transactron.testing import PysimSimulator, TestGen
+from transactron.testing import PysimSimulator, TestGen, profiler_process, Profile
 from transactron.utils.dependencies import DependencyContext, DependencyManager
 from transactron.lib.metrics import HardwareMetricsManager
 from ..peripherals.test_wishbone import WishboneInterfaceWrapper
@@ -142,6 +145,12 @@ async def run(self, mem_model: CoreMemoryModel, timeout_cycles: int = 5000) -> S
             sim.add_sync_process(self._wishbone_slave(mem_model, wb_instr_ctrl, is_instr_bus=True))
             sim.add_sync_process(self._wishbone_slave(mem_model, wb_data_ctrl, is_instr_bus=False))
 
+            profile = None
+            if "__TRANSACTRON_PROFILE" in os.environ:
+                transaction_manager = DependencyContext.get().get_dependency(TransactionManagerKey())
+                profile = Profile()
+                sim.add_sync_process(profiler_process(transaction_manager, profile))
+
             metric_values: dict[str, dict[str, int]] = {}
 
             def on_sim_finish():
@@ -160,7 +169,7 @@ def on_sim_finish():
             if self.verbose:
                 self.pretty_dump_metrics(metric_values)
 
-            return SimulationExecutionResult(success, metric_values)
+            return SimulationExecutionResult(success, metric_values, profile)
 
     def stop(self):
         self.running = False

test/regression/test_regression.py

@@ -1,6 +1,6 @@
 from .memory import *
 from .common import SimulationBackend
-from .conftest import riscv_tests_dir
+from .conftest import riscv_tests_dir, profile_dir
 from test.regression.pysim import PySimulation
 import xml.etree.ElementTree as eT
 import asyncio
@@ -48,6 +48,10 @@ async def run_test(sim_backend: SimulationBackend, test_name: str):
 
     result = await sim_backend.run(mem_model, timeout_cycles=5000)
 
+    if result.profile is not None:
+        os.makedirs(profile_dir, exist_ok=True)
+        result.profile.encode(f"{profile_dir}/test.regression.{test_name}.json")
+
     if not result.success:
         raise RuntimeError("Simulation timed out")