multiprocess exec_fixtures

commands.md

@@ -112,6 +112,7 @@ $ solana-test-suite exec-fixtures [OPTIONS]
 * `-t, --target PATH`: Shared object (.so) target file path to execute  [default: impl/firedancer/build/native/clang/lib/libfd_exec_sol_compat.so]
 * `-r, --randomize-output-buffer`: Randomizes bytes in output buffer before shared library execution
 * `-l, --log-level INTEGER`: FD logging level  [default: 2]
+* `-p, --num-processes INTEGER`: Number of processes to use  [default: 4]
 * `--help`: Show this message and exit.
 
 ## `solana-test-suite execute`

src/test_suite/multiprocessing_utils.py

@@ -1,7 +1,7 @@
 from dataclasses import dataclass, field
 from test_suite.constants import OUTPUT_BUFFER_SIZE
 from test_suite.fuzz_context import ENTRYPOINT_HARNESS_MAP, HarnessCtx
-from test_suite.fuzz_interface import ContextType
+from test_suite.fuzz_interface import ContextType, EffectsType
 import test_suite.invoke_pb2 as invoke_pb
 import test_suite.metadata_pb2 as metadata_pb2
 import ctypes
@@ -372,3 +372,21 @@ def run_test(test_file: Path) -> tuple[str, int, dict | None]:
     results = process_single_test_case(harness_ctx, context)
     pruned_results = harness_ctx.prune_effects_fn(context, results)
     return test_file.stem, *build_test_results(harness_ctx, pruned_results)
+
+
+def execute_fixture(test_file: Path) -> tuple[str, int | None]:
+    if test_file.suffix != ".fix":
+        print(f"File {test_file} is not a fixture")
+        return test_file.stem, None
+
+    fn_entrypoint = extract_metadata(test_file).fn_entrypoint
+    harness_ctx = ENTRYPOINT_HARNESS_MAP[fn_entrypoint]
+    fixture = read_fixture(test_file)
+    context = fixture.input
+    output = fixture.output
+
+    effects = process_target(
+        harness_ctx, globals.target_libraries[globals.reference_shared_library], context
+    )
+
+    return test_file.stem, effects == output

src/test_suite/test_suite.py

@@ -19,6 +19,7 @@
 )
 from test_suite.multiprocessing_utils import (
     decode_single_test_case,
+    execute_fixture,
     extract_metadata,
     read_fixture,
     initialize_process_output_buffers,
@@ -94,7 +95,7 @@ def execute(
     except:
         set_ld_preload_asan()
 
-    files_to_exec = input.iterdir() if input.is_dir() else [input]
+    files_to_exec = list(input.iterdir()) if input.is_dir() else [input]
     for file in files_to_exec:
         print(f"Handling {file}...")
         if file.suffix == ".fix":
@@ -162,7 +163,7 @@ def fix_to_ctx(
         shutil.rmtree(globals.output_dir)
     globals.output_dir.mkdir(parents=True, exist_ok=True)
 
-    test_cases = input.iterdir() if input.is_dir() else [input]
+    test_cases = list(input.iterdir()) if input.is_dir() else [input]
     num_test_cases = len(test_cases)
 
     print(f"Converting to Fixture messages...")
@@ -955,6 +956,9 @@ def exec_fixtures(
         "-l",
         help="FD logging level",
     ),
+    num_processes: int = typer.Option(
+        4, "--num-processes", "-p", help="Number of processes to use"
+    ),
 ):
     # Initialize output buffers and shared library
     initialize_process_output_buffers(randomize_output_buffer=randomize_output_buffer)
@@ -966,39 +970,36 @@ def exec_fixtures(
     except:
         set_ld_preload_asan()
 
+    test_cases = list(input.iterdir()) if input.is_dir() else [input]
+    num_test_cases = len(test_cases)
+    print("Running tests...")
+    results = []
+    with Pool(
+        processes=num_processes,
+        initializer=initialize_process_output_buffers,
+        initargs=(randomize_output_buffer,),
+    ) as pool:
+        for result in tqdm.tqdm(
+            pool.imap(execute_fixture, test_cases),
+            total=num_test_cases,
+        ):
+            results.append(result)
+
     passed = 0
     failed = 0
     skipped = 0
     failed_tests = []
     skipped_tests = []
 
-    files_to_exec = input.iterdir() if input.is_dir() else [input]
-    for file in files_to_exec:
-        if file.suffix == ".fix":
-            fn_entrypoint = extract_metadata(file).fn_entrypoint
-            harness_ctx = ENTRYPOINT_HARNESS_MAP[fn_entrypoint]
-            fixture = read_fixture(file)
-            context = fixture.input
-            output = fixture.output
-        else:
-            print(f"File {file} is not a fixture")
+    for file, status in results:
+        if status == None:
             skipped += 1
-            skipped_tests.append(file.stem)
-            continue
-
-        effects = process_target(harness_ctx, lib, context)
-
-        if not effects:
-            print(f"No {harness_ctx.effects_type.__name__} returned for file {file}")
-            skipped += 1
-            skipped_tests.append(file.stem)
-            continue
-
-        if output == effects:
+            skipped_tests.append(file)
+        elif status == 1:
             passed += 1
         else:
             failed += 1
-            failed_tests.append(file.stem)
+            failed_tests.append(file)
 
     print(f"Total test cases: {passed + failed + skipped}")
     print(f"Passed: {passed}, Failed: {failed}, Skipped: {skipped}")