Allow generation of test fixture messages

README.md

@@ -64,12 +64,12 @@ solana-test-suite run-tests --input-dir <input_dir> --solana-target <solana_targ
 You can pick out a single test case and run it to view the instruction effects via output with the following command:
 
 ```sh
-solana-test-suite execute-single-instruction --input-dir <input_dir> --target <shared_lib>
+solana-test-suite execute-single-instruction --input <input_file> --target <shared_lib>
 ```
 
 | Argument        | Description                                                                                         |
 |-----------------|-----------------------------------------------------------------------------------------------------|
-| `--input`      | Input file |
+| `--input`      | Input file containing instruction context message |
 | `--target`      | Shared object (.so) target file path to debug  |
 
 
@@ -78,31 +78,47 @@ solana-test-suite execute-single-instruction --input-dir <input_dir> --target <s
 For failing test cases, it may be useful to analyze what could have differed between Solana and Firedancer. You can execute a Protobuf message (human-readable or binary) through the desired client as such:
 
 ```sh
-solana-test-suite debug-instruction --input-dir <input_dir> --target <shared_lib> --debugger <gdb,rust-gdb,etc>
+solana-test-suite debug-instruction --input <input_file> --target <shared_lib> --debugger <gdb,rust-gdb,etc>
 ```
 
 | Argument        | Description                                                                                         |
 |-----------------|-----------------------------------------------------------------------------------------------------|
-| `--input`      | Input file |
+| `--input`      | Input file containing instruction context message |
 | `--target`      | Shared object (.so) target file path to debug  |
 | `--debugger`  | Debugger to use (gdb, rust-gdb) |
 
-Recommended usage is opening two terminals side by side, and running the above command on both with one having `--executable-path` for Solana (`impl/lib/libsolfuzz_agave_v2.0.so`) and another for Firedancer (`impl/lib/libsolfuzz_firedancer.so`), and then stepping through the debugger for each corresponding test case.
+Recommended usage is opening two terminals side by side, and running the above command on both with one having `--target` for Solana (`impl/lib/libsolfuzz_agave_v2.0.so`) and another for Firedancer (`impl/lib/libsolfuzz_firedancer.so`), and then stepping through the debugger for each corresponding test case.
 
 
 ### Minimizing
 
 Prunes extra fields in the input (e.g. feature set) and produces a minimal test case such that the output does not change.
 
 ```sh
-solana-test-suite minimize-tests --input-dir <input_dir> --solana-target <solana_target.so> --output-dir <log_output_dir> --num-processes <num_processes>
+solana-test-suite minimize-tests --input-dir <input_dir> --solana-target <solana_target.so> --output-dir <pruned_ctx_output_dir> --num-processes <num_processes>
 ```
 
 | Argument        | Description                                                                                         |
 |-----------------|-----------------------------------------------------------------------------------------------------|
 | `--input-dir`   | Input directory containing instruction context messages |
 | `--solana-target` | Path to Solana Agave shared object (.so) target file            |
-| `--output-dir`  | Log output directory for test results |
+| `--output-dir`  | Pruned instruction context dumping directory |
+| `--num-processes`  | Number of processes to use |
+
+
+### Creating Fixtures
+
+Create full test fixtures containing both instruction context and effects. Effects are computed by running instruction context through `--solana-target`. Fixtures with `None` values for instruction context/effects are not included.
+
+```sh
+solana-test-suite create-fixtures --input-dir <input_dir> --solana-target <solana_target.so> --output-dir <fixtures_output_dir> --num-processes <num_processes>
+```
+
+| Argument        | Description                                                                                         |
+|-----------------|-----------------------------------------------------------------------------------------------------|
+| `--input-dir`   | Input directory containing instruction context messages |
+| `--solana-target` | Path to Solana Agave shared object (.so) target file            |
+| `--output-dir`  | Instruction fixtures dumping directory |
 | `--num-processes`  | Number of processes to use |
 
 

src/test_suite/fixture_utils.py

@@ -0,0 +1,68 @@
+from test_suite.multiprocessing_utils import prune_execution_result
+import test_suite.globals as globals
+import test_suite.invoke_pb2 as pb
+
+
+def create_fixture(
+    file_serialized_instruction_context: tuple[str, dict],
+    file_serialized_instruction_effects: tuple[str, dict[str, str | None]],
+) -> tuple[str, str | None]:
+    """
+    Create instruction fixture for an instruction context and effects.
+
+    Args:
+        - file_serialized_instruction_context (tuple[str, str]): Tuple of file stem and serialized instruction context.
+        - file_serialized_instruction_effects (tuple[str, dict[str, str | None]]): Tuple of file stem and dictionary of target library names and serialized instruction effects.
+
+    Returns:
+        - tuple[str, str | None]: Tuple of file stem and instruction fixture.
+    """
+
+    file_stem, serialized_instruction_context = file_serialized_instruction_context
+    file_stem_2, serialized_instruction_effects = file_serialized_instruction_effects
+
+    assert file_stem == file_stem_2, f"{file_stem} != {file_stem_2}"
+
+    # Both instruction context and instruction effects should not be None
+    if serialized_instruction_context is None or serialized_instruction_effects is None:
+        return file_stem, None
+
+    _, targets_to_serialized_pruned_instruction_effects = prune_execution_result(
+        file_serialized_instruction_context, file_serialized_instruction_effects
+    )
+
+    pruned_instruction_effects = targets_to_serialized_pruned_instruction_effects[
+        globals.solana_shared_library
+    ]
+
+    # Create instruction fixture
+    instr_context = pb.InstrContext()
+    instr_context.ParseFromString(serialized_instruction_context)
+    instr_effects = pb.InstrEffects()
+    instr_effects.ParseFromString(pruned_instruction_effects)
+
+    fixture = pb.InstrFixture()
+    fixture.input.MergeFrom(instr_context)
+    fixture.output.MergeFrom(instr_effects)
+
+    return file_stem, fixture.SerializeToString(deterministic=True)
+
+
+def write_fixture_to_disk(file_stem: str, serialized_instruction_fixture: str) -> int:
+    """
+    Writes instruction fixtures to disk.
+
+    Args:
+        - file_stem (str): File stem
+        - serialized_instruction_fixture (str): Serialized instruction fixture
+
+    Returns:
+        - int: 0 on failure, 1 on success
+    """
+    if serialized_instruction_fixture is None:
+        return 0
+
+    with open(f"{globals.output_dir}/{file_stem}.bin", "wb") as f:
+        f.write(serialized_instruction_fixture)
+
+    return 1

src/test_suite/minimize_utils.py

@@ -7,10 +7,16 @@
 )
 
 
-def minimize_single_test_case(test_file: Path):
+def minimize_single_test_case(test_file: Path) -> int:
     """
     Minimize a single test case by pruning any additional accounts / features that do not
     affect output.
+
+    Args:
+        test_file (Path): The test file to minimize
+
+    Returns:
+        int: 0 on failure, 1 on success
     """
     _, serialized_instruction_context = generate_test_case(test_file)
 

src/test_suite/multiprocessing_utils.py

@@ -191,7 +191,7 @@ def merge_results_over_iterations(results: tuple) -> tuple[str, dict]:
 
 def prune_execution_result(
     file_serialized_instruction_context: tuple[str, dict],
-    file_serialized_instruction_effects,
+    file_serialized_instruction_effects: tuple[str, dict[str, str | None]],
 ) -> tuple[str, dict]:
     """
     Prune execution result to only include actually modified accounts.

src/test_suite/test_suite.py

@@ -7,6 +7,7 @@
 from pathlib import Path
 from google.protobuf import text_format
 from test_suite.constants import LOG_FILE_SEPARATOR_LENGTH
+from test_suite.fixture_utils import create_fixture, write_fixture_to_disk
 import test_suite.invoke_pb2 as pb
 from test_suite.codec_utils import decode_input, encode_input, encode_output
 from test_suite.minimize_utils import minimize_single_test_case
@@ -57,9 +58,12 @@ def execute_single_instruction(
     lib.sol_compat_init()
 
     # Execute and cleanup
-    instruction_effects = process_instruction(
-        lib, instruction_context
-    ).SerializeToString(deterministic=True)
+    instruction_effects = process_instruction(lib, instruction_context)
+
+    if not instruction_effects:
+        return None
+
+    instruction_effects = instruction_effects.SerializeToString(deterministic=True)
 
     # Prune execution results
     _, pruned_instruction_effects = prune_execution_result(
@@ -273,6 +277,78 @@ def minimize_tests(
     print(f"{sum(minimize_results)} files successfully minimized")
 
 
+@app.command()
+def create_fixtures(
+    input_dir: Path = typer.Option(
+        Path("corpus8"),
+        "--input-dir",
+        "-i",
+        help="Input directory containing instruction context messages",
+    ),
+    solana_shared_library: Path = typer.Option(
+        Path("impl/lib/libsolfuzz_agave_v2.0.so"),
+        "--solana-target",
+        "-s",
+        help="Solana (or ground truth) shared object (.so) target file path",
+    ),
+    output_dir: Path = typer.Option(
+        Path("test_fixtures"),
+        "--output-dir",
+        "-o",
+        help="Output directory for fixtures",
+    ),
+    num_processes: int = typer.Option(
+        4, "--num-processes", "-p", help="Number of processes to use"
+    ),
+):
+    # Specify globals
+    globals.output_dir = output_dir
+    globals.solana_shared_library = solana_shared_library
+
+    # Create the output directory, if necessary
+    if globals.output_dir.exists():
+        shutil.rmtree(globals.output_dir)
+    globals.output_dir.mkdir(parents=True, exist_ok=True)
+
+    # Initialize shared library
+    globals.solana_shared_library = solana_shared_library
+    lib = ctypes.CDLL(solana_shared_library)
+    lib.sol_compat_init()
+    globals.target_libraries[solana_shared_library] = lib
+
+    # Generate the test cases in parallel from files on disk
+    print("Reading test files...")
+    with Pool(processes=num_processes) as pool:
+        execution_contexts = pool.map(generate_test_case, input_dir.iterdir())
+
+    # Process the test cases in parallel through shared libraries
+    print("Executing tests...")
+    with Pool(
+        processes=num_processes, initializer=initialize_process_output_buffers
+    ) as pool:
+        execution_results = pool.starmap(process_single_test_case, execution_contexts)
+
+    print("Creating fixtures...")
+    # Prune effects and create fixtures
+    with Pool(processes=num_processes) as pool:
+        execution_fixtures = pool.starmap(
+            create_fixture, zip(execution_contexts, execution_results)
+        )
+
+    # Write fixtures to disk
+    print("Writing results to disk...")
+    with Pool(processes=num_processes) as pool:
+        write_results = pool.starmap(write_fixture_to_disk, execution_fixtures)
+
+    # Clean up
+    print("Cleaning up...")
+    lib.sol_compat_fini()
+
+    print("-" * LOG_FILE_SEPARATOR_LENGTH)
+    print(f"{len(write_results)} total files seen")
+    print(f"{sum(write_results)} files successfully written")
+
+
 @app.command()
 def run_tests(
     input_dir: Path = typer.Option(