diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7c49805..73a7c31 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,3 +9,12 @@ repos: # pre-commit's default_language_version, see # https://pre-commit.com/#top_level-default_language_version language_version: python3.11 + - repo: local + hooks: + - id: gen-cmd-docs + name: Generate command documentation + language: python + entry: typer src/test_suite/test_suite.py utils docs --name solana-test-suite --output commands.md + files: src/test_suite/test_suite.py + pass_filenames: false + diff --git a/README.md b/README.md index 1efa135..699ea3f 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,13 @@ Clone this repository and run: source install.sh ``` +### (HIGHLY Recommended) Install auto-completion + +```sh +solana-test-suite --install-completion +``` +You will need to reload your shell + the `test_suite_env` venv to see the changes. + ## Protobuf Each target must contain a `sol_compat_instr_execute_v1` function that takes in a `InstrContext` message and outputs a `InstrEffects` message (see [`proto/invoke.proto`](https://github.com/firedancer-io/protosol/blob/main/proto/invoke.proto)). See `utils.py:process_instruction` to see how the program interacts with shared libraries. @@ -26,6 +33,14 @@ All message definitions are defined in [Protosol](https://github.com/firedancer- ``` ## Usage +Run `solana-test-suite --help` to see the available commands. +Or, refer to the [commands.md](commands.md) for a list of available commands. + +### Note on [commands.md](commands.md) +`commands.md` is automatically generated from the `typer utils docs` module. +Help messages are dynamically generated based on the [currently set harness type](#selecting-the-correct-harness). Thus, descriptions specific to harness type (typically the Context, Effects, and Fixtures names) in `commands.md` refer to the harness type set during time of generation. + +When the desired [harness type is set](#selecting-the-correct-harness), the `--help` output in the CLI will reflect the correct names. Try it! ### Selecting the correct harness The harness type should be specified by an environment variable `HARNESS_TYPE` and supports the following values (default is `InstrHarness` if not provided): @@ -38,73 +53,15 @@ The harness type should be specified by an environment variable `HARNESS_TYPE` a ### Data Preparation -Before running tests, `InstrContext` messages may be converted into Protobuf's text format, with all `bytes` fields base58-encoded (for human readability). Run the following command to do this: - -```sh -solana-test-suite decode-protobuf --input-dir --output-dir --num-processes -``` - -| Argument | Description | -|----------------|-----------------------------------------------------------------------------------------------| -| `--input-dir` | Input directory containing instruction context messages in binary format | -| `--output-dir` | Output directory for encoded, human-readable instruction context messages | -| `--num-processes` | Number of processes to use | - - -Optionally, instruction context messages may also be left in the original Protobuf binary-encoded format. - - -### Test Suite - -To run the test suite, use the following command: - -```sh -solana-test-suite run-tests --input-dir --solana-target --target [--target ...] --output-dir --num-processes --chunk-size [--randomize-output-buffer] [--verbose] [--consensus-mode] [--failures-only] [--save-failures] -``` - -You can provide both `InstrContext` and `InstrFixture` within `--input-dir` - parsing is taken care of depending on the file extension `.bin` for `InstrContext` and `.fix` for `InstrFixture`. - -| Argument | Description | -|-----------------|-----------------------------------------------------------------------------------------------------| -| `--input-dir` | Input directory containing instruction context or fixture messages | -| `--solana-target` | Path to Solana Agave shared object (.so) target file | -| `--target` | Additional shared object (.so) target file paths | -| `--output-dir` | Log output directory for test results | -| `--num-processes` | Number of processes to use | -| `--randomize-output-buffer`| Randomizes bytes in output buffer before shared library execution | -| `--chunk-size` | Number of test results per log file | -| `--verbose` | Verbose output: log failed test cases | -| `--consensus-mode` | Only fail on consensus failures. One such effect is to normalize error codes when comparing results | -| `--failures-only` | Only log failed test cases | -| `--save-failures` | Saves failed test cases to results directory | - - -**Note:** Each `.so` target file name should be unique. - - -### Single instruction - -You can pick out a single test case and run it to view the instruction effects via output with the following command: - -```sh -solana-test-suite exec-instr --input --target -``` - -For flexibility, `--input` can be either a file or directory and will execute on one or more files based on what's provided. +Before running tests, Context messages may be converted into Protobuf's text format, with all `bytes` fields base58-encoded (for human readability). This can be done with [decode-protobuf](commands.md#solana-test-suite-decode-protobuf) command. -| Argument | Description | -|-----------------|-----------------------------------------------------------------------------------------------------| -| `--input` | Input file OR directory of input files containing instruction context messages | -| `--target` | Shared object (.so) target file path to debug | +Optionally, context messages may also be left in the original Protobuf binary-encoded format. ### Debugging -For failing test cases, it may be useful to analyze what could have differed between Solana and Firedancer. You can execute a Protobuf message (human-readable or binary) through the desired client as such: +For failing test cases, it may be useful to analyze what could have differed between Solana and Firedancer. You can execute a Protobuf message (human-readable or binary) through the desired client with the [`debug-instr`](commands.md#solana-test-suite-debug-instr) command. -```sh -solana-test-suite debug-instr --input --target --debugger -``` #### Alternative (and preferred) debugging solution @@ -113,69 +70,11 @@ Use the following command instead if you want the ability to directly restart th --args python3.11 -m test_suite.test_suite exec-instr --input --target ``` -| Argument | Description | -|-----------------|-----------------------------------------------------------------------------------------------------| -| `--input` | Input file containing instruction context message | -| `--target` | Shared object (.so) target file path to debug | -| `--debugger` | Debugger to use (gdb, rust-gdb) | +Refer to [`exec-instr`](commands.md#solana-test-suite-exec-instr) command for more information. Recommended usage is opening two terminals side by side, and running the above command on both with one having `--target` for Solana (`impl/lib/libsolfuzz_agave_v2.0.so`) and another for Firedancer (`impl/lib/libsolfuzz_firedancer.so`), and then stepping through the debugger for each corresponding test case. -### Minimizing - -Prunes extra fields in the input (e.g. feature set) and produces a minimal test case such that the output does not change. - -```sh -solana-test-suite minimize-tests --input-dir --solana-target --output-dir --num-processes -``` - -| Argument | Description | -|-----------------|-----------------------------------------------------------------------------------------------------| -| `--input-dir` | Input directory containing instruction context messages | -| `--solana-target` | Path to Solana Agave shared object (.so) target file | -| `--output-dir` | Pruned instruction context dumping directory | -| `--num-processes` | Number of processes to use | - - -### Creating Fixtures from Instruction Context - -Create full test fixtures containing both instruction context and effects. Effects are computed by running instruction context through `--solana-target`. Fixtures with `None` values for instruction context/effects are not included. - -```sh -solana-test-suite create-fixtures --input-dir --solana-target --target [--target ...] --output-dir --num-processes [--readable] [--keep-passing] [--group-by-program] -``` - -You have an additional option to produce fixtures for only passing test cases (makes it easier to produce fixtures from larger batches of new-passing mismatches). - - -| Argument | Description | -|-----------------|-----------------------------------------------------------------------------------------------------| -| `--input-dir` | Input directory containing instruction context messages | -| `--solana-target` | Path to Solana Agave shared object (.so) target file | -| `--target` | Shared object (.so) target file paths (pairs with `--keep-passing`) -| `--output-dir` | Instruction fixtures dumping directory | -| `--num-processes` | Number of processes to use | -| `--readable` | Output fixtures in human-readable format | -| `--keep-passing` | Only keep passing test cases | -| `--group-by-program` | Group fixture output by program type | - - -### Create Instruction Context from Fixtures - -Opposite as above. Does not require a target. - -```sh -solana-test-suite instr-from-fixtures --input-dir --solana-target --output-dir --num-processes [--readable] -``` - -| Argument | Description | -|-----------------|-----------------------------------------------------------------------------------------------------| -| `--input-dir` | Input directory containing instruction fixture messages | -| `--output-dir` | Output directory for instr contexts | -| `--num-processes` | Number of processes to use | - - ### Uninstalling ```sh diff --git a/commands.md b/commands.md new file mode 100644 index 0000000..dfae3f1 --- /dev/null +++ b/commands.md @@ -0,0 +1,143 @@ +# `solana-test-suite` + +Validate instruction effects from clients using instruction context Protobuf messages. + +**Usage**: + +```console +$ solana-test-suite [OPTIONS] COMMAND [ARGS]... +``` + +**Options**: + +* `--install-completion`: Install completion for the current shell. +* `--show-completion`: Show completion for the current shell, to copy it or customize the installation. +* `--help`: Show this message and exit. + +**Commands**: + +* `create-fixtures`: Create test fixtures from a directory of... +* `debug-instr` +* `decode-protobuf`: Convert InstrContext messages to... +* `exec-instr`: Execute InstrContext message(s) and print... +* `instr-from-fixtures`: Extract InstrContext messages from fixtures. +* `run-tests`: Run tests on a set of targets with a... + +## `solana-test-suite create-fixtures` + +Create test fixtures from a directory of InstrContext messages. +Effects are generated by the target passed in with --solana-target or -s. +You can also pass in additional targets with --target or -t +and use --keep-passing or -k to only generate effects for test cases that match. + +**Usage**: + +```console +$ solana-test-suite create-fixtures [OPTIONS] +``` + +**Options**: + +* `-i, --input-dir PATH`: Input directory containing InstrContext messages [default: corpus8] +* `-s, --solana-target PATH`: Solana (or ground truth) shared object (.so) target file path [default: impl/lib/libsolfuzz_agave_v2.0.so] +* `-t, --target PATH`: Shared object (.so) target file paths (pairs with --keep-passing). Targets must have sol_compat_instr_execute_v1 defined +* `-o, --output-dir PATH`: Output directory for fixtures [default: test_fixtures] +* `-p, --num-processes INTEGER`: Number of processes to use [default: 4] +* `-r, --readable`: Output fixtures in human-readable format +* `-k, --keep-passing`: Only keep passing test cases +* `-g, --group-by-program`: Group fixture output by program type +* `--help`: Show this message and exit. + +## `solana-test-suite debug-instr` + +**Usage**: + +```console +$ solana-test-suite debug-instr [OPTIONS] +``` + +**Options**: + +* `-i, --input PATH`: Input file +* `-t, --target PATH`: Shared object (.so) target file path to debug [default: impl/lib/libsolfuzz_firedancer.so] +* `-d, --debugger TEXT`: Debugger to use (gdb, rust-gdb) [default: gdb] +* `--help`: Show this message and exit. + +## `solana-test-suite decode-protobuf` + +Convert InstrContext messages to human-readable format. + +**Usage**: + +```console +$ solana-test-suite decode-protobuf [OPTIONS] +``` + +**Options**: + +* `-i, --input-dir PATH`: Input directory containing InstrContext message(s) [default: raw_instruction_context] +* `-o, --output-dir PATH`: Output directory for base58-encoded, human-readable InstrContext messages [default: readable_instruction_context] +* `-p, --num-processes INTEGER`: Number of processes to use [default: 4] +* `--help`: Show this message and exit. + +## `solana-test-suite exec-instr` + +Execute InstrContext message(s) and print the effects. + +**Usage**: + +```console +$ solana-test-suite exec-instr [OPTIONS] +``` + +**Options**: + +* `-i, --input PATH`: Input InstrContext file or directory of files +* `-t, --target PATH`: Shared object (.so) target file path to execute [default: impl/firedancer/build/native/clang/lib/libfd_exec_sol_compat.so] +* `-r, --randomize-output-buffer`: Randomizes bytes in output buffer before shared library execution +* `--help`: Show this message and exit. + +## `solana-test-suite instr-from-fixtures` + +Extract InstrContext messages from fixtures. + +**Usage**: + +```console +$ solana-test-suite instr-from-fixtures [OPTIONS] +``` + +**Options**: + +* `-i, --input-dir PATH`: Input directory containing InstrFixture messages [default: fixtures] +* `-o, --output-dir PATH`: Output directory for InstrContext messages [default: instr] +* `-p, --num-processes INTEGER`: Number of processes to use [default: 4] +* `--help`: Show this message and exit. + +## `solana-test-suite run-tests` + +Run tests on a set of targets with a directory of InstrContext +or InstrFixture messages. + +Note: each `.so` target filename must be unique. + +**Usage**: + +```console +$ solana-test-suite run-tests [OPTIONS] +``` + +**Options**: + +* `-i, --input-dir PATH`: Input directory containing InstrContext or InstrFixture messages [default: corpus8] +* `-s, --solana-target PATH`: Solana (or ground truth) shared object (.so) target file path [default: impl/lib/libsolfuzz_agave_v2.0.so] +* `-t, --target PATH`: Shared object (.so) target file paths [default: impl/lib/libsolfuzz_firedancer.so] +* `-o, --output-dir PATH`: Output directory for test results [default: test_results] +* `-p, --num-processes INTEGER`: Number of processes to use [default: 4] +* `-r, --randomize-output-buffer`: Randomizes bytes in output buffer before shared library execution +* `-c, --chunk-size INTEGER`: Number of test results per file [default: 10000] +* `-v, --verbose`: Verbose output: log failed test cases +* `-c, --consensus-mode`: Only fail on consensus failures. One such effect is to normalize error codes when comparing results +* `-f, --failures-only`: Only log failed test cases +* `-sf, --save-failures`: Saves failed test cases to results directory +* `--help`: Show this message and exit. diff --git a/install.sh b/install.sh index bb6a2c2..0292d9f 100755 --- a/install.sh +++ b/install.sh @@ -6,3 +6,4 @@ source test_suite_env/bin/activate sudo dnf install -y python3.11-devel || true make -j -C impl pip install -e ".[dev]" +pre-commit install diff --git a/install_lite.sh b/install_lite.sh index 195d81f..deca583 100755 --- a/install_lite.sh +++ b/install_lite.sh @@ -3,3 +3,4 @@ python3.11 -m venv test_suite_env source test_suite_env/bin/activate sudo dnf install -y python3.11-devel || true pip install -e ".[dev]" +pre-commit install diff --git a/pyproject.toml b/pyproject.toml index 672ac21..44d885e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ dependencies = [ "fd58~=0.1.0", "loguru~=0.7.0", "protobuf~=3.19.0", - "typer~=0.9.0", + "typer~=0.12.3", "Cython>=3.0.9", "tqdm~=4.66.0", "protoletariat~=3.2.0", diff --git a/src/test_suite/test_suite.py b/src/test_suite/test_suite.py index e291d99..4933718 100644 --- a/src/test_suite/test_suite.py +++ b/src/test_suite/test_suite.py @@ -43,7 +43,9 @@ ) -@app.command() +@app.command( + help=f"Execute {globals.harness_ctx.context_type.__name__} message(s) and print the effects." +) def exec_instr( file_or_dir: Path = typer.Option( None, @@ -64,7 +66,6 @@ def exec_instr( help="Randomizes bytes in output buffer before shared library execution", ), ): - print(globals.harness_ctx) # Initialize output buffers and shared library initialize_process_output_buffers(randomize_output_buffer=randomize_output_buffer) try: @@ -128,7 +129,9 @@ def debug_instr( debug_host(shared_library, instruction_context, gdb=debugger) -@app.command() +@app.command( + help=f"Extract {globals.harness_ctx.context_type.__name__} messages from fixtures." +) def instr_from_fixtures( input_dir: Path = typer.Option( Path("fixtures"), @@ -171,7 +174,14 @@ def instr_from_fixtures( print(f"{sum(results)} files successfully written") -@app.command() +@app.command( + help=f""" + Create test fixtures from a directory of {globals.harness_ctx.context_type.__name__} messages. + Effects are generated by the target passed in with --solana-target or -s. + You can also pass in additional targets with --target or -t + and use --keep-passing or -k to only generate effects for test cases that match. + """ +) def create_fixtures( input_dir: Path = typer.Option( Path("corpus8"), @@ -261,7 +271,14 @@ def create_fixtures( print(f"{sum(write_results)} files successfully written") -@app.command() +@app.command( + help=f""" + Run tests on a set of targets with a directory of {globals.harness_ctx.context_type.__name__} + or {globals.harness_ctx.fixture_type.__name__} messages. + + Note: each `.so` target filename must be unique. + """ +) def run_tests( input_dir: Path = typer.Option( Path("corpus8"), @@ -458,13 +475,15 @@ def diff_effect_wrapper(a, b): print(f"Skipped tests: {skipped_tests}") -@app.command() +@app.command( + help=f"Convert {globals.harness_ctx.context_type.__name__} messages to human-readable format." +) def decode_protobuf( input_dir: Path = typer.Option( Path("raw_instruction_context"), "--input-dir", "-i", - help=f"Input directory containing {globals.harness_ctx.context_type.__name__} messages", + help=f"Input directory containing {globals.harness_ctx.context_type.__name__} message(s)", ), output_dir: Path = typer.Option( Path("readable_instruction_context"),