diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index e9e1e3a2f7..b507d1c9e9 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -6,22 +6,27 @@ "context": "..", // Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename. "dockerFile": "../Dockerfile", - // Set *default* container specific settings.json values on container create. - "settings": {}, - // Add the IDs of extensions you want installed when the container is created. - "extensions": [ - "matklad.rust-analyzer" - ], + "customizations": { + "vscode": { + // Add the IDs of extensions you want installed when the container is created. + "extensions": ["matklad.rust-analyzer", "microsoft.Docker"], + // Set *default* container specific settings.json values on container create. + "settings": { + "rust-analyzer.cargo.noDefaultFeatures": true + } + } + }, // Use 'forwardPorts' to make a list of ports inside the container available locally. // "forwardPorts": [], // Uncomment the next line to run commands after the container is created - for example installing curl. - // "postCreateCommand": "apt-get update && apt-get install -y curl", + // Install development components that shouldn't be in the main Dockerfile + "postCreateCommand": "rustup component add --toolchain nightly rustfmt clippy llvm-tools-preview && cargo install --locked cargo-make", // Uncomment when using a ptrace-based debugger like C++, Go, and Rust "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined" - ], + ] // Uncomment to use the Docker CLI from inside the container. See https://aka.ms/vscode-remote/samples/docker-from-docker. // "mounts": [ "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind" ], // Uncomment to connect as a non-root user if you've added one. See https://aka.ms/vscode-remote/containers/non-root. diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index cc25331022..0a923db03c 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -2,18 +2,22 @@ name: build and test on: push: - branches: [ main, 'pr/**' ] + branches: [ main, "pr/**" ] pull_request: branches: [ main ] workflow_dispatch: + merge_group: env: CARGO_TERM_COLOR: always +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: common: strategy: matrix: - os: [ubuntu-latest, windows-latest, macOS-latest] + os: [ ubuntu-latest, windows-latest, macOS-latest ] runs-on: ${{ matrix.os }} steps: - name: Install mimetype @@ -37,6 +41,10 @@ jobs: crate: mdbook-linkcheck - uses: actions/checkout@v3 - uses: Swatinem/rust-cache@v2 + with: { shared-key: "ubuntu" } + if: runner.os == 'Linux' + - uses: Swatinem/rust-cache@v2 + if: runner.os != 'Linux' - name: Check for binary blobs if: runner.os == 'Linux' run: ./scripts/check_for_blobs.sh @@ -62,7 +70,7 @@ jobs: continue-on-error: true strategy: matrix: - llvm-version: ["11", "12", "13", "14", "16", "17"] + llvm-version: [ "16", "17" ] # Add 18 when KyleMayes/install-llvm-action enables it steps: - name: Remove Dotnet & Haskell run: rm -rf /usr/share/dotnet && rm -rf /opt/ghc @@ -72,603 +80,486 @@ jobs: toolchain: stable - uses: actions/checkout@v3 - uses: Swatinem/rust-cache@v2 + with: { shared-key: "llvm-tester" } - name: Install LLVM and Clang - uses: KyleMayes/install-llvm-action@v1 + uses: KyleMayes/install-llvm-action@v2 with: version: "${{matrix.llvm-version}}" - name: Build and test with llvm-${{ matrix.llvm-version }} run: pwd && ls & cd libafl_cc && cargo build --release + ubuntu-doc-build: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + - uses: ./.github/workflows/ubuntu-prepare + - uses: Swatinem/rust-cache@v2 + # ---- doc check ---- + - name: Build Docs + run: RUSTFLAGS="--cfg docsrs" cargo +nightly doc --all-features --no-deps + + ubuntu-doc-test: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + - uses: ./.github/workflows/ubuntu-prepare + - uses: Swatinem/rust-cache@v2 + # ---- doc check ---- + - name: Test Docs + run: RUSTFLAGS="--cfg docsrs" cargo +nightly test --doc --all-features + + ubuntu-miri: + runs-on: ubuntu-22.04 + needs: ubuntu + steps: + - uses: actions/checkout@v3 + - uses: ./.github/workflows/ubuntu-prepare + - uses: Swatinem/rust-cache@v2 + - name: Add nightly rustfmt and clippy + run: rustup toolchain install nightly --component miri --allow-downgrade + # --- miri undefined behavior test -- + - name: Run miri tests + run: RUST_BACKTRACE=1 MIRIFLAGS="-Zmiri-disable-isolation" cargo +nightly miri test ubuntu: runs-on: ubuntu-22.04 steps: - - name: Remove Dotnet & Haskell - run: rm -rf /usr/share/dotnet && rm -rf /opt/ghc - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - - name: Remove existing clang and LLVM - run: sudo apt purge llvm* clang* lld* lldb* opt* - - name: Install and cache deps - run: sudo apt update && sudo apt install ninja-build shellcheck libgtk-3-dev gcc-arm-linux-gnueabi g++-arm-linux-gnueabi libslirp-dev libz3-dev - - name: Add nightly rustfmt and clippy - run: rustup toolchain install nightly --component rustfmt --component clippy --component miri --allow-downgrade - - name: Install ucd-generate - run: cargo install -f ucd-generate - - uses: actions/checkout@v3 - - uses: Swatinem/rust-cache@v2 - - name: Install LLVM and Clang - uses: KyleMayes/install-llvm-action@v1 - with: - directory: ${{ runner.temp }}/llvm - version: 17 - # ---- format check ---- - # pcguard edges and pcguard hitcounts are not compatible and we need to build them seperately - - name: Check pcguard edges - run: cargo check --features=sancov_pcguard_edges - - name: Format - run: cargo fmt -- --check - - name: Cleanup - run: cargo clean - - name: Run clang-format style check for C/C++ programs. - run: clang-format -n -Werror --style=file $(find . -type f \( -name '*.cpp' -o -iname '*.hpp' -o -name '*.cc' -o -name '*.cxx' -o -name '*.cc' -o -name '*.h' \) | grep -v '/target/' | grep -v 'libpng-1\.6\.37' | grep -v 'stb_image\.h' | grep -v 'dlmalloc\.c' | grep -v 'QEMU-Nyx') - - name: run shellcheck - run: shellcheck ./scripts/*.sh - - # ---- doc check ---- - - name: Build Docs - run: RUSTFLAGS="--cfg docsrs" cargo +nightly doc --all-features - - name: Test Docs - run: RUSTFLAGS="--cfg docsrs" cargo +nightly test --doc --all-features - # ---- build normal and examples ---- - - name: Run a normal build - run: cargo build --verbose - - name: Build examples - run: cargo build --examples --verbose - - # --- miri undefined behavior test -- - - name: Run miri tests - run: RUST_BACKTRACE=1 MIRIFLAGS="-Zmiri-disable-isolation" cargo +nightly miri test + - name: Remove Dotnet & Haskell + run: rm -rf /usr/share/dotnet && rm -rf /opt/ghc + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + - name: Remove existing clang and LLVM + run: sudo apt purge llvm* clang* lld* lldb* opt* + - name: Install and cache deps + run: sudo apt update && sudo apt install ninja-build shellcheck libgtk-3-dev gcc-arm-linux-gnueabi g++-arm-linux-gnueabi libslirp-dev libz3-dev + - name: Add nightly rustfmt and clippy + run: rustup toolchain install nightly --component rustfmt --component clippy --component miri --allow-downgrade + - uses: actions/checkout@v3 + - uses: Swatinem/rust-cache@v2 + with: { shared-key: "ubuntu" } + - name: Install LLVM and Clang + uses: KyleMayes/install-llvm-action@v1 + with: + directory: ${{ runner.temp }}/llvm + version: 17 + # ---- format check ---- + # pcguard edges and pcguard hitcounts are not compatible and we need to build them seperately + - name: Check pcguard edges + run: cargo check --features=sancov_pcguard_edges + - name: Run clang-format style check for C/C++ programs. + run: clang-format -n -Werror --style=file $(find . -type f \( -name '*.cpp' -o -iname '*.hpp' -o -name '*.cc' -o -name '*.cxx' -o -name '*.cc' -o -name '*.h' \) | grep -v '/target/' | grep -v 'libpng-1\.6\.37' | grep -v 'stb_image\.h' | grep -v 'dlmalloc\.c' | grep -v 'QEMU-Nyx') + - name: run shellcheck + run: shellcheck ./scripts/*.sh + # ---- build normal and examples ---- + - name: Run a normal build + run: cargo build --verbose + - name: Build examples + run: cargo build --examples --verbose ubuntu-clippy: runs-on: ubuntu-22.04 steps: - - name: Remove Dotnet & Haskell - run: rm -rf /usr/share/dotnet && rm -rf /opt/ghc - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - - - name: Install and cache deps - run: sudo apt update && sudo apt install ninja-build shellcheck libgtk-3-dev gcc-arm-linux-gnueabi g++-arm-linux-gnueabi libslirp-dev libz3-dev - - name: Add nightly rustfmt and clippy - run: rustup toolchain install nightly --component clippy --allow-downgrade && rustup default nightly - - uses: actions/checkout@v3 - - uses: Swatinem/rust-cache@v2 - - name: Install LLVM and Clang - uses: KyleMayes/install-llvm-action@v1 - with: - directory: ${{ runner.temp }}/llvm - version: 17 - - name: Run clippy - run: ./scripts/clippy.sh - - # Clean up files to save up disk space - - name: Cleanup - run: cargo clean + - name: Remove Dotnet & Haskell + run: rm -rf /usr/share/dotnet && rm -rf /opt/ghc + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + + - name: Install and cache deps + run: sudo apt update && sudo apt install ninja-build shellcheck libgtk-3-dev gcc-arm-linux-gnueabi g++-arm-linux-gnueabi libslirp-dev libz3-dev + - name: Add nightly rustfmt and clippy + run: rustup toolchain install nightly --component clippy --allow-downgrade && rustup default nightly + - uses: actions/checkout@v3 + - uses: Swatinem/rust-cache@v2 + with: { shared-key: "ubuntu" } + - name: Install LLVM and Clang + uses: KyleMayes/install-llvm-action@v1 + with: + directory: ${{ runner.temp }}/llvm + version: 17 + - name: Run clippy + run: ./scripts/clippy.sh # --- test embedding the libafl_libfuzzer_runtime library # Fix me plz # - name: Test Build libafl_libfuzzer with embed # run: cargo +nightly test --features=embed-runtime --manifest-path libafl_libfuzzer/Cargo.toml + ubuntu-check-nightly: + runs-on: ubuntu-22.04 + needs: ubuntu + steps: + - uses: actions/checkout@v3 + - uses: ./.github/workflows/ubuntu-prepare + - uses: Swatinem/rust-cache@v2 + with: { shared-key: "ubuntu" } + # ---- build and feature check ---- + # cargo-hack's --feature-powerset would be nice here but libafl has a too many knobs + - name: Check nightly features + run: cargo +nightly check --features=agpl && cargo +nightly check --features=nautilus + ubuntu-check: runs-on: ubuntu-22.04 + needs: ubuntu + strategy: + matrix: + instance_idx: [ "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17" ] steps: - - name: Remove Dotnet & Haskell - run: rm -rf /usr/share/dotnet && rm -rf /opt/ghc - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - components: llvm-tools - - name: Remove existing clang and LLVM - run: sudo apt purge llvm* clang* - - name: Install and cache deps - run: sudo apt update && sudo apt install ninja-build clang-format shellcheck libgtk-3-dev gcc-arm-linux-gnueabi g++-arm-linux-gnueabi libslirp-dev libz3-dev - - name: Install cargo-hack - run: curl -LsSf https://github.com/taiki-e/cargo-hack/releases/latest/download/cargo-hack-x86_64-unknown-linux-gnu.tar.gz | tar xzf - -C ~/.cargo/bin - - name: Install ucd-generate - run: cargo install -f ucd-generate - - name: Add nightly - run: rustup toolchain install nightly --allow-downgrade - - uses: actions/checkout@v3 - - uses: Swatinem/rust-cache@v2 - - name: Install LLVM and Clang - uses: KyleMayes/install-llvm-action@v1 - with: - directory: ${{ runner.temp }}/llvm - version: 17 - # ---- build and feature check ---- - # cargo-hack's --feature-powerset would be nice here but libafl has a too many knobs - - name: Check each feature - # Skipping `python` as it has to be built with the `maturin` tool - # `agpl`, `nautilus` require nightly - # `sancov_pcguard_edges` is tested seperatelyc - run: LLVM_CONFIG=llvm-config cargo hack check --workspace --each-feature --clean-per-run --exclude-features=prelude,agpl,nautilus,python,sancov_pcguard_edges,arm,aarch64,i386,be,systemmode,whole_archive --no-dev-deps --exclude libafl_libfuzzer - - name: Check nightly features - run: cargo +nightly check --features=agpl && cargo +nightly check --features=nautilus + - uses: actions/checkout@v3 + - uses: ./.github/workflows/ubuntu-prepare + - uses: Swatinem/rust-cache@v2 + with: { shared-key: "ubuntu" } + # ---- build and feature check ---- + # cargo-hack's --feature-powerset would be nice here but libafl has a too many knobs + - name: Check each feature + # Skipping `python` as it has to be built with the `maturin` tool + # `agpl`, `nautilus` require nightly + # `sancov_pcguard_edges` is tested seperatelyc + run: python3 ./scripts/parallellize_cargo_check.py ${{ matrix.instance_idx }} ubuntu-concolic: runs-on: ubuntu-latest + needs: ubuntu steps: - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - - uses: actions/checkout@v3 - - uses: Swatinem/rust-cache@v2 - - name: Install smoke test deps - run: sudo ./libafl_concolic/test/smoke_test_ubuntu_deps.sh - - name: Run smoke test - run: ./libafl_concolic/test/smoke_test.sh - - bindings: + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + - uses: actions/checkout@v3 + - uses: Swatinem/rust-cache@v2 + with: { shared-key: "ubuntu" } + - name: Install smoke test deps + run: sudo ./libafl_concolic/test/smoke_test_ubuntu_deps.sh + - name: Run smoke test + run: ./libafl_concolic/test/smoke_test.sh + + python-bindings: runs-on: ubuntu-latest steps: - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - - name: Remove existing clang and LLVM - run: sudo apt purge llvm* clang* - - name: Install LLVM and Clang - uses: KyleMayes/install-llvm-action@v1 - with: - directory: ${{ runner.temp }}/llvm - version: 17 - - name: Install deps - run: sudo apt-get install -y ninja-build python3-dev python3-pip python3-venv libz3-dev - - name: Install maturin - run: python3 -m pip install maturin - - uses: actions/checkout@v3 - - uses: Swatinem/rust-cache@v2 - - name: Run a maturin build - run: export LLVM_CONFIG=llvm-config-16 && cd ./bindings/pylibafl && python3 -m venv .env && . .env/bin/activate && pip install --upgrade --force-reinstall . && ./test.sh - - name: Run python test - run: . ./bindings/pylibafl/.env/bin/activate && cd ./fuzzers/baby_fuzzer && python3 baby_fuzzer.py 2>&1 | grep "Bye" + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + - name: Remove existing clang and LLVM + run: sudo apt purge llvm* clang* + - name: Install LLVM and Clang + uses: KyleMayes/install-llvm-action@v1 + with: + directory: ${{ runner.temp }}/llvm + version: 17 + - name: Install deps + run: sudo apt-get install -y ninja-build python3-dev python3-pip python3-venv libz3-dev + - name: Install maturin + run: python3 -m pip install maturin + - uses: actions/checkout@v3 + - uses: Swatinem/rust-cache@v2 + - name: Run a maturin build + run: export LLVM_CONFIG=llvm-config-16 && cd ./bindings/pylibafl && python3 -m venv .env && . .env/bin/activate && pip install --upgrade --force-reinstall . && ./test.sh + - name: Run python test + run: . ./bindings/pylibafl/.env/bin/activate # && cd ./fuzzers/python_qemu/ && python3 fuzzer.py 2>&1 | grep "Bye" - fuzzers: - strategy: - matrix: - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} + cargo-fmt: + runs-on: ubuntu-latest steps: - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - with: - # this might remove tools that are actually needed, - # if set to "true" but frees about 6 GB - tool-cache: false - - # all of these default to true, but feel free to set to - # "false" if necessary for your workflow - android: true - dotnet: true - haskell: true - large-packages: false - docker-images: true - swap-storage: true - - name: Add nightly rustfmt and clippy - run: rustup toolchain install nightly --component rustfmt --component clippy --allow-downgrade - - name: Add no_std toolchain - run: rustup toolchain install nightly-x86_64-unknown-linux-gnu ; rustup component add rust-src --toolchain nightly-x86_64-unknown-linux-gnu - - name: Add wasm target - run: rustup target add wasm32-unknown-unknown - - name: Install ucd-generate - run: cargo install -f ucd-generate - - name: Remove obsolete llvm (Linux) - run: sudo apt purge llvm* clang* - - name: Install LLVM and Clang - uses: KyleMayes/install-llvm-action@v1 - with: - directory: ${{ runner.temp }}/llvm - version: 17 - - name: Install deps - run: sudo apt update && sudo apt install nasm ninja-build gcc-arm-linux-gnueabi g++-arm-linux-gnueabi gcc-aarch64-linux-gnu g++-aarch64-linux-gnu gcc-mipsel-linux-gnu g++-mipsel-linux-gnu gcc-powerpc-linux-gnu g++-powerpc-linux-gnu libc6-dev-i386-cross libc6-dev libc6-dev-i386 lib32gcc-11-dev lib32stdc++-11-dev libgtk-3-dev pax-utils libz3-dev - - name: pip install - run: python3 -m pip install msgpack jinja2 find_libpython - # Note that nproc needs to have coreutils installed on macOS, so the order of CI commands matters. - - name: enable mult-thread for `make` - run: export MAKEFLAGS="-j$(expr $(nproc) \+ 1)" - - name: install cargo-make - uses: baptiste0928/cargo-install@v1.3.0 - with: - crate: cargo-make - - name: install wasm-pack - uses: baptiste0928/cargo-install@v1.3.0 - with: - crate: wasm-pack - - name: install chrome - uses: browser-actions/setup-chrome@v1 - with: - chrome-version: stable - - uses: actions/checkout@v3 - with: - submodules: true # recursively checkout submodules - fetch-depth: 0 # to diff with origin/main - - uses: Swatinem/rust-cache@v2 - - name: Symlink Headers - if: runner.os == 'Linux' - # We can't install gcc-multilib which would usually do this for us due to collisions with other packages - run: sudo ln -s /usr/include/asm-generic /usr/include/asm - - name: Build and run example fuzzers (Linux) - if: runner.os == 'Linux' - run: RUN_ON_CI=1 LLVM_CONFIG=llvm-config ./scripts/test_all_fuzzers.sh - - qemu_fuzzers: - strategy: - matrix: - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: nightly + override: true + components: rustfmt + - uses: actions/checkout@v3 + - name: Format Check + run: cargo fmt -- --check + + fuzzers-preflight: + runs-on: ubuntu-latest steps: - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - - name: Free Disk Space (Ubuntu) - if: runner.os == 'Linux' - uses: jlumbroso/free-disk-space@main - with: - # this might remove tools that are actually needed, - # if set to "true" but frees about 6 GB - tool-cache: false - - # all of these default to true, but feel free to set to - # "false" if necessary for your workflow - android: true - dotnet: true - haskell: true - large-packages: false - docker-images: true - swap-storage: true - - name: Add nightly rustfmt and clippy - run: rustup toolchain install nightly --component rustfmt --component clippy --allow-downgrade - - name: Add no_std toolchain - run: rustup toolchain install nightly-x86_64-unknown-linux-gnu ; rustup component add rust-src --toolchain nightly-x86_64-unknown-linux-gnu - - name: Add wasm target - run: rustup target add wasm32-unknown-unknown - - name: Install ucd-generate - run: cargo install -f ucd-generate - - name: Remove obsolete llvm (Linux) - if: runner.os == 'Linux' - run: sudo apt purge llvm* clang* - - name: Install LLVM and Clang - uses: KyleMayes/install-llvm-action@v1 - with: - directory: ${{ runner.temp }}/llvm - version: 17 - - name: Install deps - run: sudo apt update && sudo apt install nasm ninja-build gcc-arm-linux-gnueabi g++-arm-linux-gnueabi gcc-aarch64-linux-gnu g++-aarch64-linux-gnu gcc-mipsel-linux-gnu g++-mipsel-linux-gnu gcc-powerpc-linux-gnu g++-powerpc-linux-gnu libc6-dev-i386-cross libc6-dev libc6-dev-i386 lib32gcc-11-dev lib32stdc++-11-dev libgtk-3-dev pax-utils libz3-dev - - name: pip install - run: python3 -m pip install msgpack jinja2 find_libpython - # Note that nproc needs to have coreutils installed on macOS, so the order of CI commands matters. - - name: enable mult-thread for `make` - run: export MAKEFLAGS="-j$(expr $(nproc) \+ 1)" - - name: install cargo-make - uses: baptiste0928/cargo-install@v1.3.0 - with: - crate: cargo-make - - name: install wasm-pack - uses: baptiste0928/cargo-install@v1.3.0 - with: - crate: wasm-pack - - name: install chrome - uses: browser-actions/setup-chrome@v1 - with: - chrome-version: stable - - uses: actions/checkout@v3 - with: - submodules: true # recursively checkout submodules - fetch-depth: 0 # to diff with origin/main - - uses: Swatinem/rust-cache@v2 - - name: Symlink Headers - if: runner.os == 'Linux' - # We can't install gcc-multilib which would usually do this for us due to collisions with other packages - run: sudo ln -s /usr/include/asm-generic /usr/include/asm - - name: Build and run example fuzzers (Linux) - if: runner.os == 'Linux' - run: RUN_ON_CI=1 RUN_QEMU_FUZZER=1 LLVM_CONFIG=llvm-config ./scripts/test_all_fuzzers.sh - - baby_fuzzers: + - uses: actions/checkout@v3 + - name: Fuzzer in CI Check + run: ./scripts/check_tested_fuzzers.sh + + fuzzers: + needs: + - ubuntu + - fuzzers-preflight strategy: matrix: - os: [ubuntu-latest] + os: [ ubuntu-latest ] + fuzzer: + - ./fuzzers/cargo_fuzz + - ./fuzzers/fuzzbench_fork_qemu + - ./fuzzers/libfuzzer_stb_image_sugar + - ./fuzzers/nyx_libxml2_standalone + - ./fuzzers/baby_fuzzer_gramatron + - ./fuzzers/tinyinst_simple + - ./fuzzers/baby_fuzzer_with_forkexecutor + - ./fuzzers/baby_no_std + - ./fuzzers/baby_fuzzer_swap_differential + - ./fuzzers/baby_fuzzer_grimoire + - ./fuzzers/baby_fuzzer + - ./fuzzers/libfuzzer_libpng_launcher + - ./fuzzers/libfuzzer_libpng_accounting + - ./fuzzers/forkserver_libafl_cc + - ./fuzzers/libfuzzer_libpng_tcp_manager + - ./fuzzers/backtrace_baby_fuzzers + - ./fuzzers/fuzzbench_qemu + - ./fuzzers/nyx_libxml2_parallel + # - ./fuzzers/qemu_launcher + - ./fuzzers/frida_gdiplus + - ./fuzzers/libfuzzer_stb_image_concolic + - ./fuzzers/nautilus_sync + # - ./fuzzers/qemu_cmin + # - ./fuzzers/qemu_systemmode + - ./fuzzers/push_harness + - ./fuzzers/libfuzzer_libpng_centralized + - ./fuzzers/baby_fuzzer_nautilus + - ./fuzzers/fuzzbench_text + - ./fuzzers/libfuzzer_libpng_cmin + - ./fuzzers/forkserver_simple + - ./fuzzers/baby_fuzzer_unicode + - ./fuzzers/libfuzzer_libpng_norestart + - ./fuzzers/baby_fuzzer_multi + - ./fuzzers/libafl_atheris + - ./fuzzers/frida_libpng + - ./fuzzers/fuzzbench_ctx + - ./fuzzers/fuzzbench_forkserver_cmplog + - ./fuzzers/push_stage_harness + - ./fuzzers/libfuzzer_libmozjpeg + - ./fuzzers/libfuzzer_libpng_aflpp_ui + - ./fuzzers/libfuzzer_libpng + - ./fuzzers/baby_fuzzer_wasm + - ./fuzzers/fuzzbench + - ./fuzzers/libfuzzer_stb_image + - ./fuzzers/fuzzbench_forkserver + # - ./fuzzers/libfuzzer_windows_asan + - ./fuzzers/baby_fuzzer_minimizing + # - ./fuzzers/qemu_coverage + - ./fuzzers/frida_executable_libpng + - ./fuzzers/tutorial + - ./fuzzers/baby_fuzzer_tokens + - ./fuzzers/backtrace_baby_fuzzers/rust_code_with_inprocess_executor + - ./fuzzers/backtrace_baby_fuzzers/c_code_with_fork_executor + - ./fuzzers/backtrace_baby_fuzzers/command_executor + - ./fuzzers/backtrace_baby_fuzzers/forkserver_executor + - ./fuzzers/backtrace_baby_fuzzers/c_code_with_inprocess_executor + - ./fuzzers/backtrace_baby_fuzzers/rust_code_with_fork_executor runs-on: ${{ matrix.os }} steps: - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - - name: Free Disk Space (Ubuntu) - if: runner.os == 'Linux' - uses: jlumbroso/free-disk-space@main - with: - # this might remove tools that are actually needed, - # if set to "true" but frees about 6 GB - tool-cache: false - - # all of these default to true, but feel free to set to - # "false" if necessary for your workflow - android: true - dotnet: true - haskell: true - large-packages: false - docker-images: true - swap-storage: true - - name: Add nightly rustfmt and clippy - run: rustup toolchain install nightly --component rustfmt --component clippy --allow-downgrade - - name: Add no_std toolchain - run: rustup toolchain install nightly-x86_64-unknown-linux-gnu ; rustup component add rust-src --toolchain nightly-x86_64-unknown-linux-gnu - - name: Add wasm target - run: rustup target add wasm32-unknown-unknown - - name: Install ucd-generate - run: cargo install -f ucd-generate - - name: Remove obsolete llvm (Linux) - if: runner.os == 'Linux' - run: sudo apt purge llvm* clang* - - name: Install LLVM and Clang - uses: KyleMayes/install-llvm-action@v1 - with: - directory: ${{ runner.temp }}/llvm - version: 17 - - name: Install deps - run: sudo apt update && sudo apt install nasm ninja-build gcc-arm-linux-gnueabi g++-arm-linux-gnueabi gcc-aarch64-linux-gnu g++-aarch64-linux-gnu gcc-mipsel-linux-gnu g++-mipsel-linux-gnu gcc-powerpc-linux-gnu g++-powerpc-linux-gnu libc6-dev-i386-cross libc6-dev libc6-dev-i386 lib32gcc-11-dev lib32stdc++-11-dev libgtk-3-dev pax-utils libz3-dev - - name: pip install - run: python3 -m pip install msgpack jinja2 find_libpython - # Note that nproc needs to have coreutils installed on macOS, so the order of CI commands matters. - - name: enable mult-thread for `make` - run: export MAKEFLAGS="-j$(expr $(nproc) \+ 1)" - - name: install cargo-make - uses: baptiste0928/cargo-install@v1.3.0 - with: - crate: cargo-make - - name: install wasm-pack - uses: baptiste0928/cargo-install@v1.3.0 - with: - crate: wasm-pack - - name: install chrome - uses: browser-actions/setup-chrome@v1 - with: - chrome-version: stable - - uses: actions/checkout@v3 - with: - submodules: true # recursively checkout submodules - fetch-depth: 0 # to diff with origin/main - - uses: Swatinem/rust-cache@v2 - - name: Symlink Headers - if: runner.os == 'Linux' - # We can't install gcc-multilib which would usually do this for us due to collisions with other packages - run: sudo ln -s /usr/include/asm-generic /usr/include/asm - - name: Build and run example fuzzers (Linux) - if: runner.os == 'Linux' - run: RUN_ON_CI=1 RUN_BABY_FUZZER=1 LLVM_CONFIG=llvm-config ./scripts/test_all_fuzzers.sh - - libpng_fuzzers: + - uses: actions/checkout@v3 + - uses: ./.github/workflows/fuzzer-tester-prepare + - name: Symlink Headers + if: runner.os == 'Linux' + shell: bash + run: sudo ln -s /usr/include/asm-generic /usr/include/asm + - name: Build and run example fuzzers (Linux) + if: runner.os == 'Linux' + shell: bash + run: RUN_ON_CI=1 LLVM_CONFIG=llvm-config ./scripts/test_fuzzer.sh ${{ matrix.fuzzer }} + + changes: + runs-on: ubuntu-latest + permissions: + pull-requests: read + outputs: + qemu: ${{ steps.filter.outputs.qemu }} + steps: + - uses: actions/checkout@v3 + - uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + qemu: + - 'libafl_qemu/**' + - 'fuzzers/*qemu*/**' + + fuzzers-qemu: + needs: changes + if: ${{ needs.changes.outputs.qemu == 'true' }} strategy: matrix: - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} + os: [ubuntu-latest] + fuzzer: + - ./fuzzers/qemu_cmin + - ./fuzzers/qemu_systemmode + - ./fuzzers/qemu_coverage + - ./fuzzers/qemu_launcher + + runs-on: [ self-hosted, qemu ] + container: registry.gitlab.com/qemu-project/qemu/qemu/ubuntu2204:latest steps: - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - - name: Free Disk Space (Ubuntu) - if: runner.os == 'Linux' - uses: jlumbroso/free-disk-space@main - with: - # this might remove tools that are actually needed, - # if set to "true" but frees about 6 GB - tool-cache: false - - # all of these default to true, but feel free to set to - # "false" if necessary for your workflow - android: true - dotnet: true - haskell: true - large-packages: false - docker-images: true - swap-storage: true - - name: Add nightly rustfmt and clippy - run: rustup toolchain install nightly --component rustfmt --component clippy --allow-downgrade - - name: Add no_std toolchain - run: rustup toolchain install nightly-x86_64-unknown-linux-gnu ; rustup component add rust-src --toolchain nightly-x86_64-unknown-linux-gnu - - name: Add wasm target - run: rustup target add wasm32-unknown-unknown - - name: Install ucd-generate - run: cargo install -f ucd-generate - - name: Remove obsolete llvm (Linux) - if: runner.os == 'Linux' - run: sudo apt purge llvm* clang* - - name: Install LLVM and Clang - uses: KyleMayes/install-llvm-action@v1 - with: - directory: ${{ runner.temp }}/llvm - version: 17 - - name: Install deps - run: sudo apt update && sudo apt install nasm ninja-build gcc-arm-linux-gnueabi g++-arm-linux-gnueabi gcc-aarch64-linux-gnu g++-aarch64-linux-gnu gcc-mipsel-linux-gnu g++-mipsel-linux-gnu gcc-powerpc-linux-gnu g++-powerpc-linux-gnu libc6-dev-i386-cross libc6-dev libc6-dev-i386 lib32gcc-11-dev lib32stdc++-11-dev libgtk-3-dev pax-utils libz3-dev - - name: pip install - run: python3 -m pip install msgpack jinja2 find_libpython - # Note that nproc needs to have coreutils installed on macOS, so the order of CI commands matters. - - name: enable mult-thread for `make` - run: export MAKEFLAGS="-j$(expr $(nproc) \+ 1)" - - name: install cargo-make - uses: baptiste0928/cargo-install@v1.3.0 - with: - crate: cargo-make - - name: install wasm-pack - uses: baptiste0928/cargo-install@v1.3.0 - with: - crate: wasm-pack - - name: install chrome - uses: browser-actions/setup-chrome@v1 - with: - chrome-version: stable - - uses: actions/checkout@v3 - with: - submodules: true # recursively checkout submodules - fetch-depth: 0 # to diff with origin/main - - uses: Swatinem/rust-cache@v2 - - name: Symlink Headers - if: runner.os == 'Linux' - # We can't install gcc-multilib which would usually do this for us due to collisions with other packages - run: sudo ln -s /usr/include/asm-generic /usr/include/asm - - name: Build and run example fuzzers (Linux) - if: runner.os == 'Linux' - run: RUN_ON_CI=1 RUN_LIBPNG_FUZZER=1 LLVM_CONFIG=llvm-config ./scripts/test_all_fuzzers.sh + - uses: actions/checkout@v3 + - uses: ./.github/workflows/qemu-fuzzer-tester-prepare + - uses: ./.github/workflows/fuzzer-tester-prepare + - name: Symlink Headers + if: runner.os == 'Linux' + shell: bash + run: sudo ln -s /usr/include/asm-generic /usr/include/asm + - name: Build and run example QEMU fuzzers (Linux) + if: runner.os == 'Linux' + shell: bash + run: RUN_ON_CI=1 LLVM_CONFIG=llvm-config ./scripts/test_fuzzer.sh ${{ matrix.fuzzer }} nostd-build: runs-on: ubuntu-latest steps: - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: nightly - override: true - components: rustfmt, rust-src - - uses: actions/checkout@v3 - - uses: Swatinem/rust-cache@v2 - - name: Add targets - run: rustup target add arm-linux-androideabi && rustup target add thumbv6m-none-eabi - - name: Build aarch64-unknown-none - run: cd ./fuzzers/baby_no_std && cargo +nightly build -Zbuild-std=core,alloc --target aarch64-unknown-none -v --release && cd ../.. - - name: run x86_64 until panic! - run: cd ./fuzzers/baby_no_std && cargo +nightly run || test $? -ne 0 || exit 1 - - name: no_std tests - run: cd ./libafl && cargo test --no-default-features + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: nightly + override: true + components: rustfmt, rust-src + - uses: actions/checkout@v3 + - uses: Swatinem/rust-cache@v2 + - name: Add targets + run: rustup target add arm-linux-androideabi && rustup target add thumbv6m-none-eabi + - name: Build aarch64-unknown-none + run: cd ./fuzzers/baby_no_std && cargo +nightly build -Zbuild-std=core,alloc --target aarch64-unknown-none -v --release && cd ../.. + - name: run x86_64 until panic! + run: cd ./fuzzers/baby_no_std && cargo +nightly run || test $? -ne 0 || exit 1 + - name: no_std tests + run: cd ./libafl && cargo test --no-default-features nostd-clippy: runs-on: ubuntu-latest steps: - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: nightly - override: true - components: rustfmt, clippy, rust-src - - uses: actions/checkout@v3 - - uses: Swatinem/rust-cache@v2 - - name: Add targets - run: rustup target add arm-linux-androideabi && rustup target add thumbv6m-none-eabi - - name: libafl armv6m-none-eabi (32 bit no_std) clippy - run: cd ./libafl && cargo clippy --target thumbv6m-none-eabi --no-default-features - - name: Build no_std no_alloc bolts - run: cd ./libafl_bolts && cargo +nightly build -Zbuild-std=core --target aarch64-unknown-none --no-default-features -v --release && cd ../ + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: nightly + override: true + components: rustfmt, clippy, rust-src + - uses: actions/checkout@v3 + - uses: Swatinem/rust-cache@v2 + - name: Add targets + run: rustup target add arm-linux-androideabi && rustup target add thumbv6m-none-eabi + - name: libafl armv6m-none-eabi (32 bit no_std) clippy + run: cd ./libafl && cargo clippy --target thumbv6m-none-eabi --no-default-features + - name: Build no_std no_alloc bolts + run: cd ./libafl_bolts && cargo +nightly build -Zbuild-std=core --target aarch64-unknown-none --no-default-features -v --release && cd ../ build-docker: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Build docker - run: docker build -t libafl . + - uses: actions/checkout@v3 + - name: Build docker + run: docker build -t libafl . + + windows-frida-libpng: + runs-on: windows-latest + needs: + - common + steps: + - uses: actions/checkout@v3 + - uses: ./.github/workflows/windows-tester-prepare + - name: Build fuzzers/frida_libpng + run: cd fuzzers/frida_libpng/ && cargo make test + + windows-frida-libfuzzer-stb-image: + runs-on: windows-latest + needs: + - common + steps: + - uses: actions/checkout@v3 + - uses: ./.github/workflows/windows-tester-prepare + - name: Build fuzzers/libfuzzer_stb_image + run: cd fuzzers/libfuzzer_stb_image && cargo build --release + + windows-frida-gdiplus: + runs-on: windows-latest + needs: + - common + steps: + - uses: actions/checkout@v3 + - uses: ./.github/workflows/windows-tester-prepare + - name: Build fuzzers/frida_gdiplus + run: cd fuzzers/frida_gdiplus/ && cargo make test && cargo make test_cmplog - windows: + windows-tinyinst-simple: runs-on: windows-latest + needs: + - common steps: - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - - uses: actions/checkout@v3 - - uses: Swatinem/rust-cache@v2 - - name: Windows Build - run: cargo build --verbose - - name: Build docs - run: cargo doc - - name: Set LIBCLANG_PATH - run: echo "LIBCLANG_PATH=$((gcm clang).source -replace "clang.exe")" >> $env:GITHUB_ENV - - name: install cargo-make - run: cargo install --force cargo-make - - uses: ilammy/msvc-dev-cmd@v1 - - name: install cxx bridge - run: cargo install cxxbridge-cmd - - name: Build fuzzers/libfuzzer_stb_image - run: cd fuzzers/libfuzzer_stb_image && cargo build --release - - name: Build fuzzers/frida_libpng - run: cd fuzzers/frida_libpng/ && cargo make test - - name: Build fuzzers/frida_gdiplus - run: cd fuzzers/frida_gdiplus/ && cargo make test && cargo make test_cmplog - - name: Build fuzzers/tinyinst_simple - run: cd fuzzers/tinyinst_simple/ && cargo make test + - uses: actions/checkout@v3 + - uses: ./.github/workflows/windows-tester-prepare + - name: install cxx bridge + run: cargo install cxxbridge-cmd + - name: Build fuzzers/tinyinst_simple + run: cd fuzzers/tinyinst_simple/ && cargo make test windows-clippy: runs-on: windows-latest + needs: + - common steps: - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - - uses: actions/checkout@v3 - - uses: Swatinem/rust-cache@v2 - - name: Run clippy - uses: actions-rs/cargo@v1 - with: - command: clippy + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + - uses: actions/checkout@v3 + - uses: Swatinem/rust-cache@v2 + - name: Run clippy + uses: actions-rs/cargo@v1 + with: + command: clippy macos: runs-on: macOS-latest steps: - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - - name: Add nightly rustfmt and clippy - run: rustup toolchain install nightly --component rustfmt --component clippy --allow-downgrade && rustup default nightly - - name: Install ucd-generate - run: cargo install -f ucd-generate - - name: Install deps - run: brew install z3 gtk+3 - - name: Install cxxbridge - run: cargo install cxxbridge-cmd - - uses: actions/checkout@v3 - - uses: Swatinem/rust-cache@v2 - - name: MacOS Build - run: cargo build --verbose - - name: Increase map sizes - run: ./scripts/shmem_limits_macos.sh - - name: Run Tests - run: cargo test - - other_targets: + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + - name: Add nightly rustfmt and clippy + run: rustup toolchain install nightly --component rustfmt --component clippy --allow-downgrade && rustup default nightly + - name: Install deps + run: brew install z3 gtk+3 + - name: Install cxxbridge + run: cargo install cxxbridge-cmd + - uses: actions/checkout@v3 + - uses: Swatinem/rust-cache@v2 + - name: MacOS Build + run: cargo build --verbose + - name: Increase map sizes + run: ./scripts/shmem_limits_macos.sh + - name: Run Tests + run: cargo test + + ios: runs-on: macOS-latest steps: - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - - uses: nttld/setup-ndk@v1 - with: - ndk-version: r25b - - name: install ios - run: rustup target add aarch64-apple-ios - - name: install android - run: rustup target add aarch64-linux-android - - name: install cargo ndk - run: cargo install cargo-ndk - - uses: actions/checkout@v3 - - uses: Swatinem/rust-cache@v2 - - name: Build iOS - run: cargo build --target aarch64-apple-ios && cd libafl_frida && cargo build --target aarch64-apple-ios && cd .. - - name: Build Android - run: cargo ndk -t arm64-v8a build --release + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + - name: install ios + run: rustup target add aarch64-apple-ios + - uses: actions/checkout@v3 + - uses: Swatinem/rust-cache@v2 + - name: Build iOS + run: cargo build --target aarch64-apple-ios && cd libafl_frida && cargo build --target aarch64-apple-ios && cd .. + + android: + runs-on: ubuntu-22.04 + steps: + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + - uses: nttld/setup-ndk@v1 + with: + ndk-version: r25b + - name: install android + run: rustup target add aarch64-linux-android + - name: install cargo ndk + run: cargo install cargo-ndk + - uses: actions/checkout@v3 + - uses: Swatinem/rust-cache@v2 + - name: Build Android + run: cargo ndk -t arm64-v8a build --release + #run: cargo build --target aarch64-linux-android # TODO: Figure out how to properly build stuff with clang #- name: Add clang path to $PATH env @@ -683,32 +574,31 @@ jobs: runs-on: ubuntu-22.04 name: Simple build in FreeBSD steps: - - uses: actions/checkout@v3 - - name: Test in FreeBSD - id: test - uses: vmactions/freebsd-vm@v1 - with: - usesh: true - sync: rsync - copyback: false - mem: 2048 - release: 13.2 - prepare: | - pkg install -y curl bash sudo llvm16 - curl https://sh.rustup.rs -sSf | sh -s -- -y - - run: | - freebsd-version - . "$HOME/.cargo/env" - rustup toolchain install nightly - cargo install -f ucd-generate - export LLVM_CONFIG=/usr/local/bin/llvm-config16 - pwd - ls -lah - echo "local/bin" - ls -lah /usr/local/bin/ - which llvm-config - chmod +x ./scripts/clippy.sh - bash ./scripts/shmem_limits_fbsd.sh - bash ./scripts/clippy.sh - cargo test + - uses: actions/checkout@v3 + - name: Test in FreeBSD + id: test + uses: vmactions/freebsd-vm@v1 + with: + usesh: true + sync: rsync + copyback: false + mem: 2048 + release: 13.2 + prepare: | + pkg install -y curl bash sudo llvm16 + curl https://sh.rustup.rs -sSf | sh -s -- -y + + run: | + freebsd-version + . "$HOME/.cargo/env" + rustup toolchain install nightly + export LLVM_CONFIG=/usr/local/bin/llvm-config16 + pwd + ls -lah + echo "local/bin" + ls -lah /usr/local/bin/ + which llvm-config + chmod +x ./scripts/clippy.sh + bash ./scripts/shmem_limits_fbsd.sh + bash ./scripts/clippy.sh + cargo test diff --git a/.github/workflows/fuzzer-tester-prepare/action.yml b/.github/workflows/fuzzer-tester-prepare/action.yml new file mode 100644 index 0000000000..c161a74607 --- /dev/null +++ b/.github/workflows/fuzzer-tester-prepare/action.yml @@ -0,0 +1,65 @@ +name: Setup Rust Environment +description: Sets up the Rust environment for the CI workflow +runs: + using: composite + steps: + - uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 0 + - uses: Swatinem/rust-cache@v2 + with: { shared-key: "${{ runner.os }}-shared-fuzzer-cache" } + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + - name: Add stable rustfmt and clippy + shell: bash + run: rustup toolchain install stable --component rustfmt --component clippy --allow-downgrade + - name: Add nightly rustfmt and clippy + shell: bash + run: rustup toolchain install nightly --component rustfmt --component clippy --allow-downgrade + - name: Add no_std toolchain + shell: bash + run: rustup toolchain install nightly-x86_64-unknown-linux-gnu ; rustup component add rust-src --toolchain nightly-x86_64-unknown-linux-gnu + - name: Add wasm target + shell: bash + run: rustup target add wasm32-unknown-unknown + - name: Remove obsolete llvm (Linux) + if: runner.os == 'Linux' + shell: bash + run: sudo apt purge -y llvm* clang* + - name: Install LLVM and Clang + uses: KyleMayes/install-llvm-action@v1 + with: + directory: ${{ runner.temp }}/llvm + version: 17 + - name: Install deps + shell: bash + run: sudo apt update && sudo apt install -y nasm ninja-build gcc-arm-linux-gnueabi g++-arm-linux-gnueabi gcc-aarch64-linux-gnu g++-aarch64-linux-gnu gcc-mipsel-linux-gnu g++-mipsel-linux-gnu gcc-powerpc-linux-gnu g++-powerpc-linux-gnu libc6-dev-i386-cross libc6-dev libc6-dev-i386 lib32gcc-11-dev lib32stdc++-11-dev libgtk-3-dev pax-utils libz3-dev + - name: pip install + shell: bash + run: python3 -m pip install msgpack jinja2 find_libpython + - name: enable mult-thread for `make` + shell: bash + run: export MAKEFLAGS="-j$(expr $(nproc) \+ 1)" + - name: install cargo-make + uses: baptiste0928/cargo-install@v1.3.0 + with: + crate: cargo-make + - name: install wasm-pack + uses: baptiste0928/cargo-install@v1.3.0 + with: + crate: wasm-pack + - name: install cxxbridge-cmd + uses: baptiste0928/cargo-install@v1.3.0 + with: + crate: cxxbridge-cmd + - name: install chrome + uses: browser-actions/setup-chrome@v1 + with: + chrome-version: stable + - name: Symlink Headers + if: runner.os == 'Linux' + shell: bash + run: sudo ln -s /usr/include/asm-generic /usr/include/asm diff --git a/.github/workflows/qemu-fuzzer-tester-prepare/action.yml b/.github/workflows/qemu-fuzzer-tester-prepare/action.yml new file mode 100644 index 0000000000..337c68d1ac --- /dev/null +++ b/.github/workflows/qemu-fuzzer-tester-prepare/action.yml @@ -0,0 +1,8 @@ +name: Setup QEMU Fuzzers environment +description: Sets up the QEMU fuzzers environment +runs: + using: composite + steps: + - name: Install sudo + shell: bash + run: apt update && apt install -y sudo wget qemu-utils libsqlite3-dev gcc-arm-none-eabi \ No newline at end of file diff --git a/.github/workflows/ubuntu-prepare/action.yml b/.github/workflows/ubuntu-prepare/action.yml new file mode 100644 index 0000000000..999c50d4c4 --- /dev/null +++ b/.github/workflows/ubuntu-prepare/action.yml @@ -0,0 +1,27 @@ +name: Setup Rust Environment +description: Sets up the Rust environment for the CI workflow +runs: + using: composite + steps: + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + components: llvm-tools + - name: Remove existing clang and LLVM + shell: bash + run: sudo apt purge llvm* clang* + - name: Install and cache deps + shell: bash + run: sudo apt update && sudo apt install ninja-build clang-format shellcheck libgtk-3-dev gcc-arm-linux-gnueabi g++-arm-linux-gnueabi libslirp-dev libz3-dev + - name: Install cargo-hack + shell: bash + run: curl -LsSf https://github.com/taiki-e/cargo-hack/releases/latest/download/cargo-hack-x86_64-unknown-linux-gnu.tar.gz | tar xzf - -C ~/.cargo/bin + - name: Add nightly + shell: bash + run: rustup toolchain install nightly --allow-downgrade + - name: Install LLVM and Clang + uses: KyleMayes/install-llvm-action@v1 + with: + directory: ${{ runner.temp }}/llvm + version: 17 \ No newline at end of file diff --git a/.github/workflows/windows-tester-prepare/action.yml b/.github/workflows/windows-tester-prepare/action.yml new file mode 100644 index 0000000000..3ea5ec482e --- /dev/null +++ b/.github/workflows/windows-tester-prepare/action.yml @@ -0,0 +1,24 @@ +name: Setup Rust Environment +description: Sets up the Rust environment for the CI workflow +runs: + using: composite + steps: + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + - uses: actions/checkout@v3 + - uses: Swatinem/rust-cache@v2 + - name: Windows Build + shell: pwsh + run: cargo build --verbose + - name: Build docs + shell: pwsh + run: cargo doc + - uses: ilammy/msvc-dev-cmd@v1 + - name: Set LIBCLANG_PATH + shell: pwsh + run: echo "LIBCLANG_PATH=$((gcm clang).source -replace "clang.exe")" >> $env:GITHUB_ENV + - name: install cargo-make + shell: pwsh + run: cargo install --force cargo-make \ No newline at end of file diff --git a/.gitignore b/.gitignore index 4ea9818e41..c1b1522943 100644 --- a/.gitignore +++ b/.gitignore @@ -68,3 +68,10 @@ libafl_nyx/packer *.ll *.tar.gz + +# common harness names +harness +program +fuzzer +fuzzer_libpng* +forkserver_simple diff --git a/Cargo.toml b/Cargo.toml index 96a1034caf..0c49a0b6df 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,7 +39,7 @@ exclude = [ ] [workspace.package] -version = "0.11.2" +version = "0.12.0" [profile.release] lto = true diff --git a/Dockerfile b/Dockerfile index 0265192687..128b722afa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ LABEL "maintainer"="afl++ team " LABEL "about"="LibAFL Docker image" # install sccache to cache subsequent builds of dependencies -RUN cargo install sccache +RUN cargo install --locked sccache ENV HOME=/root ENV SCCACHE_CACHE_SIZE="1G" @@ -16,12 +16,18 @@ RUN sh -c 'echo set encoding=utf-8 > /root/.vimrc' \ mkdir ~/.cargo && \ echo "[build]\nrustc-wrapper = \"${RUSTC_WRAPPER}\"" >> ~/.cargo/config -RUN rustup component add rustfmt clippy RUN rustup default nightly +RUN rustup component add rustfmt clippy -# Install clang 11, common build tools -RUN apt update && apt install -y build-essential gdb git wget python3-venv ninja-build lsb-release software-properties-common gnupg -RUN wget https://apt.llvm.org/llvm.sh && chmod +x llvm.sh && ./llvm.sh 15 +# Install clang 18, common build tools +ENV LLVM_VERSION=18 +RUN apt update && apt install -y build-essential gdb git wget python3-venv ninja-build lsb-release software-properties-common gnupg cmake +# Workaround until https://github.com/llvm/llvm-project/issues/62475 is resolved +RUN set -ex &&\ + echo "deb http://apt.llvm.org/bookworm/ llvm-toolchain-bookworm-${LLVM_VERSION} main" > /etc/apt/sources.list.d/apt.llvm.org.list &&\ + wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc &&\ + apt update &&\ + apt-get install -y clang-${LLVM_VERSION} lldb-${LLVM_VERSION} lld-${LLVM_VERSION} clangd-${LLVM_VERSION} clang-tidy-${LLVM_VERSION} clang-format-${LLVM_VERSION} clang-tools-${LLVM_VERSION} llvm-${LLVM_VERSION}-dev lld-${LLVM_VERSION} lldb-${LLVM_VERSION} llvm-${LLVM_VERSION}-tools libomp-${LLVM_VERSION}-dev libc++-${LLVM_VERSION}-dev libc++abi-${LLVM_VERSION}-dev libclang-common-${LLVM_VERSION}-dev libclang-${LLVM_VERSION}-dev libclang-cpp${LLVM_VERSION}-dev libunwind-${LLVM_VERSION}-dev libclang-rt-${LLVM_VERSION}-dev libpolly-${LLVM_VERSION}-dev # Copy a dummy.rs and Cargo.toml first, so that dependencies are cached WORKDIR /libafl @@ -41,13 +47,13 @@ COPY libafl_frida/Cargo.toml libafl_frida/build.rs libafl_frida/ COPY scripts/dummy.rs libafl_frida/src/lib.rs COPY libafl_frida/src/gettls.c libafl_frida/src/gettls.c -COPY libafl_qemu/Cargo.toml libafl_qemu/build.rs libafl_qemu/ +COPY libafl_qemu/Cargo.toml libafl_qemu/build.rs libafl_qemu/build_linux.rs libafl_qemu/ COPY scripts/dummy.rs libafl_qemu/src/lib.rs COPY libafl_qemu/libafl_qemu_build/Cargo.toml libafl_qemu/libafl_qemu_build/ COPY scripts/dummy.rs libafl_qemu/libafl_qemu_build/src/lib.rs -COPY libafl_qemu/libafl_qemu_sys/Cargo.toml libafl_qemu/libafl_qemu_sys/build.rs libafl_qemu/libafl_qemu_sys/ +COPY libafl_qemu/libafl_qemu_sys/Cargo.toml libafl_qemu/libafl_qemu_sys/build.rs libafl_qemu/libafl_qemu_sys/build_linux.rs libafl_qemu/libafl_qemu_sys/ COPY scripts/dummy.rs libafl_qemu/libafl_qemu_sys/src/lib.rs COPY libafl_sugar/Cargo.toml libafl_sugar/ @@ -74,7 +80,7 @@ COPY scripts/dummy.rs libafl_concolic/symcc_runtime/src/lib.rs COPY libafl_concolic/symcc_libafl/Cargo.toml libafl_concolic/symcc_libafl/ COPY scripts/dummy.rs libafl_concolic/symcc_libafl/src/lib.rs -COPY libafl_nyx/Cargo.toml libafl_nyx/build.rs libafl_nyx/ +COPY libafl_nyx/Cargo.toml libafl_nyx/build.rs libafl_nyx/build_nyx_support.sh libafl_nyx/ COPY scripts/dummy.rs libafl_nyx/src/lib.rs COPY libafl_tinyinst/Cargo.toml libafl_tinyinst/ @@ -115,6 +121,8 @@ RUN touch libafl_qemu/libafl_qemu_build/src/lib.rs COPY libafl_qemu/libafl_qemu_build/src libafl_qemu/libafl_qemu_build/src RUN touch libafl_qemu/libafl_qemu_sys/src/lib.rs COPY libafl_qemu/libafl_qemu_sys/src libafl_qemu/libafl_qemu_sys/src +COPY libafl_qemu/runtime libafl_qemu/runtime +COPY libafl_qemu/libqasan libafl_qemu/libqasan RUN touch libafl_qemu/src/lib.rs COPY libafl_qemu/src libafl_qemu/src RUN touch libafl_frida/src/lib.rs @@ -132,6 +140,7 @@ RUN cargo build && cargo build --release # Copy fuzzers over COPY fuzzers fuzzers -# RUN ./scripts/test_all_fuzzers.sh --no-fmt +# RUN ./scripts/test_fuzzer.sh --no-fmt -ENTRYPOINT [ "/bin/bash" ] +ENTRYPOINT [ "/bin/bash", "-c" ] +CMD ["/bin/bash"] diff --git a/README.md b/README.md index 0765abd1e6..4e38b68408 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,10 @@ We highly recommend *not* to use e.g. your Linux distribition package as this is Rust directly, instructions can be found [here](https://www.rust-lang.org/tools/install). - LLVM tools -The LLVM tools (including clang, clang++) are needed (newer than LLVM 11.0.0 up to LLVM 17.0.0) +The LLVM tools (including clang, clang++) are needed (newer than LLVM 15.0.0 up to LLVM 18.1.3) +If you are using Debian/Ubuntu, again, we highly recommmend that you install the package from [here](https://apt.llvm.org/) + +(In `libafl_concolic`, we only support LLVM version newer than 18) - Cargo-make We use cargo-make to build the fuzzers in `fuzzers/` directory. You can install it with @@ -117,7 +120,7 @@ For bugs, feel free to open issues or contact us directly. Thank you for your su Even though we will gladly assist you in finishing up your PR, try to - keep all the crates compiling with *stable* rust (hide the eventual non-stable code under [`cfg`s](https://github.com/AFLplusplus/LibAFL/blob/main/libafl/build.rs#L26)) -- run `cargo fmt` on your code before pushing +- run `cargo +nightly fmt` on your code before pushing - check the output of `cargo clippy --all` or `./clippy.sh` - run `cargo build --no-default-features` to check for `no_std` compatibility (and possibly add `#[cfg(feature = "std")]`) to hide parts of your code. diff --git a/bindings/pylibafl/Cargo.toml b/bindings/pylibafl/Cargo.toml index 3880bc6e07..f3497c50b8 100644 --- a/bindings/pylibafl/Cargo.toml +++ b/bindings/pylibafl/Cargo.toml @@ -1,15 +1,16 @@ [package] name = "pylibafl" -version = "0.11.2" +version = "0.12.0" edition = "2021" [dependencies] pyo3 = { version = "0.18.3", features = ["extension-module"] } pyo3-log = "0.8.1" -libafl_qemu = { path = "../../libafl_qemu", version = "0.11.2", features = ["python"] } -libafl_sugar = { path = "../../libafl_sugar", version = "0.11.2", features = ["python"] } -libafl = { path = "../../libafl", version = "0.11.2", features = ["python"] } -libafl_bolts = { path = "../../libafl_bolts", version = "0.11.2", features = ["python"] } +libafl_sugar = { path = "../../libafl_sugar", version = "0.12.0", features = ["python"] } +libafl_bolts = { path = "../../libafl_bolts", version = "0.12.0", features = ["python"] } + +[target.'cfg(target_os = "linux")'.dependencies] +libafl_qemu = { path = "../../libafl_qemu", version = "0.12.0", features = ["python"] } [build-dependencies] pyo3-build-config = { version = "0.17" } diff --git a/bindings/pylibafl/src/lib.rs b/bindings/pylibafl/src/lib.rs index f5047ea544..017dae5211 100644 --- a/bindings/pylibafl/src/lib.rs +++ b/bindings/pylibafl/src/lib.rs @@ -1,87 +1,9 @@ -use libafl; -use libafl_bolts; -#[cfg(target_os = "linux")] -use libafl_qemu; -use libafl_sugar; -use pyo3::{prelude::*, types::PyDict}; - -const LIBAFL_CODE: &str = r#" -class BaseObserver: - def flush(self): - pass - def pre_exec(self, state, input): - pass - def post_exec(self, state, input, exit_kind): - pass - def pre_exec_child(self, state, input): - pass - def post_exec_child(self, state, input, exit_kind): - pass - def name(self): - return type(self).__name__ - def as_observer(self): - return Observer.new_py(self) - -class BaseFeedback: - def init_state(self, state): - pass - def is_interesting(self, state, mgr, input, observers, exit_kind) -> bool: - return False - def append_metadata(self, state, observers, testcase): - pass - def discard_metadata(self, state, input): - pass - def name(self): - return type(self).__name__ - def as_feedback(self): - return Feedback.new_py(self) - -class BaseExecutor: - def observers(self) -> ObserversTuple: - raise NotImplementedError('Implement this yourself') - def run_target(self, fuzzer, state, mgr, input) -> ExitKind: - raise NotImplementedError('Implement this yourself') - def as_executor(self): - return Executor.new_py(self) - -class BaseStage: - def perform(self, fuzzer, executor, state, manager, corpus_idx): - pass - def as_stage(self): - return Stage.new_py(self) - -class BaseMutator: - def mutate(self, state, input, stage_idx): - pass - def post_exec(self, state, stage_idx, corpus_idx): - pass - def as_mutator(self): - return Mutator.new_py(self) - -class FnStage(BaseStage): - def __init__(self, fn): - self.fn = fn - def __call__(self, fuzzer, executor, state, manager, corpus_idx): - self.fn(fuzzer, executor, state, manager, corpus_idx) - def perform(self, fuzzer, executor, state, manager, corpus_idx): - self.fn(fuzzer, executor, state, manager, corpus_idx) - -def feedback_not(a): - return NotFeedback(a).as_feedback() - -def feedback_and(a, b): - return EagerAndFeedback(a, b).as_feedback() - -def feedback_and_fast(a, b): - return FastAndFeedback(a, b).as_feedback() - -def feedback_or(a, b): - return EagerOrFeedback(a, b).as_feedback() - -def feedback_or_fast(a, b): - return FastOrFeedback(a, b).as_feedback() -"#; +use pyo3::prelude::*; +/// Setup python modules for `libafl_qemu` and `libafl_sugar`. +/// +/// # Errors +/// Returns error if python libafl setup failed. #[pymodule] #[pyo3(name = "pylibafl")] pub fn python_module(py: Python, m: &PyModule) -> PyResult<()> { @@ -107,19 +29,5 @@ pub fn python_module(py: Python, m: &PyModule) -> PyResult<()> { m.add_submodule(bolts_module)?; modules.set_item("pylibafl.libafl_bolts", bolts_module)?; - let libafl_module = PyModule::new(py, "libafl")?; - libafl::pybind::python_module(py, libafl_module)?; - - libafl_module.add("__builtins__", py.import("builtins")?)?; - - let locals = PyDict::new(py); - py.run(LIBAFL_CODE, Some(libafl_module.dict()), Some(locals))?; - for (key, val) in locals.iter() { - libafl_module.add(key.extract::<&str>()?, val)?; - } - - m.add_submodule(libafl_module)?; - modules.set_item("pylibafl.libafl", libafl_module)?; - Ok(()) } diff --git a/bindings/pylibafl/test.py b/bindings/pylibafl/test.py index ecfe35ec90..41d90c9e3e 100644 --- a/bindings/pylibafl/test.py +++ b/bindings/pylibafl/test.py @@ -1,109 +1,7 @@ -from pylibafl.libafl import * +import pylibafl.sugar as sugar import ctypes import platform -MAP_SIZE = 4096 - - -class FooObserver(BaseObserver): - def __init__(self): - self.n = 0 - - def name(self): - return "Foo" - - def pre_exec(self, state, input): - if self.n % 10000 == 0: - print("FOO!", self.n, input) - self.n += 1 - - -class FooFeedback(BaseFeedback): - def is_interesting(self, state, mgr, input, observers, exit_kind): - ob = observers.match_name("Foo").unwrap_py() - return ob.n % 10000 == 0 - - -class FooExecutor(BaseExecutor): - def __init__(self, harness, observers: ObserversTuple): - self.h = harness - self.o = observers - - def observers(self): - return self.o - - def run_target(self, fuzzer, state, mgr, input) -> ExitKind: - return (self.h)(input) - - -if platform.system() == "Darwin": - libc = ctypes.cdll.LoadLibrary("libc.dylib") -else: - libc = ctypes.cdll.LoadLibrary("libc.so.6") - -# Get a buffer to use for our map observer -libc.calloc.restype = ctypes.c_void_p -area_ptr = libc.calloc(1, MAP_SIZE) - -observer = StdMapObserverI8("mymap", area_ptr, MAP_SIZE) - -m = observer.as_map_observer() - -observers = ObserversTuple( - [observer.as_map_observer().as_observer(), FooObserver().as_observer()] -) - -feedback = feedback_or(MaxMapFeedbackI8(m).as_feedback(), FooFeedback().as_feedback()) - -objective = feedback_and_fast( - CrashFeedback().as_feedback(), MaxMapFeedbackI8(m).as_feedback() -) - -fuzzer = StdFuzzer(feedback, objective) - -rand = StdRand.with_current_nanos() - -state = StdState( - rand.as_rand(), - InMemoryCorpus().as_corpus(), - InMemoryCorpus().as_corpus(), - feedback, - objective, -) - -monitor = SimpleMonitor(lambda s: print(s)) - -mgr = SimpleEventManager(monitor.as_monitor()) - - -def harness(buf) -> ExitKind: - """ - The harness fn that the fuzzer will execute in a loop - """ - # print(buf) - - # set the observer map byte from python - m[0] = 1 - if len(buf) > 0 and buf[0] == ord("a"): - m[1] = 1 - if len(buf) > 1 and buf[1] == ord("b"): - m[2] = 1 - if len(buf) > 2 and buf[2] == ord("c"): - m[3] = 1 - return ExitKind.crash() - return ExitKind.ok() - - -# executor = InProcessExecutor(harness, observers, fuzzer, state, mgr.as_manager()) - -executor = FooExecutor(harness, observers) - -stage = StdMutationalStage(StdHavocMutator().as_mutator()) - -stage_tuple_list = StagesTuple([stage.as_stage()]) - -fuzzer.add_input(state, executor.as_executor(), mgr.as_manager(), b"\0\0") - print("Starting to fuzz from python!") - -fuzzer.fuzz_loop(executor.as_executor(), state, mgr.as_manager(), stage_tuple_list) +fuzzer = sugar.InMemoryBytesCoverageSugar(input_dirs=["./in"], output_dir="out", broker_port=1337, cores=[0,1]) +fuzzer.run(lambda b: print("foo")) \ No newline at end of file diff --git a/bindings/pylibafl/test.sh b/bindings/pylibafl/test.sh index 720488b14c..e02a03da8b 100755 --- a/bindings/pylibafl/test.sh +++ b/bindings/pylibafl/test.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash +mkdir in || true +echo "a" > ./in/a + timeout 10 python3 ./test.py export exit_code=$? if [ $exit_code -eq 124 ]; then diff --git a/docs/listings/baby_fuzzer/listing-04/src/main.rs b/docs/listings/baby_fuzzer/listing-04/src/main.rs index 2e514fc5a6..dd64597244 100644 --- a/docs/listings/baby_fuzzer/listing-04/src/main.rs +++ b/docs/listings/baby_fuzzer/listing-04/src/main.rs @@ -15,7 +15,7 @@ use libafl::{ schedulers::QueueScheduler, state::StdState, }; -use libafl_bolts::{current_nanos, rands::StdRand, tuples::tuple_list, AsSlice}; +use libafl_bolts::{rands::StdRand, tuples::tuple_list, AsSlice}; /* ANCHOR_END: use */ fn main() { @@ -40,7 +40,7 @@ fn main() { // create a State from scratch let mut state = StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryCorpus::new(), // Corpus in which we store solutions (crashes in this example), @@ -71,14 +71,8 @@ fn main() { /* ANCHOR: executor */ // Create the executor for an in-process function - let mut executor = InProcessExecutor::new( - &mut harness, - (), - &mut fuzzer, - &mut state, - &mut mgr, - ) - .expect("Failed to create the Executor"); + let mut executor = InProcessExecutor::new(&mut harness, (), &mut fuzzer, &mut state, &mut mgr) + .expect("Failed to create the Executor"); /* ANCHOR_END: executor */ /* ANCHOR: generator */ diff --git a/docs/listings/baby_fuzzer/listing-05/src/main.rs b/docs/listings/baby_fuzzer/listing-05/src/main.rs index f7fec1fb3b..1377243fb7 100644 --- a/docs/listings/baby_fuzzer/listing-05/src/main.rs +++ b/docs/listings/baby_fuzzer/listing-05/src/main.rs @@ -15,7 +15,7 @@ use libafl::{ schedulers::QueueScheduler, state::StdState, }; -use libafl_bolts::{current_nanos, rands::StdRand, tuples::tuple_list, AsSlice}; +use libafl_bolts::{rands::StdRand, tuples::tuple_list, AsSlice}; use std::path::PathBuf; /* ANCHOR_END: use */ @@ -65,7 +65,7 @@ fn main() { // create a State from scratch let mut state = StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryCorpus::new(), // Corpus in which we store solutions (crashes in this example), diff --git a/docs/listings/baby_fuzzer/listing-06/src/main.rs b/docs/listings/baby_fuzzer/listing-06/src/main.rs index 02b27c7e92..6faff74382 100644 --- a/docs/listings/baby_fuzzer/listing-06/src/main.rs +++ b/docs/listings/baby_fuzzer/listing-06/src/main.rs @@ -17,7 +17,7 @@ use libafl::{ stages::mutational::StdMutationalStage, state::StdState, }; -use libafl_bolts::{current_nanos, rands::StdRand, tuples::tuple_list, AsSlice}; +use libafl_bolts::{rands::StdRand, tuples::tuple_list, AsSlice}; use std::path::PathBuf; /* ANCHOR_END: use */ @@ -62,7 +62,7 @@ fn main() { // create a State from scratch let mut state = StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryCorpus::new(), // Corpus in which we store solutions (crashes in this example), diff --git a/docs/src/DEBUGGING.md b/docs/src/DEBUGGING.md new file mode 100644 index 0000000000..d6063f968b --- /dev/null +++ b/docs/src/DEBUGGING.md @@ -0,0 +1,32 @@ +# General debugging tips +This file answers some commmon questions that arise when you are writing a fuzzer using LibAFL. + +## Q. My fuzzer crashed but the stack trace is useless. +You can enable the `errors_backtrace` feature of the `libafl` crate. With this the stacktrace is meaningful. + +## Q. I started the fuzzer but the corpus count is 0. +Unless the initial corpus is loaded with the "load_initial_inputs_forced" function, we only store the interesting inputs, which is the inputs that triggered the feedback. So this usually means that your input was not interesting or your target was simply not properly implemented. +Either way, what you can do is attach to the executable with gdb and set a breakpoint at where the new edges should be reported. If no instrumentation code is executed, then the the problem is in the instrumentation. If the instrumentation code is hit, but still the your input is not instrumented, then the problem could be that you are not passign the observer/feedback correctly to the fuzzer. + +## Q. I started the fuzzer but the coverage is 0. +This could mean two things. Perhaps your target was not properly instrumented, or you are not using the correct observer, feedback feature. +In this case, again, what usually should do is to run the fuzzer with gdb and set a breakpoint at where the coverage is recorded (e.g. __sanitizer_coverage_trace_pcguard), and validate that the target is giving the feedback to the fuzzer. + +## Q. I started the fuzzer but there's no output. +First, verify that your stdout and stderr are not redirected to `/dev/null`. If you get the log, then it should either fall into the previous 2 cases. Either the fuzzer crashed because you didn't have the initial seeds, or the coverage feedback is not working. + +## Q. My fuzzer is slow. +Try running the fuzzer with the `introspection` feature of the `libafl`. This will show how much time is spent on each module of your fuzzer. Also you might be using a wrong size of the coverage map. If you see `2621440` for the size of the coverage map, you are doing it wrong. One possible mistake is the misuse of `libafl_targets::coverage::EDGES_MAP` +``` +let map = StdMapObserver::from_mut_ptr("edges", EDGES_MAP.as_mut_ptr(), EDGES_MAP.len()); +``` +You should *never* use the `EDGES_MAP`'s size as this is just the size of the allocated size of the coverage map. Consider using something smaller or our default value `libafl_targets::LIBAFL_EDGES_MAP_SIZE_IN_USE`. + +## Q. I still have problems with my fuzzer. +Finally, if you really have no idea what is going on, run your fuzzer with logging enabled. (You can use `env_logger`, `SimpleStdoutLogger`, `SimpleStderrLogger` from `libafl_bolts`. `fuzzbench_text` has an example to show how to use it.) (Don't forget to enable stdout and stderr), and you can open an issue or ask us in Discord. + +## Q. My fuzzer died of ``Storing state in crashed fuzzer instance did not work''. +If the exit code is zero, then this is because either your harness exited or you are using fuzzer_loop_for and forgot to add `mgr.on_restart` at the end of the fuzzer. In the first case, you should patch your harness not to exit. (or use `utils/deexit`). + +## Q. I can't leave the TUI screen +Type `q` then you leave TUI. \ No newline at end of file diff --git a/docs/src/advanced_features/frida.md b/docs/src/advanced_features/frida.md index b538a99b0b..5d1532de9f 100644 --- a/docs/src/advanced_features/frida.md +++ b/docs/src/advanced_features/frida.md @@ -73,7 +73,7 @@ You can then link this observer to `FridaInProcessExecutor` as follows: tuple_list!( edges_observer, time_observer, - AsanErrorsObserver::new(addr_of!(ASAN_ERRORS)) + AsanErrorsObserver::from_static_asan_errors() ), &mut fuzzer, &mut state, diff --git a/docs/src/core_concepts/executor.md b/docs/src/core_concepts/executor.md index 009dc1cab0..1a677b6ef9 100644 --- a/docs/src/core_concepts/executor.md +++ b/docs/src/core_concepts/executor.md @@ -31,7 +31,7 @@ As you can see from the forkserver example, let mut shmem = StdShMemProvider::new().unwrap().new_shmem(MAP_SIZE).unwrap(); //let the forkserver know the shmid shmem.write_to_env("__AFL_SHM_ID").unwrap(); -let mut shmem_buf = shmem.as_mut_slice(); +let mut shmem_buf = shmem.as_slice_mut(); ``` Here we make a shared memory region; `shmem`, and write this to environmental variable `__AFL_SHM_ID`. Then the instrumented binary, or the forkserver, finds this shared memory region (from the aforementioned env var) to record its coverage. On your fuzzer side, you can pass this shmem map to your `Observer` to obtain coverage feedbacks combined with any `Feedback`. @@ -57,9 +57,9 @@ On your fuzzer side, you can allocate a shared memory region and make the `EDGES ```rust,ignore let mut shmem; unsafe{ - shmem = StdShMemProvider::new().unwrap().new_shmem(MAX_EDGES_NUM).unwrap(); + shmem = StdShMemProvider::new().unwrap().new_shmem(EDGES_MAP_SIZE_IN_USE).unwrap(); } -let shmem_buf = shmem.as_mut_slice(); +let shmem_buf = shmem.as_slice_mut(); unsafe{ EDGES_PTR = shmem_buf.as_ptr(); } diff --git a/docs/src/design/migration-0.12.md b/docs/src/design/migration-0.12.md new file mode 100644 index 0000000000..582b91b0c7 --- /dev/null +++ b/docs/src/design/migration-0.12.md @@ -0,0 +1,9 @@ +# Migrating from <0.12 to 0.12 + +We deleted `TimeoutExecutor` and `TimeoutForkserverExecutor` and make it mandatory for `InProcessExecutor` and `ForkserverExecutor` to have the timeout. Now `InProcessExecutor` and `ForkserverExecutor` have the default timeout of 5 seconds. + +## Reason for This Change. +In 99% of the case, it is advised to have the timeout for the fuzzer. This is because we do not want the fuzzer to stop forever just because the target has hit a path that resulted in a infinite-loop. + +## What changed +You do not have to wrap the executor with `TimeoutExecutor` anymore. You can just use `InProcessExecutor::new()` to instantiate the executor with the default timeout or use `InProcessExecutor::timeout(duration)` to start the executor with the customized duration of timeout. \ No newline at end of file diff --git a/docs/src/design/migration-0.9.md b/docs/src/design/migration-0.9.md index 64c961c53f..b48c3e7711 100644 --- a/docs/src/design/migration-0.9.md +++ b/docs/src/design/migration-0.9.md @@ -169,7 +169,7 @@ from libafl_target's `EDGES_MAP`. In the future, instead of using: ```rust,ignore -let edges = unsafe { &mut EDGES_MAP[0..MAX_EDGES_NUM] }; +let edges = unsafe { &mut EDGES_MAP[0..EDGES_MAP_SIZE_IN_USE] }; let edges_observer = StdMapObserver::new("edges", edges); ``` diff --git a/fuzzers/baby_fuzzer/.gitignore b/fuzzers/baby_fuzzer/.gitignore index a977a2ca5b..e0921f291e 100644 --- a/fuzzers/baby_fuzzer/.gitignore +++ b/fuzzers/baby_fuzzer/.gitignore @@ -1 +1,2 @@ -libpng-* \ No newline at end of file +libpng-* +corpus \ No newline at end of file diff --git a/fuzzers/baby_fuzzer/baby_fuzzer.py b/fuzzers/baby_fuzzer/baby_fuzzer.py deleted file mode 100644 index f904a452a9..0000000000 --- a/fuzzers/baby_fuzzer/baby_fuzzer.py +++ /dev/null @@ -1,89 +0,0 @@ -from pylibafl import libafl - -# LIBRARY WRAPPER - -def map_observer_wrapper(map_observer): - if type(map_observer).__name__ == "OwnedMapObserverI32": - return libafl.MapObserverI32.new_owned(map_observer) - -def executor_wrapper(executor): - if type(executor).__name__ == "InProcessExecutor": - return libafl.Executor.new_inprocess(executor) - -def generator_wrapper(generator): - if type(generator).__name__ == "RandPrintablesGenerator": - return libafl.Generator.new_rand_printables(generator) - -def monitor_wrapper(monitor): - return monitor.as_monitor() - -def event_manager_wrapper(event_manager): - return event_manager.as_manager() - -def corpus_wrapper(corpus): - if type(corpus).__name__ == "InMemoryCorpus": - return libafl.Corpus.new_in_memory(corpus) - if type(corpus).__name__ == "OnDiskCorpus": - return libafl.Corpus.new_on_disk(corpus) - -def rand_wrapper(rand): - if type(rand).__name__ == "StdRand": - return libafl.Rand.new_std(rand) - -def mutator_wrapper(mutator): - if type(mutator).__name__ == "StdHavocMutator": - return libafl.Mutator.new_std_havoc(mutator) - -def stage_wrapper(stage): - if type(stage).__name__ == "StdMutationalStage": - return libafl.Stage.new_std_mutational(stage) - -# CODE WRITTEN BY USER -import logging -logging.basicConfig(level=logging.INFO) - -map_observer = libafl.OwnedMapObserverI32("signals", [0] * 16) - -def harness(inp): - #print(inp) - map_observer[0] = 1 - if len(inp) > 0 and inp[0] == ord('a'): - map_observer[1] = 1 - if len(inp) > 1 and inp[1] == ord('b'): - map_observer[2] = 1 - if len(inp) > 2 and inp[2] == ord('c'): - map_observer[3] = 1 - raise Exception("NOOOOOO =)") - -feedback = libafl.MaxMapFeedbackI32(map_observer_wrapper(map_observer)) -objective = libafl.CrashFeedback() - -state = libafl.StdState( - rand_wrapper(libafl.StdRand.with_current_nanos()), - corpus_wrapper(libafl.InMemoryCorpus()), - corpus_wrapper(libafl.OnDiskCorpus("./crashes")), - feedback.as_feedback(), - objective.as_feedback(), -) - -monitor = libafl.SimpleMonitor(lambda x: print(x)) - -mgr = libafl.SimpleEventManager(monitor_wrapper(monitor)) - -fuzzer = libafl.StdFuzzer(feedback.as_feedback(), objective.as_feedback()) - -observers = libafl.ObserversTuple([libafl.Observer.new_map_i32(map_observer_wrapper(map_observer))]) - -executor = libafl.InProcessExecutor(harness, observers, fuzzer, state, event_manager_wrapper(mgr)) - -generator = libafl.RandPrintablesGenerator(32) - -state.generate_initial_inputs(fuzzer, executor_wrapper(executor), generator_wrapper(generator), event_manager_wrapper(mgr), 3) - -mutator = libafl.StdHavocMutator() - -stage = libafl.StdMutationalStage(mutator_wrapper(mutator)) - -stages = libafl.StagesTuple([stage_wrapper(stage)]) - -fuzzer.fuzz_loop(executor_wrapper(executor), state, event_manager_wrapper(mgr), stages) diff --git a/fuzzers/baby_fuzzer_gramatron/Cargo.toml b/fuzzers/baby_fuzzer_gramatron/Cargo.toml index c9f89b26ae..cebd91675e 100644 --- a/fuzzers/baby_fuzzer_gramatron/Cargo.toml +++ b/fuzzers/baby_fuzzer_gramatron/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "baby_fuzzer_gramatron" -version = "0.11.2" +version = "0.12.0" authors = ["Andrea Fioraldi ", "Dominik Maier "] edition = "2021" diff --git a/fuzzers/baby_fuzzer_gramatron/auto.json b/fuzzers/baby_fuzzer_gramatron/auto.json index efd00fd65c..80256a9b84 100644 --- a/fuzzers/baby_fuzzer_gramatron/auto.json +++ b/fuzzers/baby_fuzzer_gramatron/auto.json @@ -1 +1 @@ -{"init_state": 0, "final_state": 37, "pda": [[["0_1", 1, ""]], [["25_1", 30, ";"]], [["26_1", 31, "("]], [["27_1", 18, "->"]], [["28_1", 1, ";"]], [["29_1", 32, " "], ["29_2", 32, "$a"], ["29_3", 32, "$b"], ["29_4", 32, "$c"], ["29_5", 32, "$d"], ["29_6", 33, "$a"], ["29_7", 33, "$b"], ["29_8", 33, "$c"], ["29_9", 33, "$d"]], [["30_1", 34, " "]], [["31_1", 35, " "], ["31_2", 35, "$a"], ["31_3", 35, "$b"], ["31_4", 35, "$c"], ["31_5", 35, "$d"], ["31_6", 36, "$a"], ["31_7", 36, "$b"], ["31_8", 36, "$c"], ["31_9", 36, "$d"]], [["32_1", 25, ")"]], [["33_1", 29, ","]], [["34_1", 37, "?>"]], [["35_1", 28, ")"]], [["36_1", 31, ","]]]} \ No newline at end of file +{"final_state":6,"init_state":0,"pda":[[{"dest":1,"term":"a"},{"dest":1,"term":"b"},{"dest":1,"term":"c"},{"dest":1,"term":"d"},{"dest":2,"term":"a"},{"dest":2,"term":"b"},{"dest":2,"term":"c"},{"dest":2,"term":"d"},{"dest":3,"term":"a"},{"dest":3,"term":"b"},{"dest":3,"term":"c"},{"dest":3,"term":"d"},{"dest":4,"term":"a"},{"dest":4,"term":"b"},{"dest":4,"term":"c"},{"dest":4,"term":"d"},{"dest":5,"term":"return"},{"dest":5,"term":"yield"},{"dest":5,"term":"continue"},{"dest":5,"term":"break"},{"dest":5,"term":"next"},{"dest":6,"term":" "}],[{"dest":7,"term":"="}],[{"dest":8,"term":"="}],[{"dest":9,"term":"="}],[{"dest":10,"term":"="}],[{"dest":11,"term":" "}],[],[{"dest":12,"term":"a"},{"dest":12,"term":"b"},{"dest":12,"term":"c"},{"dest":12,"term":"d"}],[{"dest":13,"term":"abcdef0123456789ABCDEF"},{"dest":13,"term":"abcdefghijklmnopqrstuvwxyz"},{"dest":13,"term":"abort"},{"dest":13,"term":"abs"},{"dest":13,"term":"accept"},{"dest":13,"term":"acos"},{"dest":13,"term":"acosh"},{"dest":13,"term":"address"},{"dest":13,"term":"alias"},{"dest":13,"term":"alias_method"},{"dest":13,"term":"allocation"},{"dest":13,"term":"all_symbols"},{"dest":13,"term":"ancestors"},{"dest":13,"term":"and"},{"dest":13,"term":"anum"},{"dest":13,"term":"append"},{"dest":13,"term":"append_features"},{"dest":13,"term":"Apr"},{"dest":13,"term":"aref_args"},{"dest":13,"term":"arg"},{"dest":13,"term":"arg0"},{"dest":13,"term":"arg1"},{"dest":13,"term":"arg2"},{"dest":13,"term":"arg_rhs"},{"dest":13,"term":"args"},{"dest":13,"term":"argument"},{"dest":13,"term":"ArgumentError"},{"dest":13,"term":"arguments"},{"dest":13,"term":"argv"},{"dest":13,"term":"ARGV"},{"dest":13,"term":"arity"},{"dest":13,"term":"array"},{"dest":13,"term":"Array"},{"dest":13,"term":"ary"},{"dest":13,"term":"__ary_cmp"},{"dest":13,"term":"ary_concat"},{"dest":13,"term":"__ary_eq"},{"dest":13,"term":"ary_F"},{"dest":13,"term":"__ary_index"},{"dest":13,"term":"ary_replace"},{"dest":13,"term":"ary_T"},{"dest":13,"term":"asctime"},{"dest":13,"term":"asin"},{"dest":13,"term":"asinh"},{"dest":13,"term":"__assert_fail"},{"dest":13,"term":"assignment"},{"dest":13,"term":"assoc"},{"dest":13,"term":"assoc_list"},{"dest":13,"term":"assocs"},{"dest":13,"term":"assumed"},{"dest":13,"term":"at"},{"dest":13,"term":"atan"},{"dest":13,"term":"atan2"},{"dest":13,"term":"atanh"},{"dest":13,"term":"__attached__"},{"dest":13,"term":"attr"},{"dest":13,"term":"attr_accessor"},{"dest":13,"term":"attr_reader"},{"dest":13,"term":"attrsym"},{"dest":13,"term":"attr_writer"},{"dest":13,"term":"available"},{"dest":13,"term":"backref"},{"dest":13,"term":"backtrace"},{"dest":13,"term":"Backtrace"},{"dest":13,"term":"BasicObject"},{"dest":13,"term":"basic_symbol"},{"dest":13,"term":"beg"},{"dest":13,"term":"begin"},{"dest":13,"term":"BEGIN"},{"dest":13,"term":"big"},{"dest":13,"term":"BIT"},{"dest":13,"term":"blkarg_mark"},{"dest":13,"term":"block"},{"dest":13,"term":"block_arg"},{"dest":13,"term":"block_call"},{"dest":13,"term":"block_command"},{"dest":13,"term":"block_param"},{"dest":13,"term":"block_param_def"},{"dest":13,"term":"BMATZ0000IREP"},{"dest":13,"term":"body"},{"dest":13,"term":"bodystmt"},{"dest":13,"term":"boundary"},{"dest":13,"term":"brace_block"},{"dest":13,"term":"break"},{"dest":13,"term":"bsearch"},{"dest":13,"term":"bsearch_index"},{"dest":13,"term":"buf"},{"dest":13,"term":"bvar"},{"dest":13,"term":"bv_decls"},{"dest":13,"term":"byte"},{"dest":13,"term":"bytes"},{"dest":13,"term":"bytesize"},{"dest":13,"term":"byteslice"},{"dest":13,"term":"call"},{"dest":13,"term":"call_args"},{"dest":13,"term":"caller"},{"dest":13,"term":"call_op"},{"dest":13,"term":"call_op2"},{"dest":13,"term":"capitalize"},{"dest":13,"term":"case"},{"dest":13,"term":"case_body"},{"dest":13,"term":"casecmp"},{"dest":13,"term":"__case_eqq"},{"dest":13,"term":"cases"},{"dest":13,"term":"cbrt"},{"dest":13,"term":"cdr"},{"dest":13,"term":"ceil"},{"dest":13,"term":"change_gen_gc_mode"},{"dest":13,"term":"character"},{"dest":13,"term":"chars"},{"dest":13,"term":"chomp"},{"dest":13,"term":"chop"},{"dest":13,"term":"chr"},{"dest":13,"term":"clamp"},{"dest":13,"term":"Class"},{"dest":13,"term":"class_eval"},{"dest":13,"term":"__classname__"},{"dest":13,"term":"class_variable_get"},{"dest":13,"term":"class_variables"},{"dest":13,"term":"class_variable_set"},{"dest":13,"term":"clause"},{"dest":13,"term":"clear_all_old"},{"dest":13,"term":"clone"},{"dest":13,"term":"closure"},{"dest":13,"term":"cLVAR"},{"dest":13,"term":"cmd_brace_block"},{"dest":13,"term":"cmp"},{"dest":13,"term":"cname"},{"dest":13,"term":"codegen"},{"dest":13,"term":"codepoints"},{"dest":13,"term":"collect"},{"dest":13,"term":"collect_concat"},{"dest":13,"term":"color"},{"dest":13,"term":"column_count"},{"dest":13,"term":"column_index"},{"dest":13,"term":"combination"},{"dest":13,"term":"comma"},{"dest":13,"term":"command"},{"dest":13,"term":"command_args"},{"dest":13,"term":"command_asgn"},{"dest":13,"term":"command_call"},{"dest":13,"term":"command_rhs"},{"dest":13,"term":"compact"},{"dest":13,"term":"Comparable"},{"dest":13,"term":"compile"},{"dest":13,"term":"compstmt"},{"dest":13,"term":"concat"},{"dest":13,"term":"constant"},{"dest":13,"term":"CONSTANT"},{"dest":13,"term":"constants"},{"dest":13,"term":"const_get"},{"dest":13,"term":"const_missing"},{"dest":13,"term":"const_set"},{"dest":13,"term":"cont"},{"dest":13,"term":"context"},{"dest":13,"term":"copyright"},{"dest":13,"term":"corrupted"},{"dest":13,"term":"cos"},{"dest":13,"term":"cosh"},{"dest":13,"term":"count"},{"dest":13,"term":"count_objects"},{"dest":13,"term":"cpath"},{"dest":13,"term":"ctime"},{"dest":13,"term":"__ctype_b_loc"},{"dest":13,"term":"curr"},{"dest":13,"term":"current"},{"dest":13,"term":"curry"},{"dest":13,"term":"cycle"},{"dest":13,"term":"Data"},{"dest":13,"term":"day"},{"dest":13,"term":"debug_info"},{"dest":13,"term":"Dec"},{"dest":13,"term":"deep"},{"dest":13,"term":"def"},{"dest":13,"term":"default"},{"dest":13,"term":"DEFAULT"},{"dest":13,"term":"default_proc"},{"dest":13,"term":"defined"},{"dest":13,"term":"define_method"},{"dest":13,"term":"define_singleton_method"},{"dest":13,"term":"__delete"},{"dest":13,"term":"delete"},{"dest":13,"term":"delete_at"},{"dest":13,"term":"delete_if"},{"dest":13,"term":"delete_prefix"},{"dest":13,"term":"delete_suffix"},{"dest":13,"term":"Deleting"},{"dest":13,"term":"depth"},{"dest":13,"term":"detect"},{"dest":13,"term":"detected"},{"dest":13,"term":"developers"},{"dest":13,"term":"differs"},{"dest":13,"term":"digit"},{"dest":13,"term":"digits"},{"dest":13,"term":"disable"},{"dest":13,"term":"disabled"},{"dest":13,"term":"discarding"},{"dest":13,"term":"div"},{"dest":13,"term":"divmod"},{"dest":13,"term":"do"},{"dest":13,"term":"do_block"},{"dest":13,"term":"DomainError"},{"dest":13,"term":"dot"},{"dest":13,"term":"dot_or_colon"},{"dest":13,"term":"downcase"},{"dest":13,"term":"downto"},{"dest":13,"term":"drop"},{"dest":13,"term":"dropped"},{"dest":13,"term":"dropping"},{"dest":13,"term":"drop_while"},{"dest":13,"term":"dump"},{"dest":13,"term":"dup"},{"dest":13,"term":"each"},{"dest":13,"term":"each_byte"},{"dest":13,"term":"each_char"},{"dest":13,"term":"each_codepoint"},{"dest":13,"term":"each_cons"},{"dest":13,"term":"each_index"},{"dest":13,"term":"each_key"},{"dest":13,"term":"each_line"},{"dest":13,"term":"each_object"},{"dest":13,"term":"each_pair"},{"dest":13,"term":"each_slice"},{"dest":13,"term":"each_value"},{"dest":13,"term":"each_with_index"},{"dest":13,"term":"each_with_object"},{"dest":13,"term":"ecall"},{"dest":13,"term":"elem"},{"dest":13,"term":"else"},{"dest":13,"term":"elsif"},{"dest":13,"term":"en"},{"dest":13,"term":"enable"},{"dest":13,"term":"__ENCODING__"},{"dest":13,"term":"end"},{"dest":13,"term":"__END__"},{"dest":13,"term":"END"},{"dest":13,"term":"ensure"},{"dest":13,"term":"entries"},{"dest":13,"term":"Enumerable"},{"dest":13,"term":"enumerator"},{"dest":13,"term":"Enumerator"},{"dest":13,"term":"enumerator_block_call"},{"dest":13,"term":"enum_for"},{"dest":13,"term":"enums"},{"dest":13,"term":"env"},{"dest":13,"term":"erf"},{"dest":13,"term":"erfc"},{"dest":13,"term":"__errno_location"},{"dest":13,"term":"error"},{"dest":13,"term":"escape"},{"dest":13,"term":"ETIR"},{"dest":13,"term":"ETIR0004Ci"},{"dest":13,"term":"exception"},{"dest":13,"term":"Exception"},{"dest":13,"term":"exc_list"},{"dest":13,"term":"exc_var"},{"dest":13,"term":"exhausted"},{"dest":13,"term":"exp"},{"dest":13,"term":"expected"},{"dest":13,"term":"expr"},{"dest":13,"term":"expression"},{"dest":13,"term":"expr_value"},{"dest":13,"term":"extend"},{"dest":13,"term":"extended"},{"dest":13,"term":"extend_object"},{"dest":13,"term":"fail"},{"dest":13,"term":"failed"},{"dest":13,"term":"failure"},{"dest":13,"term":"false"},{"dest":13,"term":"FalseClass"},{"dest":13,"term":"f_arg"},{"dest":13,"term":"f_arg_item"},{"dest":13,"term":"f_arglist"},{"dest":13,"term":"f_args"},{"dest":13,"term":"f_bad_arg"},{"dest":13,"term":"f_block_arg"},{"dest":13,"term":"f_block_opt"},{"dest":13,"term":"f_block_optarg"},{"dest":13,"term":"fclose"},{"dest":13,"term":"Feb"},{"dest":13,"term":"feed"},{"dest":13,"term":"feedvalue"},{"dest":13,"term":"feof"},{"dest":13,"term":"fetch"},{"dest":13,"term":"fetch_values"},{"dest":13,"term":"fflush"},{"dest":13,"term":"fgetc"},{"dest":13,"term":"fib"},{"dest":13,"term":"fiber"},{"dest":13,"term":"Fiber"},{"dest":13,"term":"fiber_check"},{"dest":13,"term":"FiberError"},{"dest":13,"term":"field"},{"dest":13,"term":"file"},{"dest":13,"term":"File"},{"dest":13,"term":"__FILE__"},{"dest":13,"term":"filename"},{"dest":13,"term":"filenames_len"},{"dest":13,"term":"fill"},{"dest":13,"term":"final_marking_phase"},{"dest":13,"term":"find"},{"dest":13,"term":"find_all"},{"dest":13,"term":"find_index"},{"dest":13,"term":"first"},{"dest":13,"term":"fish"},{"dest":13,"term":"Fixnum"},{"dest":13,"term":"flag"},{"dest":13,"term":"f_larglist"},{"dest":13,"term":"flat_map"},{"dest":13,"term":"flatten"},{"dest":13,"term":"Float"},{"dest":13,"term":"FloatDomainError"},{"dest":13,"term":"floor"},{"dest":13,"term":"f_marg"},{"dest":13,"term":"f_marg_list"},{"dest":13,"term":"f_margs"},{"dest":13,"term":"fmod"},{"dest":13,"term":"fn"},{"dest":13,"term":"Fn"},{"dest":13,"term":"fname"},{"dest":13,"term":"f_norm_arg"},{"dest":13,"term":"fopen"},{"dest":13,"term":"f_opt"},{"dest":13,"term":"f_optarg"},{"dest":13,"term":"f_opt_asgn"},{"dest":13,"term":"for"},{"dest":13,"term":"force"},{"dest":13,"term":"format"},{"dest":13,"term":"for_var"},{"dest":13,"term":"found"},{"dest":13,"term":"fprintf"},{"dest":13,"term":"fputc"},{"dest":13,"term":"fread"},{"dest":13,"term":"free"},{"dest":13,"term":"FREE"},{"dest":13,"term":"freeze"},{"dest":13,"term":"f_rest_arg"},{"dest":13,"term":"frexp"},{"dest":13,"term":"Fri"},{"dest":13,"term":"FrozenError"},{"dest":13,"term":"FsC"},{"dest":13,"term":"fsym"},{"dest":13,"term":"fwrite"},{"dest":13,"term":"games"},{"dest":13,"term":"GB"},{"dest":13,"term":"GC"},{"dest":13,"term":"gc_mark_children"},{"dest":13,"term":"_gc_root_"},{"dest":13,"term":"generational_mode"},{"dest":13,"term":"Generator"},{"dest":13,"term":"getbyte"},{"dest":13,"term":"get_file"},{"dest":13,"term":"getgm"},{"dest":13,"term":"getlocal"},{"dest":13,"term":"gettimeofday"},{"dest":13,"term":"getutc"},{"dest":13,"term":"given"},{"dest":13,"term":"given_args"},{"dest":13,"term":"global_variables"},{"dest":13,"term":"__gmon_start__"},{"dest":13,"term":"gmtime"},{"dest":13,"term":"gmtime_r"},{"dest":13,"term":"gn"},{"dest":13,"term":"gnu"},{"dest":13,"term":"GNU"},{"dest":13,"term":"go"},{"dest":13,"term":"grep"},{"dest":13,"term":"group_by"},{"dest":13,"term":"gsub"},{"dest":13,"term":"h0"},{"dest":13,"term":"h2"},{"dest":13,"term":"H3"},{"dest":13,"term":"h4"},{"dest":13,"term":"h5"},{"dest":13,"term":"H5"},{"dest":13,"term":"h6"},{"dest":13,"term":"H6"},{"dest":13,"term":"h7"},{"dest":13,"term":"h8"},{"dest":13,"term":"hA"},{"dest":13,"term":"hash"},{"dest":13,"term":"Hash"},{"dest":13,"term":"head"},{"dest":13,"term":"heredoc"},{"dest":13,"term":"heredoc_bodies"},{"dest":13,"term":"heredoc_body"},{"dest":13,"term":"heredoc_string_interp"},{"dest":13,"term":"heredoc_string_rep"},{"dest":13,"term":"heredoc_treat_nextline"},{"dest":13,"term":"hex"},{"dest":13,"term":"high"},{"dest":13,"term":"hour"},{"dest":13,"term":"hypot"},{"dest":13,"term":"i2"},{"dest":13,"term":"iClass"},{"dest":13,"term":"__id__"},{"dest":13,"term":"id2name"},{"dest":13,"term":"identifier"},{"dest":13,"term":"idx"},{"dest":13,"term":"idx2"},{"dest":13,"term":"if"},{"dest":13,"term":"ifnone"},{"dest":13,"term":"if_tail"},{"dest":13,"term":"implemented"},{"dest":13,"term":"in"},{"dest":13,"term":"include"},{"dest":13,"term":"included"},{"dest":13,"term":"included_modules"},{"dest":13,"term":"incremental_gc"},{"dest":13,"term":"index"},{"dest":13,"term":"IndexError"},{"dest":13,"term":"inf"},{"dest":13,"term":"Inf"},{"dest":13,"term":"INF"},{"dest":13,"term":"Infinity"},{"dest":13,"term":"INFINITY"},{"dest":13,"term":"inherited"},{"dest":13,"term":"initialize"},{"dest":13,"term":"initialize_copy"},{"dest":13,"term":"inject"},{"dest":13,"term":"in_lower_half"},{"dest":13,"term":"input"},{"dest":13,"term":"insert"},{"dest":13,"term":"_inspect"},{"dest":13,"term":"inspect"},{"dest":13,"term":"instance_eval"},{"dest":13,"term":"instance_exec"},{"dest":13,"term":"instance_methods"},{"dest":13,"term":"instance_variable_get"},{"dest":13,"term":"instance_variables"},{"dest":13,"term":"instance_variable_set"},{"dest":13,"term":"int"},{"dest":13,"term":"integer"},{"dest":13,"term":"Integer"},{"dest":13,"term":"Integral"},{"dest":13,"term":"intern"},{"dest":13,"term":"interval_ratio"},{"dest":13,"term":"invert"},{"dest":13,"term":"io"},{"dest":13,"term":"Io"},{"dest":13,"term":"_IO_putc"},{"dest":13,"term":"ip"},{"dest":13,"term":"Ip"},{"dest":13,"term":"irep"},{"dest":13,"term":"IREP"},{"dest":13,"term":"isz"},{"dest":13,"term":"iterate"},{"dest":13,"term":"_ITM_deregisterTMCloneTable"},{"dest":13,"term":"_ITM_registerTMCloneTable"},{"dest":13,"term":"itself"},{"dest":13,"term":"Jan"},{"dest":13,"term":"join"},{"dest":13,"term":"_Jv_RegisterClasses"},{"dest":13,"term":"keep_if"},{"dest":13,"term":"Kernel"},{"dest":13,"term":"key"},{"dest":13,"term":"KeyError"},{"dest":13,"term":"keys"},{"dest":13,"term":"keyword_alias"},{"dest":13,"term":"keyword_and"},{"dest":13,"term":"keyword_begin"},{"dest":13,"term":"keyword_BEGIN"},{"dest":13,"term":"keyword_break"},{"dest":13,"term":"keyword_case"},{"dest":13,"term":"keyword_class"},{"dest":13,"term":"keyword_def"},{"dest":13,"term":"keyword_do"},{"dest":13,"term":"keyword_do_block"},{"dest":13,"term":"keyword_do_cond"},{"dest":13,"term":"keyword_do_LAMBDA"},{"dest":13,"term":"keyword_else"},{"dest":13,"term":"keyword_elsif"},{"dest":13,"term":"keyword__ENCODING__"},{"dest":13,"term":"keyword_end"},{"dest":13,"term":"keyword_END"},{"dest":13,"term":"keyword_ensure"},{"dest":13,"term":"keyword_false"},{"dest":13,"term":"keyword__FILE__"},{"dest":13,"term":"keyword_for"},{"dest":13,"term":"keyword_if"},{"dest":13,"term":"keyword_in"},{"dest":13,"term":"keyword__LINE__"},{"dest":13,"term":"keyword_module"},{"dest":13,"term":"keyword_next"},{"dest":13,"term":"keyword_nil"},{"dest":13,"term":"keyword_not"},{"dest":13,"term":"keyword_or"},{"dest":13,"term":"keyword_redo"},{"dest":13,"term":"keyword_rescue"},{"dest":13,"term":"keyword_retry"},{"dest":13,"term":"keyword_return"},{"dest":13,"term":"keyword_self"},{"dest":13,"term":"keyword_super"},{"dest":13,"term":"keyword_then"},{"dest":13,"term":"keyword_true"},{"dest":13,"term":"keyword_undef"},{"dest":13,"term":"keyword_unless"},{"dest":13,"term":"keyword_until"},{"dest":13,"term":"keyword_when"},{"dest":13,"term":"keyword_while"},{"dest":13,"term":"keyword_yield"},{"dest":13,"term":"kh_del_ht"},{"dest":13,"term":"kh_del_iv"},{"dest":13,"term":"kh_del_mt"},{"dest":13,"term":"kh_del_n2s"},{"dest":13,"term":"kh_del_st"},{"dest":13,"term":"KLVAR"},{"dest":13,"term":"lambda"},{"dest":13,"term":"lambda_body"},{"dest":13,"term":"last"},{"dest":13,"term":"lazy"},{"dest":13,"term":"Lazy"},{"dest":13,"term":"LC"},{"dest":13,"term":"ld"},{"dest":13,"term":"LD"},{"dest":13,"term":"ldexp"},{"dest":13,"term":"left"},{"dest":13,"term":"len"},{"dest":13,"term":"length"},{"dest":13,"term":"level"},{"dest":13,"term":"lfD"},{"dest":13,"term":"lhs"},{"dest":13,"term":"__libc_start_main"},{"dest":13,"term":"LII"},{"dest":13,"term":"lIJ"},{"dest":13,"term":"lim"},{"dest":13,"term":"line"},{"dest":13,"term":"__LINE__"},{"dest":13,"term":"LINE"},{"dest":13,"term":"lines"},{"dest":13,"term":"literal"},{"dest":13,"term":"literals"},{"dest":13,"term":"live_after_mark"},{"dest":13,"term":"ljust"},{"dest":13,"term":"ln"},{"dest":13,"term":"Ln"},{"dest":13,"term":"lo"},{"dest":13,"term":"local"},{"dest":13,"term":"LOCAL"},{"dest":13,"term":"LocalJumpError"},{"dest":13,"term":"localtime"},{"dest":13,"term":"localtime_r"},{"dest":13,"term":"local_variables"},{"dest":13,"term":"log"},{"dest":13,"term":"log10"},{"dest":13,"term":"log2"},{"dest":13,"term":"long"},{"dest":13,"term":"longjmp"},{"dest":13,"term":"lookahead"},{"dest":13,"term":"loop"},{"dest":13,"term":"low"},{"dest":13,"term":"lround"},{"dest":13,"term":"LS"},{"dest":13,"term":"lstrip"},{"dest":13,"term":"LVAR"},{"dest":13,"term":"machine"},{"dest":13,"term":"main"},{"dest":13,"term":"make_curry"},{"dest":13,"term":"map"},{"dest":13,"term":"match"},{"dest":13,"term":"matched"},{"dest":13,"term":"Math"},{"dest":13,"term":"max"},{"dest":13,"term":"max_by"},{"dest":13,"term":"max_cmp"},{"dest":13,"term":"May"},{"dest":13,"term":"mday"},{"dest":13,"term":"member"},{"dest":13,"term":"__members__"},{"dest":13,"term":"members"},{"dest":13,"term":"memchr"},{"dest":13,"term":"memcmp"},{"dest":13,"term":"memcpy"},{"dest":13,"term":"memmove"},{"dest":13,"term":"memory"},{"dest":13,"term":"memset"},{"dest":13,"term":"merge"},{"dest":13,"term":"mesg"},{"dest":13,"term":"message"},{"dest":13,"term":"meth"},{"dest":13,"term":"__method__"},{"dest":13,"term":"method"},{"dest":13,"term":"method_call"},{"dest":13,"term":"method_missing"},{"dest":13,"term":"method_removed"},{"dest":13,"term":"methods"},{"dest":13,"term":"mid"},{"dest":13,"term":"min"},{"dest":13,"term":"min_by"},{"dest":13,"term":"min_cmp"},{"dest":13,"term":"minmax"},{"dest":13,"term":"minmax_by"},{"dest":13,"term":"mktime"},{"dest":13,"term":"mlhs_basic"},{"dest":13,"term":"mlhs_inner"},{"dest":13,"term":"mlhs_item"},{"dest":13,"term":"mlhs_list"},{"dest":13,"term":"mlhs_node"},{"dest":13,"term":"mlhs_post"},{"dest":13,"term":"mode"},{"dest":13,"term":"modified"},{"dest":13,"term":"modifier_if"},{"dest":13,"term":"modifier_rescue"},{"dest":13,"term":"modifier_unless"},{"dest":13,"term":"modifier_until"},{"dest":13,"term":"modifier_while"},{"dest":13,"term":"module"},{"dest":13,"term":"Module"},{"dest":13,"term":"module_eval"},{"dest":13,"term":"module_function"},{"dest":13,"term":"modules"},{"dest":13,"term":"mon"},{"dest":13,"term":"Mon"},{"dest":13,"term":"month"},{"dest":13,"term":"mrb_ary_delete_at"},{"dest":13,"term":"mrb_ary_new_from_values"},{"dest":13,"term":"mrb_ary_plus"},{"dest":13,"term":"mrb_ary_pop"},{"dest":13,"term":"mrb_ary_push"},{"dest":13,"term":"mrb_ary_push_m"},{"dest":13,"term":"mrb_ary_resize"},{"dest":13,"term":"mrb_ary_reverse"},{"dest":13,"term":"mrb_ary_set"},{"dest":13,"term":"mrb_ary_shift"},{"dest":13,"term":"mrb_ary_splice"},{"dest":13,"term":"mrb_ary_times"},{"dest":13,"term":"mrb_ary_unshift"},{"dest":13,"term":"mrb_ary_unshift_m"},{"dest":13,"term":"mrb_assoc_new"},{"dest":13,"term":"mrb_data_init"},{"dest":13,"term":"mrb_debug_get_line"},{"dest":13,"term":"mrb_debug_info_alloc"},{"dest":13,"term":"mrb_debug_info_append_file"},{"dest":13,"term":"mrb_debug_info_free"},{"dest":13,"term":"mrb_field_write_barrier"},{"dest":13,"term":"mrb_gc_mark"},{"dest":13,"term":"MRB_GC_STATE_ROOT"},{"dest":13,"term":"MRB_GC_STATE_SWEEP"},{"dest":13,"term":"mrb_gc_unregister"},{"dest":13,"term":"mrb_i_mt_state"},{"dest":13,"term":"mrb_incremental_gc"},{"dest":13,"term":"mrb_malloc"},{"dest":13,"term":"mrb_mod_s_nesting"},{"dest":13,"term":"mrb_obj_value"},{"dest":13,"term":"mrb_random_init"},{"dest":13,"term":"mrb_random_srand"},{"dest":13,"term":"mrb_realloc"},{"dest":13,"term":"mrb_str_format"},{"dest":13,"term":"MRB_TT_DATA"},{"dest":13,"term":"MRB_TT_FIBER"},{"dest":13,"term":"MRB_TT_FREE"},{"dest":13,"term":"mrb_vm_const_get"},{"dest":13,"term":"mrb_vm_exec"},{"dest":13,"term":"mrb_write_barrier"},{"dest":13,"term":"mrhs"},{"dest":13,"term":"mruby"},{"dest":13,"term":"MRUBY_COPYRIGHT"},{"dest":13,"term":"MRUBY_DESCRIPTION"},{"dest":13,"term":"MRUBY_RELEASE_DATE"},{"dest":13,"term":"MRUBY_RELEASE_NO"},{"dest":13,"term":"MRUBY_VERSION"},{"dest":13,"term":"name"},{"dest":13,"term":"named"},{"dest":13,"term":"NameError"},{"dest":13,"term":"names"},{"dest":13,"term":"nan"},{"dest":13,"term":"NaN"},{"dest":13,"term":"NAN"},{"dest":13,"term":"nesting"},{"dest":13,"term":"new"},{"dest":13,"term":"new_args"},{"dest":13,"term":"new_key"},{"dest":13,"term":"new_msym"},{"dest":13,"term":"next"},{"dest":13,"term":"next_values"},{"dest":13,"term":"nil"},{"dest":13,"term":"NilClass"},{"dest":13,"term":"nl"},{"dest":13,"term":"nlocals"},{"dest":13,"term":"nLVAR"},{"dest":13,"term":"nMATZ0000IREP"},{"dest":13,"term":"NODE_DREGX"},{"dest":13,"term":"NODE_DSTR"},{"dest":13,"term":"NODE_DXSTR"},{"dest":13,"term":"NODE_FALSE"},{"dest":13,"term":"NODE_NEGATE"},{"dest":13,"term":"NODE_NIL"},{"dest":13,"term":"NODE_REDO"},{"dest":13,"term":"NODE_RETRY"},{"dest":13,"term":"NODE_SELF"},{"dest":13,"term":"NODE_TRUE"},{"dest":13,"term":"NODE_UNDEF"},{"dest":13,"term":"NODE_ZSUPER"},{"dest":13,"term":"NoMemoryError"},{"dest":13,"term":"NoMethodError"},{"dest":13,"term":"none"},{"dest":13,"term":"NONE"},{"dest":13,"term":"norm"},{"dest":13,"term":"not"},{"dest":13,"term":"NotImplementedError"},{"dest":13,"term":"Nov"},{"dest":13,"term":"now"},{"dest":13,"term":"Np"},{"dest":13,"term":"nregs"},{"dest":13,"term":"num"},{"dest":13,"term":"number"},{"dest":13,"term":"numbered"},{"dest":13,"term":"numeric"},{"dest":13,"term":"Numeric"},{"dest":13,"term":"obj"},{"dest":13,"term":"object"},{"dest":13,"term":"Object"},{"dest":13,"term":"object_id"},{"dest":13,"term":"ObjectSpace"},{"dest":13,"term":"oct"},{"dest":13,"term":"Oct"},{"dest":13,"term":"offset"},{"dest":13,"term":"on"},{"dest":13,"term":"On"},{"dest":13,"term":"only"},{"dest":13,"term":"Oo"},{"dest":13,"term":"op"},{"dest":13,"term":"Op"},{"dest":13,"term":"operation"},{"dest":13,"term":"operation2"},{"dest":13,"term":"operation3"},{"dest":13,"term":"OP_NOP"},{"dest":13,"term":"OP_STOP"},{"dest":13,"term":"opt_block_arg"},{"dest":13,"term":"opt_block_param"},{"dest":13,"term":"opt_bv_decl"},{"dest":13,"term":"opt_call_args"},{"dest":13,"term":"opt_else"},{"dest":13,"term":"opt_ensure"},{"dest":13,"term":"opt_f_block_arg"},{"dest":13,"term":"opt_nl"},{"dest":13,"term":"opt_paren_args"},{"dest":13,"term":"opt_rescue"},{"dest":13,"term":"opt_terms"},{"dest":13,"term":"or"},{"dest":13,"term":"ord"},{"dest":13,"term":"orig"},{"dest":13,"term":"other"},{"dest":13,"term":"__outer__"},{"dest":13,"term":"P9o"},{"dest":13,"term":"padding"},{"dest":13,"term":"pad_repetitions"},{"dest":13,"term":"padstr"},{"dest":13,"term":"parameters"},{"dest":13,"term":"paren_args"},{"dest":13,"term":"partition"},{"dest":13,"term":"pattern"},{"dest":13,"term":"PC"},{"dest":13,"term":"peek"},{"dest":13,"term":"peek_values"},{"dest":13,"term":"permutation"},{"dest":13,"term":"plen"},{"dest":13,"term":"point"},{"dest":13,"term":"pop"},{"dest":13,"term":"popping"},{"dest":13,"term":"pos"},{"dest":13,"term":"posnum"},{"dest":13,"term":"post"},{"dest":13,"term":"pow"},{"dest":13,"term":"pp"},{"dest":13,"term":"pproc"},{"dest":13,"term":"pre"},{"dest":13,"term":"precision"},{"dest":13,"term":"prefix"},{"dest":13,"term":"prepend"},{"dest":13,"term":"prepended"},{"dest":13,"term":"prepend_features"},{"dest":13,"term":"primary"},{"dest":13,"term":"primary_value"},{"dest":13,"term":"print"},{"dest":13,"term":"printf"},{"dest":13,"term":"__printstr__"},{"dest":13,"term":"private"},{"dest":13,"term":"private_methods"},{"dest":13,"term":"prl"},{"dest":13,"term":"proc"},{"dest":13,"term":"Proc"},{"dest":13,"term":"program"},{"dest":13,"term":"protected"},{"dest":13,"term":"protected_methods"},{"dest":13,"term":"ps"},{"dest":13,"term":"public"},{"dest":13,"term":"public_methods"},{"dest":13,"term":"push"},{"dest":13,"term":"putchar"},{"dest":13,"term":"puts"},{"dest":13,"term":"quo"},{"dest":13,"term":"raise"},{"dest":13,"term":"rand"},{"dest":13,"term":"Random"},{"dest":13,"term":"range"},{"dest":13,"term":"Range"},{"dest":13,"term":"RangeError"},{"dest":13,"term":"rassoc"},{"dest":13,"term":"rb"},{"dest":13,"term":"RB"},{"dest":13,"term":"rbracket"},{"dest":13,"term":"RC"},{"dest":13,"term":"read_debug_record"},{"dest":13,"term":"readint_mrb_int"},{"dest":13,"term":"read_irep_record_1"},{"dest":13,"term":"read_lv_record"},{"dest":13,"term":"read_section_debug"},{"dest":13,"term":"read_section_lv"},{"dest":13,"term":"realloc"},{"dest":13,"term":"redo"},{"dest":13,"term":"reduce"},{"dest":13,"term":"reg"},{"dest":13,"term":"regexp"},{"dest":13,"term":"Regexp"},{"dest":13,"term":"RegexpError"},{"dest":13,"term":"rehash"},{"dest":13,"term":"reject"},{"dest":13,"term":"remove_class_variable"},{"dest":13,"term":"remove_const"},{"dest":13,"term":"remove_instance_variable"},{"dest":13,"term":"remove_method"},{"dest":13,"term":"replace"},{"dest":13,"term":"req"},{"dest":13,"term":"required"},{"dest":13,"term":"res"},{"dest":13,"term":"rescue"},{"dest":13,"term":"resize_capa"},{"dest":13,"term":"rest"},{"dest":13,"term":"restarg_mark"},{"dest":13,"term":"result"},{"dest":13,"term":"resume"},{"dest":13,"term":"reswords"},{"dest":13,"term":"ret"},{"dest":13,"term":"retry"},{"dest":13,"term":"return"},{"dest":13,"term":"reverse"},{"dest":13,"term":"reverse_each"},{"dest":13,"term":"rewind"},{"dest":13,"term":"right"},{"dest":13,"term":"rindex"},{"dest":13,"term":"rjust"},{"dest":13,"term":"rotate"},{"dest":13,"term":"round"},{"dest":13,"term":"row"},{"dest":13,"term":"rparen"},{"dest":13,"term":"rpartition"},{"dest":13,"term":"rs_len"},{"dest":13,"term":"rstrip"},{"dest":13,"term":"RUBY_ENGINE"},{"dest":13,"term":"RUBY_ENGINE_VERSION"},{"dest":13,"term":"RUBY_VERSION"},{"dest":13,"term":"RuntimeError"},{"dest":13,"term":"sample"},{"dest":13,"term":"Sat"},{"dest":13,"term":"satisfied"},{"dest":13,"term":"scan"},{"dest":13,"term":"SClass"},{"dest":13,"term":"scope"},{"dest":13,"term":"scope_new"},{"dest":13,"term":"script"},{"dest":13,"term":"ScriptError"},{"dest":13,"term":"sec"},{"dest":13,"term":"select"},{"dest":13,"term":"self"},{"dest":13,"term":"self_arity"},{"dest":13,"term":"__send__"},{"dest":13,"term":"send"},{"dest":13,"term":"sep"},{"dest":13,"term":"Sep"},{"dest":13,"term":"sequence"},{"dest":13,"term":"set"},{"dest":13,"term":"set_backtrace"},{"dest":13,"term":"setbyte"},{"dest":13,"term":"_setjmp"},{"dest":13,"term":"shift"},{"dest":13,"term":"shuffle"},{"dest":13,"term":"sin"},{"dest":13,"term":"singleton"},{"dest":13,"term":"singleton_class"},{"dest":13,"term":"singleton_methods"},{"dest":13,"term":"sinh"},{"dest":13,"term":"size"},{"dest":13,"term":"sl"},{"dest":13,"term":"slice"},{"dest":13,"term":"snprintf"},{"dest":13,"term":"so"},{"dest":13,"term":"So"},{"dest":13,"term":"sort"},{"dest":13,"term":"sort_by"},{"dest":13,"term":"__sort_sub__"},{"dest":13,"term":"source_location"},{"dest":13,"term":"Sp"},{"dest":13,"term":"spaces"},{"dest":13,"term":"specifier"},{"dest":13,"term":"splice"},{"dest":13,"term":"split"},{"dest":13,"term":"sprintf"},{"dest":13,"term":"sqrt"},{"dest":13,"term":"srand"},{"dest":13,"term":"__stack_chk_fail"},{"dest":13,"term":"StandardError"},{"dest":13,"term":"start"},{"dest":13,"term":"state"},{"dest":13,"term":"stderr"},{"dest":13,"term":"stdin"},{"dest":13,"term":"stdout"},{"dest":13,"term":"step"},{"dest":13,"term":"step_ratio"},{"dest":13,"term":"stmt"},{"dest":13,"term":"stmts"},{"dest":13,"term":"stop_exc"},{"dest":13,"term":"StopIteration"},{"dest":13,"term":"store"},{"dest":13,"term":"str"},{"dest":13,"term":"str2"},{"dest":13,"term":"strchr"},{"dest":13,"term":"strcmp"},{"dest":13,"term":"str_each"},{"dest":13,"term":"string"},{"dest":13,"term":"String"},{"dest":13,"term":"string_interp"},{"dest":13,"term":"string_rep"},{"dest":13,"term":"strip"},{"dest":13,"term":"strlen"},{"dest":13,"term":"str_make_shared"},{"dest":13,"term":"strncmp"},{"dest":13,"term":"strncpy"},{"dest":13,"term":"strtoul"},{"dest":13,"term":"struct"},{"dest":13,"term":"Struct"},{"dest":13,"term":"sub"},{"dest":13,"term":"__sub_replace"},{"dest":13,"term":"succ"},{"dest":13,"term":"Sun"},{"dest":13,"term":"super"},{"dest":13,"term":"superclass"},{"dest":13,"term":"supported"},{"dest":13,"term":"__svalue"},{"dest":13,"term":"SVD"},{"dest":13,"term":"swapcase"},{"dest":13,"term":"sym"},{"dest":13,"term":"symbol"},{"dest":13,"term":"Symbol"},{"dest":13,"term":"symbols"},{"dest":13,"term":"sym_inspect"},{"dest":13,"term":"syntax"},{"dest":13,"term":"SyntaxError"},{"dest":13,"term":"_sys_fail"},{"dest":13,"term":"SystemCallError"},{"dest":13,"term":"SystemStackError"},{"dest":13,"term":"TA"},{"dest":13,"term":"tail"},{"dest":13,"term":"take"},{"dest":13,"term":"taken"},{"dest":13,"term":"take_while"},{"dest":13,"term":"tAMPER"},{"dest":13,"term":"tan"},{"dest":13,"term":"tANDDOT"},{"dest":13,"term":"tANDOP"},{"dest":13,"term":"tanh"},{"dest":13,"term":"tap"},{"dest":13,"term":"tAREF"},{"dest":13,"term":"T_ARRAY"},{"dest":13,"term":"tASET"},{"dest":13,"term":"tASSOC"},{"dest":13,"term":"TB"},{"dest":13,"term":"tBACK_REF"},{"dest":13,"term":"TbG"},{"dest":13,"term":"T_CLASS"},{"dest":13,"term":"tCMP"},{"dest":13,"term":"tCOLON2"},{"dest":13,"term":"tCOLON3"},{"dest":13,"term":"tCONSTANT"},{"dest":13,"term":"T_CPTR"},{"dest":13,"term":"tCVAR"},{"dest":13,"term":"T_DATA"},{"dest":13,"term":"tDOT2"},{"dest":13,"term":"tDOT3"},{"dest":13,"term":"TeD"},{"dest":13,"term":"T_ENV"},{"dest":13,"term":"tEQ"},{"dest":13,"term":"tEQQ"},{"dest":13,"term":"term"},{"dest":13,"term":"terms"},{"dest":13,"term":"T_EXCEPTION"},{"dest":13,"term":"T_FALSE"},{"dest":13,"term":"T_FIBER"},{"dest":13,"term":"tFID"},{"dest":13,"term":"T_FILE"},{"dest":13,"term":"T_FIXNUM"},{"dest":13,"term":"tFLOAT"},{"dest":13,"term":"T_FLOAT"},{"dest":13,"term":"T_FREE"},{"dest":13,"term":"tGEQ"},{"dest":13,"term":"tGVAR"},{"dest":13,"term":"T_HASH"},{"dest":13,"term":"tHD_LITERAL_DELIM"},{"dest":13,"term":"tHD_STRING_MID"},{"dest":13,"term":"tHD_STRING_PART"},{"dest":13,"term":"then"},{"dest":13,"term":"tHEREDOC_BEG"},{"dest":13,"term":"tHEREDOC_END"},{"dest":13,"term":"this"},{"dest":13,"term":"T_ICLASS"},{"dest":13,"term":"tIDENTIFIER"},{"dest":13,"term":"time"},{"dest":13,"term":"Time"},{"dest":13,"term":"times"},{"dest":13,"term":"tINTEGER"},{"dest":13,"term":"tIVAR"},{"dest":13,"term":"tLABEL"},{"dest":13,"term":"tLABEL_END"},{"dest":13,"term":"tLAMBDA"},{"dest":13,"term":"tLAMBEG"},{"dest":13,"term":"tLAST_TOKEN"},{"dest":13,"term":"tLBRACE"},{"dest":13,"term":"tLBRACE_ARG"},{"dest":13,"term":"tLBRACK"},{"dest":13,"term":"tLEQ"},{"dest":13,"term":"tLITERAL_DELIM"},{"dest":13,"term":"tLOWEST"},{"dest":13,"term":"tLPAREN"},{"dest":13,"term":"tLPAREN_ARG"},{"dest":13,"term":"tLSHFT"},{"dest":13,"term":"tMATCH"},{"dest":13,"term":"T_MODULE"},{"dest":13,"term":"tmp"},{"dest":13,"term":"tNEQ"},{"dest":13,"term":"tNMATCH"},{"dest":13,"term":"tNTH_REF"},{"dest":13,"term":"to_ary"},{"dest":13,"term":"T_OBJECT"},{"dest":13,"term":"to_enum"},{"dest":13,"term":"to_h"},{"dest":13,"term":"to_hash"},{"dest":13,"term":"to_i"},{"dest":13,"term":"to_int"},{"dest":13,"term":"TOJ"},{"dest":13,"term":"TOLERANCE"},{"dest":13,"term":"tolower"},{"dest":13,"term":"tOP_ASGN"},{"dest":13,"term":"top_compstmt"},{"dest":13,"term":"to_proc"},{"dest":13,"term":"top_stmt"},{"dest":13,"term":"top_stmts"},{"dest":13,"term":"tOROP"},{"dest":13,"term":"to_s"},{"dest":13,"term":"to_str"},{"dest":13,"term":"to_sym"},{"dest":13,"term":"TOTAL"},{"dest":13,"term":"toupper"},{"dest":13,"term":"tPOW"},{"dest":13,"term":"T_PROC"},{"dest":13,"term":"trailer"},{"dest":13,"term":"T_RANGE"},{"dest":13,"term":"transfer"},{"dest":13,"term":"transform_keys"},{"dest":13,"term":"transform_values"},{"dest":13,"term":"transpose"},{"dest":13,"term":"tREGEXP"},{"dest":13,"term":"tREGEXP_BEG"},{"dest":13,"term":"tREGEXP_END"},{"dest":13,"term":"tRPAREN"},{"dest":13,"term":"tRSHFT"},{"dest":13,"term":"true"},{"dest":13,"term":"TrueClass"},{"dest":13,"term":"truncate"},{"dest":13,"term":"try_convert"},{"dest":13,"term":"T_SCLASS"},{"dest":13,"term":"tSTAR"},{"dest":13,"term":"tSTRING"},{"dest":13,"term":"T_STRING"},{"dest":13,"term":"tSTRING_BEG"},{"dest":13,"term":"tSTRING_DVAR"},{"dest":13,"term":"tSTRING_MID"},{"dest":13,"term":"tSTRING_PART"},{"dest":13,"term":"tSYMBEG"},{"dest":13,"term":"T_SYMBOL"},{"dest":13,"term":"tSYMBOLS_BEG"},{"dest":13,"term":"tt"},{"dest":13,"term":"T_TRUE"},{"dest":13,"term":"Tue"},{"dest":13,"term":"tUMINUS"},{"dest":13,"term":"tUMINUS_NUM"},{"dest":13,"term":"T_UNDEF"},{"dest":13,"term":"tUPLUS"},{"dest":13,"term":"twice"},{"dest":13,"term":"tWORDS_BEG"},{"dest":13,"term":"tXSTRING"},{"dest":13,"term":"tXSTRING_BEG"},{"dest":13,"term":"type"},{"dest":13,"term":"TypeError"},{"dest":13,"term":"umrb_obj_value"},{"dest":13,"term":"undef"},{"dest":13,"term":"undefined"},{"dest":13,"term":"undef_list"},{"dest":13,"term":"undef_method"},{"dest":13,"term":"uniq"},{"dest":13,"term":"unless"},{"dest":13,"term":"unshift"},{"dest":13,"term":"until"},{"dest":13,"term":"upcase"},{"dest":13,"term":"__update"},{"dest":13,"term":"update"},{"dest":13,"term":"upto"},{"dest":13,"term":"usec"},{"dest":13,"term":"useless"},{"dest":13,"term":"utc"},{"dest":13,"term":"v0000"},{"dest":13,"term":"val"},{"dest":13,"term":"validated"},{"dest":13,"term":"vals"},{"dest":13,"term":"value"},{"dest":13,"term":"values"},{"dest":13,"term":"values_at"},{"dest":13,"term":"variable"},{"dest":13,"term":"var_lhs"},{"dest":13,"term":"var_ref"},{"dest":13,"term":"verbose"},{"dest":13,"term":"version"},{"dest":13,"term":"vm"},{"dest":13,"term":"Vm"},{"dest":13,"term":"warn"},{"dest":13,"term":"wday"},{"dest":13,"term":"Wed"},{"dest":13,"term":"when"},{"dest":13,"term":"while"},{"dest":13,"term":"width"},{"dest":13,"term":"with_index"},{"dest":13,"term":"with_object"},{"dest":13,"term":"words"},{"dest":13,"term":"x86_64"},{"dest":13,"term":"xstring"},{"dest":13,"term":"yday"},{"dest":13,"term":"year"},{"dest":13,"term":"yield"},{"dest":13,"term":"yielder"},{"dest":13,"term":"Yielder"},{"dest":13,"term":"yield_self"},{"dest":13,"term":"zip"},{"dest":13,"term":"zone"}],[{"dest":14,"term":"1"},{"dest":14,"term":"0"},{"dest":14,"term":"0.0"},{"dest":14,"term":"\"foo\""},{"dest":14,"term":"\"asdfasdf\""},{"dest":14,"term":"\"o\""},{"dest":14,"term":"nil"},{"dest":14,"term":"true"},{"dest":14,"term":"false"},{"dest":14,"term":"/foo/"},{"dest":14,"term":"[]"},{"dest":14,"term":"[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,nil]"}],[{"dest":15,"term":"1"},{"dest":15,"term":"0"},{"dest":15,"term":"0.0"},{"dest":15,"term":"\"foo\""},{"dest":15,"term":"\"asdfasdf\""},{"dest":15,"term":"\"o\""},{"dest":15,"term":"nil"},{"dest":15,"term":"true"},{"dest":15,"term":"false"},{"dest":15,"term":"/foo/"},{"dest":15,"term":"[]"},{"dest":15,"term":"[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,nil]"}],[{"dest":15,"term":"a"},{"dest":15,"term":"b"},{"dest":15,"term":"c"},{"dest":15,"term":"d"}],[{"dest":16,"term":"."}],[{"dest":17,"term":"."}],[{"dest":18,"term":"."}],[{"dest":19,"term":"\n"}],[{"dest":20,"term":"abcdef0123456789ABCDEF"},{"dest":20,"term":"abcdefghijklmnopqrstuvwxyz"},{"dest":20,"term":"abort"},{"dest":20,"term":"abs"},{"dest":20,"term":"accept"},{"dest":20,"term":"acos"},{"dest":20,"term":"acosh"},{"dest":20,"term":"address"},{"dest":20,"term":"alias"},{"dest":20,"term":"alias_method"},{"dest":20,"term":"allocation"},{"dest":20,"term":"all_symbols"},{"dest":20,"term":"ancestors"},{"dest":20,"term":"and"},{"dest":20,"term":"anum"},{"dest":20,"term":"append"},{"dest":20,"term":"append_features"},{"dest":20,"term":"Apr"},{"dest":20,"term":"aref_args"},{"dest":20,"term":"arg"},{"dest":20,"term":"arg0"},{"dest":20,"term":"arg1"},{"dest":20,"term":"arg2"},{"dest":20,"term":"arg_rhs"},{"dest":20,"term":"args"},{"dest":20,"term":"argument"},{"dest":20,"term":"ArgumentError"},{"dest":20,"term":"arguments"},{"dest":20,"term":"argv"},{"dest":20,"term":"ARGV"},{"dest":20,"term":"arity"},{"dest":20,"term":"array"},{"dest":20,"term":"Array"},{"dest":20,"term":"ary"},{"dest":20,"term":"__ary_cmp"},{"dest":20,"term":"ary_concat"},{"dest":20,"term":"__ary_eq"},{"dest":20,"term":"ary_F"},{"dest":20,"term":"__ary_index"},{"dest":20,"term":"ary_replace"},{"dest":20,"term":"ary_T"},{"dest":20,"term":"asctime"},{"dest":20,"term":"asin"},{"dest":20,"term":"asinh"},{"dest":20,"term":"__assert_fail"},{"dest":20,"term":"assignment"},{"dest":20,"term":"assoc"},{"dest":20,"term":"assoc_list"},{"dest":20,"term":"assocs"},{"dest":20,"term":"assumed"},{"dest":20,"term":"at"},{"dest":20,"term":"atan"},{"dest":20,"term":"atan2"},{"dest":20,"term":"atanh"},{"dest":20,"term":"__attached__"},{"dest":20,"term":"attr"},{"dest":20,"term":"attr_accessor"},{"dest":20,"term":"attr_reader"},{"dest":20,"term":"attrsym"},{"dest":20,"term":"attr_writer"},{"dest":20,"term":"available"},{"dest":20,"term":"backref"},{"dest":20,"term":"backtrace"},{"dest":20,"term":"Backtrace"},{"dest":20,"term":"BasicObject"},{"dest":20,"term":"basic_symbol"},{"dest":20,"term":"beg"},{"dest":20,"term":"begin"},{"dest":20,"term":"BEGIN"},{"dest":20,"term":"big"},{"dest":20,"term":"BIT"},{"dest":20,"term":"blkarg_mark"},{"dest":20,"term":"block"},{"dest":20,"term":"block_arg"},{"dest":20,"term":"block_call"},{"dest":20,"term":"block_command"},{"dest":20,"term":"block_param"},{"dest":20,"term":"block_param_def"},{"dest":20,"term":"BMATZ0000IREP"},{"dest":20,"term":"body"},{"dest":20,"term":"bodystmt"},{"dest":20,"term":"boundary"},{"dest":20,"term":"brace_block"},{"dest":20,"term":"break"},{"dest":20,"term":"bsearch"},{"dest":20,"term":"bsearch_index"},{"dest":20,"term":"buf"},{"dest":20,"term":"bvar"},{"dest":20,"term":"bv_decls"},{"dest":20,"term":"byte"},{"dest":20,"term":"bytes"},{"dest":20,"term":"bytesize"},{"dest":20,"term":"byteslice"},{"dest":20,"term":"call"},{"dest":20,"term":"call_args"},{"dest":20,"term":"caller"},{"dest":20,"term":"call_op"},{"dest":20,"term":"call_op2"},{"dest":20,"term":"capitalize"},{"dest":20,"term":"case"},{"dest":20,"term":"case_body"},{"dest":20,"term":"casecmp"},{"dest":20,"term":"__case_eqq"},{"dest":20,"term":"cases"},{"dest":20,"term":"cbrt"},{"dest":20,"term":"cdr"},{"dest":20,"term":"ceil"},{"dest":20,"term":"change_gen_gc_mode"},{"dest":20,"term":"character"},{"dest":20,"term":"chars"},{"dest":20,"term":"chomp"},{"dest":20,"term":"chop"},{"dest":20,"term":"chr"},{"dest":20,"term":"clamp"},{"dest":20,"term":"Class"},{"dest":20,"term":"class_eval"},{"dest":20,"term":"__classname__"},{"dest":20,"term":"class_variable_get"},{"dest":20,"term":"class_variables"},{"dest":20,"term":"class_variable_set"},{"dest":20,"term":"clause"},{"dest":20,"term":"clear_all_old"},{"dest":20,"term":"clone"},{"dest":20,"term":"closure"},{"dest":20,"term":"cLVAR"},{"dest":20,"term":"cmd_brace_block"},{"dest":20,"term":"cmp"},{"dest":20,"term":"cname"},{"dest":20,"term":"codegen"},{"dest":20,"term":"codepoints"},{"dest":20,"term":"collect"},{"dest":20,"term":"collect_concat"},{"dest":20,"term":"color"},{"dest":20,"term":"column_count"},{"dest":20,"term":"column_index"},{"dest":20,"term":"combination"},{"dest":20,"term":"comma"},{"dest":20,"term":"command"},{"dest":20,"term":"command_args"},{"dest":20,"term":"command_asgn"},{"dest":20,"term":"command_call"},{"dest":20,"term":"command_rhs"},{"dest":20,"term":"compact"},{"dest":20,"term":"Comparable"},{"dest":20,"term":"compile"},{"dest":20,"term":"compstmt"},{"dest":20,"term":"concat"},{"dest":20,"term":"constant"},{"dest":20,"term":"CONSTANT"},{"dest":20,"term":"constants"},{"dest":20,"term":"const_get"},{"dest":20,"term":"const_missing"},{"dest":20,"term":"const_set"},{"dest":20,"term":"cont"},{"dest":20,"term":"context"},{"dest":20,"term":"copyright"},{"dest":20,"term":"corrupted"},{"dest":20,"term":"cos"},{"dest":20,"term":"cosh"},{"dest":20,"term":"count"},{"dest":20,"term":"count_objects"},{"dest":20,"term":"cpath"},{"dest":20,"term":"ctime"},{"dest":20,"term":"__ctype_b_loc"},{"dest":20,"term":"curr"},{"dest":20,"term":"current"},{"dest":20,"term":"curry"},{"dest":20,"term":"cycle"},{"dest":20,"term":"Data"},{"dest":20,"term":"day"},{"dest":20,"term":"debug_info"},{"dest":20,"term":"Dec"},{"dest":20,"term":"deep"},{"dest":20,"term":"def"},{"dest":20,"term":"default"},{"dest":20,"term":"DEFAULT"},{"dest":20,"term":"default_proc"},{"dest":20,"term":"defined"},{"dest":20,"term":"define_method"},{"dest":20,"term":"define_singleton_method"},{"dest":20,"term":"__delete"},{"dest":20,"term":"delete"},{"dest":20,"term":"delete_at"},{"dest":20,"term":"delete_if"},{"dest":20,"term":"delete_prefix"},{"dest":20,"term":"delete_suffix"},{"dest":20,"term":"Deleting"},{"dest":20,"term":"depth"},{"dest":20,"term":"detect"},{"dest":20,"term":"detected"},{"dest":20,"term":"developers"},{"dest":20,"term":"differs"},{"dest":20,"term":"digit"},{"dest":20,"term":"digits"},{"dest":20,"term":"disable"},{"dest":20,"term":"disabled"},{"dest":20,"term":"discarding"},{"dest":20,"term":"div"},{"dest":20,"term":"divmod"},{"dest":20,"term":"do"},{"dest":20,"term":"do_block"},{"dest":20,"term":"DomainError"},{"dest":20,"term":"dot"},{"dest":20,"term":"dot_or_colon"},{"dest":20,"term":"downcase"},{"dest":20,"term":"downto"},{"dest":20,"term":"drop"},{"dest":20,"term":"dropped"},{"dest":20,"term":"dropping"},{"dest":20,"term":"drop_while"},{"dest":20,"term":"dump"},{"dest":20,"term":"dup"},{"dest":20,"term":"each"},{"dest":20,"term":"each_byte"},{"dest":20,"term":"each_char"},{"dest":20,"term":"each_codepoint"},{"dest":20,"term":"each_cons"},{"dest":20,"term":"each_index"},{"dest":20,"term":"each_key"},{"dest":20,"term":"each_line"},{"dest":20,"term":"each_object"},{"dest":20,"term":"each_pair"},{"dest":20,"term":"each_slice"},{"dest":20,"term":"each_value"},{"dest":20,"term":"each_with_index"},{"dest":20,"term":"each_with_object"},{"dest":20,"term":"ecall"},{"dest":20,"term":"elem"},{"dest":20,"term":"else"},{"dest":20,"term":"elsif"},{"dest":20,"term":"en"},{"dest":20,"term":"enable"},{"dest":20,"term":"__ENCODING__"},{"dest":20,"term":"end"},{"dest":20,"term":"__END__"},{"dest":20,"term":"END"},{"dest":20,"term":"ensure"},{"dest":20,"term":"entries"},{"dest":20,"term":"Enumerable"},{"dest":20,"term":"enumerator"},{"dest":20,"term":"Enumerator"},{"dest":20,"term":"enumerator_block_call"},{"dest":20,"term":"enum_for"},{"dest":20,"term":"enums"},{"dest":20,"term":"env"},{"dest":20,"term":"erf"},{"dest":20,"term":"erfc"},{"dest":20,"term":"__errno_location"},{"dest":20,"term":"error"},{"dest":20,"term":"escape"},{"dest":20,"term":"ETIR"},{"dest":20,"term":"ETIR0004Ci"},{"dest":20,"term":"exception"},{"dest":20,"term":"Exception"},{"dest":20,"term":"exc_list"},{"dest":20,"term":"exc_var"},{"dest":20,"term":"exhausted"},{"dest":20,"term":"exp"},{"dest":20,"term":"expected"},{"dest":20,"term":"expr"},{"dest":20,"term":"expression"},{"dest":20,"term":"expr_value"},{"dest":20,"term":"extend"},{"dest":20,"term":"extended"},{"dest":20,"term":"extend_object"},{"dest":20,"term":"fail"},{"dest":20,"term":"failed"},{"dest":20,"term":"failure"},{"dest":20,"term":"false"},{"dest":20,"term":"FalseClass"},{"dest":20,"term":"f_arg"},{"dest":20,"term":"f_arg_item"},{"dest":20,"term":"f_arglist"},{"dest":20,"term":"f_args"},{"dest":20,"term":"f_bad_arg"},{"dest":20,"term":"f_block_arg"},{"dest":20,"term":"f_block_opt"},{"dest":20,"term":"f_block_optarg"},{"dest":20,"term":"fclose"},{"dest":20,"term":"Feb"},{"dest":20,"term":"feed"},{"dest":20,"term":"feedvalue"},{"dest":20,"term":"feof"},{"dest":20,"term":"fetch"},{"dest":20,"term":"fetch_values"},{"dest":20,"term":"fflush"},{"dest":20,"term":"fgetc"},{"dest":20,"term":"fib"},{"dest":20,"term":"fiber"},{"dest":20,"term":"Fiber"},{"dest":20,"term":"fiber_check"},{"dest":20,"term":"FiberError"},{"dest":20,"term":"field"},{"dest":20,"term":"file"},{"dest":20,"term":"File"},{"dest":20,"term":"__FILE__"},{"dest":20,"term":"filename"},{"dest":20,"term":"filenames_len"},{"dest":20,"term":"fill"},{"dest":20,"term":"final_marking_phase"},{"dest":20,"term":"find"},{"dest":20,"term":"find_all"},{"dest":20,"term":"find_index"},{"dest":20,"term":"first"},{"dest":20,"term":"fish"},{"dest":20,"term":"Fixnum"},{"dest":20,"term":"flag"},{"dest":20,"term":"f_larglist"},{"dest":20,"term":"flat_map"},{"dest":20,"term":"flatten"},{"dest":20,"term":"Float"},{"dest":20,"term":"FloatDomainError"},{"dest":20,"term":"floor"},{"dest":20,"term":"f_marg"},{"dest":20,"term":"f_marg_list"},{"dest":20,"term":"f_margs"},{"dest":20,"term":"fmod"},{"dest":20,"term":"fn"},{"dest":20,"term":"Fn"},{"dest":20,"term":"fname"},{"dest":20,"term":"f_norm_arg"},{"dest":20,"term":"fopen"},{"dest":20,"term":"f_opt"},{"dest":20,"term":"f_optarg"},{"dest":20,"term":"f_opt_asgn"},{"dest":20,"term":"for"},{"dest":20,"term":"force"},{"dest":20,"term":"format"},{"dest":20,"term":"for_var"},{"dest":20,"term":"found"},{"dest":20,"term":"fprintf"},{"dest":20,"term":"fputc"},{"dest":20,"term":"fread"},{"dest":20,"term":"free"},{"dest":20,"term":"FREE"},{"dest":20,"term":"freeze"},{"dest":20,"term":"f_rest_arg"},{"dest":20,"term":"frexp"},{"dest":20,"term":"Fri"},{"dest":20,"term":"FrozenError"},{"dest":20,"term":"FsC"},{"dest":20,"term":"fsym"},{"dest":20,"term":"fwrite"},{"dest":20,"term":"games"},{"dest":20,"term":"GB"},{"dest":20,"term":"GC"},{"dest":20,"term":"gc_mark_children"},{"dest":20,"term":"_gc_root_"},{"dest":20,"term":"generational_mode"},{"dest":20,"term":"Generator"},{"dest":20,"term":"getbyte"},{"dest":20,"term":"get_file"},{"dest":20,"term":"getgm"},{"dest":20,"term":"getlocal"},{"dest":20,"term":"gettimeofday"},{"dest":20,"term":"getutc"},{"dest":20,"term":"given"},{"dest":20,"term":"given_args"},{"dest":20,"term":"global_variables"},{"dest":20,"term":"__gmon_start__"},{"dest":20,"term":"gmtime"},{"dest":20,"term":"gmtime_r"},{"dest":20,"term":"gn"},{"dest":20,"term":"gnu"},{"dest":20,"term":"GNU"},{"dest":20,"term":"go"},{"dest":20,"term":"grep"},{"dest":20,"term":"group_by"},{"dest":20,"term":"gsub"},{"dest":20,"term":"h0"},{"dest":20,"term":"h2"},{"dest":20,"term":"H3"},{"dest":20,"term":"h4"},{"dest":20,"term":"h5"},{"dest":20,"term":"H5"},{"dest":20,"term":"h6"},{"dest":20,"term":"H6"},{"dest":20,"term":"h7"},{"dest":20,"term":"h8"},{"dest":20,"term":"hA"},{"dest":20,"term":"hash"},{"dest":20,"term":"Hash"},{"dest":20,"term":"head"},{"dest":20,"term":"heredoc"},{"dest":20,"term":"heredoc_bodies"},{"dest":20,"term":"heredoc_body"},{"dest":20,"term":"heredoc_string_interp"},{"dest":20,"term":"heredoc_string_rep"},{"dest":20,"term":"heredoc_treat_nextline"},{"dest":20,"term":"hex"},{"dest":20,"term":"high"},{"dest":20,"term":"hour"},{"dest":20,"term":"hypot"},{"dest":20,"term":"i2"},{"dest":20,"term":"iClass"},{"dest":20,"term":"__id__"},{"dest":20,"term":"id2name"},{"dest":20,"term":"identifier"},{"dest":20,"term":"idx"},{"dest":20,"term":"idx2"},{"dest":20,"term":"if"},{"dest":20,"term":"ifnone"},{"dest":20,"term":"if_tail"},{"dest":20,"term":"implemented"},{"dest":20,"term":"in"},{"dest":20,"term":"include"},{"dest":20,"term":"included"},{"dest":20,"term":"included_modules"},{"dest":20,"term":"incremental_gc"},{"dest":20,"term":"index"},{"dest":20,"term":"IndexError"},{"dest":20,"term":"inf"},{"dest":20,"term":"Inf"},{"dest":20,"term":"INF"},{"dest":20,"term":"Infinity"},{"dest":20,"term":"INFINITY"},{"dest":20,"term":"inherited"},{"dest":20,"term":"initialize"},{"dest":20,"term":"initialize_copy"},{"dest":20,"term":"inject"},{"dest":20,"term":"in_lower_half"},{"dest":20,"term":"input"},{"dest":20,"term":"insert"},{"dest":20,"term":"_inspect"},{"dest":20,"term":"inspect"},{"dest":20,"term":"instance_eval"},{"dest":20,"term":"instance_exec"},{"dest":20,"term":"instance_methods"},{"dest":20,"term":"instance_variable_get"},{"dest":20,"term":"instance_variables"},{"dest":20,"term":"instance_variable_set"},{"dest":20,"term":"int"},{"dest":20,"term":"integer"},{"dest":20,"term":"Integer"},{"dest":20,"term":"Integral"},{"dest":20,"term":"intern"},{"dest":20,"term":"interval_ratio"},{"dest":20,"term":"invert"},{"dest":20,"term":"io"},{"dest":20,"term":"Io"},{"dest":20,"term":"_IO_putc"},{"dest":20,"term":"ip"},{"dest":20,"term":"Ip"},{"dest":20,"term":"irep"},{"dest":20,"term":"IREP"},{"dest":20,"term":"isz"},{"dest":20,"term":"iterate"},{"dest":20,"term":"_ITM_deregisterTMCloneTable"},{"dest":20,"term":"_ITM_registerTMCloneTable"},{"dest":20,"term":"itself"},{"dest":20,"term":"Jan"},{"dest":20,"term":"join"},{"dest":20,"term":"_Jv_RegisterClasses"},{"dest":20,"term":"keep_if"},{"dest":20,"term":"Kernel"},{"dest":20,"term":"key"},{"dest":20,"term":"KeyError"},{"dest":20,"term":"keys"},{"dest":20,"term":"keyword_alias"},{"dest":20,"term":"keyword_and"},{"dest":20,"term":"keyword_begin"},{"dest":20,"term":"keyword_BEGIN"},{"dest":20,"term":"keyword_break"},{"dest":20,"term":"keyword_case"},{"dest":20,"term":"keyword_class"},{"dest":20,"term":"keyword_def"},{"dest":20,"term":"keyword_do"},{"dest":20,"term":"keyword_do_block"},{"dest":20,"term":"keyword_do_cond"},{"dest":20,"term":"keyword_do_LAMBDA"},{"dest":20,"term":"keyword_else"},{"dest":20,"term":"keyword_elsif"},{"dest":20,"term":"keyword__ENCODING__"},{"dest":20,"term":"keyword_end"},{"dest":20,"term":"keyword_END"},{"dest":20,"term":"keyword_ensure"},{"dest":20,"term":"keyword_false"},{"dest":20,"term":"keyword__FILE__"},{"dest":20,"term":"keyword_for"},{"dest":20,"term":"keyword_if"},{"dest":20,"term":"keyword_in"},{"dest":20,"term":"keyword__LINE__"},{"dest":20,"term":"keyword_module"},{"dest":20,"term":"keyword_next"},{"dest":20,"term":"keyword_nil"},{"dest":20,"term":"keyword_not"},{"dest":20,"term":"keyword_or"},{"dest":20,"term":"keyword_redo"},{"dest":20,"term":"keyword_rescue"},{"dest":20,"term":"keyword_retry"},{"dest":20,"term":"keyword_return"},{"dest":20,"term":"keyword_self"},{"dest":20,"term":"keyword_super"},{"dest":20,"term":"keyword_then"},{"dest":20,"term":"keyword_true"},{"dest":20,"term":"keyword_undef"},{"dest":20,"term":"keyword_unless"},{"dest":20,"term":"keyword_until"},{"dest":20,"term":"keyword_when"},{"dest":20,"term":"keyword_while"},{"dest":20,"term":"keyword_yield"},{"dest":20,"term":"kh_del_ht"},{"dest":20,"term":"kh_del_iv"},{"dest":20,"term":"kh_del_mt"},{"dest":20,"term":"kh_del_n2s"},{"dest":20,"term":"kh_del_st"},{"dest":20,"term":"KLVAR"},{"dest":20,"term":"lambda"},{"dest":20,"term":"lambda_body"},{"dest":20,"term":"last"},{"dest":20,"term":"lazy"},{"dest":20,"term":"Lazy"},{"dest":20,"term":"LC"},{"dest":20,"term":"ld"},{"dest":20,"term":"LD"},{"dest":20,"term":"ldexp"},{"dest":20,"term":"left"},{"dest":20,"term":"len"},{"dest":20,"term":"length"},{"dest":20,"term":"level"},{"dest":20,"term":"lfD"},{"dest":20,"term":"lhs"},{"dest":20,"term":"__libc_start_main"},{"dest":20,"term":"LII"},{"dest":20,"term":"lIJ"},{"dest":20,"term":"lim"},{"dest":20,"term":"line"},{"dest":20,"term":"__LINE__"},{"dest":20,"term":"LINE"},{"dest":20,"term":"lines"},{"dest":20,"term":"literal"},{"dest":20,"term":"literals"},{"dest":20,"term":"live_after_mark"},{"dest":20,"term":"ljust"},{"dest":20,"term":"ln"},{"dest":20,"term":"Ln"},{"dest":20,"term":"lo"},{"dest":20,"term":"local"},{"dest":20,"term":"LOCAL"},{"dest":20,"term":"LocalJumpError"},{"dest":20,"term":"localtime"},{"dest":20,"term":"localtime_r"},{"dest":20,"term":"local_variables"},{"dest":20,"term":"log"},{"dest":20,"term":"log10"},{"dest":20,"term":"log2"},{"dest":20,"term":"long"},{"dest":20,"term":"longjmp"},{"dest":20,"term":"lookahead"},{"dest":20,"term":"loop"},{"dest":20,"term":"low"},{"dest":20,"term":"lround"},{"dest":20,"term":"LS"},{"dest":20,"term":"lstrip"},{"dest":20,"term":"LVAR"},{"dest":20,"term":"machine"},{"dest":20,"term":"main"},{"dest":20,"term":"make_curry"},{"dest":20,"term":"map"},{"dest":20,"term":"match"},{"dest":20,"term":"matched"},{"dest":20,"term":"Math"},{"dest":20,"term":"max"},{"dest":20,"term":"max_by"},{"dest":20,"term":"max_cmp"},{"dest":20,"term":"May"},{"dest":20,"term":"mday"},{"dest":20,"term":"member"},{"dest":20,"term":"__members__"},{"dest":20,"term":"members"},{"dest":20,"term":"memchr"},{"dest":20,"term":"memcmp"},{"dest":20,"term":"memcpy"},{"dest":20,"term":"memmove"},{"dest":20,"term":"memory"},{"dest":20,"term":"memset"},{"dest":20,"term":"merge"},{"dest":20,"term":"mesg"},{"dest":20,"term":"message"},{"dest":20,"term":"meth"},{"dest":20,"term":"__method__"},{"dest":20,"term":"method"},{"dest":20,"term":"method_call"},{"dest":20,"term":"method_missing"},{"dest":20,"term":"method_removed"},{"dest":20,"term":"methods"},{"dest":20,"term":"mid"},{"dest":20,"term":"min"},{"dest":20,"term":"min_by"},{"dest":20,"term":"min_cmp"},{"dest":20,"term":"minmax"},{"dest":20,"term":"minmax_by"},{"dest":20,"term":"mktime"},{"dest":20,"term":"mlhs_basic"},{"dest":20,"term":"mlhs_inner"},{"dest":20,"term":"mlhs_item"},{"dest":20,"term":"mlhs_list"},{"dest":20,"term":"mlhs_node"},{"dest":20,"term":"mlhs_post"},{"dest":20,"term":"mode"},{"dest":20,"term":"modified"},{"dest":20,"term":"modifier_if"},{"dest":20,"term":"modifier_rescue"},{"dest":20,"term":"modifier_unless"},{"dest":20,"term":"modifier_until"},{"dest":20,"term":"modifier_while"},{"dest":20,"term":"module"},{"dest":20,"term":"Module"},{"dest":20,"term":"module_eval"},{"dest":20,"term":"module_function"},{"dest":20,"term":"modules"},{"dest":20,"term":"mon"},{"dest":20,"term":"Mon"},{"dest":20,"term":"month"},{"dest":20,"term":"mrb_ary_delete_at"},{"dest":20,"term":"mrb_ary_new_from_values"},{"dest":20,"term":"mrb_ary_plus"},{"dest":20,"term":"mrb_ary_pop"},{"dest":20,"term":"mrb_ary_push"},{"dest":20,"term":"mrb_ary_push_m"},{"dest":20,"term":"mrb_ary_resize"},{"dest":20,"term":"mrb_ary_reverse"},{"dest":20,"term":"mrb_ary_set"},{"dest":20,"term":"mrb_ary_shift"},{"dest":20,"term":"mrb_ary_splice"},{"dest":20,"term":"mrb_ary_times"},{"dest":20,"term":"mrb_ary_unshift"},{"dest":20,"term":"mrb_ary_unshift_m"},{"dest":20,"term":"mrb_assoc_new"},{"dest":20,"term":"mrb_data_init"},{"dest":20,"term":"mrb_debug_get_line"},{"dest":20,"term":"mrb_debug_info_alloc"},{"dest":20,"term":"mrb_debug_info_append_file"},{"dest":20,"term":"mrb_debug_info_free"},{"dest":20,"term":"mrb_field_write_barrier"},{"dest":20,"term":"mrb_gc_mark"},{"dest":20,"term":"MRB_GC_STATE_ROOT"},{"dest":20,"term":"MRB_GC_STATE_SWEEP"},{"dest":20,"term":"mrb_gc_unregister"},{"dest":20,"term":"mrb_i_mt_state"},{"dest":20,"term":"mrb_incremental_gc"},{"dest":20,"term":"mrb_malloc"},{"dest":20,"term":"mrb_mod_s_nesting"},{"dest":20,"term":"mrb_obj_value"},{"dest":20,"term":"mrb_random_init"},{"dest":20,"term":"mrb_random_srand"},{"dest":20,"term":"mrb_realloc"},{"dest":20,"term":"mrb_str_format"},{"dest":20,"term":"MRB_TT_DATA"},{"dest":20,"term":"MRB_TT_FIBER"},{"dest":20,"term":"MRB_TT_FREE"},{"dest":20,"term":"mrb_vm_const_get"},{"dest":20,"term":"mrb_vm_exec"},{"dest":20,"term":"mrb_write_barrier"},{"dest":20,"term":"mrhs"},{"dest":20,"term":"mruby"},{"dest":20,"term":"MRUBY_COPYRIGHT"},{"dest":20,"term":"MRUBY_DESCRIPTION"},{"dest":20,"term":"MRUBY_RELEASE_DATE"},{"dest":20,"term":"MRUBY_RELEASE_NO"},{"dest":20,"term":"MRUBY_VERSION"},{"dest":20,"term":"name"},{"dest":20,"term":"named"},{"dest":20,"term":"NameError"},{"dest":20,"term":"names"},{"dest":20,"term":"nan"},{"dest":20,"term":"NaN"},{"dest":20,"term":"NAN"},{"dest":20,"term":"nesting"},{"dest":20,"term":"new"},{"dest":20,"term":"new_args"},{"dest":20,"term":"new_key"},{"dest":20,"term":"new_msym"},{"dest":20,"term":"next"},{"dest":20,"term":"next_values"},{"dest":20,"term":"nil"},{"dest":20,"term":"NilClass"},{"dest":20,"term":"nl"},{"dest":20,"term":"nlocals"},{"dest":20,"term":"nLVAR"},{"dest":20,"term":"nMATZ0000IREP"},{"dest":20,"term":"NODE_DREGX"},{"dest":20,"term":"NODE_DSTR"},{"dest":20,"term":"NODE_DXSTR"},{"dest":20,"term":"NODE_FALSE"},{"dest":20,"term":"NODE_NEGATE"},{"dest":20,"term":"NODE_NIL"},{"dest":20,"term":"NODE_REDO"},{"dest":20,"term":"NODE_RETRY"},{"dest":20,"term":"NODE_SELF"},{"dest":20,"term":"NODE_TRUE"},{"dest":20,"term":"NODE_UNDEF"},{"dest":20,"term":"NODE_ZSUPER"},{"dest":20,"term":"NoMemoryError"},{"dest":20,"term":"NoMethodError"},{"dest":20,"term":"none"},{"dest":20,"term":"NONE"},{"dest":20,"term":"norm"},{"dest":20,"term":"not"},{"dest":20,"term":"NotImplementedError"},{"dest":20,"term":"Nov"},{"dest":20,"term":"now"},{"dest":20,"term":"Np"},{"dest":20,"term":"nregs"},{"dest":20,"term":"num"},{"dest":20,"term":"number"},{"dest":20,"term":"numbered"},{"dest":20,"term":"numeric"},{"dest":20,"term":"Numeric"},{"dest":20,"term":"obj"},{"dest":20,"term":"object"},{"dest":20,"term":"Object"},{"dest":20,"term":"object_id"},{"dest":20,"term":"ObjectSpace"},{"dest":20,"term":"oct"},{"dest":20,"term":"Oct"},{"dest":20,"term":"offset"},{"dest":20,"term":"on"},{"dest":20,"term":"On"},{"dest":20,"term":"only"},{"dest":20,"term":"Oo"},{"dest":20,"term":"op"},{"dest":20,"term":"Op"},{"dest":20,"term":"operation"},{"dest":20,"term":"operation2"},{"dest":20,"term":"operation3"},{"dest":20,"term":"OP_NOP"},{"dest":20,"term":"OP_STOP"},{"dest":20,"term":"opt_block_arg"},{"dest":20,"term":"opt_block_param"},{"dest":20,"term":"opt_bv_decl"},{"dest":20,"term":"opt_call_args"},{"dest":20,"term":"opt_else"},{"dest":20,"term":"opt_ensure"},{"dest":20,"term":"opt_f_block_arg"},{"dest":20,"term":"opt_nl"},{"dest":20,"term":"opt_paren_args"},{"dest":20,"term":"opt_rescue"},{"dest":20,"term":"opt_terms"},{"dest":20,"term":"or"},{"dest":20,"term":"ord"},{"dest":20,"term":"orig"},{"dest":20,"term":"other"},{"dest":20,"term":"__outer__"},{"dest":20,"term":"P9o"},{"dest":20,"term":"padding"},{"dest":20,"term":"pad_repetitions"},{"dest":20,"term":"padstr"},{"dest":20,"term":"parameters"},{"dest":20,"term":"paren_args"},{"dest":20,"term":"partition"},{"dest":20,"term":"pattern"},{"dest":20,"term":"PC"},{"dest":20,"term":"peek"},{"dest":20,"term":"peek_values"},{"dest":20,"term":"permutation"},{"dest":20,"term":"plen"},{"dest":20,"term":"point"},{"dest":20,"term":"pop"},{"dest":20,"term":"popping"},{"dest":20,"term":"pos"},{"dest":20,"term":"posnum"},{"dest":20,"term":"post"},{"dest":20,"term":"pow"},{"dest":20,"term":"pp"},{"dest":20,"term":"pproc"},{"dest":20,"term":"pre"},{"dest":20,"term":"precision"},{"dest":20,"term":"prefix"},{"dest":20,"term":"prepend"},{"dest":20,"term":"prepended"},{"dest":20,"term":"prepend_features"},{"dest":20,"term":"primary"},{"dest":20,"term":"primary_value"},{"dest":20,"term":"print"},{"dest":20,"term":"printf"},{"dest":20,"term":"__printstr__"},{"dest":20,"term":"private"},{"dest":20,"term":"private_methods"},{"dest":20,"term":"prl"},{"dest":20,"term":"proc"},{"dest":20,"term":"Proc"},{"dest":20,"term":"program"},{"dest":20,"term":"protected"},{"dest":20,"term":"protected_methods"},{"dest":20,"term":"ps"},{"dest":20,"term":"public"},{"dest":20,"term":"public_methods"},{"dest":20,"term":"push"},{"dest":20,"term":"putchar"},{"dest":20,"term":"puts"},{"dest":20,"term":"quo"},{"dest":20,"term":"raise"},{"dest":20,"term":"rand"},{"dest":20,"term":"Random"},{"dest":20,"term":"range"},{"dest":20,"term":"Range"},{"dest":20,"term":"RangeError"},{"dest":20,"term":"rassoc"},{"dest":20,"term":"rb"},{"dest":20,"term":"RB"},{"dest":20,"term":"rbracket"},{"dest":20,"term":"RC"},{"dest":20,"term":"read_debug_record"},{"dest":20,"term":"readint_mrb_int"},{"dest":20,"term":"read_irep_record_1"},{"dest":20,"term":"read_lv_record"},{"dest":20,"term":"read_section_debug"},{"dest":20,"term":"read_section_lv"},{"dest":20,"term":"realloc"},{"dest":20,"term":"redo"},{"dest":20,"term":"reduce"},{"dest":20,"term":"reg"},{"dest":20,"term":"regexp"},{"dest":20,"term":"Regexp"},{"dest":20,"term":"RegexpError"},{"dest":20,"term":"rehash"},{"dest":20,"term":"reject"},{"dest":20,"term":"remove_class_variable"},{"dest":20,"term":"remove_const"},{"dest":20,"term":"remove_instance_variable"},{"dest":20,"term":"remove_method"},{"dest":20,"term":"replace"},{"dest":20,"term":"req"},{"dest":20,"term":"required"},{"dest":20,"term":"res"},{"dest":20,"term":"rescue"},{"dest":20,"term":"resize_capa"},{"dest":20,"term":"rest"},{"dest":20,"term":"restarg_mark"},{"dest":20,"term":"result"},{"dest":20,"term":"resume"},{"dest":20,"term":"reswords"},{"dest":20,"term":"ret"},{"dest":20,"term":"retry"},{"dest":20,"term":"return"},{"dest":20,"term":"reverse"},{"dest":20,"term":"reverse_each"},{"dest":20,"term":"rewind"},{"dest":20,"term":"right"},{"dest":20,"term":"rindex"},{"dest":20,"term":"rjust"},{"dest":20,"term":"rotate"},{"dest":20,"term":"round"},{"dest":20,"term":"row"},{"dest":20,"term":"rparen"},{"dest":20,"term":"rpartition"},{"dest":20,"term":"rs_len"},{"dest":20,"term":"rstrip"},{"dest":20,"term":"RUBY_ENGINE"},{"dest":20,"term":"RUBY_ENGINE_VERSION"},{"dest":20,"term":"RUBY_VERSION"},{"dest":20,"term":"RuntimeError"},{"dest":20,"term":"sample"},{"dest":20,"term":"Sat"},{"dest":20,"term":"satisfied"},{"dest":20,"term":"scan"},{"dest":20,"term":"SClass"},{"dest":20,"term":"scope"},{"dest":20,"term":"scope_new"},{"dest":20,"term":"script"},{"dest":20,"term":"ScriptError"},{"dest":20,"term":"sec"},{"dest":20,"term":"select"},{"dest":20,"term":"self"},{"dest":20,"term":"self_arity"},{"dest":20,"term":"__send__"},{"dest":20,"term":"send"},{"dest":20,"term":"sep"},{"dest":20,"term":"Sep"},{"dest":20,"term":"sequence"},{"dest":20,"term":"set"},{"dest":20,"term":"set_backtrace"},{"dest":20,"term":"setbyte"},{"dest":20,"term":"_setjmp"},{"dest":20,"term":"shift"},{"dest":20,"term":"shuffle"},{"dest":20,"term":"sin"},{"dest":20,"term":"singleton"},{"dest":20,"term":"singleton_class"},{"dest":20,"term":"singleton_methods"},{"dest":20,"term":"sinh"},{"dest":20,"term":"size"},{"dest":20,"term":"sl"},{"dest":20,"term":"slice"},{"dest":20,"term":"snprintf"},{"dest":20,"term":"so"},{"dest":20,"term":"So"},{"dest":20,"term":"sort"},{"dest":20,"term":"sort_by"},{"dest":20,"term":"__sort_sub__"},{"dest":20,"term":"source_location"},{"dest":20,"term":"Sp"},{"dest":20,"term":"spaces"},{"dest":20,"term":"specifier"},{"dest":20,"term":"splice"},{"dest":20,"term":"split"},{"dest":20,"term":"sprintf"},{"dest":20,"term":"sqrt"},{"dest":20,"term":"srand"},{"dest":20,"term":"__stack_chk_fail"},{"dest":20,"term":"StandardError"},{"dest":20,"term":"start"},{"dest":20,"term":"state"},{"dest":20,"term":"stderr"},{"dest":20,"term":"stdin"},{"dest":20,"term":"stdout"},{"dest":20,"term":"step"},{"dest":20,"term":"step_ratio"},{"dest":20,"term":"stmt"},{"dest":20,"term":"stmts"},{"dest":20,"term":"stop_exc"},{"dest":20,"term":"StopIteration"},{"dest":20,"term":"store"},{"dest":20,"term":"str"},{"dest":20,"term":"str2"},{"dest":20,"term":"strchr"},{"dest":20,"term":"strcmp"},{"dest":20,"term":"str_each"},{"dest":20,"term":"string"},{"dest":20,"term":"String"},{"dest":20,"term":"string_interp"},{"dest":20,"term":"string_rep"},{"dest":20,"term":"strip"},{"dest":20,"term":"strlen"},{"dest":20,"term":"str_make_shared"},{"dest":20,"term":"strncmp"},{"dest":20,"term":"strncpy"},{"dest":20,"term":"strtoul"},{"dest":20,"term":"struct"},{"dest":20,"term":"Struct"},{"dest":20,"term":"sub"},{"dest":20,"term":"__sub_replace"},{"dest":20,"term":"succ"},{"dest":20,"term":"Sun"},{"dest":20,"term":"super"},{"dest":20,"term":"superclass"},{"dest":20,"term":"supported"},{"dest":20,"term":"__svalue"},{"dest":20,"term":"SVD"},{"dest":20,"term":"swapcase"},{"dest":20,"term":"sym"},{"dest":20,"term":"symbol"},{"dest":20,"term":"Symbol"},{"dest":20,"term":"symbols"},{"dest":20,"term":"sym_inspect"},{"dest":20,"term":"syntax"},{"dest":20,"term":"SyntaxError"},{"dest":20,"term":"_sys_fail"},{"dest":20,"term":"SystemCallError"},{"dest":20,"term":"SystemStackError"},{"dest":20,"term":"TA"},{"dest":20,"term":"tail"},{"dest":20,"term":"take"},{"dest":20,"term":"taken"},{"dest":20,"term":"take_while"},{"dest":20,"term":"tAMPER"},{"dest":20,"term":"tan"},{"dest":20,"term":"tANDDOT"},{"dest":20,"term":"tANDOP"},{"dest":20,"term":"tanh"},{"dest":20,"term":"tap"},{"dest":20,"term":"tAREF"},{"dest":20,"term":"T_ARRAY"},{"dest":20,"term":"tASET"},{"dest":20,"term":"tASSOC"},{"dest":20,"term":"TB"},{"dest":20,"term":"tBACK_REF"},{"dest":20,"term":"TbG"},{"dest":20,"term":"T_CLASS"},{"dest":20,"term":"tCMP"},{"dest":20,"term":"tCOLON2"},{"dest":20,"term":"tCOLON3"},{"dest":20,"term":"tCONSTANT"},{"dest":20,"term":"T_CPTR"},{"dest":20,"term":"tCVAR"},{"dest":20,"term":"T_DATA"},{"dest":20,"term":"tDOT2"},{"dest":20,"term":"tDOT3"},{"dest":20,"term":"TeD"},{"dest":20,"term":"T_ENV"},{"dest":20,"term":"tEQ"},{"dest":20,"term":"tEQQ"},{"dest":20,"term":"term"},{"dest":20,"term":"terms"},{"dest":20,"term":"T_EXCEPTION"},{"dest":20,"term":"T_FALSE"},{"dest":20,"term":"T_FIBER"},{"dest":20,"term":"tFID"},{"dest":20,"term":"T_FILE"},{"dest":20,"term":"T_FIXNUM"},{"dest":20,"term":"tFLOAT"},{"dest":20,"term":"T_FLOAT"},{"dest":20,"term":"T_FREE"},{"dest":20,"term":"tGEQ"},{"dest":20,"term":"tGVAR"},{"dest":20,"term":"T_HASH"},{"dest":20,"term":"tHD_LITERAL_DELIM"},{"dest":20,"term":"tHD_STRING_MID"},{"dest":20,"term":"tHD_STRING_PART"},{"dest":20,"term":"then"},{"dest":20,"term":"tHEREDOC_BEG"},{"dest":20,"term":"tHEREDOC_END"},{"dest":20,"term":"this"},{"dest":20,"term":"T_ICLASS"},{"dest":20,"term":"tIDENTIFIER"},{"dest":20,"term":"time"},{"dest":20,"term":"Time"},{"dest":20,"term":"times"},{"dest":20,"term":"tINTEGER"},{"dest":20,"term":"tIVAR"},{"dest":20,"term":"tLABEL"},{"dest":20,"term":"tLABEL_END"},{"dest":20,"term":"tLAMBDA"},{"dest":20,"term":"tLAMBEG"},{"dest":20,"term":"tLAST_TOKEN"},{"dest":20,"term":"tLBRACE"},{"dest":20,"term":"tLBRACE_ARG"},{"dest":20,"term":"tLBRACK"},{"dest":20,"term":"tLEQ"},{"dest":20,"term":"tLITERAL_DELIM"},{"dest":20,"term":"tLOWEST"},{"dest":20,"term":"tLPAREN"},{"dest":20,"term":"tLPAREN_ARG"},{"dest":20,"term":"tLSHFT"},{"dest":20,"term":"tMATCH"},{"dest":20,"term":"T_MODULE"},{"dest":20,"term":"tmp"},{"dest":20,"term":"tNEQ"},{"dest":20,"term":"tNMATCH"},{"dest":20,"term":"tNTH_REF"},{"dest":20,"term":"to_ary"},{"dest":20,"term":"T_OBJECT"},{"dest":20,"term":"to_enum"},{"dest":20,"term":"to_h"},{"dest":20,"term":"to_hash"},{"dest":20,"term":"to_i"},{"dest":20,"term":"to_int"},{"dest":20,"term":"TOJ"},{"dest":20,"term":"TOLERANCE"},{"dest":20,"term":"tolower"},{"dest":20,"term":"tOP_ASGN"},{"dest":20,"term":"top_compstmt"},{"dest":20,"term":"to_proc"},{"dest":20,"term":"top_stmt"},{"dest":20,"term":"top_stmts"},{"dest":20,"term":"tOROP"},{"dest":20,"term":"to_s"},{"dest":20,"term":"to_str"},{"dest":20,"term":"to_sym"},{"dest":20,"term":"TOTAL"},{"dest":20,"term":"toupper"},{"dest":20,"term":"tPOW"},{"dest":20,"term":"T_PROC"},{"dest":20,"term":"trailer"},{"dest":20,"term":"T_RANGE"},{"dest":20,"term":"transfer"},{"dest":20,"term":"transform_keys"},{"dest":20,"term":"transform_values"},{"dest":20,"term":"transpose"},{"dest":20,"term":"tREGEXP"},{"dest":20,"term":"tREGEXP_BEG"},{"dest":20,"term":"tREGEXP_END"},{"dest":20,"term":"tRPAREN"},{"dest":20,"term":"tRSHFT"},{"dest":20,"term":"true"},{"dest":20,"term":"TrueClass"},{"dest":20,"term":"truncate"},{"dest":20,"term":"try_convert"},{"dest":20,"term":"T_SCLASS"},{"dest":20,"term":"tSTAR"},{"dest":20,"term":"tSTRING"},{"dest":20,"term":"T_STRING"},{"dest":20,"term":"tSTRING_BEG"},{"dest":20,"term":"tSTRING_DVAR"},{"dest":20,"term":"tSTRING_MID"},{"dest":20,"term":"tSTRING_PART"},{"dest":20,"term":"tSYMBEG"},{"dest":20,"term":"T_SYMBOL"},{"dest":20,"term":"tSYMBOLS_BEG"},{"dest":20,"term":"tt"},{"dest":20,"term":"T_TRUE"},{"dest":20,"term":"Tue"},{"dest":20,"term":"tUMINUS"},{"dest":20,"term":"tUMINUS_NUM"},{"dest":20,"term":"T_UNDEF"},{"dest":20,"term":"tUPLUS"},{"dest":20,"term":"twice"},{"dest":20,"term":"tWORDS_BEG"},{"dest":20,"term":"tXSTRING"},{"dest":20,"term":"tXSTRING_BEG"},{"dest":20,"term":"type"},{"dest":20,"term":"TypeError"},{"dest":20,"term":"umrb_obj_value"},{"dest":20,"term":"undef"},{"dest":20,"term":"undefined"},{"dest":20,"term":"undef_list"},{"dest":20,"term":"undef_method"},{"dest":20,"term":"uniq"},{"dest":20,"term":"unless"},{"dest":20,"term":"unshift"},{"dest":20,"term":"until"},{"dest":20,"term":"upcase"},{"dest":20,"term":"__update"},{"dest":20,"term":"update"},{"dest":20,"term":"upto"},{"dest":20,"term":"usec"},{"dest":20,"term":"useless"},{"dest":20,"term":"utc"},{"dest":20,"term":"v0000"},{"dest":20,"term":"val"},{"dest":20,"term":"validated"},{"dest":20,"term":"vals"},{"dest":20,"term":"value"},{"dest":20,"term":"values"},{"dest":20,"term":"values_at"},{"dest":20,"term":"variable"},{"dest":20,"term":"var_lhs"},{"dest":20,"term":"var_ref"},{"dest":20,"term":"verbose"},{"dest":20,"term":"version"},{"dest":20,"term":"vm"},{"dest":20,"term":"Vm"},{"dest":20,"term":"warn"},{"dest":20,"term":"wday"},{"dest":20,"term":"Wed"},{"dest":20,"term":"when"},{"dest":20,"term":"while"},{"dest":20,"term":"width"},{"dest":20,"term":"with_index"},{"dest":20,"term":"with_object"},{"dest":20,"term":"words"},{"dest":20,"term":"x86_64"},{"dest":20,"term":"xstring"},{"dest":20,"term":"yday"},{"dest":20,"term":"year"},{"dest":20,"term":"yield"},{"dest":20,"term":"yielder"},{"dest":20,"term":"Yielder"},{"dest":20,"term":"yield_self"},{"dest":20,"term":"zip"},{"dest":20,"term":"zone"}],[{"dest":21,"term":"abcdef0123456789ABCDEF"},{"dest":21,"term":"abcdefghijklmnopqrstuvwxyz"},{"dest":21,"term":"abort"},{"dest":21,"term":"abs"},{"dest":21,"term":"accept"},{"dest":21,"term":"acos"},{"dest":21,"term":"acosh"},{"dest":21,"term":"address"},{"dest":21,"term":"alias"},{"dest":21,"term":"alias_method"},{"dest":21,"term":"allocation"},{"dest":21,"term":"all_symbols"},{"dest":21,"term":"ancestors"},{"dest":21,"term":"and"},{"dest":21,"term":"anum"},{"dest":21,"term":"append"},{"dest":21,"term":"append_features"},{"dest":21,"term":"Apr"},{"dest":21,"term":"aref_args"},{"dest":21,"term":"arg"},{"dest":21,"term":"arg0"},{"dest":21,"term":"arg1"},{"dest":21,"term":"arg2"},{"dest":21,"term":"arg_rhs"},{"dest":21,"term":"args"},{"dest":21,"term":"argument"},{"dest":21,"term":"ArgumentError"},{"dest":21,"term":"arguments"},{"dest":21,"term":"argv"},{"dest":21,"term":"ARGV"},{"dest":21,"term":"arity"},{"dest":21,"term":"array"},{"dest":21,"term":"Array"},{"dest":21,"term":"ary"},{"dest":21,"term":"__ary_cmp"},{"dest":21,"term":"ary_concat"},{"dest":21,"term":"__ary_eq"},{"dest":21,"term":"ary_F"},{"dest":21,"term":"__ary_index"},{"dest":21,"term":"ary_replace"},{"dest":21,"term":"ary_T"},{"dest":21,"term":"asctime"},{"dest":21,"term":"asin"},{"dest":21,"term":"asinh"},{"dest":21,"term":"__assert_fail"},{"dest":21,"term":"assignment"},{"dest":21,"term":"assoc"},{"dest":21,"term":"assoc_list"},{"dest":21,"term":"assocs"},{"dest":21,"term":"assumed"},{"dest":21,"term":"at"},{"dest":21,"term":"atan"},{"dest":21,"term":"atan2"},{"dest":21,"term":"atanh"},{"dest":21,"term":"__attached__"},{"dest":21,"term":"attr"},{"dest":21,"term":"attr_accessor"},{"dest":21,"term":"attr_reader"},{"dest":21,"term":"attrsym"},{"dest":21,"term":"attr_writer"},{"dest":21,"term":"available"},{"dest":21,"term":"backref"},{"dest":21,"term":"backtrace"},{"dest":21,"term":"Backtrace"},{"dest":21,"term":"BasicObject"},{"dest":21,"term":"basic_symbol"},{"dest":21,"term":"beg"},{"dest":21,"term":"begin"},{"dest":21,"term":"BEGIN"},{"dest":21,"term":"big"},{"dest":21,"term":"BIT"},{"dest":21,"term":"blkarg_mark"},{"dest":21,"term":"block"},{"dest":21,"term":"block_arg"},{"dest":21,"term":"block_call"},{"dest":21,"term":"block_command"},{"dest":21,"term":"block_param"},{"dest":21,"term":"block_param_def"},{"dest":21,"term":"BMATZ0000IREP"},{"dest":21,"term":"body"},{"dest":21,"term":"bodystmt"},{"dest":21,"term":"boundary"},{"dest":21,"term":"brace_block"},{"dest":21,"term":"break"},{"dest":21,"term":"bsearch"},{"dest":21,"term":"bsearch_index"},{"dest":21,"term":"buf"},{"dest":21,"term":"bvar"},{"dest":21,"term":"bv_decls"},{"dest":21,"term":"byte"},{"dest":21,"term":"bytes"},{"dest":21,"term":"bytesize"},{"dest":21,"term":"byteslice"},{"dest":21,"term":"call"},{"dest":21,"term":"call_args"},{"dest":21,"term":"caller"},{"dest":21,"term":"call_op"},{"dest":21,"term":"call_op2"},{"dest":21,"term":"capitalize"},{"dest":21,"term":"case"},{"dest":21,"term":"case_body"},{"dest":21,"term":"casecmp"},{"dest":21,"term":"__case_eqq"},{"dest":21,"term":"cases"},{"dest":21,"term":"cbrt"},{"dest":21,"term":"cdr"},{"dest":21,"term":"ceil"},{"dest":21,"term":"change_gen_gc_mode"},{"dest":21,"term":"character"},{"dest":21,"term":"chars"},{"dest":21,"term":"chomp"},{"dest":21,"term":"chop"},{"dest":21,"term":"chr"},{"dest":21,"term":"clamp"},{"dest":21,"term":"Class"},{"dest":21,"term":"class_eval"},{"dest":21,"term":"__classname__"},{"dest":21,"term":"class_variable_get"},{"dest":21,"term":"class_variables"},{"dest":21,"term":"class_variable_set"},{"dest":21,"term":"clause"},{"dest":21,"term":"clear_all_old"},{"dest":21,"term":"clone"},{"dest":21,"term":"closure"},{"dest":21,"term":"cLVAR"},{"dest":21,"term":"cmd_brace_block"},{"dest":21,"term":"cmp"},{"dest":21,"term":"cname"},{"dest":21,"term":"codegen"},{"dest":21,"term":"codepoints"},{"dest":21,"term":"collect"},{"dest":21,"term":"collect_concat"},{"dest":21,"term":"color"},{"dest":21,"term":"column_count"},{"dest":21,"term":"column_index"},{"dest":21,"term":"combination"},{"dest":21,"term":"comma"},{"dest":21,"term":"command"},{"dest":21,"term":"command_args"},{"dest":21,"term":"command_asgn"},{"dest":21,"term":"command_call"},{"dest":21,"term":"command_rhs"},{"dest":21,"term":"compact"},{"dest":21,"term":"Comparable"},{"dest":21,"term":"compile"},{"dest":21,"term":"compstmt"},{"dest":21,"term":"concat"},{"dest":21,"term":"constant"},{"dest":21,"term":"CONSTANT"},{"dest":21,"term":"constants"},{"dest":21,"term":"const_get"},{"dest":21,"term":"const_missing"},{"dest":21,"term":"const_set"},{"dest":21,"term":"cont"},{"dest":21,"term":"context"},{"dest":21,"term":"copyright"},{"dest":21,"term":"corrupted"},{"dest":21,"term":"cos"},{"dest":21,"term":"cosh"},{"dest":21,"term":"count"},{"dest":21,"term":"count_objects"},{"dest":21,"term":"cpath"},{"dest":21,"term":"ctime"},{"dest":21,"term":"__ctype_b_loc"},{"dest":21,"term":"curr"},{"dest":21,"term":"current"},{"dest":21,"term":"curry"},{"dest":21,"term":"cycle"},{"dest":21,"term":"Data"},{"dest":21,"term":"day"},{"dest":21,"term":"debug_info"},{"dest":21,"term":"Dec"},{"dest":21,"term":"deep"},{"dest":21,"term":"def"},{"dest":21,"term":"default"},{"dest":21,"term":"DEFAULT"},{"dest":21,"term":"default_proc"},{"dest":21,"term":"defined"},{"dest":21,"term":"define_method"},{"dest":21,"term":"define_singleton_method"},{"dest":21,"term":"__delete"},{"dest":21,"term":"delete"},{"dest":21,"term":"delete_at"},{"dest":21,"term":"delete_if"},{"dest":21,"term":"delete_prefix"},{"dest":21,"term":"delete_suffix"},{"dest":21,"term":"Deleting"},{"dest":21,"term":"depth"},{"dest":21,"term":"detect"},{"dest":21,"term":"detected"},{"dest":21,"term":"developers"},{"dest":21,"term":"differs"},{"dest":21,"term":"digit"},{"dest":21,"term":"digits"},{"dest":21,"term":"disable"},{"dest":21,"term":"disabled"},{"dest":21,"term":"discarding"},{"dest":21,"term":"div"},{"dest":21,"term":"divmod"},{"dest":21,"term":"do"},{"dest":21,"term":"do_block"},{"dest":21,"term":"DomainError"},{"dest":21,"term":"dot"},{"dest":21,"term":"dot_or_colon"},{"dest":21,"term":"downcase"},{"dest":21,"term":"downto"},{"dest":21,"term":"drop"},{"dest":21,"term":"dropped"},{"dest":21,"term":"dropping"},{"dest":21,"term":"drop_while"},{"dest":21,"term":"dump"},{"dest":21,"term":"dup"},{"dest":21,"term":"each"},{"dest":21,"term":"each_byte"},{"dest":21,"term":"each_char"},{"dest":21,"term":"each_codepoint"},{"dest":21,"term":"each_cons"},{"dest":21,"term":"each_index"},{"dest":21,"term":"each_key"},{"dest":21,"term":"each_line"},{"dest":21,"term":"each_object"},{"dest":21,"term":"each_pair"},{"dest":21,"term":"each_slice"},{"dest":21,"term":"each_value"},{"dest":21,"term":"each_with_index"},{"dest":21,"term":"each_with_object"},{"dest":21,"term":"ecall"},{"dest":21,"term":"elem"},{"dest":21,"term":"else"},{"dest":21,"term":"elsif"},{"dest":21,"term":"en"},{"dest":21,"term":"enable"},{"dest":21,"term":"__ENCODING__"},{"dest":21,"term":"end"},{"dest":21,"term":"__END__"},{"dest":21,"term":"END"},{"dest":21,"term":"ensure"},{"dest":21,"term":"entries"},{"dest":21,"term":"Enumerable"},{"dest":21,"term":"enumerator"},{"dest":21,"term":"Enumerator"},{"dest":21,"term":"enumerator_block_call"},{"dest":21,"term":"enum_for"},{"dest":21,"term":"enums"},{"dest":21,"term":"env"},{"dest":21,"term":"erf"},{"dest":21,"term":"erfc"},{"dest":21,"term":"__errno_location"},{"dest":21,"term":"error"},{"dest":21,"term":"escape"},{"dest":21,"term":"ETIR"},{"dest":21,"term":"ETIR0004Ci"},{"dest":21,"term":"exception"},{"dest":21,"term":"Exception"},{"dest":21,"term":"exc_list"},{"dest":21,"term":"exc_var"},{"dest":21,"term":"exhausted"},{"dest":21,"term":"exp"},{"dest":21,"term":"expected"},{"dest":21,"term":"expr"},{"dest":21,"term":"expression"},{"dest":21,"term":"expr_value"},{"dest":21,"term":"extend"},{"dest":21,"term":"extended"},{"dest":21,"term":"extend_object"},{"dest":21,"term":"fail"},{"dest":21,"term":"failed"},{"dest":21,"term":"failure"},{"dest":21,"term":"false"},{"dest":21,"term":"FalseClass"},{"dest":21,"term":"f_arg"},{"dest":21,"term":"f_arg_item"},{"dest":21,"term":"f_arglist"},{"dest":21,"term":"f_args"},{"dest":21,"term":"f_bad_arg"},{"dest":21,"term":"f_block_arg"},{"dest":21,"term":"f_block_opt"},{"dest":21,"term":"f_block_optarg"},{"dest":21,"term":"fclose"},{"dest":21,"term":"Feb"},{"dest":21,"term":"feed"},{"dest":21,"term":"feedvalue"},{"dest":21,"term":"feof"},{"dest":21,"term":"fetch"},{"dest":21,"term":"fetch_values"},{"dest":21,"term":"fflush"},{"dest":21,"term":"fgetc"},{"dest":21,"term":"fib"},{"dest":21,"term":"fiber"},{"dest":21,"term":"Fiber"},{"dest":21,"term":"fiber_check"},{"dest":21,"term":"FiberError"},{"dest":21,"term":"field"},{"dest":21,"term":"file"},{"dest":21,"term":"File"},{"dest":21,"term":"__FILE__"},{"dest":21,"term":"filename"},{"dest":21,"term":"filenames_len"},{"dest":21,"term":"fill"},{"dest":21,"term":"final_marking_phase"},{"dest":21,"term":"find"},{"dest":21,"term":"find_all"},{"dest":21,"term":"find_index"},{"dest":21,"term":"first"},{"dest":21,"term":"fish"},{"dest":21,"term":"Fixnum"},{"dest":21,"term":"flag"},{"dest":21,"term":"f_larglist"},{"dest":21,"term":"flat_map"},{"dest":21,"term":"flatten"},{"dest":21,"term":"Float"},{"dest":21,"term":"FloatDomainError"},{"dest":21,"term":"floor"},{"dest":21,"term":"f_marg"},{"dest":21,"term":"f_marg_list"},{"dest":21,"term":"f_margs"},{"dest":21,"term":"fmod"},{"dest":21,"term":"fn"},{"dest":21,"term":"Fn"},{"dest":21,"term":"fname"},{"dest":21,"term":"f_norm_arg"},{"dest":21,"term":"fopen"},{"dest":21,"term":"f_opt"},{"dest":21,"term":"f_optarg"},{"dest":21,"term":"f_opt_asgn"},{"dest":21,"term":"for"},{"dest":21,"term":"force"},{"dest":21,"term":"format"},{"dest":21,"term":"for_var"},{"dest":21,"term":"found"},{"dest":21,"term":"fprintf"},{"dest":21,"term":"fputc"},{"dest":21,"term":"fread"},{"dest":21,"term":"free"},{"dest":21,"term":"FREE"},{"dest":21,"term":"freeze"},{"dest":21,"term":"f_rest_arg"},{"dest":21,"term":"frexp"},{"dest":21,"term":"Fri"},{"dest":21,"term":"FrozenError"},{"dest":21,"term":"FsC"},{"dest":21,"term":"fsym"},{"dest":21,"term":"fwrite"},{"dest":21,"term":"games"},{"dest":21,"term":"GB"},{"dest":21,"term":"GC"},{"dest":21,"term":"gc_mark_children"},{"dest":21,"term":"_gc_root_"},{"dest":21,"term":"generational_mode"},{"dest":21,"term":"Generator"},{"dest":21,"term":"getbyte"},{"dest":21,"term":"get_file"},{"dest":21,"term":"getgm"},{"dest":21,"term":"getlocal"},{"dest":21,"term":"gettimeofday"},{"dest":21,"term":"getutc"},{"dest":21,"term":"given"},{"dest":21,"term":"given_args"},{"dest":21,"term":"global_variables"},{"dest":21,"term":"__gmon_start__"},{"dest":21,"term":"gmtime"},{"dest":21,"term":"gmtime_r"},{"dest":21,"term":"gn"},{"dest":21,"term":"gnu"},{"dest":21,"term":"GNU"},{"dest":21,"term":"go"},{"dest":21,"term":"grep"},{"dest":21,"term":"group_by"},{"dest":21,"term":"gsub"},{"dest":21,"term":"h0"},{"dest":21,"term":"h2"},{"dest":21,"term":"H3"},{"dest":21,"term":"h4"},{"dest":21,"term":"h5"},{"dest":21,"term":"H5"},{"dest":21,"term":"h6"},{"dest":21,"term":"H6"},{"dest":21,"term":"h7"},{"dest":21,"term":"h8"},{"dest":21,"term":"hA"},{"dest":21,"term":"hash"},{"dest":21,"term":"Hash"},{"dest":21,"term":"head"},{"dest":21,"term":"heredoc"},{"dest":21,"term":"heredoc_bodies"},{"dest":21,"term":"heredoc_body"},{"dest":21,"term":"heredoc_string_interp"},{"dest":21,"term":"heredoc_string_rep"},{"dest":21,"term":"heredoc_treat_nextline"},{"dest":21,"term":"hex"},{"dest":21,"term":"high"},{"dest":21,"term":"hour"},{"dest":21,"term":"hypot"},{"dest":21,"term":"i2"},{"dest":21,"term":"iClass"},{"dest":21,"term":"__id__"},{"dest":21,"term":"id2name"},{"dest":21,"term":"identifier"},{"dest":21,"term":"idx"},{"dest":21,"term":"idx2"},{"dest":21,"term":"if"},{"dest":21,"term":"ifnone"},{"dest":21,"term":"if_tail"},{"dest":21,"term":"implemented"},{"dest":21,"term":"in"},{"dest":21,"term":"include"},{"dest":21,"term":"included"},{"dest":21,"term":"included_modules"},{"dest":21,"term":"incremental_gc"},{"dest":21,"term":"index"},{"dest":21,"term":"IndexError"},{"dest":21,"term":"inf"},{"dest":21,"term":"Inf"},{"dest":21,"term":"INF"},{"dest":21,"term":"Infinity"},{"dest":21,"term":"INFINITY"},{"dest":21,"term":"inherited"},{"dest":21,"term":"initialize"},{"dest":21,"term":"initialize_copy"},{"dest":21,"term":"inject"},{"dest":21,"term":"in_lower_half"},{"dest":21,"term":"input"},{"dest":21,"term":"insert"},{"dest":21,"term":"_inspect"},{"dest":21,"term":"inspect"},{"dest":21,"term":"instance_eval"},{"dest":21,"term":"instance_exec"},{"dest":21,"term":"instance_methods"},{"dest":21,"term":"instance_variable_get"},{"dest":21,"term":"instance_variables"},{"dest":21,"term":"instance_variable_set"},{"dest":21,"term":"int"},{"dest":21,"term":"integer"},{"dest":21,"term":"Integer"},{"dest":21,"term":"Integral"},{"dest":21,"term":"intern"},{"dest":21,"term":"interval_ratio"},{"dest":21,"term":"invert"},{"dest":21,"term":"io"},{"dest":21,"term":"Io"},{"dest":21,"term":"_IO_putc"},{"dest":21,"term":"ip"},{"dest":21,"term":"Ip"},{"dest":21,"term":"irep"},{"dest":21,"term":"IREP"},{"dest":21,"term":"isz"},{"dest":21,"term":"iterate"},{"dest":21,"term":"_ITM_deregisterTMCloneTable"},{"dest":21,"term":"_ITM_registerTMCloneTable"},{"dest":21,"term":"itself"},{"dest":21,"term":"Jan"},{"dest":21,"term":"join"},{"dest":21,"term":"_Jv_RegisterClasses"},{"dest":21,"term":"keep_if"},{"dest":21,"term":"Kernel"},{"dest":21,"term":"key"},{"dest":21,"term":"KeyError"},{"dest":21,"term":"keys"},{"dest":21,"term":"keyword_alias"},{"dest":21,"term":"keyword_and"},{"dest":21,"term":"keyword_begin"},{"dest":21,"term":"keyword_BEGIN"},{"dest":21,"term":"keyword_break"},{"dest":21,"term":"keyword_case"},{"dest":21,"term":"keyword_class"},{"dest":21,"term":"keyword_def"},{"dest":21,"term":"keyword_do"},{"dest":21,"term":"keyword_do_block"},{"dest":21,"term":"keyword_do_cond"},{"dest":21,"term":"keyword_do_LAMBDA"},{"dest":21,"term":"keyword_else"},{"dest":21,"term":"keyword_elsif"},{"dest":21,"term":"keyword__ENCODING__"},{"dest":21,"term":"keyword_end"},{"dest":21,"term":"keyword_END"},{"dest":21,"term":"keyword_ensure"},{"dest":21,"term":"keyword_false"},{"dest":21,"term":"keyword__FILE__"},{"dest":21,"term":"keyword_for"},{"dest":21,"term":"keyword_if"},{"dest":21,"term":"keyword_in"},{"dest":21,"term":"keyword__LINE__"},{"dest":21,"term":"keyword_module"},{"dest":21,"term":"keyword_next"},{"dest":21,"term":"keyword_nil"},{"dest":21,"term":"keyword_not"},{"dest":21,"term":"keyword_or"},{"dest":21,"term":"keyword_redo"},{"dest":21,"term":"keyword_rescue"},{"dest":21,"term":"keyword_retry"},{"dest":21,"term":"keyword_return"},{"dest":21,"term":"keyword_self"},{"dest":21,"term":"keyword_super"},{"dest":21,"term":"keyword_then"},{"dest":21,"term":"keyword_true"},{"dest":21,"term":"keyword_undef"},{"dest":21,"term":"keyword_unless"},{"dest":21,"term":"keyword_until"},{"dest":21,"term":"keyword_when"},{"dest":21,"term":"keyword_while"},{"dest":21,"term":"keyword_yield"},{"dest":21,"term":"kh_del_ht"},{"dest":21,"term":"kh_del_iv"},{"dest":21,"term":"kh_del_mt"},{"dest":21,"term":"kh_del_n2s"},{"dest":21,"term":"kh_del_st"},{"dest":21,"term":"KLVAR"},{"dest":21,"term":"lambda"},{"dest":21,"term":"lambda_body"},{"dest":21,"term":"last"},{"dest":21,"term":"lazy"},{"dest":21,"term":"Lazy"},{"dest":21,"term":"LC"},{"dest":21,"term":"ld"},{"dest":21,"term":"LD"},{"dest":21,"term":"ldexp"},{"dest":21,"term":"left"},{"dest":21,"term":"len"},{"dest":21,"term":"length"},{"dest":21,"term":"level"},{"dest":21,"term":"lfD"},{"dest":21,"term":"lhs"},{"dest":21,"term":"__libc_start_main"},{"dest":21,"term":"LII"},{"dest":21,"term":"lIJ"},{"dest":21,"term":"lim"},{"dest":21,"term":"line"},{"dest":21,"term":"__LINE__"},{"dest":21,"term":"LINE"},{"dest":21,"term":"lines"},{"dest":21,"term":"literal"},{"dest":21,"term":"literals"},{"dest":21,"term":"live_after_mark"},{"dest":21,"term":"ljust"},{"dest":21,"term":"ln"},{"dest":21,"term":"Ln"},{"dest":21,"term":"lo"},{"dest":21,"term":"local"},{"dest":21,"term":"LOCAL"},{"dest":21,"term":"LocalJumpError"},{"dest":21,"term":"localtime"},{"dest":21,"term":"localtime_r"},{"dest":21,"term":"local_variables"},{"dest":21,"term":"log"},{"dest":21,"term":"log10"},{"dest":21,"term":"log2"},{"dest":21,"term":"long"},{"dest":21,"term":"longjmp"},{"dest":21,"term":"lookahead"},{"dest":21,"term":"loop"},{"dest":21,"term":"low"},{"dest":21,"term":"lround"},{"dest":21,"term":"LS"},{"dest":21,"term":"lstrip"},{"dest":21,"term":"LVAR"},{"dest":21,"term":"machine"},{"dest":21,"term":"main"},{"dest":21,"term":"make_curry"},{"dest":21,"term":"map"},{"dest":21,"term":"match"},{"dest":21,"term":"matched"},{"dest":21,"term":"Math"},{"dest":21,"term":"max"},{"dest":21,"term":"max_by"},{"dest":21,"term":"max_cmp"},{"dest":21,"term":"May"},{"dest":21,"term":"mday"},{"dest":21,"term":"member"},{"dest":21,"term":"__members__"},{"dest":21,"term":"members"},{"dest":21,"term":"memchr"},{"dest":21,"term":"memcmp"},{"dest":21,"term":"memcpy"},{"dest":21,"term":"memmove"},{"dest":21,"term":"memory"},{"dest":21,"term":"memset"},{"dest":21,"term":"merge"},{"dest":21,"term":"mesg"},{"dest":21,"term":"message"},{"dest":21,"term":"meth"},{"dest":21,"term":"__method__"},{"dest":21,"term":"method"},{"dest":21,"term":"method_call"},{"dest":21,"term":"method_missing"},{"dest":21,"term":"method_removed"},{"dest":21,"term":"methods"},{"dest":21,"term":"mid"},{"dest":21,"term":"min"},{"dest":21,"term":"min_by"},{"dest":21,"term":"min_cmp"},{"dest":21,"term":"minmax"},{"dest":21,"term":"minmax_by"},{"dest":21,"term":"mktime"},{"dest":21,"term":"mlhs_basic"},{"dest":21,"term":"mlhs_inner"},{"dest":21,"term":"mlhs_item"},{"dest":21,"term":"mlhs_list"},{"dest":21,"term":"mlhs_node"},{"dest":21,"term":"mlhs_post"},{"dest":21,"term":"mode"},{"dest":21,"term":"modified"},{"dest":21,"term":"modifier_if"},{"dest":21,"term":"modifier_rescue"},{"dest":21,"term":"modifier_unless"},{"dest":21,"term":"modifier_until"},{"dest":21,"term":"modifier_while"},{"dest":21,"term":"module"},{"dest":21,"term":"Module"},{"dest":21,"term":"module_eval"},{"dest":21,"term":"module_function"},{"dest":21,"term":"modules"},{"dest":21,"term":"mon"},{"dest":21,"term":"Mon"},{"dest":21,"term":"month"},{"dest":21,"term":"mrb_ary_delete_at"},{"dest":21,"term":"mrb_ary_new_from_values"},{"dest":21,"term":"mrb_ary_plus"},{"dest":21,"term":"mrb_ary_pop"},{"dest":21,"term":"mrb_ary_push"},{"dest":21,"term":"mrb_ary_push_m"},{"dest":21,"term":"mrb_ary_resize"},{"dest":21,"term":"mrb_ary_reverse"},{"dest":21,"term":"mrb_ary_set"},{"dest":21,"term":"mrb_ary_shift"},{"dest":21,"term":"mrb_ary_splice"},{"dest":21,"term":"mrb_ary_times"},{"dest":21,"term":"mrb_ary_unshift"},{"dest":21,"term":"mrb_ary_unshift_m"},{"dest":21,"term":"mrb_assoc_new"},{"dest":21,"term":"mrb_data_init"},{"dest":21,"term":"mrb_debug_get_line"},{"dest":21,"term":"mrb_debug_info_alloc"},{"dest":21,"term":"mrb_debug_info_append_file"},{"dest":21,"term":"mrb_debug_info_free"},{"dest":21,"term":"mrb_field_write_barrier"},{"dest":21,"term":"mrb_gc_mark"},{"dest":21,"term":"MRB_GC_STATE_ROOT"},{"dest":21,"term":"MRB_GC_STATE_SWEEP"},{"dest":21,"term":"mrb_gc_unregister"},{"dest":21,"term":"mrb_i_mt_state"},{"dest":21,"term":"mrb_incremental_gc"},{"dest":21,"term":"mrb_malloc"},{"dest":21,"term":"mrb_mod_s_nesting"},{"dest":21,"term":"mrb_obj_value"},{"dest":21,"term":"mrb_random_init"},{"dest":21,"term":"mrb_random_srand"},{"dest":21,"term":"mrb_realloc"},{"dest":21,"term":"mrb_str_format"},{"dest":21,"term":"MRB_TT_DATA"},{"dest":21,"term":"MRB_TT_FIBER"},{"dest":21,"term":"MRB_TT_FREE"},{"dest":21,"term":"mrb_vm_const_get"},{"dest":21,"term":"mrb_vm_exec"},{"dest":21,"term":"mrb_write_barrier"},{"dest":21,"term":"mrhs"},{"dest":21,"term":"mruby"},{"dest":21,"term":"MRUBY_COPYRIGHT"},{"dest":21,"term":"MRUBY_DESCRIPTION"},{"dest":21,"term":"MRUBY_RELEASE_DATE"},{"dest":21,"term":"MRUBY_RELEASE_NO"},{"dest":21,"term":"MRUBY_VERSION"},{"dest":21,"term":"name"},{"dest":21,"term":"named"},{"dest":21,"term":"NameError"},{"dest":21,"term":"names"},{"dest":21,"term":"nan"},{"dest":21,"term":"NaN"},{"dest":21,"term":"NAN"},{"dest":21,"term":"nesting"},{"dest":21,"term":"new"},{"dest":21,"term":"new_args"},{"dest":21,"term":"new_key"},{"dest":21,"term":"new_msym"},{"dest":21,"term":"next"},{"dest":21,"term":"next_values"},{"dest":21,"term":"nil"},{"dest":21,"term":"NilClass"},{"dest":21,"term":"nl"},{"dest":21,"term":"nlocals"},{"dest":21,"term":"nLVAR"},{"dest":21,"term":"nMATZ0000IREP"},{"dest":21,"term":"NODE_DREGX"},{"dest":21,"term":"NODE_DSTR"},{"dest":21,"term":"NODE_DXSTR"},{"dest":21,"term":"NODE_FALSE"},{"dest":21,"term":"NODE_NEGATE"},{"dest":21,"term":"NODE_NIL"},{"dest":21,"term":"NODE_REDO"},{"dest":21,"term":"NODE_RETRY"},{"dest":21,"term":"NODE_SELF"},{"dest":21,"term":"NODE_TRUE"},{"dest":21,"term":"NODE_UNDEF"},{"dest":21,"term":"NODE_ZSUPER"},{"dest":21,"term":"NoMemoryError"},{"dest":21,"term":"NoMethodError"},{"dest":21,"term":"none"},{"dest":21,"term":"NONE"},{"dest":21,"term":"norm"},{"dest":21,"term":"not"},{"dest":21,"term":"NotImplementedError"},{"dest":21,"term":"Nov"},{"dest":21,"term":"now"},{"dest":21,"term":"Np"},{"dest":21,"term":"nregs"},{"dest":21,"term":"num"},{"dest":21,"term":"number"},{"dest":21,"term":"numbered"},{"dest":21,"term":"numeric"},{"dest":21,"term":"Numeric"},{"dest":21,"term":"obj"},{"dest":21,"term":"object"},{"dest":21,"term":"Object"},{"dest":21,"term":"object_id"},{"dest":21,"term":"ObjectSpace"},{"dest":21,"term":"oct"},{"dest":21,"term":"Oct"},{"dest":21,"term":"offset"},{"dest":21,"term":"on"},{"dest":21,"term":"On"},{"dest":21,"term":"only"},{"dest":21,"term":"Oo"},{"dest":21,"term":"op"},{"dest":21,"term":"Op"},{"dest":21,"term":"operation"},{"dest":21,"term":"operation2"},{"dest":21,"term":"operation3"},{"dest":21,"term":"OP_NOP"},{"dest":21,"term":"OP_STOP"},{"dest":21,"term":"opt_block_arg"},{"dest":21,"term":"opt_block_param"},{"dest":21,"term":"opt_bv_decl"},{"dest":21,"term":"opt_call_args"},{"dest":21,"term":"opt_else"},{"dest":21,"term":"opt_ensure"},{"dest":21,"term":"opt_f_block_arg"},{"dest":21,"term":"opt_nl"},{"dest":21,"term":"opt_paren_args"},{"dest":21,"term":"opt_rescue"},{"dest":21,"term":"opt_terms"},{"dest":21,"term":"or"},{"dest":21,"term":"ord"},{"dest":21,"term":"orig"},{"dest":21,"term":"other"},{"dest":21,"term":"__outer__"},{"dest":21,"term":"P9o"},{"dest":21,"term":"padding"},{"dest":21,"term":"pad_repetitions"},{"dest":21,"term":"padstr"},{"dest":21,"term":"parameters"},{"dest":21,"term":"paren_args"},{"dest":21,"term":"partition"},{"dest":21,"term":"pattern"},{"dest":21,"term":"PC"},{"dest":21,"term":"peek"},{"dest":21,"term":"peek_values"},{"dest":21,"term":"permutation"},{"dest":21,"term":"plen"},{"dest":21,"term":"point"},{"dest":21,"term":"pop"},{"dest":21,"term":"popping"},{"dest":21,"term":"pos"},{"dest":21,"term":"posnum"},{"dest":21,"term":"post"},{"dest":21,"term":"pow"},{"dest":21,"term":"pp"},{"dest":21,"term":"pproc"},{"dest":21,"term":"pre"},{"dest":21,"term":"precision"},{"dest":21,"term":"prefix"},{"dest":21,"term":"prepend"},{"dest":21,"term":"prepended"},{"dest":21,"term":"prepend_features"},{"dest":21,"term":"primary"},{"dest":21,"term":"primary_value"},{"dest":21,"term":"print"},{"dest":21,"term":"printf"},{"dest":21,"term":"__printstr__"},{"dest":21,"term":"private"},{"dest":21,"term":"private_methods"},{"dest":21,"term":"prl"},{"dest":21,"term":"proc"},{"dest":21,"term":"Proc"},{"dest":21,"term":"program"},{"dest":21,"term":"protected"},{"dest":21,"term":"protected_methods"},{"dest":21,"term":"ps"},{"dest":21,"term":"public"},{"dest":21,"term":"public_methods"},{"dest":21,"term":"push"},{"dest":21,"term":"putchar"},{"dest":21,"term":"puts"},{"dest":21,"term":"quo"},{"dest":21,"term":"raise"},{"dest":21,"term":"rand"},{"dest":21,"term":"Random"},{"dest":21,"term":"range"},{"dest":21,"term":"Range"},{"dest":21,"term":"RangeError"},{"dest":21,"term":"rassoc"},{"dest":21,"term":"rb"},{"dest":21,"term":"RB"},{"dest":21,"term":"rbracket"},{"dest":21,"term":"RC"},{"dest":21,"term":"read_debug_record"},{"dest":21,"term":"readint_mrb_int"},{"dest":21,"term":"read_irep_record_1"},{"dest":21,"term":"read_lv_record"},{"dest":21,"term":"read_section_debug"},{"dest":21,"term":"read_section_lv"},{"dest":21,"term":"realloc"},{"dest":21,"term":"redo"},{"dest":21,"term":"reduce"},{"dest":21,"term":"reg"},{"dest":21,"term":"regexp"},{"dest":21,"term":"Regexp"},{"dest":21,"term":"RegexpError"},{"dest":21,"term":"rehash"},{"dest":21,"term":"reject"},{"dest":21,"term":"remove_class_variable"},{"dest":21,"term":"remove_const"},{"dest":21,"term":"remove_instance_variable"},{"dest":21,"term":"remove_method"},{"dest":21,"term":"replace"},{"dest":21,"term":"req"},{"dest":21,"term":"required"},{"dest":21,"term":"res"},{"dest":21,"term":"rescue"},{"dest":21,"term":"resize_capa"},{"dest":21,"term":"rest"},{"dest":21,"term":"restarg_mark"},{"dest":21,"term":"result"},{"dest":21,"term":"resume"},{"dest":21,"term":"reswords"},{"dest":21,"term":"ret"},{"dest":21,"term":"retry"},{"dest":21,"term":"return"},{"dest":21,"term":"reverse"},{"dest":21,"term":"reverse_each"},{"dest":21,"term":"rewind"},{"dest":21,"term":"right"},{"dest":21,"term":"rindex"},{"dest":21,"term":"rjust"},{"dest":21,"term":"rotate"},{"dest":21,"term":"round"},{"dest":21,"term":"row"},{"dest":21,"term":"rparen"},{"dest":21,"term":"rpartition"},{"dest":21,"term":"rs_len"},{"dest":21,"term":"rstrip"},{"dest":21,"term":"RUBY_ENGINE"},{"dest":21,"term":"RUBY_ENGINE_VERSION"},{"dest":21,"term":"RUBY_VERSION"},{"dest":21,"term":"RuntimeError"},{"dest":21,"term":"sample"},{"dest":21,"term":"Sat"},{"dest":21,"term":"satisfied"},{"dest":21,"term":"scan"},{"dest":21,"term":"SClass"},{"dest":21,"term":"scope"},{"dest":21,"term":"scope_new"},{"dest":21,"term":"script"},{"dest":21,"term":"ScriptError"},{"dest":21,"term":"sec"},{"dest":21,"term":"select"},{"dest":21,"term":"self"},{"dest":21,"term":"self_arity"},{"dest":21,"term":"__send__"},{"dest":21,"term":"send"},{"dest":21,"term":"sep"},{"dest":21,"term":"Sep"},{"dest":21,"term":"sequence"},{"dest":21,"term":"set"},{"dest":21,"term":"set_backtrace"},{"dest":21,"term":"setbyte"},{"dest":21,"term":"_setjmp"},{"dest":21,"term":"shift"},{"dest":21,"term":"shuffle"},{"dest":21,"term":"sin"},{"dest":21,"term":"singleton"},{"dest":21,"term":"singleton_class"},{"dest":21,"term":"singleton_methods"},{"dest":21,"term":"sinh"},{"dest":21,"term":"size"},{"dest":21,"term":"sl"},{"dest":21,"term":"slice"},{"dest":21,"term":"snprintf"},{"dest":21,"term":"so"},{"dest":21,"term":"So"},{"dest":21,"term":"sort"},{"dest":21,"term":"sort_by"},{"dest":21,"term":"__sort_sub__"},{"dest":21,"term":"source_location"},{"dest":21,"term":"Sp"},{"dest":21,"term":"spaces"},{"dest":21,"term":"specifier"},{"dest":21,"term":"splice"},{"dest":21,"term":"split"},{"dest":21,"term":"sprintf"},{"dest":21,"term":"sqrt"},{"dest":21,"term":"srand"},{"dest":21,"term":"__stack_chk_fail"},{"dest":21,"term":"StandardError"},{"dest":21,"term":"start"},{"dest":21,"term":"state"},{"dest":21,"term":"stderr"},{"dest":21,"term":"stdin"},{"dest":21,"term":"stdout"},{"dest":21,"term":"step"},{"dest":21,"term":"step_ratio"},{"dest":21,"term":"stmt"},{"dest":21,"term":"stmts"},{"dest":21,"term":"stop_exc"},{"dest":21,"term":"StopIteration"},{"dest":21,"term":"store"},{"dest":21,"term":"str"},{"dest":21,"term":"str2"},{"dest":21,"term":"strchr"},{"dest":21,"term":"strcmp"},{"dest":21,"term":"str_each"},{"dest":21,"term":"string"},{"dest":21,"term":"String"},{"dest":21,"term":"string_interp"},{"dest":21,"term":"string_rep"},{"dest":21,"term":"strip"},{"dest":21,"term":"strlen"},{"dest":21,"term":"str_make_shared"},{"dest":21,"term":"strncmp"},{"dest":21,"term":"strncpy"},{"dest":21,"term":"strtoul"},{"dest":21,"term":"struct"},{"dest":21,"term":"Struct"},{"dest":21,"term":"sub"},{"dest":21,"term":"__sub_replace"},{"dest":21,"term":"succ"},{"dest":21,"term":"Sun"},{"dest":21,"term":"super"},{"dest":21,"term":"superclass"},{"dest":21,"term":"supported"},{"dest":21,"term":"__svalue"},{"dest":21,"term":"SVD"},{"dest":21,"term":"swapcase"},{"dest":21,"term":"sym"},{"dest":21,"term":"symbol"},{"dest":21,"term":"Symbol"},{"dest":21,"term":"symbols"},{"dest":21,"term":"sym_inspect"},{"dest":21,"term":"syntax"},{"dest":21,"term":"SyntaxError"},{"dest":21,"term":"_sys_fail"},{"dest":21,"term":"SystemCallError"},{"dest":21,"term":"SystemStackError"},{"dest":21,"term":"TA"},{"dest":21,"term":"tail"},{"dest":21,"term":"take"},{"dest":21,"term":"taken"},{"dest":21,"term":"take_while"},{"dest":21,"term":"tAMPER"},{"dest":21,"term":"tan"},{"dest":21,"term":"tANDDOT"},{"dest":21,"term":"tANDOP"},{"dest":21,"term":"tanh"},{"dest":21,"term":"tap"},{"dest":21,"term":"tAREF"},{"dest":21,"term":"T_ARRAY"},{"dest":21,"term":"tASET"},{"dest":21,"term":"tASSOC"},{"dest":21,"term":"TB"},{"dest":21,"term":"tBACK_REF"},{"dest":21,"term":"TbG"},{"dest":21,"term":"T_CLASS"},{"dest":21,"term":"tCMP"},{"dest":21,"term":"tCOLON2"},{"dest":21,"term":"tCOLON3"},{"dest":21,"term":"tCONSTANT"},{"dest":21,"term":"T_CPTR"},{"dest":21,"term":"tCVAR"},{"dest":21,"term":"T_DATA"},{"dest":21,"term":"tDOT2"},{"dest":21,"term":"tDOT3"},{"dest":21,"term":"TeD"},{"dest":21,"term":"T_ENV"},{"dest":21,"term":"tEQ"},{"dest":21,"term":"tEQQ"},{"dest":21,"term":"term"},{"dest":21,"term":"terms"},{"dest":21,"term":"T_EXCEPTION"},{"dest":21,"term":"T_FALSE"},{"dest":21,"term":"T_FIBER"},{"dest":21,"term":"tFID"},{"dest":21,"term":"T_FILE"},{"dest":21,"term":"T_FIXNUM"},{"dest":21,"term":"tFLOAT"},{"dest":21,"term":"T_FLOAT"},{"dest":21,"term":"T_FREE"},{"dest":21,"term":"tGEQ"},{"dest":21,"term":"tGVAR"},{"dest":21,"term":"T_HASH"},{"dest":21,"term":"tHD_LITERAL_DELIM"},{"dest":21,"term":"tHD_STRING_MID"},{"dest":21,"term":"tHD_STRING_PART"},{"dest":21,"term":"then"},{"dest":21,"term":"tHEREDOC_BEG"},{"dest":21,"term":"tHEREDOC_END"},{"dest":21,"term":"this"},{"dest":21,"term":"T_ICLASS"},{"dest":21,"term":"tIDENTIFIER"},{"dest":21,"term":"time"},{"dest":21,"term":"Time"},{"dest":21,"term":"times"},{"dest":21,"term":"tINTEGER"},{"dest":21,"term":"tIVAR"},{"dest":21,"term":"tLABEL"},{"dest":21,"term":"tLABEL_END"},{"dest":21,"term":"tLAMBDA"},{"dest":21,"term":"tLAMBEG"},{"dest":21,"term":"tLAST_TOKEN"},{"dest":21,"term":"tLBRACE"},{"dest":21,"term":"tLBRACE_ARG"},{"dest":21,"term":"tLBRACK"},{"dest":21,"term":"tLEQ"},{"dest":21,"term":"tLITERAL_DELIM"},{"dest":21,"term":"tLOWEST"},{"dest":21,"term":"tLPAREN"},{"dest":21,"term":"tLPAREN_ARG"},{"dest":21,"term":"tLSHFT"},{"dest":21,"term":"tMATCH"},{"dest":21,"term":"T_MODULE"},{"dest":21,"term":"tmp"},{"dest":21,"term":"tNEQ"},{"dest":21,"term":"tNMATCH"},{"dest":21,"term":"tNTH_REF"},{"dest":21,"term":"to_ary"},{"dest":21,"term":"T_OBJECT"},{"dest":21,"term":"to_enum"},{"dest":21,"term":"to_h"},{"dest":21,"term":"to_hash"},{"dest":21,"term":"to_i"},{"dest":21,"term":"to_int"},{"dest":21,"term":"TOJ"},{"dest":21,"term":"TOLERANCE"},{"dest":21,"term":"tolower"},{"dest":21,"term":"tOP_ASGN"},{"dest":21,"term":"top_compstmt"},{"dest":21,"term":"to_proc"},{"dest":21,"term":"top_stmt"},{"dest":21,"term":"top_stmts"},{"dest":21,"term":"tOROP"},{"dest":21,"term":"to_s"},{"dest":21,"term":"to_str"},{"dest":21,"term":"to_sym"},{"dest":21,"term":"TOTAL"},{"dest":21,"term":"toupper"},{"dest":21,"term":"tPOW"},{"dest":21,"term":"T_PROC"},{"dest":21,"term":"trailer"},{"dest":21,"term":"T_RANGE"},{"dest":21,"term":"transfer"},{"dest":21,"term":"transform_keys"},{"dest":21,"term":"transform_values"},{"dest":21,"term":"transpose"},{"dest":21,"term":"tREGEXP"},{"dest":21,"term":"tREGEXP_BEG"},{"dest":21,"term":"tREGEXP_END"},{"dest":21,"term":"tRPAREN"},{"dest":21,"term":"tRSHFT"},{"dest":21,"term":"true"},{"dest":21,"term":"TrueClass"},{"dest":21,"term":"truncate"},{"dest":21,"term":"try_convert"},{"dest":21,"term":"T_SCLASS"},{"dest":21,"term":"tSTAR"},{"dest":21,"term":"tSTRING"},{"dest":21,"term":"T_STRING"},{"dest":21,"term":"tSTRING_BEG"},{"dest":21,"term":"tSTRING_DVAR"},{"dest":21,"term":"tSTRING_MID"},{"dest":21,"term":"tSTRING_PART"},{"dest":21,"term":"tSYMBEG"},{"dest":21,"term":"T_SYMBOL"},{"dest":21,"term":"tSYMBOLS_BEG"},{"dest":21,"term":"tt"},{"dest":21,"term":"T_TRUE"},{"dest":21,"term":"Tue"},{"dest":21,"term":"tUMINUS"},{"dest":21,"term":"tUMINUS_NUM"},{"dest":21,"term":"T_UNDEF"},{"dest":21,"term":"tUPLUS"},{"dest":21,"term":"twice"},{"dest":21,"term":"tWORDS_BEG"},{"dest":21,"term":"tXSTRING"},{"dest":21,"term":"tXSTRING_BEG"},{"dest":21,"term":"type"},{"dest":21,"term":"TypeError"},{"dest":21,"term":"umrb_obj_value"},{"dest":21,"term":"undef"},{"dest":21,"term":"undefined"},{"dest":21,"term":"undef_list"},{"dest":21,"term":"undef_method"},{"dest":21,"term":"uniq"},{"dest":21,"term":"unless"},{"dest":21,"term":"unshift"},{"dest":21,"term":"until"},{"dest":21,"term":"upcase"},{"dest":21,"term":"__update"},{"dest":21,"term":"update"},{"dest":21,"term":"upto"},{"dest":21,"term":"usec"},{"dest":21,"term":"useless"},{"dest":21,"term":"utc"},{"dest":21,"term":"v0000"},{"dest":21,"term":"val"},{"dest":21,"term":"validated"},{"dest":21,"term":"vals"},{"dest":21,"term":"value"},{"dest":21,"term":"values"},{"dest":21,"term":"values_at"},{"dest":21,"term":"variable"},{"dest":21,"term":"var_lhs"},{"dest":21,"term":"var_ref"},{"dest":21,"term":"verbose"},{"dest":21,"term":"version"},{"dest":21,"term":"vm"},{"dest":21,"term":"Vm"},{"dest":21,"term":"warn"},{"dest":21,"term":"wday"},{"dest":21,"term":"Wed"},{"dest":21,"term":"when"},{"dest":21,"term":"while"},{"dest":21,"term":"width"},{"dest":21,"term":"with_index"},{"dest":21,"term":"with_object"},{"dest":21,"term":"words"},{"dest":21,"term":"x86_64"},{"dest":21,"term":"xstring"},{"dest":21,"term":"yday"},{"dest":21,"term":"year"},{"dest":21,"term":"yield"},{"dest":21,"term":"yielder"},{"dest":21,"term":"Yielder"},{"dest":21,"term":"yield_self"},{"dest":21,"term":"zip"},{"dest":21,"term":"zone"}],[{"dest":22,"term":"abcdef0123456789ABCDEF"},{"dest":22,"term":"abcdefghijklmnopqrstuvwxyz"},{"dest":22,"term":"abort"},{"dest":22,"term":"abs"},{"dest":22,"term":"accept"},{"dest":22,"term":"acos"},{"dest":22,"term":"acosh"},{"dest":22,"term":"address"},{"dest":22,"term":"alias"},{"dest":22,"term":"alias_method"},{"dest":22,"term":"allocation"},{"dest":22,"term":"all_symbols"},{"dest":22,"term":"ancestors"},{"dest":22,"term":"and"},{"dest":22,"term":"anum"},{"dest":22,"term":"append"},{"dest":22,"term":"append_features"},{"dest":22,"term":"Apr"},{"dest":22,"term":"aref_args"},{"dest":22,"term":"arg"},{"dest":22,"term":"arg0"},{"dest":22,"term":"arg1"},{"dest":22,"term":"arg2"},{"dest":22,"term":"arg_rhs"},{"dest":22,"term":"args"},{"dest":22,"term":"argument"},{"dest":22,"term":"ArgumentError"},{"dest":22,"term":"arguments"},{"dest":22,"term":"argv"},{"dest":22,"term":"ARGV"},{"dest":22,"term":"arity"},{"dest":22,"term":"array"},{"dest":22,"term":"Array"},{"dest":22,"term":"ary"},{"dest":22,"term":"__ary_cmp"},{"dest":22,"term":"ary_concat"},{"dest":22,"term":"__ary_eq"},{"dest":22,"term":"ary_F"},{"dest":22,"term":"__ary_index"},{"dest":22,"term":"ary_replace"},{"dest":22,"term":"ary_T"},{"dest":22,"term":"asctime"},{"dest":22,"term":"asin"},{"dest":22,"term":"asinh"},{"dest":22,"term":"__assert_fail"},{"dest":22,"term":"assignment"},{"dest":22,"term":"assoc"},{"dest":22,"term":"assoc_list"},{"dest":22,"term":"assocs"},{"dest":22,"term":"assumed"},{"dest":22,"term":"at"},{"dest":22,"term":"atan"},{"dest":22,"term":"atan2"},{"dest":22,"term":"atanh"},{"dest":22,"term":"__attached__"},{"dest":22,"term":"attr"},{"dest":22,"term":"attr_accessor"},{"dest":22,"term":"attr_reader"},{"dest":22,"term":"attrsym"},{"dest":22,"term":"attr_writer"},{"dest":22,"term":"available"},{"dest":22,"term":"backref"},{"dest":22,"term":"backtrace"},{"dest":22,"term":"Backtrace"},{"dest":22,"term":"BasicObject"},{"dest":22,"term":"basic_symbol"},{"dest":22,"term":"beg"},{"dest":22,"term":"begin"},{"dest":22,"term":"BEGIN"},{"dest":22,"term":"big"},{"dest":22,"term":"BIT"},{"dest":22,"term":"blkarg_mark"},{"dest":22,"term":"block"},{"dest":22,"term":"block_arg"},{"dest":22,"term":"block_call"},{"dest":22,"term":"block_command"},{"dest":22,"term":"block_param"},{"dest":22,"term":"block_param_def"},{"dest":22,"term":"BMATZ0000IREP"},{"dest":22,"term":"body"},{"dest":22,"term":"bodystmt"},{"dest":22,"term":"boundary"},{"dest":22,"term":"brace_block"},{"dest":22,"term":"break"},{"dest":22,"term":"bsearch"},{"dest":22,"term":"bsearch_index"},{"dest":22,"term":"buf"},{"dest":22,"term":"bvar"},{"dest":22,"term":"bv_decls"},{"dest":22,"term":"byte"},{"dest":22,"term":"bytes"},{"dest":22,"term":"bytesize"},{"dest":22,"term":"byteslice"},{"dest":22,"term":"call"},{"dest":22,"term":"call_args"},{"dest":22,"term":"caller"},{"dest":22,"term":"call_op"},{"dest":22,"term":"call_op2"},{"dest":22,"term":"capitalize"},{"dest":22,"term":"case"},{"dest":22,"term":"case_body"},{"dest":22,"term":"casecmp"},{"dest":22,"term":"__case_eqq"},{"dest":22,"term":"cases"},{"dest":22,"term":"cbrt"},{"dest":22,"term":"cdr"},{"dest":22,"term":"ceil"},{"dest":22,"term":"change_gen_gc_mode"},{"dest":22,"term":"character"},{"dest":22,"term":"chars"},{"dest":22,"term":"chomp"},{"dest":22,"term":"chop"},{"dest":22,"term":"chr"},{"dest":22,"term":"clamp"},{"dest":22,"term":"Class"},{"dest":22,"term":"class_eval"},{"dest":22,"term":"__classname__"},{"dest":22,"term":"class_variable_get"},{"dest":22,"term":"class_variables"},{"dest":22,"term":"class_variable_set"},{"dest":22,"term":"clause"},{"dest":22,"term":"clear_all_old"},{"dest":22,"term":"clone"},{"dest":22,"term":"closure"},{"dest":22,"term":"cLVAR"},{"dest":22,"term":"cmd_brace_block"},{"dest":22,"term":"cmp"},{"dest":22,"term":"cname"},{"dest":22,"term":"codegen"},{"dest":22,"term":"codepoints"},{"dest":22,"term":"collect"},{"dest":22,"term":"collect_concat"},{"dest":22,"term":"color"},{"dest":22,"term":"column_count"},{"dest":22,"term":"column_index"},{"dest":22,"term":"combination"},{"dest":22,"term":"comma"},{"dest":22,"term":"command"},{"dest":22,"term":"command_args"},{"dest":22,"term":"command_asgn"},{"dest":22,"term":"command_call"},{"dest":22,"term":"command_rhs"},{"dest":22,"term":"compact"},{"dest":22,"term":"Comparable"},{"dest":22,"term":"compile"},{"dest":22,"term":"compstmt"},{"dest":22,"term":"concat"},{"dest":22,"term":"constant"},{"dest":22,"term":"CONSTANT"},{"dest":22,"term":"constants"},{"dest":22,"term":"const_get"},{"dest":22,"term":"const_missing"},{"dest":22,"term":"const_set"},{"dest":22,"term":"cont"},{"dest":22,"term":"context"},{"dest":22,"term":"copyright"},{"dest":22,"term":"corrupted"},{"dest":22,"term":"cos"},{"dest":22,"term":"cosh"},{"dest":22,"term":"count"},{"dest":22,"term":"count_objects"},{"dest":22,"term":"cpath"},{"dest":22,"term":"ctime"},{"dest":22,"term":"__ctype_b_loc"},{"dest":22,"term":"curr"},{"dest":22,"term":"current"},{"dest":22,"term":"curry"},{"dest":22,"term":"cycle"},{"dest":22,"term":"Data"},{"dest":22,"term":"day"},{"dest":22,"term":"debug_info"},{"dest":22,"term":"Dec"},{"dest":22,"term":"deep"},{"dest":22,"term":"def"},{"dest":22,"term":"default"},{"dest":22,"term":"DEFAULT"},{"dest":22,"term":"default_proc"},{"dest":22,"term":"defined"},{"dest":22,"term":"define_method"},{"dest":22,"term":"define_singleton_method"},{"dest":22,"term":"__delete"},{"dest":22,"term":"delete"},{"dest":22,"term":"delete_at"},{"dest":22,"term":"delete_if"},{"dest":22,"term":"delete_prefix"},{"dest":22,"term":"delete_suffix"},{"dest":22,"term":"Deleting"},{"dest":22,"term":"depth"},{"dest":22,"term":"detect"},{"dest":22,"term":"detected"},{"dest":22,"term":"developers"},{"dest":22,"term":"differs"},{"dest":22,"term":"digit"},{"dest":22,"term":"digits"},{"dest":22,"term":"disable"},{"dest":22,"term":"disabled"},{"dest":22,"term":"discarding"},{"dest":22,"term":"div"},{"dest":22,"term":"divmod"},{"dest":22,"term":"do"},{"dest":22,"term":"do_block"},{"dest":22,"term":"DomainError"},{"dest":22,"term":"dot"},{"dest":22,"term":"dot_or_colon"},{"dest":22,"term":"downcase"},{"dest":22,"term":"downto"},{"dest":22,"term":"drop"},{"dest":22,"term":"dropped"},{"dest":22,"term":"dropping"},{"dest":22,"term":"drop_while"},{"dest":22,"term":"dump"},{"dest":22,"term":"dup"},{"dest":22,"term":"each"},{"dest":22,"term":"each_byte"},{"dest":22,"term":"each_char"},{"dest":22,"term":"each_codepoint"},{"dest":22,"term":"each_cons"},{"dest":22,"term":"each_index"},{"dest":22,"term":"each_key"},{"dest":22,"term":"each_line"},{"dest":22,"term":"each_object"},{"dest":22,"term":"each_pair"},{"dest":22,"term":"each_slice"},{"dest":22,"term":"each_value"},{"dest":22,"term":"each_with_index"},{"dest":22,"term":"each_with_object"},{"dest":22,"term":"ecall"},{"dest":22,"term":"elem"},{"dest":22,"term":"else"},{"dest":22,"term":"elsif"},{"dest":22,"term":"en"},{"dest":22,"term":"enable"},{"dest":22,"term":"__ENCODING__"},{"dest":22,"term":"end"},{"dest":22,"term":"__END__"},{"dest":22,"term":"END"},{"dest":22,"term":"ensure"},{"dest":22,"term":"entries"},{"dest":22,"term":"Enumerable"},{"dest":22,"term":"enumerator"},{"dest":22,"term":"Enumerator"},{"dest":22,"term":"enumerator_block_call"},{"dest":22,"term":"enum_for"},{"dest":22,"term":"enums"},{"dest":22,"term":"env"},{"dest":22,"term":"erf"},{"dest":22,"term":"erfc"},{"dest":22,"term":"__errno_location"},{"dest":22,"term":"error"},{"dest":22,"term":"escape"},{"dest":22,"term":"ETIR"},{"dest":22,"term":"ETIR0004Ci"},{"dest":22,"term":"exception"},{"dest":22,"term":"Exception"},{"dest":22,"term":"exc_list"},{"dest":22,"term":"exc_var"},{"dest":22,"term":"exhausted"},{"dest":22,"term":"exp"},{"dest":22,"term":"expected"},{"dest":22,"term":"expr"},{"dest":22,"term":"expression"},{"dest":22,"term":"expr_value"},{"dest":22,"term":"extend"},{"dest":22,"term":"extended"},{"dest":22,"term":"extend_object"},{"dest":22,"term":"fail"},{"dest":22,"term":"failed"},{"dest":22,"term":"failure"},{"dest":22,"term":"false"},{"dest":22,"term":"FalseClass"},{"dest":22,"term":"f_arg"},{"dest":22,"term":"f_arg_item"},{"dest":22,"term":"f_arglist"},{"dest":22,"term":"f_args"},{"dest":22,"term":"f_bad_arg"},{"dest":22,"term":"f_block_arg"},{"dest":22,"term":"f_block_opt"},{"dest":22,"term":"f_block_optarg"},{"dest":22,"term":"fclose"},{"dest":22,"term":"Feb"},{"dest":22,"term":"feed"},{"dest":22,"term":"feedvalue"},{"dest":22,"term":"feof"},{"dest":22,"term":"fetch"},{"dest":22,"term":"fetch_values"},{"dest":22,"term":"fflush"},{"dest":22,"term":"fgetc"},{"dest":22,"term":"fib"},{"dest":22,"term":"fiber"},{"dest":22,"term":"Fiber"},{"dest":22,"term":"fiber_check"},{"dest":22,"term":"FiberError"},{"dest":22,"term":"field"},{"dest":22,"term":"file"},{"dest":22,"term":"File"},{"dest":22,"term":"__FILE__"},{"dest":22,"term":"filename"},{"dest":22,"term":"filenames_len"},{"dest":22,"term":"fill"},{"dest":22,"term":"final_marking_phase"},{"dest":22,"term":"find"},{"dest":22,"term":"find_all"},{"dest":22,"term":"find_index"},{"dest":22,"term":"first"},{"dest":22,"term":"fish"},{"dest":22,"term":"Fixnum"},{"dest":22,"term":"flag"},{"dest":22,"term":"f_larglist"},{"dest":22,"term":"flat_map"},{"dest":22,"term":"flatten"},{"dest":22,"term":"Float"},{"dest":22,"term":"FloatDomainError"},{"dest":22,"term":"floor"},{"dest":22,"term":"f_marg"},{"dest":22,"term":"f_marg_list"},{"dest":22,"term":"f_margs"},{"dest":22,"term":"fmod"},{"dest":22,"term":"fn"},{"dest":22,"term":"Fn"},{"dest":22,"term":"fname"},{"dest":22,"term":"f_norm_arg"},{"dest":22,"term":"fopen"},{"dest":22,"term":"f_opt"},{"dest":22,"term":"f_optarg"},{"dest":22,"term":"f_opt_asgn"},{"dest":22,"term":"for"},{"dest":22,"term":"force"},{"dest":22,"term":"format"},{"dest":22,"term":"for_var"},{"dest":22,"term":"found"},{"dest":22,"term":"fprintf"},{"dest":22,"term":"fputc"},{"dest":22,"term":"fread"},{"dest":22,"term":"free"},{"dest":22,"term":"FREE"},{"dest":22,"term":"freeze"},{"dest":22,"term":"f_rest_arg"},{"dest":22,"term":"frexp"},{"dest":22,"term":"Fri"},{"dest":22,"term":"FrozenError"},{"dest":22,"term":"FsC"},{"dest":22,"term":"fsym"},{"dest":22,"term":"fwrite"},{"dest":22,"term":"games"},{"dest":22,"term":"GB"},{"dest":22,"term":"GC"},{"dest":22,"term":"gc_mark_children"},{"dest":22,"term":"_gc_root_"},{"dest":22,"term":"generational_mode"},{"dest":22,"term":"Generator"},{"dest":22,"term":"getbyte"},{"dest":22,"term":"get_file"},{"dest":22,"term":"getgm"},{"dest":22,"term":"getlocal"},{"dest":22,"term":"gettimeofday"},{"dest":22,"term":"getutc"},{"dest":22,"term":"given"},{"dest":22,"term":"given_args"},{"dest":22,"term":"global_variables"},{"dest":22,"term":"__gmon_start__"},{"dest":22,"term":"gmtime"},{"dest":22,"term":"gmtime_r"},{"dest":22,"term":"gn"},{"dest":22,"term":"gnu"},{"dest":22,"term":"GNU"},{"dest":22,"term":"go"},{"dest":22,"term":"grep"},{"dest":22,"term":"group_by"},{"dest":22,"term":"gsub"},{"dest":22,"term":"h0"},{"dest":22,"term":"h2"},{"dest":22,"term":"H3"},{"dest":22,"term":"h4"},{"dest":22,"term":"h5"},{"dest":22,"term":"H5"},{"dest":22,"term":"h6"},{"dest":22,"term":"H6"},{"dest":22,"term":"h7"},{"dest":22,"term":"h8"},{"dest":22,"term":"hA"},{"dest":22,"term":"hash"},{"dest":22,"term":"Hash"},{"dest":22,"term":"head"},{"dest":22,"term":"heredoc"},{"dest":22,"term":"heredoc_bodies"},{"dest":22,"term":"heredoc_body"},{"dest":22,"term":"heredoc_string_interp"},{"dest":22,"term":"heredoc_string_rep"},{"dest":22,"term":"heredoc_treat_nextline"},{"dest":22,"term":"hex"},{"dest":22,"term":"high"},{"dest":22,"term":"hour"},{"dest":22,"term":"hypot"},{"dest":22,"term":"i2"},{"dest":22,"term":"iClass"},{"dest":22,"term":"__id__"},{"dest":22,"term":"id2name"},{"dest":22,"term":"identifier"},{"dest":22,"term":"idx"},{"dest":22,"term":"idx2"},{"dest":22,"term":"if"},{"dest":22,"term":"ifnone"},{"dest":22,"term":"if_tail"},{"dest":22,"term":"implemented"},{"dest":22,"term":"in"},{"dest":22,"term":"include"},{"dest":22,"term":"included"},{"dest":22,"term":"included_modules"},{"dest":22,"term":"incremental_gc"},{"dest":22,"term":"index"},{"dest":22,"term":"IndexError"},{"dest":22,"term":"inf"},{"dest":22,"term":"Inf"},{"dest":22,"term":"INF"},{"dest":22,"term":"Infinity"},{"dest":22,"term":"INFINITY"},{"dest":22,"term":"inherited"},{"dest":22,"term":"initialize"},{"dest":22,"term":"initialize_copy"},{"dest":22,"term":"inject"},{"dest":22,"term":"in_lower_half"},{"dest":22,"term":"input"},{"dest":22,"term":"insert"},{"dest":22,"term":"_inspect"},{"dest":22,"term":"inspect"},{"dest":22,"term":"instance_eval"},{"dest":22,"term":"instance_exec"},{"dest":22,"term":"instance_methods"},{"dest":22,"term":"instance_variable_get"},{"dest":22,"term":"instance_variables"},{"dest":22,"term":"instance_variable_set"},{"dest":22,"term":"int"},{"dest":22,"term":"integer"},{"dest":22,"term":"Integer"},{"dest":22,"term":"Integral"},{"dest":22,"term":"intern"},{"dest":22,"term":"interval_ratio"},{"dest":22,"term":"invert"},{"dest":22,"term":"io"},{"dest":22,"term":"Io"},{"dest":22,"term":"_IO_putc"},{"dest":22,"term":"ip"},{"dest":22,"term":"Ip"},{"dest":22,"term":"irep"},{"dest":22,"term":"IREP"},{"dest":22,"term":"isz"},{"dest":22,"term":"iterate"},{"dest":22,"term":"_ITM_deregisterTMCloneTable"},{"dest":22,"term":"_ITM_registerTMCloneTable"},{"dest":22,"term":"itself"},{"dest":22,"term":"Jan"},{"dest":22,"term":"join"},{"dest":22,"term":"_Jv_RegisterClasses"},{"dest":22,"term":"keep_if"},{"dest":22,"term":"Kernel"},{"dest":22,"term":"key"},{"dest":22,"term":"KeyError"},{"dest":22,"term":"keys"},{"dest":22,"term":"keyword_alias"},{"dest":22,"term":"keyword_and"},{"dest":22,"term":"keyword_begin"},{"dest":22,"term":"keyword_BEGIN"},{"dest":22,"term":"keyword_break"},{"dest":22,"term":"keyword_case"},{"dest":22,"term":"keyword_class"},{"dest":22,"term":"keyword_def"},{"dest":22,"term":"keyword_do"},{"dest":22,"term":"keyword_do_block"},{"dest":22,"term":"keyword_do_cond"},{"dest":22,"term":"keyword_do_LAMBDA"},{"dest":22,"term":"keyword_else"},{"dest":22,"term":"keyword_elsif"},{"dest":22,"term":"keyword__ENCODING__"},{"dest":22,"term":"keyword_end"},{"dest":22,"term":"keyword_END"},{"dest":22,"term":"keyword_ensure"},{"dest":22,"term":"keyword_false"},{"dest":22,"term":"keyword__FILE__"},{"dest":22,"term":"keyword_for"},{"dest":22,"term":"keyword_if"},{"dest":22,"term":"keyword_in"},{"dest":22,"term":"keyword__LINE__"},{"dest":22,"term":"keyword_module"},{"dest":22,"term":"keyword_next"},{"dest":22,"term":"keyword_nil"},{"dest":22,"term":"keyword_not"},{"dest":22,"term":"keyword_or"},{"dest":22,"term":"keyword_redo"},{"dest":22,"term":"keyword_rescue"},{"dest":22,"term":"keyword_retry"},{"dest":22,"term":"keyword_return"},{"dest":22,"term":"keyword_self"},{"dest":22,"term":"keyword_super"},{"dest":22,"term":"keyword_then"},{"dest":22,"term":"keyword_true"},{"dest":22,"term":"keyword_undef"},{"dest":22,"term":"keyword_unless"},{"dest":22,"term":"keyword_until"},{"dest":22,"term":"keyword_when"},{"dest":22,"term":"keyword_while"},{"dest":22,"term":"keyword_yield"},{"dest":22,"term":"kh_del_ht"},{"dest":22,"term":"kh_del_iv"},{"dest":22,"term":"kh_del_mt"},{"dest":22,"term":"kh_del_n2s"},{"dest":22,"term":"kh_del_st"},{"dest":22,"term":"KLVAR"},{"dest":22,"term":"lambda"},{"dest":22,"term":"lambda_body"},{"dest":22,"term":"last"},{"dest":22,"term":"lazy"},{"dest":22,"term":"Lazy"},{"dest":22,"term":"LC"},{"dest":22,"term":"ld"},{"dest":22,"term":"LD"},{"dest":22,"term":"ldexp"},{"dest":22,"term":"left"},{"dest":22,"term":"len"},{"dest":22,"term":"length"},{"dest":22,"term":"level"},{"dest":22,"term":"lfD"},{"dest":22,"term":"lhs"},{"dest":22,"term":"__libc_start_main"},{"dest":22,"term":"LII"},{"dest":22,"term":"lIJ"},{"dest":22,"term":"lim"},{"dest":22,"term":"line"},{"dest":22,"term":"__LINE__"},{"dest":22,"term":"LINE"},{"dest":22,"term":"lines"},{"dest":22,"term":"literal"},{"dest":22,"term":"literals"},{"dest":22,"term":"live_after_mark"},{"dest":22,"term":"ljust"},{"dest":22,"term":"ln"},{"dest":22,"term":"Ln"},{"dest":22,"term":"lo"},{"dest":22,"term":"local"},{"dest":22,"term":"LOCAL"},{"dest":22,"term":"LocalJumpError"},{"dest":22,"term":"localtime"},{"dest":22,"term":"localtime_r"},{"dest":22,"term":"local_variables"},{"dest":22,"term":"log"},{"dest":22,"term":"log10"},{"dest":22,"term":"log2"},{"dest":22,"term":"long"},{"dest":22,"term":"longjmp"},{"dest":22,"term":"lookahead"},{"dest":22,"term":"loop"},{"dest":22,"term":"low"},{"dest":22,"term":"lround"},{"dest":22,"term":"LS"},{"dest":22,"term":"lstrip"},{"dest":22,"term":"LVAR"},{"dest":22,"term":"machine"},{"dest":22,"term":"main"},{"dest":22,"term":"make_curry"},{"dest":22,"term":"map"},{"dest":22,"term":"match"},{"dest":22,"term":"matched"},{"dest":22,"term":"Math"},{"dest":22,"term":"max"},{"dest":22,"term":"max_by"},{"dest":22,"term":"max_cmp"},{"dest":22,"term":"May"},{"dest":22,"term":"mday"},{"dest":22,"term":"member"},{"dest":22,"term":"__members__"},{"dest":22,"term":"members"},{"dest":22,"term":"memchr"},{"dest":22,"term":"memcmp"},{"dest":22,"term":"memcpy"},{"dest":22,"term":"memmove"},{"dest":22,"term":"memory"},{"dest":22,"term":"memset"},{"dest":22,"term":"merge"},{"dest":22,"term":"mesg"},{"dest":22,"term":"message"},{"dest":22,"term":"meth"},{"dest":22,"term":"__method__"},{"dest":22,"term":"method"},{"dest":22,"term":"method_call"},{"dest":22,"term":"method_missing"},{"dest":22,"term":"method_removed"},{"dest":22,"term":"methods"},{"dest":22,"term":"mid"},{"dest":22,"term":"min"},{"dest":22,"term":"min_by"},{"dest":22,"term":"min_cmp"},{"dest":22,"term":"minmax"},{"dest":22,"term":"minmax_by"},{"dest":22,"term":"mktime"},{"dest":22,"term":"mlhs_basic"},{"dest":22,"term":"mlhs_inner"},{"dest":22,"term":"mlhs_item"},{"dest":22,"term":"mlhs_list"},{"dest":22,"term":"mlhs_node"},{"dest":22,"term":"mlhs_post"},{"dest":22,"term":"mode"},{"dest":22,"term":"modified"},{"dest":22,"term":"modifier_if"},{"dest":22,"term":"modifier_rescue"},{"dest":22,"term":"modifier_unless"},{"dest":22,"term":"modifier_until"},{"dest":22,"term":"modifier_while"},{"dest":22,"term":"module"},{"dest":22,"term":"Module"},{"dest":22,"term":"module_eval"},{"dest":22,"term":"module_function"},{"dest":22,"term":"modules"},{"dest":22,"term":"mon"},{"dest":22,"term":"Mon"},{"dest":22,"term":"month"},{"dest":22,"term":"mrb_ary_delete_at"},{"dest":22,"term":"mrb_ary_new_from_values"},{"dest":22,"term":"mrb_ary_plus"},{"dest":22,"term":"mrb_ary_pop"},{"dest":22,"term":"mrb_ary_push"},{"dest":22,"term":"mrb_ary_push_m"},{"dest":22,"term":"mrb_ary_resize"},{"dest":22,"term":"mrb_ary_reverse"},{"dest":22,"term":"mrb_ary_set"},{"dest":22,"term":"mrb_ary_shift"},{"dest":22,"term":"mrb_ary_splice"},{"dest":22,"term":"mrb_ary_times"},{"dest":22,"term":"mrb_ary_unshift"},{"dest":22,"term":"mrb_ary_unshift_m"},{"dest":22,"term":"mrb_assoc_new"},{"dest":22,"term":"mrb_data_init"},{"dest":22,"term":"mrb_debug_get_line"},{"dest":22,"term":"mrb_debug_info_alloc"},{"dest":22,"term":"mrb_debug_info_append_file"},{"dest":22,"term":"mrb_debug_info_free"},{"dest":22,"term":"mrb_field_write_barrier"},{"dest":22,"term":"mrb_gc_mark"},{"dest":22,"term":"MRB_GC_STATE_ROOT"},{"dest":22,"term":"MRB_GC_STATE_SWEEP"},{"dest":22,"term":"mrb_gc_unregister"},{"dest":22,"term":"mrb_i_mt_state"},{"dest":22,"term":"mrb_incremental_gc"},{"dest":22,"term":"mrb_malloc"},{"dest":22,"term":"mrb_mod_s_nesting"},{"dest":22,"term":"mrb_obj_value"},{"dest":22,"term":"mrb_random_init"},{"dest":22,"term":"mrb_random_srand"},{"dest":22,"term":"mrb_realloc"},{"dest":22,"term":"mrb_str_format"},{"dest":22,"term":"MRB_TT_DATA"},{"dest":22,"term":"MRB_TT_FIBER"},{"dest":22,"term":"MRB_TT_FREE"},{"dest":22,"term":"mrb_vm_const_get"},{"dest":22,"term":"mrb_vm_exec"},{"dest":22,"term":"mrb_write_barrier"},{"dest":22,"term":"mrhs"},{"dest":22,"term":"mruby"},{"dest":22,"term":"MRUBY_COPYRIGHT"},{"dest":22,"term":"MRUBY_DESCRIPTION"},{"dest":22,"term":"MRUBY_RELEASE_DATE"},{"dest":22,"term":"MRUBY_RELEASE_NO"},{"dest":22,"term":"MRUBY_VERSION"},{"dest":22,"term":"name"},{"dest":22,"term":"named"},{"dest":22,"term":"NameError"},{"dest":22,"term":"names"},{"dest":22,"term":"nan"},{"dest":22,"term":"NaN"},{"dest":22,"term":"NAN"},{"dest":22,"term":"nesting"},{"dest":22,"term":"new"},{"dest":22,"term":"new_args"},{"dest":22,"term":"new_key"},{"dest":22,"term":"new_msym"},{"dest":22,"term":"next"},{"dest":22,"term":"next_values"},{"dest":22,"term":"nil"},{"dest":22,"term":"NilClass"},{"dest":22,"term":"nl"},{"dest":22,"term":"nlocals"},{"dest":22,"term":"nLVAR"},{"dest":22,"term":"nMATZ0000IREP"},{"dest":22,"term":"NODE_DREGX"},{"dest":22,"term":"NODE_DSTR"},{"dest":22,"term":"NODE_DXSTR"},{"dest":22,"term":"NODE_FALSE"},{"dest":22,"term":"NODE_NEGATE"},{"dest":22,"term":"NODE_NIL"},{"dest":22,"term":"NODE_REDO"},{"dest":22,"term":"NODE_RETRY"},{"dest":22,"term":"NODE_SELF"},{"dest":22,"term":"NODE_TRUE"},{"dest":22,"term":"NODE_UNDEF"},{"dest":22,"term":"NODE_ZSUPER"},{"dest":22,"term":"NoMemoryError"},{"dest":22,"term":"NoMethodError"},{"dest":22,"term":"none"},{"dest":22,"term":"NONE"},{"dest":22,"term":"norm"},{"dest":22,"term":"not"},{"dest":22,"term":"NotImplementedError"},{"dest":22,"term":"Nov"},{"dest":22,"term":"now"},{"dest":22,"term":"Np"},{"dest":22,"term":"nregs"},{"dest":22,"term":"num"},{"dest":22,"term":"number"},{"dest":22,"term":"numbered"},{"dest":22,"term":"numeric"},{"dest":22,"term":"Numeric"},{"dest":22,"term":"obj"},{"dest":22,"term":"object"},{"dest":22,"term":"Object"},{"dest":22,"term":"object_id"},{"dest":22,"term":"ObjectSpace"},{"dest":22,"term":"oct"},{"dest":22,"term":"Oct"},{"dest":22,"term":"offset"},{"dest":22,"term":"on"},{"dest":22,"term":"On"},{"dest":22,"term":"only"},{"dest":22,"term":"Oo"},{"dest":22,"term":"op"},{"dest":22,"term":"Op"},{"dest":22,"term":"operation"},{"dest":22,"term":"operation2"},{"dest":22,"term":"operation3"},{"dest":22,"term":"OP_NOP"},{"dest":22,"term":"OP_STOP"},{"dest":22,"term":"opt_block_arg"},{"dest":22,"term":"opt_block_param"},{"dest":22,"term":"opt_bv_decl"},{"dest":22,"term":"opt_call_args"},{"dest":22,"term":"opt_else"},{"dest":22,"term":"opt_ensure"},{"dest":22,"term":"opt_f_block_arg"},{"dest":22,"term":"opt_nl"},{"dest":22,"term":"opt_paren_args"},{"dest":22,"term":"opt_rescue"},{"dest":22,"term":"opt_terms"},{"dest":22,"term":"or"},{"dest":22,"term":"ord"},{"dest":22,"term":"orig"},{"dest":22,"term":"other"},{"dest":22,"term":"__outer__"},{"dest":22,"term":"P9o"},{"dest":22,"term":"padding"},{"dest":22,"term":"pad_repetitions"},{"dest":22,"term":"padstr"},{"dest":22,"term":"parameters"},{"dest":22,"term":"paren_args"},{"dest":22,"term":"partition"},{"dest":22,"term":"pattern"},{"dest":22,"term":"PC"},{"dest":22,"term":"peek"},{"dest":22,"term":"peek_values"},{"dest":22,"term":"permutation"},{"dest":22,"term":"plen"},{"dest":22,"term":"point"},{"dest":22,"term":"pop"},{"dest":22,"term":"popping"},{"dest":22,"term":"pos"},{"dest":22,"term":"posnum"},{"dest":22,"term":"post"},{"dest":22,"term":"pow"},{"dest":22,"term":"pp"},{"dest":22,"term":"pproc"},{"dest":22,"term":"pre"},{"dest":22,"term":"precision"},{"dest":22,"term":"prefix"},{"dest":22,"term":"prepend"},{"dest":22,"term":"prepended"},{"dest":22,"term":"prepend_features"},{"dest":22,"term":"primary"},{"dest":22,"term":"primary_value"},{"dest":22,"term":"print"},{"dest":22,"term":"printf"},{"dest":22,"term":"__printstr__"},{"dest":22,"term":"private"},{"dest":22,"term":"private_methods"},{"dest":22,"term":"prl"},{"dest":22,"term":"proc"},{"dest":22,"term":"Proc"},{"dest":22,"term":"program"},{"dest":22,"term":"protected"},{"dest":22,"term":"protected_methods"},{"dest":22,"term":"ps"},{"dest":22,"term":"public"},{"dest":22,"term":"public_methods"},{"dest":22,"term":"push"},{"dest":22,"term":"putchar"},{"dest":22,"term":"puts"},{"dest":22,"term":"quo"},{"dest":22,"term":"raise"},{"dest":22,"term":"rand"},{"dest":22,"term":"Random"},{"dest":22,"term":"range"},{"dest":22,"term":"Range"},{"dest":22,"term":"RangeError"},{"dest":22,"term":"rassoc"},{"dest":22,"term":"rb"},{"dest":22,"term":"RB"},{"dest":22,"term":"rbracket"},{"dest":22,"term":"RC"},{"dest":22,"term":"read_debug_record"},{"dest":22,"term":"readint_mrb_int"},{"dest":22,"term":"read_irep_record_1"},{"dest":22,"term":"read_lv_record"},{"dest":22,"term":"read_section_debug"},{"dest":22,"term":"read_section_lv"},{"dest":22,"term":"realloc"},{"dest":22,"term":"redo"},{"dest":22,"term":"reduce"},{"dest":22,"term":"reg"},{"dest":22,"term":"regexp"},{"dest":22,"term":"Regexp"},{"dest":22,"term":"RegexpError"},{"dest":22,"term":"rehash"},{"dest":22,"term":"reject"},{"dest":22,"term":"remove_class_variable"},{"dest":22,"term":"remove_const"},{"dest":22,"term":"remove_instance_variable"},{"dest":22,"term":"remove_method"},{"dest":22,"term":"replace"},{"dest":22,"term":"req"},{"dest":22,"term":"required"},{"dest":22,"term":"res"},{"dest":22,"term":"rescue"},{"dest":22,"term":"resize_capa"},{"dest":22,"term":"rest"},{"dest":22,"term":"restarg_mark"},{"dest":22,"term":"result"},{"dest":22,"term":"resume"},{"dest":22,"term":"reswords"},{"dest":22,"term":"ret"},{"dest":22,"term":"retry"},{"dest":22,"term":"return"},{"dest":22,"term":"reverse"},{"dest":22,"term":"reverse_each"},{"dest":22,"term":"rewind"},{"dest":22,"term":"right"},{"dest":22,"term":"rindex"},{"dest":22,"term":"rjust"},{"dest":22,"term":"rotate"},{"dest":22,"term":"round"},{"dest":22,"term":"row"},{"dest":22,"term":"rparen"},{"dest":22,"term":"rpartition"},{"dest":22,"term":"rs_len"},{"dest":22,"term":"rstrip"},{"dest":22,"term":"RUBY_ENGINE"},{"dest":22,"term":"RUBY_ENGINE_VERSION"},{"dest":22,"term":"RUBY_VERSION"},{"dest":22,"term":"RuntimeError"},{"dest":22,"term":"sample"},{"dest":22,"term":"Sat"},{"dest":22,"term":"satisfied"},{"dest":22,"term":"scan"},{"dest":22,"term":"SClass"},{"dest":22,"term":"scope"},{"dest":22,"term":"scope_new"},{"dest":22,"term":"script"},{"dest":22,"term":"ScriptError"},{"dest":22,"term":"sec"},{"dest":22,"term":"select"},{"dest":22,"term":"self"},{"dest":22,"term":"self_arity"},{"dest":22,"term":"__send__"},{"dest":22,"term":"send"},{"dest":22,"term":"sep"},{"dest":22,"term":"Sep"},{"dest":22,"term":"sequence"},{"dest":22,"term":"set"},{"dest":22,"term":"set_backtrace"},{"dest":22,"term":"setbyte"},{"dest":22,"term":"_setjmp"},{"dest":22,"term":"shift"},{"dest":22,"term":"shuffle"},{"dest":22,"term":"sin"},{"dest":22,"term":"singleton"},{"dest":22,"term":"singleton_class"},{"dest":22,"term":"singleton_methods"},{"dest":22,"term":"sinh"},{"dest":22,"term":"size"},{"dest":22,"term":"sl"},{"dest":22,"term":"slice"},{"dest":22,"term":"snprintf"},{"dest":22,"term":"so"},{"dest":22,"term":"So"},{"dest":22,"term":"sort"},{"dest":22,"term":"sort_by"},{"dest":22,"term":"__sort_sub__"},{"dest":22,"term":"source_location"},{"dest":22,"term":"Sp"},{"dest":22,"term":"spaces"},{"dest":22,"term":"specifier"},{"dest":22,"term":"splice"},{"dest":22,"term":"split"},{"dest":22,"term":"sprintf"},{"dest":22,"term":"sqrt"},{"dest":22,"term":"srand"},{"dest":22,"term":"__stack_chk_fail"},{"dest":22,"term":"StandardError"},{"dest":22,"term":"start"},{"dest":22,"term":"state"},{"dest":22,"term":"stderr"},{"dest":22,"term":"stdin"},{"dest":22,"term":"stdout"},{"dest":22,"term":"step"},{"dest":22,"term":"step_ratio"},{"dest":22,"term":"stmt"},{"dest":22,"term":"stmts"},{"dest":22,"term":"stop_exc"},{"dest":22,"term":"StopIteration"},{"dest":22,"term":"store"},{"dest":22,"term":"str"},{"dest":22,"term":"str2"},{"dest":22,"term":"strchr"},{"dest":22,"term":"strcmp"},{"dest":22,"term":"str_each"},{"dest":22,"term":"string"},{"dest":22,"term":"String"},{"dest":22,"term":"string_interp"},{"dest":22,"term":"string_rep"},{"dest":22,"term":"strip"},{"dest":22,"term":"strlen"},{"dest":22,"term":"str_make_shared"},{"dest":22,"term":"strncmp"},{"dest":22,"term":"strncpy"},{"dest":22,"term":"strtoul"},{"dest":22,"term":"struct"},{"dest":22,"term":"Struct"},{"dest":22,"term":"sub"},{"dest":22,"term":"__sub_replace"},{"dest":22,"term":"succ"},{"dest":22,"term":"Sun"},{"dest":22,"term":"super"},{"dest":22,"term":"superclass"},{"dest":22,"term":"supported"},{"dest":22,"term":"__svalue"},{"dest":22,"term":"SVD"},{"dest":22,"term":"swapcase"},{"dest":22,"term":"sym"},{"dest":22,"term":"symbol"},{"dest":22,"term":"Symbol"},{"dest":22,"term":"symbols"},{"dest":22,"term":"sym_inspect"},{"dest":22,"term":"syntax"},{"dest":22,"term":"SyntaxError"},{"dest":22,"term":"_sys_fail"},{"dest":22,"term":"SystemCallError"},{"dest":22,"term":"SystemStackError"},{"dest":22,"term":"TA"},{"dest":22,"term":"tail"},{"dest":22,"term":"take"},{"dest":22,"term":"taken"},{"dest":22,"term":"take_while"},{"dest":22,"term":"tAMPER"},{"dest":22,"term":"tan"},{"dest":22,"term":"tANDDOT"},{"dest":22,"term":"tANDOP"},{"dest":22,"term":"tanh"},{"dest":22,"term":"tap"},{"dest":22,"term":"tAREF"},{"dest":22,"term":"T_ARRAY"},{"dest":22,"term":"tASET"},{"dest":22,"term":"tASSOC"},{"dest":22,"term":"TB"},{"dest":22,"term":"tBACK_REF"},{"dest":22,"term":"TbG"},{"dest":22,"term":"T_CLASS"},{"dest":22,"term":"tCMP"},{"dest":22,"term":"tCOLON2"},{"dest":22,"term":"tCOLON3"},{"dest":22,"term":"tCONSTANT"},{"dest":22,"term":"T_CPTR"},{"dest":22,"term":"tCVAR"},{"dest":22,"term":"T_DATA"},{"dest":22,"term":"tDOT2"},{"dest":22,"term":"tDOT3"},{"dest":22,"term":"TeD"},{"dest":22,"term":"T_ENV"},{"dest":22,"term":"tEQ"},{"dest":22,"term":"tEQQ"},{"dest":22,"term":"term"},{"dest":22,"term":"terms"},{"dest":22,"term":"T_EXCEPTION"},{"dest":22,"term":"T_FALSE"},{"dest":22,"term":"T_FIBER"},{"dest":22,"term":"tFID"},{"dest":22,"term":"T_FILE"},{"dest":22,"term":"T_FIXNUM"},{"dest":22,"term":"tFLOAT"},{"dest":22,"term":"T_FLOAT"},{"dest":22,"term":"T_FREE"},{"dest":22,"term":"tGEQ"},{"dest":22,"term":"tGVAR"},{"dest":22,"term":"T_HASH"},{"dest":22,"term":"tHD_LITERAL_DELIM"},{"dest":22,"term":"tHD_STRING_MID"},{"dest":22,"term":"tHD_STRING_PART"},{"dest":22,"term":"then"},{"dest":22,"term":"tHEREDOC_BEG"},{"dest":22,"term":"tHEREDOC_END"},{"dest":22,"term":"this"},{"dest":22,"term":"T_ICLASS"},{"dest":22,"term":"tIDENTIFIER"},{"dest":22,"term":"time"},{"dest":22,"term":"Time"},{"dest":22,"term":"times"},{"dest":22,"term":"tINTEGER"},{"dest":22,"term":"tIVAR"},{"dest":22,"term":"tLABEL"},{"dest":22,"term":"tLABEL_END"},{"dest":22,"term":"tLAMBDA"},{"dest":22,"term":"tLAMBEG"},{"dest":22,"term":"tLAST_TOKEN"},{"dest":22,"term":"tLBRACE"},{"dest":22,"term":"tLBRACE_ARG"},{"dest":22,"term":"tLBRACK"},{"dest":22,"term":"tLEQ"},{"dest":22,"term":"tLITERAL_DELIM"},{"dest":22,"term":"tLOWEST"},{"dest":22,"term":"tLPAREN"},{"dest":22,"term":"tLPAREN_ARG"},{"dest":22,"term":"tLSHFT"},{"dest":22,"term":"tMATCH"},{"dest":22,"term":"T_MODULE"},{"dest":22,"term":"tmp"},{"dest":22,"term":"tNEQ"},{"dest":22,"term":"tNMATCH"},{"dest":22,"term":"tNTH_REF"},{"dest":22,"term":"to_ary"},{"dest":22,"term":"T_OBJECT"},{"dest":22,"term":"to_enum"},{"dest":22,"term":"to_h"},{"dest":22,"term":"to_hash"},{"dest":22,"term":"to_i"},{"dest":22,"term":"to_int"},{"dest":22,"term":"TOJ"},{"dest":22,"term":"TOLERANCE"},{"dest":22,"term":"tolower"},{"dest":22,"term":"tOP_ASGN"},{"dest":22,"term":"top_compstmt"},{"dest":22,"term":"to_proc"},{"dest":22,"term":"top_stmt"},{"dest":22,"term":"top_stmts"},{"dest":22,"term":"tOROP"},{"dest":22,"term":"to_s"},{"dest":22,"term":"to_str"},{"dest":22,"term":"to_sym"},{"dest":22,"term":"TOTAL"},{"dest":22,"term":"toupper"},{"dest":22,"term":"tPOW"},{"dest":22,"term":"T_PROC"},{"dest":22,"term":"trailer"},{"dest":22,"term":"T_RANGE"},{"dest":22,"term":"transfer"},{"dest":22,"term":"transform_keys"},{"dest":22,"term":"transform_values"},{"dest":22,"term":"transpose"},{"dest":22,"term":"tREGEXP"},{"dest":22,"term":"tREGEXP_BEG"},{"dest":22,"term":"tREGEXP_END"},{"dest":22,"term":"tRPAREN"},{"dest":22,"term":"tRSHFT"},{"dest":22,"term":"true"},{"dest":22,"term":"TrueClass"},{"dest":22,"term":"truncate"},{"dest":22,"term":"try_convert"},{"dest":22,"term":"T_SCLASS"},{"dest":22,"term":"tSTAR"},{"dest":22,"term":"tSTRING"},{"dest":22,"term":"T_STRING"},{"dest":22,"term":"tSTRING_BEG"},{"dest":22,"term":"tSTRING_DVAR"},{"dest":22,"term":"tSTRING_MID"},{"dest":22,"term":"tSTRING_PART"},{"dest":22,"term":"tSYMBEG"},{"dest":22,"term":"T_SYMBOL"},{"dest":22,"term":"tSYMBOLS_BEG"},{"dest":22,"term":"tt"},{"dest":22,"term":"T_TRUE"},{"dest":22,"term":"Tue"},{"dest":22,"term":"tUMINUS"},{"dest":22,"term":"tUMINUS_NUM"},{"dest":22,"term":"T_UNDEF"},{"dest":22,"term":"tUPLUS"},{"dest":22,"term":"twice"},{"dest":22,"term":"tWORDS_BEG"},{"dest":22,"term":"tXSTRING"},{"dest":22,"term":"tXSTRING_BEG"},{"dest":22,"term":"type"},{"dest":22,"term":"TypeError"},{"dest":22,"term":"umrb_obj_value"},{"dest":22,"term":"undef"},{"dest":22,"term":"undefined"},{"dest":22,"term":"undef_list"},{"dest":22,"term":"undef_method"},{"dest":22,"term":"uniq"},{"dest":22,"term":"unless"},{"dest":22,"term":"unshift"},{"dest":22,"term":"until"},{"dest":22,"term":"upcase"},{"dest":22,"term":"__update"},{"dest":22,"term":"update"},{"dest":22,"term":"upto"},{"dest":22,"term":"usec"},{"dest":22,"term":"useless"},{"dest":22,"term":"utc"},{"dest":22,"term":"v0000"},{"dest":22,"term":"val"},{"dest":22,"term":"validated"},{"dest":22,"term":"vals"},{"dest":22,"term":"value"},{"dest":22,"term":"values"},{"dest":22,"term":"values_at"},{"dest":22,"term":"variable"},{"dest":22,"term":"var_lhs"},{"dest":22,"term":"var_ref"},{"dest":22,"term":"verbose"},{"dest":22,"term":"version"},{"dest":22,"term":"vm"},{"dest":22,"term":"Vm"},{"dest":22,"term":"warn"},{"dest":22,"term":"wday"},{"dest":22,"term":"Wed"},{"dest":22,"term":"when"},{"dest":22,"term":"while"},{"dest":22,"term":"width"},{"dest":22,"term":"with_index"},{"dest":22,"term":"with_object"},{"dest":22,"term":"words"},{"dest":22,"term":"x86_64"},{"dest":22,"term":"xstring"},{"dest":22,"term":"yday"},{"dest":22,"term":"year"},{"dest":22,"term":"yield"},{"dest":22,"term":"yielder"},{"dest":22,"term":"Yielder"},{"dest":22,"term":"yield_self"},{"dest":22,"term":"zip"},{"dest":22,"term":"zone"}],[{"dest":1,"term":"a"},{"dest":1,"term":"b"},{"dest":1,"term":"c"},{"dest":1,"term":"d"},{"dest":2,"term":"a"},{"dest":2,"term":"b"},{"dest":2,"term":"c"},{"dest":2,"term":"d"},{"dest":3,"term":"a"},{"dest":3,"term":"b"},{"dest":3,"term":"c"},{"dest":3,"term":"d"},{"dest":4,"term":"a"},{"dest":4,"term":"b"},{"dest":4,"term":"c"},{"dest":4,"term":"d"},{"dest":5,"term":"return"},{"dest":5,"term":"yield"},{"dest":5,"term":"continue"},{"dest":5,"term":"break"},{"dest":5,"term":"next"},{"dest":6,"term":" "}],[{"dest":23,"term":"("}],[{"dest":24,"term":"("}],[{"dest":25,"term":"("}],[{"dest":26,"term":"a"},{"dest":26,"term":"b"},{"dest":26,"term":"c"},{"dest":26,"term":"d"},{"dest":27,"term":"a"},{"dest":27,"term":"b"},{"dest":27,"term":"c"},{"dest":27,"term":"d"},{"dest":26,"term":" "}],[{"dest":28,"term":"a"},{"dest":28,"term":"b"},{"dest":28,"term":"c"},{"dest":28,"term":"d"},{"dest":29,"term":"a"},{"dest":29,"term":"b"},{"dest":29,"term":"c"},{"dest":29,"term":"d"},{"dest":28,"term":" "}],[{"dest":30,"term":"a"},{"dest":30,"term":"b"},{"dest":30,"term":"c"},{"dest":30,"term":"d"},{"dest":31,"term":"a"},{"dest":31,"term":"b"},{"dest":31,"term":"c"},{"dest":31,"term":"d"},{"dest":30,"term":" "}],[{"dest":15,"term":")"}],[{"dest":23,"term":","}],[{"dest":15,"term":")"}],[{"dest":24,"term":","}],[{"dest":15,"term":")"}],[{"dest":25,"term":","}]]} \ No newline at end of file diff --git a/fuzzers/baby_fuzzer_gramatron/auto.postcard b/fuzzers/baby_fuzzer_gramatron/auto.postcard index c68b218ccb..e6db94f9e7 100644 Binary files a/fuzzers/baby_fuzzer_gramatron/auto.postcard and b/fuzzers/baby_fuzzer_gramatron/auto.postcard differ diff --git a/fuzzers/baby_fuzzer_gramatron/src/main.rs b/fuzzers/baby_fuzzer_gramatron/src/main.rs index ada8c1dfb2..357bd3b17a 100644 --- a/fuzzers/baby_fuzzer_gramatron/src/main.rs +++ b/fuzzers/baby_fuzzer_gramatron/src/main.rs @@ -24,7 +24,7 @@ use libafl::{ stages::mutational::StdMutationalStage, state::StdState, }; -use libafl_bolts::{current_nanos, rands::StdRand, tuples::tuple_list}; +use libafl_bolts::{rands::StdRand, tuples::tuple_list}; /// Coverage map with explicit assignments due to the lack of instrumentation static mut SIGNALS: [u8; 16] = [0; 16]; @@ -69,7 +69,7 @@ pub fn main() { // create a State from scratch let mut state = StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryCorpus::new(), // Corpus in which we store solutions (crashes in this example), diff --git a/fuzzers/baby_fuzzer_grimoire/Cargo.toml b/fuzzers/baby_fuzzer_grimoire/Cargo.toml index a5f382fb20..eff4f6da91 100644 --- a/fuzzers/baby_fuzzer_grimoire/Cargo.toml +++ b/fuzzers/baby_fuzzer_grimoire/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "baby_fuzzer_grimoire" -version = "0.11.2" +version = "0.12.0" authors = ["Andrea Fioraldi ", "Dominik Maier "] edition = "2021" diff --git a/fuzzers/baby_fuzzer_grimoire/src/main.rs b/fuzzers/baby_fuzzer_grimoire/src/main.rs index b3be156e47..60162e5919 100644 --- a/fuzzers/baby_fuzzer_grimoire/src/main.rs +++ b/fuzzers/baby_fuzzer_grimoire/src/main.rs @@ -15,12 +15,13 @@ use libafl::{ GrimoireRandomDeleteMutator, GrimoireRecursiveReplacementMutator, GrimoireStringReplacementMutator, Tokens, }, - observers::StdMapObserver, + observers::{CanTrack, StdMapObserver}, schedulers::QueueScheduler, stages::{mutational::StdMutationalStage, GeneralizationStage}, - state::{HasMetadata, StdState}, + state::StdState, + HasMetadata, }; -use libafl_bolts::{current_nanos, rands::StdRand, tuples::tuple_list, AsSlice}; +use libafl_bolts::{rands::StdRand, tuples::tuple_list, AsSlice}; /// Coverage map with explicit assignments due to the lack of instrumentation static mut SIGNALS: [u8; 16] = [0; 16]; @@ -82,9 +83,11 @@ pub fn main() { }; // Create an observation channel using the signals map - let observer = unsafe { StdMapObserver::from_mut_ptr("signals", SIGNALS_PTR, SIGNALS.len()) }; + let observer = unsafe { + StdMapObserver::from_mut_ptr("signals", SIGNALS_PTR, SIGNALS.len()).track_novelties() + }; // Feedback to rate the interestingness of an input - let mut feedback = MaxMapFeedback::tracking(&observer, false, true); + let mut feedback = MaxMapFeedback::new(&observer); // A feedback to choose if an input is a solution or not let mut objective = CrashFeedback::new(); @@ -92,7 +95,7 @@ pub fn main() { // create a State from scratch let mut state = StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryCorpus::new(), // Corpus in which we store solutions (crashes in this example), diff --git a/fuzzers/baby_fuzzer_minimizing/Cargo.toml b/fuzzers/baby_fuzzer_minimizing/Cargo.toml index 45493b5cad..3063b8f7c9 100644 --- a/fuzzers/baby_fuzzer_minimizing/Cargo.toml +++ b/fuzzers/baby_fuzzer_minimizing/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "baby_fuzzer_minimizing" -version = "0.11.2" +version = "0.12.0" authors = ["Andrea Fioraldi ", "Dominik Maier ", "Addison Crump "] edition = "2021" diff --git a/fuzzers/baby_fuzzer_minimizing/src/main.rs b/fuzzers/baby_fuzzer_minimizing/src/main.rs index 1f7fc51701..f4b23658ea 100644 --- a/fuzzers/baby_fuzzer_minimizing/src/main.rs +++ b/fuzzers/baby_fuzzer_minimizing/src/main.rs @@ -36,7 +36,7 @@ pub fn main() -> Result<(), Error> { // Create an observation channel using the signals map let observer = unsafe { StdMapObserver::from_mut_ptr("signals", SIGNALS_PTR, SIGNALS.len()) }; - let factory = MapEqualityFactory::with_observer(&observer); + let factory = MapEqualityFactory::new(&observer); // Feedback to rate the interestingness of an input let mut feedback = MaxMapFeedback::new(&observer); @@ -55,9 +55,9 @@ pub fn main() -> Result<(), Error> { // create a State from scratch let mut state = StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance - InMemoryOnDiskCorpus::new(&corpus_dir).unwrap(), + InMemoryOnDiskCorpus::new(corpus_dir).unwrap(), // Corpus in which we store solutions (crashes in this example), // on disk so the user can get them after stopping the fuzzer OnDiskCorpus::new(&solution_dir).unwrap(), @@ -108,7 +108,7 @@ pub fn main() -> Result<(), Error> { let minimized_dir = PathBuf::from("./minimized"); let mut state = StdState::new( - StdRand::with_seed(current_nanos()), + StdRand::new(), InMemoryOnDiskCorpus::new(minimized_dir).unwrap(), InMemoryCorpus::new(), &mut (), @@ -124,7 +124,7 @@ pub fn main() -> Result<(), Error> { let minimizer = StdScheduledMutator::new(havoc_mutations()); let mut stages = tuple_list!(StdTMinMutationalStage::new( minimizer, - CrashFeedbackFactory::default(), + CrashFeedback::new(), 1 << 10 )); @@ -138,7 +138,7 @@ pub fn main() -> Result<(), Error> { state.load_initial_inputs_forced(&mut fuzzer, &mut executor, &mut mgr, &[solution_dir])?; - state.set_corpus_idx(CorpusId::from(0usize))?; + state.set_corpus_idx(CorpusId::from(0_usize))?; stages.perform_all(&mut fuzzer, &mut executor, &mut state, &mut mgr)?; Ok(()) diff --git a/fuzzers/baby_fuzzer_multi/src/main.rs b/fuzzers/baby_fuzzer_multi/src/main.rs index f462c20b70..8956f8c1ac 100644 --- a/fuzzers/baby_fuzzer_multi/src/main.rs +++ b/fuzzers/baby_fuzzer_multi/src/main.rs @@ -21,7 +21,7 @@ use libafl::{ state::StdState, Evaluator, }; -use libafl_bolts::{current_nanos, rands::StdRand, tuples::tuple_list, AsSlice}; +use libafl_bolts::{rands::StdRand, tuples::tuple_list, AsSlice}; /// Coverage map with explicit assignments due to the lack of instrumentation static mut SIGNALS: [u8; 128] = [0; 128]; @@ -100,7 +100,7 @@ pub fn main() { // create a State from scratch let mut state = StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryCorpus::new(), // Corpus in which we store solutions (crashes in this example), diff --git a/fuzzers/baby_fuzzer_nautilus/Cargo.toml b/fuzzers/baby_fuzzer_nautilus/Cargo.toml index cae97594f3..6f8c94b8df 100644 --- a/fuzzers/baby_fuzzer_nautilus/Cargo.toml +++ b/fuzzers/baby_fuzzer_nautilus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "baby_fuzzer_nautilus" -version = "0.11.2" +version = "0.12.0" authors = ["Andrea Fioraldi ", "Dominik Maier "] edition = "2021" diff --git a/fuzzers/baby_fuzzer_nautilus/src/main.rs b/fuzzers/baby_fuzzer_nautilus/src/main.rs index 5c61c2bbc2..75a2fcbe9d 100644 --- a/fuzzers/baby_fuzzer_nautilus/src/main.rs +++ b/fuzzers/baby_fuzzer_nautilus/src/main.rs @@ -18,9 +18,10 @@ use libafl::{ observers::StdMapObserver, schedulers::QueueScheduler, stages::mutational::StdMutationalStage, - state::{HasMetadata, StdState}, + state::StdState, + HasMetadata, }; -use libafl_bolts::{current_nanos, rands::StdRand, tuples::tuple_list}; +use libafl_bolts::{rands::StdRand, tuples::tuple_list}; /// Coverage map with explicit assignments due to the lack of instrumentation static mut SIGNALS: [u8; 16] = [0; 16]; @@ -61,7 +62,7 @@ pub fn main() { // create a State from scratch let mut state = StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryCorpus::new(), // Corpus in which we store solutions (crashes in this example), diff --git a/fuzzers/baby_fuzzer_swap_differential/Cargo.toml b/fuzzers/baby_fuzzer_swap_differential/Cargo.toml index 08491cce07..2c1494acae 100644 --- a/fuzzers/baby_fuzzer_swap_differential/Cargo.toml +++ b/fuzzers/baby_fuzzer_swap_differential/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "baby_fuzzer_swap_differential" -version = "0.11.2" +version = "0.12.0" authors = ["Addison Crump "] edition = "2021" default-run = "fuzzer_sd" @@ -21,7 +21,7 @@ debug = true [build-dependencies] anyhow = "1" -bindgen = "0.63" +bindgen = "0.69.4" cc = "1.0" [dependencies] diff --git a/fuzzers/baby_fuzzer_swap_differential/Makefile.toml b/fuzzers/baby_fuzzer_swap_differential/Makefile.toml index 08b52d4b27..6ff335b2eb 100644 --- a/fuzzers/baby_fuzzer_swap_differential/Makefile.toml +++ b/fuzzers/baby_fuzzer_swap_differential/Makefile.toml @@ -19,6 +19,9 @@ command = "cargo" args = ["build" , "--profile", "${PROFILE}", "--bin", "${FUZZER_NAME}"] dependencies = [ "cc" ] +[tasks.build] +alias = "fuzzer" + # Run the fuzzer [tasks.run] command = "${CARGO_TARGET_DIR}/${PROFILE_DIR}/${FUZZER_NAME}" @@ -33,7 +36,7 @@ windows_alias = "unsupported" [tasks.test_unix] script_runner = "@shell" script=''' -timeout 30s ${CARGO_TARGET_DIR}/${PROFILE_DIR}/${FUZZER_NAME} >fuzz_stdout.log || true +timeout 30s ${CARGO_TARGET_DIR}/${PROFILE_DIR}/${FUZZER_NAME} | tee fuzz_stdout.log || true if grep -qa "objectives: 1" fuzz_stdout.log; then echo "Fuzzer is working" else @@ -50,4 +53,4 @@ clear = true script_runner="@shell" script=''' cargo clean -''' \ No newline at end of file +''' diff --git a/fuzzers/baby_fuzzer_swap_differential/build.rs b/fuzzers/baby_fuzzer_swap_differential/build.rs index 22c22830c7..6c34d1deda 100644 --- a/fuzzers/baby_fuzzer_swap_differential/build.rs +++ b/fuzzers/baby_fuzzer_swap_differential/build.rs @@ -12,7 +12,7 @@ fn main() -> anyhow::Result<()> { let bindings = bindgen::builder() .header("first.h") .header("second.h") - .parse_callbacks(Box::new(bindgen::CargoCallbacks)) + .parse_callbacks(Box::new(bindgen::CargoCallbacks::new())) .generate()?; // Write the generated bindings to an output file. diff --git a/fuzzers/baby_fuzzer_swap_differential/src/bin/libafl_cc.rs b/fuzzers/baby_fuzzer_swap_differential/src/bin/libafl_cc.rs index eae59d52db..ef5082c961 100644 --- a/fuzzers/baby_fuzzer_swap_differential/src/bin/libafl_cc.rs +++ b/fuzzers/baby_fuzzer_swap_differential/src/bin/libafl_cc.rs @@ -1,6 +1,6 @@ use std::env; -use libafl_cc::{ClangWrapper, CompilerWrapper, ToolWrapper}; +use libafl_cc::{ClangWrapper, ToolWrapper}; pub fn main() { let args: Vec = env::args().collect(); diff --git a/fuzzers/baby_fuzzer_swap_differential/src/main.rs b/fuzzers/baby_fuzzer_swap_differential/src/main.rs index ac4e5846ca..39edae9895 100644 --- a/fuzzers/baby_fuzzer_swap_differential/src/main.rs +++ b/fuzzers/baby_fuzzer_swap_differential/src/main.rs @@ -23,7 +23,7 @@ use libafl::{ stages::mutational::StdMutationalStage, state::{HasSolutions, StdState}, }; -use libafl_bolts::{current_nanos, rands::StdRand, tuples::tuple_list, AsSlice}; +use libafl_bolts::{rands::StdRand, tuples::tuple_list, AsSlice}; use libafl_targets::{edges_max_num, DifferentialAFLMapSwapObserver}; #[cfg(not(miri))] use mimalloc::MiMalloc; @@ -84,7 +84,7 @@ pub fn main() { } }; - let num_edges: usize = edges_max_num(); + let num_edges: usize = edges_max_num(); // upper bound #[cfg(feature = "multimap")] let ( @@ -186,7 +186,7 @@ pub fn main() { // create a State from scratch let mut state = StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryCorpus::new(), // Corpus in which we store solutions (crashes in this example), @@ -202,7 +202,7 @@ pub fn main() { // The Monitor trait define how the fuzzer stats are displayed to the user #[cfg(not(feature = "tui"))] - let mon = SimpleMonitor::new(|s| println!("{s}")); + let mon = SimpleMonitor::with_user_monitor(|s| println!("{s}")); #[cfg(feature = "tui")] let ui = TuiUI::new(String::from("Baby Fuzzer"), false); #[cfg(feature = "tui")] diff --git a/fuzzers/baby_fuzzer_tokens/Cargo.toml b/fuzzers/baby_fuzzer_tokens/Cargo.toml index dd1427e4c7..ce8461db02 100644 --- a/fuzzers/baby_fuzzer_tokens/Cargo.toml +++ b/fuzzers/baby_fuzzer_tokens/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "baby_fuzzer_tokens" -version = "0.11.2" +version = "0.12.0" authors = ["Andrea Fioraldi ", "Dominik Maier "] edition = "2021" diff --git a/fuzzers/baby_fuzzer_tokens/src/main.rs b/fuzzers/baby_fuzzer_tokens/src/main.rs index 49cd737a60..a47f76d677 100644 --- a/fuzzers/baby_fuzzer_tokens/src/main.rs +++ b/fuzzers/baby_fuzzer_tokens/src/main.rs @@ -1,6 +1,6 @@ #[cfg(windows)] use std::ptr::write_volatile; -use std::{fs, io::Read, path::PathBuf, ptr::write}; +use std::{fs, io::Read, path::PathBuf}; use libafl::{ corpus::{InMemoryCorpus, OnDiskCorpus}, @@ -16,7 +16,7 @@ use libafl::{ stages::mutational::StdMutationalStage, state::StdState, }; -use libafl_bolts::{current_nanos, rands::StdRand, tuples::tuple_list}; +use libafl_bolts::{rands::StdRand, tuples::tuple_list}; /// Coverage map with explicit assignments due to the lack of instrumentation static mut SIGNALS: [u8; 16] = [0; 16]; @@ -78,7 +78,7 @@ pub fn main() { // create a State from scratch let mut state = StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryCorpus::new(), // Corpus in which we store solutions (crashes in this example), diff --git a/fuzzers/baby_fuzzer_unicode/src/main.rs b/fuzzers/baby_fuzzer_unicode/src/main.rs index 55795b5149..f9742fea4f 100644 --- a/fuzzers/baby_fuzzer_unicode/src/main.rs +++ b/fuzzers/baby_fuzzer_unicode/src/main.rs @@ -20,7 +20,7 @@ use libafl::{ state::StdState, Evaluator, }; -use libafl_bolts::{current_nanos, rands::StdRand, tuples::tuple_list, AsSlice}; +use libafl_bolts::{rands::StdRand, tuples::tuple_list, AsSlice}; /// Coverage map with explicit assignments due to the lack of instrumentation static mut SIGNALS: [u8; 64] = [0; 64]; @@ -67,7 +67,7 @@ pub fn main() { // create a State from scratch let mut state = StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryCorpus::new(), // Corpus in which we store solutions (crashes in this example), diff --git a/fuzzers/baby_fuzzer_wasm/src/lib.rs b/fuzzers/baby_fuzzer_wasm/src/lib.rs index 434dcb7329..20f57a3072 100644 --- a/fuzzers/baby_fuzzer_wasm/src/lib.rs +++ b/fuzzers/baby_fuzzer_wasm/src/lib.rs @@ -11,19 +11,17 @@ use libafl::{ mutators::{havoc_mutations, StdScheduledMutator}, observers::StdMapObserver, schedulers::QueueScheduler, - stages::StdMutationalStage, + stages::{ExecutionCountRestartHelperMetadata, StdMutationalStage}, state::{HasSolutions, StdState}, Fuzzer, StdFuzzer, }; -use libafl_bolts::{ - current_nanos, rands::StdRand, serdeany::RegistryBuilder, tuples::tuple_list, AsSlice, -}; +use libafl_bolts::{rands::StdRand, serdeany::RegistryBuilder, tuples::tuple_list, AsSlice}; use wasm_bindgen::prelude::*; use web_sys::{Performance, Window}; use crate::utils::set_panic_hook; -// defined for internal use by libafl +// Defined for internal use by LibAFL #[no_mangle] #[allow(clippy::cast_sign_loss, clippy::cast_possible_truncation)] pub extern "C" fn external_current_millis() -> u64 { @@ -39,8 +37,14 @@ pub extern "C" fn external_current_millis() -> u64 { pub fn fuzz() { set_panic_hook(); + // We need to register the types as LibAFL doesn't support `SerdeAny` + // auto registration in non-standard environments. + // + // # Safety + // No concurrency in WASM so these accesses are not racing. unsafe { RegistryBuilder::register::>(); + RegistryBuilder::register::(); } let mut signals = [0u8; 64]; @@ -83,7 +87,7 @@ pub fn fuzz() { // create a State from scratch let mut state = StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryCorpus::new(), // In a "real" fuzzing campaign, you should stash solutions in a JS array instead diff --git a/fuzzers/baby_fuzzer_with_forkexecutor/Cargo.toml b/fuzzers/baby_fuzzer_with_forkexecutor/Cargo.toml index a109e46145..0543791e2f 100644 --- a/fuzzers/baby_fuzzer_with_forkexecutor/Cargo.toml +++ b/fuzzers/baby_fuzzer_with_forkexecutor/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "baby_fuzzer_with_forkexecutor" -version = "0.11.2" +version = "0.12.0" authors = ["Andrea Fioraldi ", "Dominik Maier "] edition = "2021" diff --git a/fuzzers/baby_fuzzer_with_forkexecutor/src/main.rs b/fuzzers/baby_fuzzer_with_forkexecutor/src/main.rs index cdf80e6fa8..7ab274bdcf 100644 --- a/fuzzers/baby_fuzzer_with_forkexecutor/src/main.rs +++ b/fuzzers/baby_fuzzer_with_forkexecutor/src/main.rs @@ -1,6 +1,6 @@ #[cfg(windows)] use std::ptr::write_volatile; -use std::{path::PathBuf, ptr::write, time::Duration}; +use std::{path::PathBuf, ptr::write}; use libafl::{ corpus::{InMemoryCorpus, OnDiskCorpus}, @@ -18,11 +18,10 @@ use libafl::{ state::StdState, }; use libafl_bolts::{ - current_nanos, rands::StdRand, shmem::{unix_shmem, ShMemProvider}, tuples::tuple_list, - AsMutSlice, AsSlice, + AsSlice, AsSliceMut, }; #[allow(clippy::similar_names)] @@ -30,7 +29,7 @@ pub fn main() { let mut shmem_provider = unix_shmem::UnixShMemProvider::new().unwrap(); let mut signals = shmem_provider.new_shmem(16).unwrap(); let signals_len = signals.as_slice().len(); - let signals_ptr = signals.as_mut_slice().as_mut_ptr(); + let signals_ptr = signals.as_slice_mut().as_mut_ptr(); let signals_set = |idx: usize| { unsafe { write(signals_ptr.add(idx), 1) }; @@ -76,7 +75,7 @@ pub fn main() { // create a State from scratch let mut state = StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryCorpus::new(), // Corpus in which we store solutions (crashes in this example), diff --git a/fuzzers/baby_no_std/Cargo.toml b/fuzzers/baby_no_std/Cargo.toml index 73ffe9d6fd..59f0fdbaf9 100644 --- a/fuzzers/baby_no_std/Cargo.toml +++ b/fuzzers/baby_no_std/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "baby_no_std" -version = "0.11.2" +version = "0.12.0" authors = ["Andrea Fioraldi ", "Dominik Maier "] edition = "2021" diff --git a/fuzzers/baby_no_std/src/main.rs b/fuzzers/baby_no_std/src/main.rs index 12b445268e..58b5acc28a 100644 --- a/fuzzers/baby_no_std/src/main.rs +++ b/fuzzers/baby_no_std/src/main.rs @@ -25,7 +25,7 @@ use libafl::{ stages::mutational::StdMutationalStage, state::StdState, }; -use libafl_bolts::{current_nanos, rands::StdRand, tuples::tuple_list, AsSlice}; +use libafl_bolts::{rands::StdRand, tuples::tuple_list, AsSlice}; #[cfg(any(windows, unix))] use libc::{abort, printf}; use static_alloc::Bump; @@ -98,7 +98,7 @@ pub extern "C" fn main(_argc: isize, _argv: *const *const u8) -> isize { // create a State from scratch let mut state = StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryCorpus::new(), // Corpus in which we store solutions (crashes in this example), @@ -118,7 +118,7 @@ pub extern "C" fn main(_argc: isize, _argv: *const *const u8) -> isize { #[cfg(any(windows, unix))] unsafe { let s = CString::new(s).unwrap(); - printf(b"%s\n\0".as_ptr().cast(), s.as_ptr()); + printf(c"%s\n".as_ptr().cast(), s.as_ptr()); } }); diff --git a/fuzzers/backtrace_baby_fuzzers/c_code_with_fork_executor/src/main.rs b/fuzzers/backtrace_baby_fuzzers/c_code_with_fork_executor/src/main.rs index 17667ce3dd..78cb2c7a2e 100644 --- a/fuzzers/backtrace_baby_fuzzers/c_code_with_fork_executor/src/main.rs +++ b/fuzzers/backtrace_baby_fuzzers/c_code_with_fork_executor/src/main.rs @@ -17,7 +17,6 @@ use libafl::{ state::StdState, }; use libafl_bolts::{ - current_nanos, ownedref::OwnedRefMut, rands::StdRand, shmem::{ShMemProvider, StdShMemProvider}, @@ -64,7 +63,7 @@ pub fn main() { // create a State from scratch let mut state = StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryCorpus::new(), // Corpus in which we store solutions (crashes in this example), diff --git a/fuzzers/backtrace_baby_fuzzers/c_code_with_inprocess_executor/src/main.rs b/fuzzers/backtrace_baby_fuzzers/c_code_with_inprocess_executor/src/main.rs index 295ddc4762..3dbe6f0ad0 100644 --- a/fuzzers/backtrace_baby_fuzzers/c_code_with_inprocess_executor/src/main.rs +++ b/fuzzers/backtrace_baby_fuzzers/c_code_with_inprocess_executor/src/main.rs @@ -16,7 +16,7 @@ use libafl::{ stages::mutational::StdMutationalStage, state::StdState, }; -use libafl_bolts::{current_nanos, rands::StdRand, tuples::tuple_list, AsSlice}; +use libafl_bolts::{rands::StdRand, tuples::tuple_list, AsSlice}; use libc::c_uchar; extern crate libc; @@ -51,7 +51,7 @@ pub fn main() { // create a State from scratch let mut state = StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryCorpus::new(), // Corpus in which we store solutions (crashes in this example), diff --git a/fuzzers/backtrace_baby_fuzzers/command_executor/src/main.rs b/fuzzers/backtrace_baby_fuzzers/command_executor/src/main.rs index f2a894ad1b..f4cb471658 100644 --- a/fuzzers/backtrace_baby_fuzzers/command_executor/src/main.rs +++ b/fuzzers/backtrace_baby_fuzzers/command_executor/src/main.rs @@ -15,7 +15,7 @@ use libafl::{ feedbacks::{CrashFeedback, MaxMapFeedback, NewHashFeedback}, fuzzer::{Fuzzer, StdFuzzer}, generators::RandPrintablesGenerator, - inputs::{HasTargetBytes, Input}, + inputs::{BytesInput, HasTargetBytes}, monitors::SimpleMonitor, mutators::scheduled::{havoc_mutations, StdScheduledMutator}, observers::{get_asan_runtime_flags, AsanBacktraceObserver, StdMapObserver}, @@ -25,11 +25,10 @@ use libafl::{ Error, }; use libafl_bolts::{ - current_nanos, rands::StdRand, shmem::{unix_shmem, ShMem, ShMemId, ShMemProvider}, tuples::tuple_list, - AsMutSlice, AsSlice, + AsSlice, AsSliceMut, }; #[allow(clippy::similar_names)] @@ -39,7 +38,7 @@ pub fn main() { let shmem_id = signals.id(); // Create an observation channel using the signals map - let observer = unsafe { StdMapObserver::new("signals", signals.as_mut_slice()) }; + let observer = unsafe { StdMapObserver::new("signals", signals.as_slice_mut()) }; // Create a stacktrace observer let bt_observer = AsanBacktraceObserver::new("AsanBacktraceObserver"); @@ -53,7 +52,7 @@ pub fn main() { // create a State from scratch let mut state = StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryCorpus::new(), // Corpus in which we store solutions (crashes in this example), @@ -86,8 +85,8 @@ pub fn main() { shmem_id: ShMemId, } - impl CommandConfigurator for MyExecutor { - fn spawn_child(&mut self, input: &I) -> Result { + impl CommandConfigurator for MyExecutor { + fn spawn_child(&mut self, input: &BytesInput) -> Result { let mut command = Command::new("./test_command"); let command = command diff --git a/fuzzers/backtrace_baby_fuzzers/forkserver_executor/src/main.rs b/fuzzers/backtrace_baby_fuzzers/forkserver_executor/src/main.rs index 6a65550ca1..86fc4b41e9 100644 --- a/fuzzers/backtrace_baby_fuzzers/forkserver_executor/src/main.rs +++ b/fuzzers/backtrace_baby_fuzzers/forkserver_executor/src/main.rs @@ -21,11 +21,10 @@ use libafl_bolts::shmem::StdShMemProvider; #[cfg(target_vendor = "apple")] use libafl_bolts::shmem::UnixShMemProvider; use libafl_bolts::{ - current_nanos, rands::StdRand, shmem::{ShMem, ShMemProvider}, tuples::tuple_list, - AsMutSlice, + AsSliceMut, }; #[allow(clippy::similar_names)] @@ -42,7 +41,7 @@ pub fn main() { let mut shmem = shmem_provider.new_shmem(MAP_SIZE).unwrap(); //let the forkserver know the shmid shmem.write_to_env("__AFL_SHM_ID").unwrap(); - let shmem_map = shmem.as_mut_slice(); + let shmem_map = shmem.as_slice_mut(); // Create an observation channel using the signals map let edges_observer = HitcountsMapObserver::new(ConstMapObserver::<_, MAP_SIZE>::new( @@ -54,7 +53,7 @@ pub fn main() { // Feedback to rate the interestingness of an input // This one is composed by two Feedbacks in OR - let mut feedback = MaxMapFeedback::tracking(&edges_observer, true, false); + let mut feedback = MaxMapFeedback::new(&edges_observer); // A feedback to choose if an input is a solution or not // We want to do the same crash deduplication that AFL does @@ -63,7 +62,7 @@ pub fn main() { // create a State from scratch let mut state = StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryCorpus::::new(), // Corpus in which we store solutions (crashes in this example), diff --git a/fuzzers/backtrace_baby_fuzzers/rust_code_with_fork_executor/src/main.rs b/fuzzers/backtrace_baby_fuzzers/rust_code_with_fork_executor/src/main.rs index a83aa1359b..1b6e2507cc 100644 --- a/fuzzers/backtrace_baby_fuzzers/rust_code_with_fork_executor/src/main.rs +++ b/fuzzers/backtrace_baby_fuzzers/rust_code_with_fork_executor/src/main.rs @@ -19,12 +19,11 @@ use libafl::{ state::StdState, }; use libafl_bolts::{ - current_nanos, ownedref::OwnedRefMut, rands::StdRand, shmem::{unix_shmem, ShMem, ShMemProvider}, tuples::tuple_list, - AsMutSlice, AsSlice, + AsSlice, AsSliceMut, }; #[allow(clippy::similar_names)] @@ -32,7 +31,7 @@ pub fn main() { let mut shmem_provider = unix_shmem::UnixShMemProvider::new().unwrap(); let mut signals = shmem_provider.new_shmem(16).unwrap(); let signals_len = signals.len(); - let signals_ptr = signals.as_mut_slice().as_mut_ptr(); + let signals_ptr = signals.as_slice_mut().as_mut_ptr(); let mut bt = shmem_provider.new_on_shmem::>(None).unwrap(); let signals_set = |idx: usize| { @@ -83,7 +82,7 @@ pub fn main() { // create a State from scratch let mut state = StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryCorpus::new(), // Corpus in which we store solutions (crashes in this example), diff --git a/fuzzers/backtrace_baby_fuzzers/rust_code_with_inprocess_executor/src/main.rs b/fuzzers/backtrace_baby_fuzzers/rust_code_with_inprocess_executor/src/main.rs index a76726f7b1..a58ef4ec6d 100644 --- a/fuzzers/backtrace_baby_fuzzers/rust_code_with_inprocess_executor/src/main.rs +++ b/fuzzers/backtrace_baby_fuzzers/rust_code_with_inprocess_executor/src/main.rs @@ -18,7 +18,7 @@ use libafl::{ stages::mutational::StdMutationalStage, state::StdState, }; -use libafl_bolts::{current_nanos, rands::StdRand, tuples::tuple_list, AsSlice}; +use libafl_bolts::{rands::StdRand, tuples::tuple_list, AsSlice}; /// Coverage map with explicit assignments due to the lack of instrumentation static mut SIGNALS: [u8; 16] = [0; 16]; @@ -75,7 +75,7 @@ pub fn main() { // create a State from scratch let mut state = StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryCorpus::new(), // Corpus in which we store solutions (crashes in this example), diff --git a/fuzzers/cargo_fuzz/Cargo.toml b/fuzzers/cargo_fuzz/Cargo.toml new file mode 100644 index 0000000000..9933834c8b --- /dev/null +++ b/fuzzers/cargo_fuzz/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "cargo_fuzz_test" +edition = "2021" +version = "0.0.0" +description = "test" +authors = ["Andrea Fioraldi ", "Dominik Maier "] +repository = "https://github.com/AFLplusplus/LibAFL/" +keywords = ["fuzzing", "testing", "compiler"] +categories = ["development-tools::testing", "emulators", "embedded", "os", "no-std"] + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/fuzzers/cargo_fuzz/Makefile.toml b/fuzzers/cargo_fuzz/Makefile.toml new file mode 100644 index 0000000000..31bf5e7221 --- /dev/null +++ b/fuzzers/cargo_fuzz/Makefile.toml @@ -0,0 +1,44 @@ +[env] + +[tasks.unsupported] +script_runner = "@shell" +script = ''' +echo "Cargo-make not integrated yet on this" +''' + +[tasks.install_llvm_tools] +command = "rustup" +args = ["toolchain", "install", "nightly", "--component", "llvm-tools-preview"] + + +[tasks.install_cargo_fuzz] +command = "cargo" +args = ["install", "cargo-fuzz"] + +# Fuzzer +[tasks.build] +command = "cargo" +args = ["+nightly", "fuzz", "build", "fuzz_target_1"] +dependencies = ["install_cargo_fuzz", "install_llvm_tools"] + +[tasks.test] +linux_alias = "test_unix" +mac_alias = "unsupported" +windows_alias = "unsupported" + +[tasks.test_unix] +script = ''' +timeout 30s cargo +nightly fuzz run fuzz_target_1 2>&1 | tee fuzz_stdout.log || true +if grep -qa "objectives: 1" fuzz_stdout.log; then + echo "Fuzzer is working" +else + echo "Fuzzer does not generate any testcases or any crashes" + exit 1 +fi +''' +dependencies = ["build"] + +# Clean +[tasks.clean] +command = "rm " +args = ["-rf", "fuzz/target"] diff --git a/fuzzers/cargo_fuzz/README.md b/fuzzers/cargo_fuzz/README.md new file mode 100644 index 0000000000..4d30ba27ef --- /dev/null +++ b/fuzzers/cargo_fuzz/README.md @@ -0,0 +1,3 @@ +# cargo-fuzz + +This is a minimalistic example how to use LibAFL with cargo-fuzz. It uses the `libafl_libfuzzer` comatability layer to be libFuzzer compatiable. diff --git a/fuzzers/cargo_fuzz/fuzz/.gitignore b/fuzzers/cargo_fuzz/fuzz/.gitignore new file mode 100644 index 0000000000..1a45eee776 --- /dev/null +++ b/fuzzers/cargo_fuzz/fuzz/.gitignore @@ -0,0 +1,4 @@ +target +corpus +artifacts +coverage diff --git a/fuzzers/cargo_fuzz/fuzz/Cargo.toml b/fuzzers/cargo_fuzz/fuzz/Cargo.toml new file mode 100644 index 0000000000..e67e8e7c5f --- /dev/null +++ b/fuzzers/cargo_fuzz/fuzz/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "libafl-fuzz" +version = "0.0.0" +publish = false +edition = "2021" + +[workspace] + +[package.metadata] +cargo-fuzz = true + +[dependencies] + +[dependencies.cargo_fuzz_test] +path = ".." + +[dependencies.libfuzzer-sys] +path = "../../../libafl_libfuzzer" +package = "libafl_libfuzzer" + +[[bin]] +name = "fuzz_target_1" +path = "fuzz_targets/fuzz_target_1.rs" +test = false +doc = false +bench = false diff --git a/fuzzers/cargo_fuzz/fuzz/fuzz_targets/fuzz_target_1.rs b/fuzzers/cargo_fuzz/fuzz/fuzz_targets/fuzz_target_1.rs new file mode 100644 index 0000000000..05b1861c5f --- /dev/null +++ b/fuzzers/cargo_fuzz/fuzz/fuzz_targets/fuzz_target_1.rs @@ -0,0 +1,6 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use cargo_fuzz_test::do_thing; + +fuzz_target!(|data: &[u8]| do_thing(data)); diff --git a/fuzzers/cargo_fuzz/src/lib.rs b/fuzzers/cargo_fuzz/src/lib.rs new file mode 100644 index 0000000000..d2da382287 --- /dev/null +++ b/fuzzers/cargo_fuzz/src/lib.rs @@ -0,0 +1,11 @@ +pub fn do_thing(data: &[u8]) { + if data.get(0) == Some(&b'a') { + if data.get(1) == Some(&b'b') { + if data.get(2) == Some(&b'c') { + if data.get(3) == Some(&b'd') { + panic!("We found the objective!"); + } + } + } + } +} diff --git a/fuzzers/forkserver_libafl_cc/Cargo.toml b/fuzzers/forkserver_libafl_cc/Cargo.toml index 2d6efbfc7b..a8c14d4bc3 100644 --- a/fuzzers/forkserver_libafl_cc/Cargo.toml +++ b/fuzzers/forkserver_libafl_cc/Cargo.toml @@ -26,7 +26,7 @@ nix = "0.27" libafl = { path = "../../libafl/" } libafl_bolts = { path = "../../libafl_bolts/" } libafl_cc = { path = "../../libafl_cc/" } -libafl_targets = { path = "../../libafl_targets/" } +libafl_targets = { path = "../../libafl_targets/", features = ["sancov_pcguard_hitcounts", "libfuzzer", "pointer_maps"] } [lib] name = "libforkserver_libafl_cc" diff --git a/fuzzers/forkserver_libafl_cc/Makefile.toml b/fuzzers/forkserver_libafl_cc/Makefile.toml index e417e0361c..94bf9daf19 100644 --- a/fuzzers/forkserver_libafl_cc/Makefile.toml +++ b/fuzzers/forkserver_libafl_cc/Makefile.toml @@ -101,6 +101,26 @@ taskset -c 1 ${CARGO_TARGET_DIR}/${PROFILE_DIR}/${CARGO_MAKE_PROJECT_NAME} ./${F ''' dependencies = [ "fuzzer_crash" ] +# Test +[tasks.test] +linux_alias = "test_unix" +mac_alias = "test_unix" +windows_alias = "unsupported" + +[tasks.test_unix] +script_runner = "@shell" +script=''' +timeout 30s ${CARGO_TARGET_DIR}/${PROFILE_DIR}/${CARGO_MAKE_PROJECT_NAME} ./${FUZZER_NAME} ./corpus/ -t 1000 | tee fuzz_stdout.log || true +if grep -qa "objectives: 1" fuzz_stdout.log; then + echo "Fuzzer is working" +else + echo "Fuzzer does not generate any testcases or any crashes" + exit 1 +fi + +''' +dependencies = [ "fuzzer" ] + # Clean up [tasks.clean] linux_alias = "clean_unix" diff --git a/fuzzers/forkserver_libafl_cc/src/bin/libafl_cc.rs b/fuzzers/forkserver_libafl_cc/src/bin/libafl_cc.rs index 1687f30265..21ab9936e2 100644 --- a/fuzzers/forkserver_libafl_cc/src/bin/libafl_cc.rs +++ b/fuzzers/forkserver_libafl_cc/src/bin/libafl_cc.rs @@ -1,6 +1,6 @@ use std::env; -use libafl_cc::{ClangWrapper, CompilerWrapper, LLVMPasses, ToolWrapper}; +use libafl_cc::{ClangWrapper, CompilerWrapper, ToolWrapper}; pub fn main() { let args: Vec = env::args().collect(); @@ -24,9 +24,7 @@ pub fn main() { .parse_args(&args) .expect("Failed to parse the command line") // Enable libafl's coverage instrumentation - .add_pass(LLVMPasses::AFLCoverage) - .add_arg("-mllvm") - .add_arg("-ctx") // Context sensitive coverage + .add_arg("-fsanitize-coverage=trace-pc-guard") // Imitate afl-cc's compile definitions .add_arg("-D__AFL_FUZZ_INIT()=int __afl_sharedmem_fuzzing = 1;extern unsigned int *__afl_fuzz_len;extern unsigned char *__afl_fuzz_ptr;unsigned char __afl_fuzz_alt[1048576];unsigned char *__afl_fuzz_alt_ptr = __afl_fuzz_alt;void libafl_start_forkserver(void)") .add_arg("-D__AFL_FUZZ_TESTCASE_BUF=(__afl_fuzz_ptr ? __afl_fuzz_ptr : __afl_fuzz_alt_ptr)") diff --git a/fuzzers/forkserver_libafl_cc/src/main.rs b/fuzzers/forkserver_libafl_cc/src/main.rs index e90f048272..f4bba8d01e 100644 --- a/fuzzers/forkserver_libafl_cc/src/main.rs +++ b/fuzzers/forkserver_libafl_cc/src/main.rs @@ -1,7 +1,7 @@ use core::time::Duration; use std::path::PathBuf; -use clap::{self, Parser}; +use clap::Parser; use libafl::{ corpus::{Corpus, InMemoryCorpus, OnDiskCorpus}, events::SimpleEventManager, @@ -12,18 +12,19 @@ use libafl::{ inputs::BytesInput, monitors::SimpleMonitor, mutators::{scheduled::havoc_mutations, tokens_mutations, StdScheduledMutator, Tokens}, - observers::{HitcountsMapObserver, StdMapObserver, TimeObserver}, + observers::{CanTrack, HitcountsMapObserver, StdMapObserver, TimeObserver}, schedulers::{IndexesLenTimeMinimizerScheduler, QueueScheduler}, stages::mutational::StdMutationalStage, - state::{HasCorpus, HasMetadata, StdState}, + state::{HasCorpus, StdState}, + HasMetadata, }; use libafl_bolts::{ - current_nanos, rands::StdRand, shmem::{ShMem, ShMemProvider, UnixShMemProvider}, - tuples::{tuple_list, MatchName, Merge}, - AsMutSlice, Truncate, + tuples::{tuple_list, Handler, MatchNameRef, Merge}, + AsSliceMut, Truncate, }; +use libafl_targets::EDGES_MAP_SIZE_IN_USE; use nix::sys::signal::Signal; /// The commandline args this fuzzer accepts @@ -84,8 +85,7 @@ struct Opt { #[allow(clippy::similar_names)] pub fn main() { - const MAP_SIZE: usize = 65536; - + const MAP_SIZE: usize = EDGES_MAP_SIZE_IN_USE; //65536; let opt = Opt::parse(); let corpus_dirs: Vec = [opt.in_dir].to_vec(); @@ -97,11 +97,14 @@ pub fn main() { let mut shmem = shmem_provider.new_shmem(MAP_SIZE).unwrap(); // let the forkserver know the shmid shmem.write_to_env("__AFL_SHM_ID").unwrap(); - let shmem_buf = shmem.as_mut_slice(); + let shmem_buf = shmem.as_slice_mut(); + // the next line is not needed + // unsafe { EDGES_MAP_PTR = shmem_buf.as_mut_ptr() }; // Create an observation channel using the signals map - let edges_observer = - unsafe { HitcountsMapObserver::new(StdMapObserver::new("shared_mem", shmem_buf)) }; + let edges_observer = unsafe { + HitcountsMapObserver::new(StdMapObserver::new("shared_mem", shmem_buf)).track_indices() + }; // Create an observation channel to keep track of the execution time let time_observer = TimeObserver::new("time"); @@ -110,9 +113,9 @@ pub fn main() { // This one is composed by two Feedbacks in OR let mut feedback = feedback_or!( // New maximization map feedback linked to the edges observer and the feedback state - MaxMapFeedback::tracking(&edges_observer, true, false), + MaxMapFeedback::new(&edges_observer), // Time feedback, this one does not need a feedback state - TimeFeedback::with_observer(&time_observer) + TimeFeedback::new(&time_observer) ); // A feedback to choose if an input is a solution or not @@ -128,7 +131,7 @@ pub fn main() { // create a State from scratch let mut state = StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryCorpus::::new(), // Corpus in which we store solutions (crashes in this example), @@ -143,14 +146,16 @@ pub fn main() { .unwrap(); // The Monitor trait define how the fuzzer stats are reported to the user - let monitor = SimpleMonitor::new(|s| println!("{s}")); + let monitor = SimpleMonitor::with_user_monitor(|s| { + println!("{s}"); + }); // The event manager handle the various events generated during the fuzzing loop // such as the notification of the addition of a new item to the corpus let mut mgr = SimpleEventManager::new(monitor); // A minimization+queue policy to get testcasess from the corpus - let scheduler = IndexesLenTimeMinimizerScheduler::new(QueueScheduler::new()); + let scheduler = IndexesLenTimeMinimizerScheduler::new(&edges_observer, QueueScheduler::new()); // A fuzzer with feedbacks and a corpus scheduler let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); @@ -161,6 +166,8 @@ pub fn main() { // Create the executor for the forkserver let args = opt.arguments; + let observer_ref = edges_observer.handle(); + let mut tokens = Tokens::new(); let mut executor = ForkserverExecutor::builder() .program(opt.executable) @@ -175,10 +182,8 @@ pub fn main() { .unwrap(); if let Some(dynamic_map_size) = executor.coverage_map_size() { - executor - .observers_mut() - .match_name_mut::>>("shared_mem") - .unwrap() + executor.observers_mut()[&observer_ref] + .as_mut() .truncate(dynamic_map_size); } diff --git a/fuzzers/forkserver_simple/Cargo.toml b/fuzzers/forkserver_simple/Cargo.toml index 21943b2a32..1854e3cc8a 100644 --- a/fuzzers/forkserver_simple/Cargo.toml +++ b/fuzzers/forkserver_simple/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "forkserver_simple" -version = "0.11.2" +version = "0.12.0" authors = ["tokatoka "] edition = "2021" diff --git a/fuzzers/forkserver_simple/src/main.rs b/fuzzers/forkserver_simple/src/main.rs index a0c752eee1..ac80e2da4f 100644 --- a/fuzzers/forkserver_simple/src/main.rs +++ b/fuzzers/forkserver_simple/src/main.rs @@ -1,7 +1,7 @@ use core::time::Duration; use std::path::PathBuf; -use clap::{self, Parser}; +use clap::Parser; use libafl::{ corpus::{Corpus, InMemoryCorpus, OnDiskCorpus}, events::SimpleEventManager, @@ -12,17 +12,18 @@ use libafl::{ inputs::BytesInput, monitors::SimpleMonitor, mutators::{scheduled::havoc_mutations, tokens_mutations, StdScheduledMutator, Tokens}, - observers::{HitcountsMapObserver, StdMapObserver, TimeObserver}, + observers::{CanTrack, HitcountsMapObserver, StdMapObserver, TimeObserver}, schedulers::{IndexesLenTimeMinimizerScheduler, QueueScheduler}, stages::mutational::StdMutationalStage, - state::{HasCorpus, HasMetadata, StdState}, + state::{HasCorpus, StdState}, + HasMetadata, }; use libafl_bolts::{ current_nanos, rands::StdRand, shmem::{ShMem, ShMemProvider, UnixShMemProvider}, - tuples::{tuple_list, MatchName, Merge}, - AsMutSlice, Truncate, + tuples::{tuple_list, Handler, Merge}, + AsSliceMut, Truncate, }; use nix::sys::signal::Signal; @@ -97,11 +98,12 @@ pub fn main() { let mut shmem = shmem_provider.new_shmem(MAP_SIZE).unwrap(); // let the forkserver know the shmid shmem.write_to_env("__AFL_SHM_ID").unwrap(); - let shmem_buf = shmem.as_mut_slice(); + let shmem_buf = shmem.as_slice_mut(); // Create an observation channel using the signals map - let edges_observer = - unsafe { HitcountsMapObserver::new(StdMapObserver::new("shared_mem", shmem_buf)) }; + let edges_observer = unsafe { + HitcountsMapObserver::new(StdMapObserver::new("shared_mem", shmem_buf)).track_indices() + }; // Create an observation channel to keep track of the execution time let time_observer = TimeObserver::new("time"); @@ -110,9 +112,9 @@ pub fn main() { // This one is composed by two Feedbacks in OR let mut feedback = feedback_or!( // New maximization map feedback linked to the edges observer and the feedback state - MaxMapFeedback::tracking(&edges_observer, true, false), + MaxMapFeedback::new(&edges_observer), // Time feedback, this one does not need a feedback state - TimeFeedback::with_observer(&time_observer) + TimeFeedback::new(&time_observer) ); // A feedback to choose if an input is a solution or not @@ -150,7 +152,7 @@ pub fn main() { let mut mgr = SimpleEventManager::new(monitor); // A minimization+queue policy to get testcasess from the corpus - let scheduler = IndexesLenTimeMinimizerScheduler::new(QueueScheduler::new()); + let scheduler = IndexesLenTimeMinimizerScheduler::new(&edges_observer, QueueScheduler::new()); // A fuzzer with feedbacks and a corpus scheduler let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); @@ -161,6 +163,8 @@ pub fn main() { // Create the executor for the forkserver let args = opt.arguments; + let observer_ref = edges_observer.handle(); + let mut tokens = Tokens::new(); let mut executor = ForkserverExecutor::builder() .program(opt.executable) @@ -175,10 +179,8 @@ pub fn main() { .unwrap(); if let Some(dynamic_map_size) = executor.coverage_map_size() { - executor - .observers_mut() - .match_name_mut::>>("shared_mem") - .unwrap() + executor.observers_mut()[&observer_ref] + .as_mut() .truncate(dynamic_map_size); } diff --git a/fuzzers/frida_executable_libpng/Cargo.toml b/fuzzers/frida_executable_libpng/Cargo.toml index da135b32d4..1b08bfc7cb 100644 --- a/fuzzers/frida_executable_libpng/Cargo.toml +++ b/fuzzers/frida_executable_libpng/Cargo.toml @@ -28,7 +28,7 @@ reqwest = { version = "0.11.4", features = ["blocking"] } [dependencies] libafl = { path = "../../libafl/", features = [ "std", "llmp_compression", "llmp_bind_public", "frida_cli" ] } #, "llmp_small_maps", "llmp_debug"]} libafl_bolts = { path = "../../libafl_bolts/" } -frida-gum = { version = "0.13.2", features = [ "auto-download", "event-sink", "invocation-listener"] } +frida-gum = { version = "0.13.6", features = [ "auto-download", "event-sink", "invocation-listener"] } libafl_frida = { path = "../../libafl_frida", features = ["cmplog"] } libafl_targets = { path = "../../libafl_targets", features = ["sancov_cmplog"] } libc = "0.2" diff --git a/fuzzers/frida_executable_libpng/src/fuzzer.rs b/fuzzers/frida_executable_libpng/src/fuzzer.rs index 21a40c08da..0cb1d15ba3 100644 --- a/fuzzers/frida_executable_libpng/src/fuzzer.rs +++ b/fuzzers/frida_executable_libpng/src/fuzzer.rs @@ -1,9 +1,5 @@ //! A libfuzzer-like fuzzer with llmp-multithreading support and restarts //! The example harness is built for libpng. -use mimalloc::MiMalloc; -#[global_allocator] -static GLOBAL: MiMalloc = MiMalloc; - use std::{path::PathBuf, ptr::null}; use frida_gum::Gum; @@ -20,17 +16,16 @@ use libafl::{ scheduled::{havoc_mutations, tokens_mutations, StdScheduledMutator}, token_mutations::{I2SRandReplace, Tokens}, }, - observers::{HitcountsMapObserver, StdMapObserver, TimeObserver}, + observers::{CanTrack, HitcountsMapObserver, StdMapObserver, TimeObserver}, schedulers::{IndexesLenTimeMinimizerScheduler, QueueScheduler}, stages::{ShadowTracingStage, StdMutationalStage}, - state::{HasCorpus, HasMetadata, StdState}, - Error, + state::{HasCorpus, StdState}, + Error, HasMetadata, }; #[cfg(unix)] use libafl::{feedback_and_fast, feedbacks::ConstFeedback}; use libafl_bolts::{ cli::{parse_args, FuzzerOptions}, - current_nanos, rands::StdRand, shmem::{ShMemProvider, StdShMemProvider}, tuples::{tuple_list, Merge}, @@ -39,7 +34,7 @@ use libafl_bolts::{ #[cfg(unix)] use libafl_frida::asan::{ asan_rt::AsanRuntime, - errors::{AsanErrorsFeedback, AsanErrorsObserver, ASAN_ERRORS}, + errors::{AsanErrorsFeedback, AsanErrorsObserver}, }; use libafl_frida::{ cmplog_rt::CmpLogRuntime, @@ -48,6 +43,10 @@ use libafl_frida::{ helper::FridaInstrumentationHelper, }; use libafl_targets::cmplog::CmpLogObserver; +use mimalloc::MiMalloc; + +#[global_allocator] +static GLOBAL: MiMalloc = MiMalloc; pub unsafe fn lib(main: extern "C" fn(i32, *const *const u8, *const *const u8) -> i32) { color_backtrace::install(); @@ -93,13 +92,13 @@ unsafe fn fuzz( let shmem_provider = StdShMemProvider::new()?; - let mut run_client = |state: Option<_>, mgr: LlmpRestartingEventManager<_, _>, core_id| { + let mut run_client = |state: Option<_>, mgr: LlmpRestartingEventManager<_, _, _>, core_id| { // The restarting state will spawn the same process again as child, then restarted it each time it crashes. // println!("{:?}", mgr.mgr_id()); if options.asan && options.asan_cores.contains(core_id) { - (|state: Option<_>, mut mgr: LlmpRestartingEventManager<_, _>, _core_id| { + (|state: Option<_>, mut mgr: LlmpRestartingEventManager<_, _, _>, _core_id| { let gum = Gum::obtain(); let coverage = CoverageRuntime::new(); @@ -118,18 +117,21 @@ unsafe fn fuzz( "edges", frida_helper.map_mut_ptr().unwrap(), MAP_SIZE, - )); + )) + .track_indices(); // Create an observation channel to keep track of the execution time let time_observer = TimeObserver::new("time"); + #[cfg(unix)] + let asan_observer = AsanErrorsObserver::from_static_asan_errors(); // Feedback to rate the interestingness of an input // This one is composed by two Feedbacks in OR let mut feedback = feedback_or!( // New maximization map feedback linked to the edges observer and the feedback state - MaxMapFeedback::tracking(&edges_observer, true, false), + MaxMapFeedback::new(&edges_observer), // Time feedback, this one does not need a feedback state - TimeFeedback::with_observer(&time_observer) + TimeFeedback::new(&time_observer) ); // Feedbacks to recognize an input as solution @@ -138,7 +140,10 @@ unsafe fn fuzz( CrashFeedback::new(), TimeoutFeedback::new(), // true enables the AsanErrorFeedback - feedback_and_fast!(ConstFeedback::from(true), AsanErrorsFeedback::new()) + feedback_and_fast!( + ConstFeedback::from(true), + AsanErrorsFeedback::new(&asan_observer) + ) ); #[cfg(windows)] let mut objective = feedback_or_fast!(CrashFeedback::new(), TimeoutFeedback::new()); @@ -147,7 +152,7 @@ unsafe fn fuzz( let mut state = state.unwrap_or_else(|| { StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance CachedOnDiskCorpus::no_meta(PathBuf::from("./corpus_discovered"), 64) .unwrap(), @@ -177,17 +182,14 @@ unsafe fn fuzz( let mutator = StdScheduledMutator::new(havoc_mutations().merge(tokens_mutations())); // A minimization+queue policy to get testcasess from the corpus - let scheduler = IndexesLenTimeMinimizerScheduler::new(QueueScheduler::new()); + let scheduler = + IndexesLenTimeMinimizerScheduler::new(&edges_observer, QueueScheduler::new()); // A fuzzer with feedbacks and a corpus scheduler let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); #[cfg(unix)] - let observers = tuple_list!( - edges_observer, - time_observer, - AsanErrorsObserver::new(&ASAN_ERRORS) - ); + let observers = tuple_list!(edges_observer, time_observer, asan_observer); #[cfg(windows)] let observers = tuple_list!(edges_observer, time_observer); @@ -221,7 +223,7 @@ unsafe fn fuzz( Ok(()) })(state, mgr, core_id) } else if options.cmplog && options.cmplog_cores.contains(core_id) { - (|state: Option<_>, mut mgr: LlmpRestartingEventManager<_, _>, _core_id| { + (|state: Option<_>, mut mgr: LlmpRestartingEventManager<_, _, _>, _core_id| { let gum = Gum::obtain(); let coverage = CoverageRuntime::new(); @@ -235,25 +237,31 @@ unsafe fn fuzz( "edges", frida_helper.map_mut_ptr().unwrap(), MAP_SIZE, - )); + )) + .track_indices(); // Create an observation channel to keep track of the execution time let time_observer = TimeObserver::new("time"); + #[cfg(unix)] + let asan_observer = AsanErrorsObserver::from_static_asan_errors(); // Feedback to rate the interestingness of an input // This one is composed by two Feedbacks in OR let mut feedback = feedback_or!( // New maximization map feedback linked to the edges observer and the feedback state - MaxMapFeedback::tracking(&edges_observer, true, false), + MaxMapFeedback::new(&edges_observer), // Time feedback, this one does not need a feedback state - TimeFeedback::with_observer(&time_observer) + TimeFeedback::new(&time_observer) ); #[cfg(unix)] let mut objective = feedback_or_fast!( CrashFeedback::new(), TimeoutFeedback::new(), - feedback_and_fast!(ConstFeedback::from(false), AsanErrorsFeedback::new()) + feedback_and_fast!( + ConstFeedback::from(false), + AsanErrorsFeedback::new(&asan_observer) + ) ); #[cfg(windows)] let mut objective = feedback_or_fast!(CrashFeedback::new(), TimeoutFeedback::new()); @@ -262,7 +270,7 @@ unsafe fn fuzz( let mut state = state.unwrap_or_else(|| { StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance CachedOnDiskCorpus::no_meta(PathBuf::from("./corpus_discovered"), 64) .unwrap(), @@ -292,19 +300,16 @@ unsafe fn fuzz( let mutator = StdScheduledMutator::new(havoc_mutations().merge(tokens_mutations())); // A minimization+queue policy to get testcasess from the corpus - let scheduler = IndexesLenTimeMinimizerScheduler::new(QueueScheduler::new()); + let scheduler = + IndexesLenTimeMinimizerScheduler::new(&edges_observer, QueueScheduler::new()); // A fuzzer with feedbacks and a corpus scheduler let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); #[cfg(unix)] - let observers = tuple_list!( - edges_observer, - time_observer, - AsanErrorsObserver::new(&ASAN_ERRORS) - ); + let observers = tuple_list!(edges_observer, time_observer, asan_observer); #[cfg(windows)] - let observers = tuple_list!(edges_observer, time_observer,); + let observers = tuple_list!(edges_observer, time_observer); // Create the executor for an in-process function with just one observer for edge coverage let mut executor = FridaInProcessExecutor::new( @@ -352,7 +357,7 @@ unsafe fn fuzz( Ok(()) })(state, mgr, core_id) } else { - (|state: Option<_>, mut mgr: LlmpRestartingEventManager<_, _>, _core_id| { + (|state: Option<_>, mut mgr: LlmpRestartingEventManager<_, _, _>, _core_id| { let gum = Gum::obtain(); let coverage = CoverageRuntime::new(); @@ -365,25 +370,31 @@ unsafe fn fuzz( "edges", frida_helper.map_mut_ptr().unwrap(), MAP_SIZE, - )); + )) + .track_indices(); // Create an observation channel to keep track of the execution time let time_observer = TimeObserver::new("time"); + #[cfg(unix)] + let asan_observer = AsanErrorsObserver::from_static_asan_errors(); // Feedback to rate the interestingness of an input // This one is composed by two Feedbacks in OR let mut feedback = feedback_or!( // New maximization map feedback linked to the edges observer and the feedback state - MaxMapFeedback::tracking(&edges_observer, true, false), + MaxMapFeedback::new(&edges_observer), // Time feedback, this one does not need a feedback state - TimeFeedback::with_observer(&time_observer) + TimeFeedback::new(&time_observer) ); #[cfg(unix)] let mut objective = feedback_or_fast!( CrashFeedback::new(), TimeoutFeedback::new(), - feedback_and_fast!(ConstFeedback::from(false), AsanErrorsFeedback::new()) + feedback_and_fast!( + ConstFeedback::from(false), + AsanErrorsFeedback::new(&asan_observer) + ) ); #[cfg(windows)] let mut objective = feedback_or_fast!(CrashFeedback::new(), TimeoutFeedback::new()); @@ -392,7 +403,7 @@ unsafe fn fuzz( let mut state = state.unwrap_or_else(|| { StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance CachedOnDiskCorpus::no_meta(PathBuf::from("./corpus_discovered"), 64) .unwrap(), @@ -422,19 +433,16 @@ unsafe fn fuzz( let mutator = StdScheduledMutator::new(havoc_mutations().merge(tokens_mutations())); // A minimization+queue policy to get testcasess from the corpus - let scheduler = IndexesLenTimeMinimizerScheduler::new(QueueScheduler::new()); + let scheduler = + IndexesLenTimeMinimizerScheduler::new(&edges_observer, QueueScheduler::new()); // A fuzzer with feedbacks and a corpus scheduler let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); #[cfg(unix)] - let observers = tuple_list!( - edges_observer, - time_observer, - AsanErrorsObserver::new(&ASAN_ERRORS) - ); + let observers = tuple_list!(edges_observer, time_observer, asan_observer); #[cfg(windows)] - let observers = tuple_list!(edges_observer, time_observer,); + let observers = tuple_list!(edges_observer, time_observer); // Create the executor for an in-process function with just one observer for edge coverage let mut executor = FridaInProcessExecutor::new( diff --git a/fuzzers/frida_executable_libpng/src/lib.rs b/fuzzers/frida_executable_libpng/src/lib.rs index 3090b533b0..da9fdeae51 100644 --- a/fuzzers/frida_executable_libpng/src/lib.rs +++ b/fuzzers/frida_executable_libpng/src/lib.rs @@ -48,7 +48,7 @@ pub unsafe extern "C" fn __libc_start_main( ORIG_MAIN = main; let orig_libc_start_main_addr: *mut c_void = - dlsym(RTLD_NEXT, "__libc_start_main\0".as_ptr().cast::()); + dlsym(RTLD_NEXT, c"__libc_start_main".as_ptr()); let orig_libc_start_main: LibcStartMainFunc = transmute(orig_libc_start_main_addr); diff --git a/fuzzers/frida_gdiplus/Cargo.toml b/fuzzers/frida_gdiplus/Cargo.toml index 10e0308b12..ce7d0d51e4 100644 --- a/fuzzers/frida_gdiplus/Cargo.toml +++ b/fuzzers/frida_gdiplus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "frida_gdiplus" -version = "0.11.2" +version = "0.12.0" authors = ["Richard Johnson "] edition = "2021" diff --git a/fuzzers/frida_gdiplus/src/fuzzer.rs b/fuzzers/frida_gdiplus/src/fuzzer.rs index 6e643d163b..33be9c26ae 100644 --- a/fuzzers/frida_gdiplus/src/fuzzer.rs +++ b/fuzzers/frida_gdiplus/src/fuzzer.rs @@ -4,7 +4,7 @@ //! is platform independent. Hence, this file contains code for other platforms //! but it's only meaningful for Windows because of the `gdiplus` target. If you //! going to make it compilable only for Windows, don't forget to modify the -//! `scripts/test_all_fuzzers.sh` to opt-out this fuzzer from that test. +//! `scripts/test_fuzzer.sh` to opt-out this fuzzer from that test. #[cfg(unix)] use mimalloc::MiMalloc; @@ -28,20 +28,26 @@ use libafl::{ scheduled::{havoc_mutations, tokens_mutations, StdScheduledMutator}, token_mutations::{I2SRandReplace, Tokens}, }, - observers::{HitcountsMapObserver, StdMapObserver, TimeObserver}, + observers::{CanTrack, HitcountsMapObserver, StdMapObserver, TimeObserver}, schedulers::{IndexesLenTimeMinimizerScheduler, QueueScheduler}, stages::{ShadowTracingStage, StdMutationalStage}, - state::{HasCorpus, HasMetadata, StdState}, - Error, + state::{HasCorpus, StdState}, + Error, HasMetadata, }; use libafl_bolts::{ cli::{parse_args, FuzzerOptions}, - current_nanos, rands::StdRand, shmem::{ShMemProvider, StdShMemProvider}, tuples::{tuple_list, Merge}, AsSlice, }; +<<<<<<< HEAD +======= +#[cfg(unix)] +use libafl_frida::asan::asan_rt::AsanRuntime; +#[cfg(unix)] +use libafl_frida::asan::errors::{AsanErrorsFeedback, AsanErrorsObserver}; +>>>>>>> main use libafl_frida::{ asan::{ asan_rt::AsanRuntime, @@ -77,7 +83,7 @@ unsafe fn fuzz(options: &FuzzerOptions) -> Result<(), Error> { let shmem_provider = StdShMemProvider::new()?; - let mut run_client = |state: Option<_>, mgr: LlmpRestartingEventManager<_, _>, core_id| { + let mut run_client = |state: Option<_>, mgr: LlmpRestartingEventManager<_, _, _>, core_id| { // The restarting state will spawn the same process again as child, then restarted it each time it crashes. // println!("{:?}", mgr.mgr_id()); @@ -95,13 +101,20 @@ unsafe fn fuzz(options: &FuzzerOptions) -> Result<(), Error> { }; if options.asan && options.asan_cores.contains(core_id) { - (|state: Option<_>, mut mgr: LlmpRestartingEventManager<_, _>, _core_id| { + (|state: Option<_>, mut mgr: LlmpRestartingEventManager<_, _, _>, _core_id| { let gum = Gum::obtain(); let coverage = CoverageRuntime::new(); +<<<<<<< HEAD let asan = AsanRuntime::new(&options); let hooks = HookRuntime::new(); +======= + #[cfg(unix)] + let asan = AsanRuntime::new(options); + + #[cfg(unix)] +>>>>>>> main let mut frida_helper = FridaInstrumentationHelper::new(&gum, options, tuple_list!(coverage, asan, hooks)); // @@ -110,18 +123,22 @@ unsafe fn fuzz(options: &FuzzerOptions) -> Result<(), Error> { "edges", frida_helper.map_mut_ptr().unwrap(), MAP_SIZE, - )); + )) + .track_indices(); // Create an observation channel to keep track of the execution time let time_observer = TimeObserver::new("time"); + #[cfg(unix)] + let asan_observer = AsanErrorsObserver::from_static_asan_errors(); + // Feedback to rate the interestingness of an input // This one is composed by two Feedbacks in OR let mut feedback = feedback_or!( // New maximization map feedback linked to the edges observer and the feedback state - MaxMapFeedback::tracking(&edges_observer, true, false), + MaxMapFeedback::new(&edges_observer), // Time feedback, this one does not need a feedback state - TimeFeedback::with_observer(&time_observer) + TimeFeedback::new(&time_observer) ); // Feedbacks to recognize an input as solution @@ -129,14 +146,17 @@ unsafe fn fuzz(options: &FuzzerOptions) -> Result<(), Error> { CrashFeedback::new(), // TimeoutFeedback::new(), // true enables the AsanErrorFeedback - feedback_and_fast!(ConstFeedback::from(true), AsanErrorsFeedback::new()) + feedback_and_fast!( + ConstFeedback::from(true), + AsanErrorsFeedback::new(&asan_observer) + ) ); // If not restarting, create a State from scratch let mut state = state.unwrap_or_else(|| { StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance CachedOnDiskCorpus::new(PathBuf::from("./corpus_discovered"), 64).unwrap(), // Corpus in which we store solutions (crashes in this example), @@ -165,16 +185,24 @@ unsafe fn fuzz(options: &FuzzerOptions) -> Result<(), Error> { let mutator = StdScheduledMutator::new(havoc_mutations().merge(tokens_mutations())); // A minimization+queue policy to get testcasess from the corpus - let scheduler = IndexesLenTimeMinimizerScheduler::new(QueueScheduler::new()); + let scheduler = + IndexesLenTimeMinimizerScheduler::new(&edges_observer, QueueScheduler::new()); // A fuzzer with feedbacks and a corpus scheduler let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); +<<<<<<< HEAD let observers = tuple_list!( edges_observer, time_observer, AsanErrorsObserver::new(&ASAN_ERRORS) ); +======= + #[cfg(unix)] + let observers = tuple_list!(edges_observer, time_observer, asan_observer); + #[cfg(windows)] + let observers = tuple_list!(edges_observer, time_observer); +>>>>>>> main // Create the executor for an in-process function with just one observer for edge coverage let mut executor = FridaInProcessExecutor::new( @@ -207,7 +235,7 @@ unsafe fn fuzz(options: &FuzzerOptions) -> Result<(), Error> { Ok(()) })(state, mgr, core_id) } else if options.cmplog && options.cmplog_cores.contains(core_id) { - (|state: Option<_>, mut mgr: LlmpRestartingEventManager<_, _>, _core_id| { + (|state: Option<_>, mut mgr: LlmpRestartingEventManager<_, _, _>, _core_id| { let gum = Gum::obtain(); let coverage = CoverageRuntime::new(); @@ -222,31 +250,42 @@ unsafe fn fuzz(options: &FuzzerOptions) -> Result<(), Error> { "edges", frida_helper.map_mut_ptr().unwrap(), MAP_SIZE, - )); + )) + .track_indices(); // Create an observation channel to keep track of the execution time let time_observer = TimeObserver::new("time"); + #[cfg(unix)] + let asan_observer = AsanErrorsObserver::from_static_asan_errors(); // Feedback to rate the interestingness of an input // This one is composed by two Feedbacks in OR let mut feedback = feedback_or!( // New maximization map feedback linked to the edges observer and the feedback state - MaxMapFeedback::tracking(&edges_observer, true, false), + MaxMapFeedback::new(&edges_observer), // Time feedback, this one does not need a feedback state - TimeFeedback::with_observer(&time_observer) + TimeFeedback::new(&time_observer) ); let mut objective = feedback_or_fast!( CrashFeedback::new(), +<<<<<<< HEAD // TimeoutFeedback::new(), feedback_and_fast!(ConstFeedback::from(false), AsanErrorsFeedback::new()) +======= + TimeoutFeedback::new(), + feedback_and_fast!( + ConstFeedback::from(false), + AsanErrorsFeedback::new(&asan_observer) + ) +>>>>>>> main ); // If not restarting, create a State from scratch let mut state = state.unwrap_or_else(|| { StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance CachedOnDiskCorpus::no_meta(PathBuf::from("./corpus_discovered"), 64) .unwrap(), @@ -276,16 +315,24 @@ unsafe fn fuzz(options: &FuzzerOptions) -> Result<(), Error> { let mutator = StdScheduledMutator::new(havoc_mutations().merge(tokens_mutations())); // A minimization+queue policy to get testcasess from the corpus - let scheduler = IndexesLenTimeMinimizerScheduler::new(QueueScheduler::new()); + let scheduler = + IndexesLenTimeMinimizerScheduler::new(&edges_observer, QueueScheduler::new()); // A fuzzer with feedbacks and a corpus scheduler let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); +<<<<<<< HEAD let observers = tuple_list!( edges_observer, time_observer, AsanErrorsObserver::new(&ASAN_ERRORS) ); +======= + #[cfg(unix)] + let observers = tuple_list!(edges_observer, time_observer, asan_observer); + #[cfg(windows)] + let observers = tuple_list!(edges_observer, time_observer,); +>>>>>>> main // Create the executor for an in-process function with just one observer for edge coverage let mut executor = FridaInProcessExecutor::new( @@ -334,7 +381,7 @@ unsafe fn fuzz(options: &FuzzerOptions) -> Result<(), Error> { Ok(()) })(state, mgr, core_id) } else { - (|state: Option<_>, mut mgr: LlmpRestartingEventManager<_, _>, _core_id| { + (|state: Option<_>, mut mgr: LlmpRestartingEventManager<_, _, _>, _core_id| { let gum = Gum::obtain(); let coverage = CoverageRuntime::new(); @@ -347,31 +394,43 @@ unsafe fn fuzz(options: &FuzzerOptions) -> Result<(), Error> { "edges", frida_helper.map_mut_ptr().unwrap(), MAP_SIZE, - )); + )) + .track_indices(); // Create an observation channel to keep track of the execution time let time_observer = TimeObserver::new("time"); + #[cfg(unix)] + let asan_observer = AsanErrorsObserver::from_static_asan_errors(); + // Feedback to rate the interestingness of an input // This one is composed by two Feedbacks in OR let mut feedback = feedback_or!( // New maximization map feedback linked to the edges observer and the feedback state - MaxMapFeedback::tracking(&edges_observer, true, false), + MaxMapFeedback::new(&edges_observer), // Time feedback, this one does not need a feedback state - TimeFeedback::with_observer(&time_observer) + TimeFeedback::new(&time_observer) ); let mut objective = feedback_or_fast!( CrashFeedback::new(), +<<<<<<< HEAD // TimeoutFeedback::new(), feedback_and_fast!(ConstFeedback::from(false), AsanErrorsFeedback::new()) +======= + TimeoutFeedback::new(), + feedback_and_fast!( + ConstFeedback::from(false), + AsanErrorsFeedback::new(&asan_observer) + ) +>>>>>>> main ); // If not restarting, create a State from scratch let mut state = state.unwrap_or_else(|| { StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance CachedOnDiskCorpus::no_meta(PathBuf::from("./corpus_discovered"), 64) .unwrap(), @@ -401,16 +460,24 @@ unsafe fn fuzz(options: &FuzzerOptions) -> Result<(), Error> { let mutator = StdScheduledMutator::new(havoc_mutations().merge(tokens_mutations())); // A minimization+queue policy to get testcasess from the corpus - let scheduler = IndexesLenTimeMinimizerScheduler::new(QueueScheduler::new()); + let scheduler = + IndexesLenTimeMinimizerScheduler::new(&edges_observer, QueueScheduler::new()); // A fuzzer with feedbacks and a corpus scheduler let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); +<<<<<<< HEAD let observers = tuple_list!( edges_observer, time_observer, AsanErrorsObserver::new(&ASAN_ERRORS) ); +======= + #[cfg(unix)] + let observers = tuple_list!(edges_observer, time_observer, asan_observer); + #[cfg(windows)] + let observers = tuple_list!(edges_observer, time_observer); +>>>>>>> main // Create the executor for an in-process function with just one observer for edge coverage let mut executor = FridaInProcessExecutor::new( @@ -445,15 +512,22 @@ unsafe fn fuzz(options: &FuzzerOptions) -> Result<(), Error> { } }; - Launcher::builder() + let builder = Launcher::builder() .configuration(EventConfig::AlwaysUnique) .shmem_provider(shmem_provider) .monitor(monitor) .run_client(&mut run_client) .cores(&options.cores) .broker_port(options.broker_port) - .stdout_file(Some(&options.stdout)) - .remote_broker_addr(options.remote_broker_addr) - .build() - .launch() + .remote_broker_addr(options.remote_broker_addr); + + #[cfg(all(unix, feature = "std"))] + { + return builder.stdout_file(Some(&options.stdout)).build().launch(); + } + + #[cfg(not(all(unix, feature = "std")))] + { + return builder.build().launch(); + } } diff --git a/fuzzers/frida_libpng/Cargo.toml b/fuzzers/frida_libpng/Cargo.toml index 0d2e8b7ab8..30ffd0e54a 100644 --- a/fuzzers/frida_libpng/Cargo.toml +++ b/fuzzers/frida_libpng/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "frida_fuzzer" -version = "0.11.2" +version = "0.12.0" authors = ["Andrea Fioraldi ", "Dominik Maier "] edition = "2021" diff --git a/fuzzers/frida_libpng/Makefile.toml b/fuzzers/frida_libpng/Makefile.toml index 1df01fd2c2..531f77445b 100644 --- a/fuzzers/frida_libpng/Makefile.toml +++ b/fuzzers/frida_libpng/Makefile.toml @@ -110,8 +110,8 @@ windows_alias = "test_windows" script_runner = "@shell" script=''' rm -rf libafl_unix_shmem_server || true -timeout 30s ./${FUZZER_NAME} -F LLVMFuzzerTestOneInput -H ./libpng-harness.so -l ./libpng-harness.so >fuzz_stdout.log 2>/dev/null || true -if grep -qa "corpus: 30" fuzz_stdout.log; then +timeout 30s ./${FUZZER_NAME} -F LLVMFuzzerTestOneInput -H ./libpng-harness.so -l ./libpng-harness.so | tee fuzz_stdout.log 2>/dev/null || true +if grep -qa "corpus: 70" fuzz_stdout.log; then echo "Fuzzer is working" else echo "Fuzzer does not generate any testcases or any crashes" @@ -125,7 +125,7 @@ dependencies = [ "fuzzer", "harness" ] script_runner = "@shell" script=''' rm -rf libafl_unix_shmem_server || true -timeout 30s ./${FUZZER_NAME} -F LLVMFuzzerTestOneInput -H ./libpng-harness.so -l ./libpng-harness.so >fuzz_stdout.log 2>/dev/null || true +timeout 30s ./${FUZZER_NAME} -F LLVMFuzzerTestOneInput -H ./libpng-harness.so -l ./libpng-harness.so | tee fuzz_stdout.log 2>/dev/null || true ''' dependencies = [ "fuzzer", "harness" ] diff --git a/fuzzers/frida_libpng/src/fuzzer.rs b/fuzzers/frida_libpng/src/fuzzer.rs index 3f38c27f0b..d7a5fc6f94 100644 --- a/fuzzers/frida_libpng/src/fuzzer.rs +++ b/fuzzers/frida_libpng/src/fuzzer.rs @@ -1,9 +1,5 @@ //! A libfuzzer-like fuzzer with llmp-multithreading support and restarts //! The example harness is built for libpng. -use mimalloc::MiMalloc; -#[global_allocator] -static GLOBAL: MiMalloc = MiMalloc; - use std::path::PathBuf; use frida_gum::Gum; @@ -20,17 +16,16 @@ use libafl::{ scheduled::{havoc_mutations, tokens_mutations, StdScheduledMutator}, token_mutations::{I2SRandReplace, Tokens}, }, - observers::{HitcountsMapObserver, StdMapObserver, TimeObserver}, + observers::{CanTrack, HitcountsMapObserver, StdMapObserver, TimeObserver}, schedulers::{IndexesLenTimeMinimizerScheduler, QueueScheduler}, stages::{ShadowTracingStage, StdMutationalStage}, - state::{HasCorpus, HasMetadata, StdState}, - Error, + state::{HasCorpus, StdState}, + Error, HasMetadata, }; #[cfg(unix)] use libafl::{feedback_and_fast, feedbacks::ConstFeedback}; use libafl_bolts::{ cli::{parse_args, FuzzerOptions}, - current_nanos, rands::StdRand, shmem::{ShMemProvider, StdShMemProvider}, tuples::{tuple_list, Merge}, @@ -39,7 +34,7 @@ use libafl_bolts::{ #[cfg(unix)] use libafl_frida::asan::{ asan_rt::AsanRuntime, - errors::{AsanErrorsFeedback, AsanErrorsObserver, ASAN_ERRORS}, + errors::{AsanErrorsFeedback, AsanErrorsObserver}, }; use libafl_frida::{ cmplog_rt::CmpLogRuntime, @@ -48,6 +43,10 @@ use libafl_frida::{ helper::FridaInstrumentationHelper, }; use libafl_targets::cmplog::CmpLogObserver; +use mimalloc::MiMalloc; + +#[global_allocator] +static GLOBAL: MiMalloc = MiMalloc; /// The main fn, usually parsing parameters, and starting the fuzzer pub fn main() { @@ -73,7 +72,7 @@ unsafe fn fuzz(options: &FuzzerOptions) -> Result<(), Error> { let shmem_provider = StdShMemProvider::new()?; - let mut run_client = |state: Option<_>, mgr: LlmpRestartingEventManager<_, _>, core_id| { + let mut run_client = |state: Option<_>, mgr: LlmpRestartingEventManager<_, _, _>, core_id| { // The restarting state will spawn the same process again as child, then restarted it each time it crashes. // println!("{:?}", mgr.mgr_id()); @@ -91,12 +90,12 @@ unsafe fn fuzz(options: &FuzzerOptions) -> Result<(), Error> { }; if options.asan && options.asan_cores.contains(core_id) { - (|state: Option<_>, mut mgr: LlmpRestartingEventManager<_, _>, _core_id| { + (|state: Option<_>, mut mgr: LlmpRestartingEventManager<_, _, _>, _core_id| { let gum = Gum::obtain(); let coverage = CoverageRuntime::new(); #[cfg(unix)] - let asan = AsanRuntime::new(&options); + let asan = AsanRuntime::new(options); #[cfg(unix)] let mut frida_helper = @@ -110,18 +109,21 @@ unsafe fn fuzz(options: &FuzzerOptions) -> Result<(), Error> { "edges", frida_helper.map_mut_ptr().unwrap(), MAP_SIZE, - )); + )) + .track_indices(); // Create an observation channel to keep track of the execution time let time_observer = TimeObserver::new("time"); + #[cfg(unix)] + let asan_observer = AsanErrorsObserver::from_static_asan_errors(); // Feedback to rate the interestingness of an input // This one is composed by two Feedbacks in OR let mut feedback = feedback_or!( // New maximization map feedback linked to the edges observer and the feedback state - MaxMapFeedback::tracking(&edges_observer, true, false), + MaxMapFeedback::new(&edges_observer), // Time feedback, this one does not need a feedback state - TimeFeedback::with_observer(&time_observer) + TimeFeedback::new(&time_observer) ); // Feedbacks to recognize an input as solution @@ -130,7 +132,10 @@ unsafe fn fuzz(options: &FuzzerOptions) -> Result<(), Error> { CrashFeedback::new(), TimeoutFeedback::new(), // true enables the AsanErrorFeedback - feedback_and_fast!(ConstFeedback::from(true), AsanErrorsFeedback::new()) + feedback_and_fast!( + ConstFeedback::from(true), + AsanErrorsFeedback::new(&asan_observer) + ) ); #[cfg(windows)] let mut objective = feedback_or_fast!(CrashFeedback::new(), TimeoutFeedback::new()); @@ -139,7 +144,7 @@ unsafe fn fuzz(options: &FuzzerOptions) -> Result<(), Error> { let mut state = state.unwrap_or_else(|| { StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance CachedOnDiskCorpus::no_meta(PathBuf::from("./corpus_discovered"), 64) .unwrap(), @@ -169,17 +174,14 @@ unsafe fn fuzz(options: &FuzzerOptions) -> Result<(), Error> { let mutator = StdScheduledMutator::new(havoc_mutations().merge(tokens_mutations())); // A minimization+queue policy to get testcasess from the corpus - let scheduler = IndexesLenTimeMinimizerScheduler::new(QueueScheduler::new()); + let scheduler = + IndexesLenTimeMinimizerScheduler::new(&edges_observer, QueueScheduler::new()); // A fuzzer with feedbacks and a corpus scheduler let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); #[cfg(unix)] - let observers = tuple_list!( - edges_observer, - time_observer, - AsanErrorsObserver::new(&ASAN_ERRORS) - ); + let observers = tuple_list!(edges_observer, time_observer, asan_observer); #[cfg(windows)] let observers = tuple_list!(edges_observer, time_observer); @@ -213,7 +215,7 @@ unsafe fn fuzz(options: &FuzzerOptions) -> Result<(), Error> { Ok(()) })(state, mgr, core_id) } else if options.cmplog && options.cmplog_cores.contains(core_id) { - (|state: Option<_>, mut mgr: LlmpRestartingEventManager<_, _>, _core_id| { + (|state: Option<_>, mut mgr: LlmpRestartingEventManager<_, _, _>, _core_id| { let gum = Gum::obtain(); let coverage = CoverageRuntime::new(); @@ -228,25 +230,31 @@ unsafe fn fuzz(options: &FuzzerOptions) -> Result<(), Error> { "edges", frida_helper.map_mut_ptr().unwrap(), MAP_SIZE, - )); + )) + .track_indices(); // Create an observation channel to keep track of the execution time let time_observer = TimeObserver::new("time"); + #[cfg(unix)] + let asan_observer = AsanErrorsObserver::from_static_asan_errors(); // Feedback to rate the interestingness of an input // This one is composed by two Feedbacks in OR let mut feedback = feedback_or!( // New maximization map feedback linked to the edges observer and the feedback state - MaxMapFeedback::tracking(&edges_observer, true, false), + MaxMapFeedback::new(&edges_observer), // Time feedback, this one does not need a feedback state - TimeFeedback::with_observer(&time_observer) + TimeFeedback::new(&time_observer) ); #[cfg(unix)] let mut objective = feedback_or_fast!( CrashFeedback::new(), TimeoutFeedback::new(), - feedback_and_fast!(ConstFeedback::from(false), AsanErrorsFeedback::new()) + feedback_and_fast!( + ConstFeedback::from(false), + AsanErrorsFeedback::new(&asan_observer) + ) ); #[cfg(windows)] let mut objective = feedback_or_fast!(CrashFeedback::new(), TimeoutFeedback::new()); @@ -255,7 +263,7 @@ unsafe fn fuzz(options: &FuzzerOptions) -> Result<(), Error> { let mut state = state.unwrap_or_else(|| { StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance CachedOnDiskCorpus::no_meta(PathBuf::from("./corpus_discovered"), 64) .unwrap(), @@ -285,19 +293,16 @@ unsafe fn fuzz(options: &FuzzerOptions) -> Result<(), Error> { let mutator = StdScheduledMutator::new(havoc_mutations().merge(tokens_mutations())); // A minimization+queue policy to get testcasess from the corpus - let scheduler = IndexesLenTimeMinimizerScheduler::new(QueueScheduler::new()); + let scheduler = + IndexesLenTimeMinimizerScheduler::new(&edges_observer, QueueScheduler::new()); // A fuzzer with feedbacks and a corpus scheduler let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); #[cfg(unix)] - let observers = tuple_list!( - edges_observer, - time_observer, - AsanErrorsObserver::new(&ASAN_ERRORS) - ); + let observers = tuple_list!(edges_observer, time_observer, asan_observer); #[cfg(windows)] - let observers = tuple_list!(edges_observer, time_observer,); + let observers = tuple_list!(edges_observer, time_observer); // Create the executor for an in-process function with just one observer for edge coverage let mut executor = FridaInProcessExecutor::new( @@ -345,7 +350,7 @@ unsafe fn fuzz(options: &FuzzerOptions) -> Result<(), Error> { Ok(()) })(state, mgr, core_id) } else { - (|state: Option<_>, mut mgr: LlmpRestartingEventManager<_, _>, _core_id| { + (|state: Option<_>, mut mgr: LlmpRestartingEventManager<_, _, _>, _core_id| { let gum = Gum::obtain(); let coverage = CoverageRuntime::new(); @@ -358,25 +363,31 @@ unsafe fn fuzz(options: &FuzzerOptions) -> Result<(), Error> { "edges", frida_helper.map_mut_ptr().unwrap(), MAP_SIZE, - )); + )) + .track_indices(); // Create an observation channel to keep track of the execution time let time_observer = TimeObserver::new("time"); + #[cfg(unix)] + let asan_observer = AsanErrorsObserver::from_static_asan_errors(); // Feedback to rate the interestingness of an input // This one is composed by two Feedbacks in OR let mut feedback = feedback_or!( // New maximization map feedback linked to the edges observer and the feedback state - MaxMapFeedback::tracking(&edges_observer, true, false), + MaxMapFeedback::new(&edges_observer), // Time feedback, this one does not need a feedback state - TimeFeedback::with_observer(&time_observer) + TimeFeedback::new(&time_observer) ); #[cfg(unix)] let mut objective = feedback_or_fast!( CrashFeedback::new(), TimeoutFeedback::new(), - feedback_and_fast!(ConstFeedback::from(false), AsanErrorsFeedback::new()) + feedback_and_fast!( + ConstFeedback::from(false), + AsanErrorsFeedback::new(&asan_observer) + ) ); #[cfg(windows)] let mut objective = feedback_or_fast!(CrashFeedback::new(), TimeoutFeedback::new()); @@ -385,7 +396,7 @@ unsafe fn fuzz(options: &FuzzerOptions) -> Result<(), Error> { let mut state = state.unwrap_or_else(|| { StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance CachedOnDiskCorpus::no_meta(PathBuf::from("./corpus_discovered"), 64) .unwrap(), @@ -415,19 +426,16 @@ unsafe fn fuzz(options: &FuzzerOptions) -> Result<(), Error> { let mutator = StdScheduledMutator::new(havoc_mutations().merge(tokens_mutations())); // A minimization+queue policy to get testcasess from the corpus - let scheduler = IndexesLenTimeMinimizerScheduler::new(QueueScheduler::new()); + let scheduler = + IndexesLenTimeMinimizerScheduler::new(&edges_observer, QueueScheduler::new()); // A fuzzer with feedbacks and a corpus scheduler let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); #[cfg(unix)] - let observers = tuple_list!( - edges_observer, - time_observer, - AsanErrorsObserver::new(&ASAN_ERRORS) - ); + let observers = tuple_list!(edges_observer, time_observer, asan_observer); #[cfg(windows)] - let observers = tuple_list!(edges_observer, time_observer,); + let observers = tuple_list!(edges_observer, time_observer); // Create the executor for an in-process function with just one observer for edge coverage let mut executor = FridaInProcessExecutor::new( diff --git a/fuzzers/fuzzbench/Cargo.toml b/fuzzers/fuzzbench/Cargo.toml index 046c7405c0..820d32d0ae 100644 --- a/fuzzers/fuzzbench/Cargo.toml +++ b/fuzzers/fuzzbench/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fuzzbench" -version = "0.11.2" +version = "0.12.0" authors = ["Andrea Fioraldi ", "Dominik Maier "] edition = "2021" diff --git a/fuzzers/fuzzbench/Makefile.toml b/fuzzers/fuzzbench/Makefile.toml index 3cc7164e4a..c3bfdb78c1 100644 --- a/fuzzers/fuzzbench/Makefile.toml +++ b/fuzzers/fuzzbench/Makefile.toml @@ -82,7 +82,7 @@ rm -rf libafl_unix_shmem_server || true mkdir in || true echo a > in/a # Allow sigterm as exit code -timeout 31s ./${FUZZER_NAME} -o out -i in >fuzz_stdout.log || true +timeout 31s ./${FUZZER_NAME} -o out -i in | tee fuzz_stdout.log || true if grep -qa "objectives: 1" fuzz_stdout.log; then echo "Fuzzer is working" else diff --git a/fuzzers/fuzzbench/src/lib.rs b/fuzzers/fuzzbench/src/lib.rs index b6cd2c869a..9b162a48e3 100644 --- a/fuzzers/fuzzbench/src/lib.rs +++ b/fuzzers/fuzzbench/src/lib.rs @@ -28,7 +28,7 @@ use libafl::{ scheduled::havoc_mutations, token_mutations::I2SRandReplace, tokens_mutations, StdMOptMutator, StdScheduledMutator, Tokens, }, - observers::{HitcountsMapObserver, TimeObserver}, + observers::{CanTrack, HitcountsMapObserver, TimeObserver}, schedulers::{ powersched::PowerSchedule, IndexesLenTimeMinimizerScheduler, StdWeightedScheduler, }, @@ -36,11 +36,11 @@ use libafl::{ calibrate::CalibrationStage, power::StdPowerMutationalStage, StdMutationalStage, TracingStage, }, - state::{HasCorpus, HasMetadata, StdState}, - Error, + state::{HasCorpus, StdState}, + Error, HasMetadata, }; use libafl_bolts::{ - current_nanos, current_time, + current_time, os::dup2, rands::StdRand, shmem::{ShMemProvider, StdShMemProvider}, @@ -53,7 +53,7 @@ use libafl_targets::{ libfuzzer_initialize, libfuzzer_test_one_input, std_edges_map_observer, CmpLogObserver, }; #[cfg(unix)] -use nix::{self, unistd::dup}; +use nix::unistd::dup; /// The fuzzer main (as `no_mangle` C function) #[no_mangle] @@ -242,14 +242,15 @@ fn fuzz( // Create an observation channel using the coverage map // We don't use the hitcounts (see the Cargo.toml, we use pcguard_edges) - let edges_observer = HitcountsMapObserver::new(unsafe { std_edges_map_observer("edges") }); + let edges_observer = + HitcountsMapObserver::new(unsafe { std_edges_map_observer("edges") }).track_indices(); // Create an observation channel to keep track of the execution time let time_observer = TimeObserver::new("time"); let cmplog_observer = CmpLogObserver::new("cmplog", true); - let map_feedback = MaxMapFeedback::tracking(&edges_observer, true, false); + let map_feedback = MaxMapFeedback::new(&edges_observer); let calibration = CalibrationStage::new(&map_feedback); @@ -259,7 +260,7 @@ fn fuzz( // New maximization map feedback linked to the edges observer and the feedback state map_feedback, // Time feedback, this one does not need a feedback state - TimeFeedback::with_observer(&time_observer) + TimeFeedback::new(&time_observer) ); // A feedback to choose if an input is a solution or not @@ -269,7 +270,7 @@ fn fuzz( let mut state = state.unwrap_or_else(|| { StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryOnDiskCorpus::new(corpus_dir).unwrap(), // Corpus in which we store solutions (crashes in this example), @@ -307,11 +308,10 @@ fn fuzz( let power = StdPowerMutationalStage::new(mutator); // A minimization+queue policy to get testcasess from the corpus - let scheduler = IndexesLenTimeMinimizerScheduler::new(StdWeightedScheduler::with_schedule( - &mut state, + let scheduler = IndexesLenTimeMinimizerScheduler::new( &edges_observer, - Some(PowerSchedule::FAST), - )); + StdWeightedScheduler::with_schedule(&mut state, &edges_observer, Some(PowerSchedule::FAST)), + ); // A fuzzer with feedbacks and a corpus scheduler let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); diff --git a/fuzzers/libfuzzer_reachability/Cargo.toml b/fuzzers/fuzzbench_ctx/Cargo.toml similarity index 64% rename from fuzzers/libfuzzer_reachability/Cargo.toml rename to fuzzers/fuzzbench_ctx/Cargo.toml index 1a197fea66..410d4cdb59 100644 --- a/fuzzers/libfuzzer_reachability/Cargo.toml +++ b/fuzzers/fuzzbench_ctx/Cargo.toml @@ -1,12 +1,13 @@ [package] -name = "libfuzzer_reachability" -version = "0.11.2" +name = "fuzzbench_ctx" +version = "0.12.0" authors = ["Andrea Fioraldi ", "Dominik Maier "] edition = "2021" [features] default = ["std"] std = [] +no_link_main = ["libafl_targets/libfuzzer_no_link_main"] [profile.release] lto = true @@ -14,6 +15,11 @@ codegen-units = 1 opt-level = 3 debug = true +[profile.release-fuzzbench] +inherits = "release" +debug = false +strip = true + [build-dependencies] cc = { version = "1.0", features = ["parallel"] } which = "4.4" @@ -21,11 +27,13 @@ which = "4.4" [dependencies] libafl = { path = "../../libafl/" } libafl_bolts = { path = "../../libafl_bolts/" } -libafl_targets = { path = "../../libafl_targets/", features = ["sancov_pcguard_hitcounts", "libfuzzer"] } +libafl_targets = { path = "../../libafl_targets/", features = ["sancov_pcguard_hitcounts", "sancov_cmplog", "libfuzzer", "sancov_ctx"] } # TODO Include it only when building cc libafl_cc = { path = "../../libafl_cc/" } +clap = { version = "4.0", features = ["default"] } +nix = { version = "0.27", features = ["fs"] } mimalloc = { version = "*", default-features = false } [lib] -name = "libfuzzer_libpng" +name = "fuzzbench" crate-type = ["staticlib"] diff --git a/fuzzers/fuzzbench_ctx/Makefile.toml b/fuzzers/fuzzbench_ctx/Makefile.toml new file mode 100644 index 0000000000..c3bfdb78c1 --- /dev/null +++ b/fuzzers/fuzzbench_ctx/Makefile.toml @@ -0,0 +1,108 @@ +[env] +PROJECT_DIR = { script = ["pwd"] } +CARGO_TARGET_DIR = { value = "${PROJECT_DIR}/target", condition = { env_not_set = ["CARGO_TARGET_DIR"] } } +FUZZER_NAME="fuzzer" +PROFILE = { value = "release", condition = {env_not_set = ["PROFILE"]} } +PROFILE_DIR = {value = "release", condition = {env_not_set = ["PROFILE_DIR"] }} + +[tasks.unsupported] +script_runner="@shell" +script=''' +echo "Cargo-make not integrated yet on this" +''' + +# Compilers +[tasks.cxx] +linux_alias = "cxx_unix" +mac_alias = "cxx_unix" +windows_alias = "unsupported" + +[tasks.cxx_unix] +command = "cargo" +args = ["build", "--profile", "${PROFILE}"] + +[tasks.cc] +linux_alias = "cc_unix" +mac_alias = "cc_unix" +windows_alias = "unsupported" + +[tasks.cc_unix] +command = "cargo" +args = ["build", "--profile", "${PROFILE}"] + +# fuzz.o File +[tasks.fuzz_o] +linux_alias = "fuzz_o_unix" +mac_alias = "fuzz_o_unix" +windows_alias = "unsupported" + +[tasks.fuzz_o_unix] +command = "${CARGO_TARGET_DIR}/${PROFILE_DIR}/libafl_cc" +args = ["--libafl-no-link", "-O3", "-c", "fuzz.c", "-o", "fuzz.o"] +dependencies = ["cc", "cxx"] + +# Fuzzer +[tasks.fuzzer] +linux_alias = "fuzzer_unix" +mac_alias = "fuzzer_unix" +windows_alias = "unsupported" + +[tasks.fuzzer_unix] +command = "${CARGO_TARGET_DIR}/${PROFILE_DIR}/libafl_cxx" +args = ["--libafl", "fuzz.o", "-o", "${FUZZER_NAME}", "-lm", "-lz"] +dependencies = ["cc", "cxx", "fuzz_o"] + +# Run +[tasks.run] +linux_alias = "run_unix" +mac_alias = "run_unix" +windows_alias = "unsupported" + +[tasks.run_unix] +script_runner="@shell" +script=''' +rm -rf libafl_unix_shmem_server || true +mkdir in || true +echo a > in/a +./${FUZZER_NAME} -o out -i in +''' +dependencies = ["fuzzer"] + + +# Test +[tasks.test] +linux_alias = "test_unix" +mac_alias = "test_unix" +windows_alias = "unsupported" + +[tasks.test_unix] +script_runner="@shell" +script=''' +rm -rf libafl_unix_shmem_server || true +mkdir in || true +echo a > in/a +# Allow sigterm as exit code +timeout 31s ./${FUZZER_NAME} -o out -i in | tee fuzz_stdout.log || true +if grep -qa "objectives: 1" fuzz_stdout.log; then + echo "Fuzzer is working" +else + echo "Fuzzer does not generate any testcases or any crashes" + exit 1 +fi +rm -rf out || true +rm -rf in || true +''' +dependencies = ["fuzzer"] + +# Clean +[tasks.clean] +linux_alias = "clean_unix" +mac_alias = "clean_unix" +windows_alias = "unsupported" + +[tasks.clean_unix] +script_runner="@shell" +script=''' +rm ./${FUZZER_NAME} || true +rm fuzz.o || true +''' diff --git a/fuzzers/fuzzbench_ctx/fuzz.c b/fuzzers/fuzzbench_ctx/fuzz.c new file mode 100644 index 0000000000..0460dd63d2 --- /dev/null +++ b/fuzzers/fuzzbench_ctx/fuzz.c @@ -0,0 +1,19 @@ +#include +#include +#include + +int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { + if (Size >= 8 && *(uint32_t *)Data == 0xaabbccdd) { abort(); } + char buf[8] = {'a', 'b', 'c', 'd'}; + + if (memcmp(Data, buf, 4) == 0) { abort(); } + return 0; +} + +/* +int main() { + + char buf [10] = {0}; + LLVMFuzzerTestOneInput(buf, 10); + +}*/ diff --git a/fuzzers/libfuzzer_libpng_ctx/src/bin/libafl_cc.rs b/fuzzers/fuzzbench_ctx/src/bin/libafl_cc.rs similarity index 66% rename from fuzzers/libfuzzer_libpng_ctx/src/bin/libafl_cc.rs rename to fuzzers/fuzzbench_ctx/src/bin/libafl_cc.rs index fc8c6c2800..9ddfe1a5e7 100644 --- a/fuzzers/libfuzzer_libpng_ctx/src/bin/libafl_cc.rs +++ b/fuzzers/fuzzbench_ctx/src/bin/libafl_cc.rs @@ -3,7 +3,7 @@ use std::env; use libafl_cc::{ClangWrapper, CompilerWrapper, LLVMPasses, ToolWrapper}; pub fn main() { - let args: Vec = env::args().collect(); + let mut args: Vec = env::args().collect(); if args.len() > 1 { let mut dir = env::current_exe().unwrap(); let wrapper_name = dir.file_name().unwrap().to_str().unwrap(); @@ -16,16 +16,25 @@ pub fn main() { dir.pop(); + // Must be always present, even without --libafl + args.push("-fsanitize-coverage=trace-pc-guard,trace-cmp".into()); + let mut cc = ClangWrapper::new(); + + #[cfg(any(target_os = "linux", target_vendor = "apple"))] + cc.add_pass(LLVMPasses::AutoTokens); + if let Some(code) = cc .cpp(is_cpp) // silence the compiler wrapper output, needed for some configure scripts. .silence(true) + // add arguments only if --libafl or --libafl-no-link are present + .need_libafl_arg(true) .parse_args(&args) .expect("Failed to parse the command line") - .add_pass(LLVMPasses::AFLCoverage) - .add_passes_arg("-ctx") // Context sensitive coverage - .link_staticlib(&dir, "libfuzzer_libpng") + .link_staticlib(&dir, "fuzzbench") + .add_pass(LLVMPasses::CmpLogRtn) + .add_pass(LLVMPasses::Ctx) .run() .expect("Failed to run the wrapped compiler") { diff --git a/fuzzers/libfuzzer_libpng_ctx/src/bin/libafl_cxx.rs b/fuzzers/fuzzbench_ctx/src/bin/libafl_cxx.rs similarity index 100% rename from fuzzers/libfuzzer_libpng_ctx/src/bin/libafl_cxx.rs rename to fuzzers/fuzzbench_ctx/src/bin/libafl_cxx.rs diff --git a/fuzzers/fuzzbench_ctx/src/lib.rs b/fuzzers/fuzzbench_ctx/src/lib.rs new file mode 100644 index 0000000000..b407fd01af --- /dev/null +++ b/fuzzers/fuzzbench_ctx/src/lib.rs @@ -0,0 +1,409 @@ +//! A singlethreaded libfuzzer-like fuzzer that can auto-restart. +use mimalloc::MiMalloc; +#[global_allocator] +static GLOBAL: MiMalloc = MiMalloc; + +use core::{cell::RefCell, time::Duration}; +#[cfg(unix)] +use std::os::unix::io::{AsRawFd, FromRawFd}; +use std::{ + env, + fs::{self, File, OpenOptions}, + io::{self, Read, Write}, + path::PathBuf, + process, +}; + +use clap::{Arg, Command}; +use libafl::{ + corpus::{Corpus, InMemoryOnDiskCorpus, OnDiskCorpus}, + events::SimpleRestartingEventManager, + executors::{ + inprocess::{HookableInProcessExecutor, InProcessExecutor}, + ExitKind, + }, + feedback_or, + feedbacks::{CrashFeedback, MaxMapFeedback, TimeFeedback}, + fuzzer::{Fuzzer, StdFuzzer}, + inputs::{BytesInput, HasTargetBytes}, + monitors::SimpleMonitor, + mutators::{ + scheduled::havoc_mutations, token_mutations::I2SRandReplace, tokens_mutations, + StdMOptMutator, StdScheduledMutator, Tokens, + }, + observers::{CanTrack, HitcountsMapObserver, StdMapObserver, TimeObserver}, + schedulers::{ + powersched::PowerSchedule, IndexesLenTimeMinimizerScheduler, StdWeightedScheduler, + }, + stages::{ + calibrate::CalibrationStage, power::StdPowerMutationalStage, StdMutationalStage, + TracingStage, + }, + state::{HasCorpus, StdState}, + Error, HasMetadata, +}; +use libafl_bolts::{ + current_time, + os::dup2, + ownedref::OwnedMutSlice, + rands::StdRand, + shmem::{ShMemProvider, StdShMemProvider}, + tuples::{tuple_list, Merge}, + AsSlice, +}; +#[cfg(any(target_os = "linux", target_vendor = "apple"))] +use libafl_targets::autotokens; +use libafl_targets::{ + edges_map_mut_ptr, libfuzzer_initialize, libfuzzer_test_one_input, CmpLogObserver, CtxHook, + EDGES_MAP_SIZE_IN_USE, +}; +#[cfg(unix)] +use nix::unistd::dup; + +/// The fuzzer main (as `no_mangle` C function) +#[no_mangle] +pub extern "C" fn libafl_main() { + // Registry the metadata types used in this fuzzer + // Needed only on no_std + // unsafe { RegistryBuilder::register::(); } + + let res = match Command::new(env!("CARGO_PKG_NAME")) + .version(env!("CARGO_PKG_VERSION")) + .author("AFLplusplus team") + .about("LibAFL-based fuzzer for Fuzzbench") + .arg( + Arg::new("out") + .short('o') + .long("output") + .help("The directory to place finds in ('corpus')"), + ) + .arg( + Arg::new("in") + .short('i') + .long("input") + .help("The directory to read initial inputs from ('seeds')"), + ) + .arg( + Arg::new("tokens") + .short('x') + .long("tokens") + .help("A file to read tokens from, to be used during fuzzing"), + ) + .arg( + Arg::new("logfile") + .short('l') + .long("logfile") + .help("Duplicates all output to this file") + .default_value("libafl.log"), + ) + .arg( + Arg::new("timeout") + .short('t') + .long("timeout") + .help("Timeout for each individual execution, in milliseconds") + .default_value("1200"), + ) + .arg(Arg::new("remaining")) + .try_get_matches() + { + Ok(res) => res, + Err(err) => { + println!( + "Syntax: {}, [-x dictionary] -o corpus_dir -i seed_dir\n{:?}", + env::current_exe() + .unwrap_or_else(|_| "fuzzer".into()) + .to_string_lossy(), + err, + ); + return; + } + }; + + println!( + "Workdir: {:?}", + env::current_dir().unwrap().to_string_lossy().to_string() + ); + + if let Some(filenames) = res.get_many::("remaining") { + let filenames: Vec<&str> = filenames.map(String::as_str).collect(); + if !filenames.is_empty() { + run_testcases(&filenames); + return; + } + } + + // For fuzzbench, crashes and finds are inside the same `corpus` directory, in the "queue" and "crashes" subdir. + let mut out_dir = PathBuf::from( + res.get_one::("out") + .expect("The --output parameter is missing") + .to_string(), + ); + if fs::create_dir(&out_dir).is_err() { + println!("Out dir at {:?} already exists.", &out_dir); + if !out_dir.is_dir() { + println!("Out dir at {:?} is not a valid directory!", &out_dir); + return; + } + } + let mut crashes = out_dir.clone(); + crashes.push("crashes"); + out_dir.push("queue"); + + let in_dir = PathBuf::from( + res.get_one::("in") + .expect("The --input parameter is missing") + .to_string(), + ); + if !in_dir.is_dir() { + println!("In dir at {:?} is not a valid directory!", &in_dir); + return; + } + + let tokens = res.get_one::("tokens").map(PathBuf::from); + + let logfile = PathBuf::from(res.get_one::("logfile").unwrap().to_string()); + + let timeout = Duration::from_millis( + res.get_one::("timeout") + .unwrap() + .to_string() + .parse() + .expect("Could not parse timeout in milliseconds"), + ); + + fuzz(out_dir, crashes, &in_dir, tokens, &logfile, timeout) + .expect("An error occurred while fuzzing"); +} + +fn run_testcases(filenames: &[&str]) { + // The actual target run starts here. + // Call LLVMFUzzerInitialize() if present. + let args: Vec = env::args().collect(); + if libfuzzer_initialize(&args) == -1 { + println!("Warning: LLVMFuzzerInitialize failed with -1"); + } + + println!( + "You are not fuzzing, just executing {} testcases", + filenames.len() + ); + for fname in filenames { + println!("Executing {fname}"); + + let mut file = File::open(fname).expect("No file found"); + let mut buffer = vec![]; + file.read_to_end(&mut buffer).expect("Buffer overflow"); + + libfuzzer_test_one_input(&buffer); + } +} + +/// The actual fuzzer +#[allow(clippy::too_many_lines)] +fn fuzz( + corpus_dir: PathBuf, + objective_dir: PathBuf, + seed_dir: &PathBuf, + tokenfile: Option, + logfile: &PathBuf, + timeout: Duration, +) -> Result<(), Error> { + let log = RefCell::new(OpenOptions::new().append(true).create(true).open(logfile)?); + + #[cfg(unix)] + let mut stdout_cpy = unsafe { + let new_fd = dup(io::stdout().as_raw_fd())?; + File::from_raw_fd(new_fd) + }; + #[cfg(unix)] + let file_null = File::open("/dev/null")?; + + // 'While the monitor are state, they are usually used in the broker - which is likely never restarted + let monitor = SimpleMonitor::new(|s| { + #[cfg(unix)] + writeln!(&mut stdout_cpy, "{s}").unwrap(); + #[cfg(windows)] + println!("{s}"); + writeln!(log.borrow_mut(), "{:?} {s}", current_time()).unwrap(); + }); + + // We need a shared map to store our state before a crash. + // This way, we are able to continue fuzzing afterwards. + let mut shmem_provider = StdShMemProvider::new()?; + + let (state, mut mgr) = match SimpleRestartingEventManager::launch(monitor, &mut shmem_provider) + { + // The restarting state will spawn the same process again as child, then restarted it each time it crashes. + Ok(res) => res, + Err(err) => match err { + Error::ShuttingDown => { + return Ok(()); + } + _ => { + panic!("Failed to setup the restarter: {err}"); + } + }, + }; + + // Create an observation channel using the coverage map + // We don't use the hitcounts (see the Cargo.toml, we use pcguard_edges) + let edges_observer = HitcountsMapObserver::new(unsafe { + StdMapObserver::from_mut_slice( + "edges", + OwnedMutSlice::from_raw_parts_mut(edges_map_mut_ptr(), EDGES_MAP_SIZE_IN_USE), + ) + }) + .track_indices(); + + // Create an observation channel to keep track of the execution time + let time_observer = TimeObserver::new("time"); + + let cmplog_observer = CmpLogObserver::new("cmplog", true); + + let map_feedback = MaxMapFeedback::new(&edges_observer); + + let calibration = CalibrationStage::new(&map_feedback); + + // Feedback to rate the interestingness of an input + // This one is composed by two Feedbacks in OR + let mut feedback = feedback_or!( + // New maximization map feedback linked to the edges observer and the feedback state + map_feedback, + // Time feedback, this one does not need a feedback state + TimeFeedback::new(&time_observer) + ); + + // A feedback to choose if an input is a solution or not + let mut objective = CrashFeedback::new(); + + // If not restarting, create a State from scratch + let mut state = state.unwrap_or_else(|| { + StdState::new( + // RNG + StdRand::new(), + // Corpus that will be evolved, we keep it in memory for performance + InMemoryOnDiskCorpus::new(corpus_dir).unwrap(), + // Corpus in which we store solutions (crashes in this example), + // on disk so the user can get them after stopping the fuzzer + OnDiskCorpus::new(objective_dir).unwrap(), + // States of the feedbacks. + // The feedbacks can report the data that should persist in the State. + &mut feedback, + // Same for objective feedbacks + &mut objective, + ) + .unwrap() + }); + + println!("Let's fuzz :)"); + + // The actual target run starts here. + // Call LLVMFUzzerInitialize() if present. + let args: Vec = env::args().collect(); + if libfuzzer_initialize(&args) == -1 { + println!("Warning: LLVMFuzzerInitialize failed with -1"); + } + + // Setup a randomic Input2State stage + let i2s = StdMutationalStage::new(StdScheduledMutator::new(tuple_list!(I2SRandReplace::new()))); + + // Setup a MOPT mutator + let mutator = StdMOptMutator::new( + &mut state, + havoc_mutations().merge(tokens_mutations()), + 7, + 5, + )?; + + let power = StdPowerMutationalStage::new(mutator); + + // A minimization+queue policy to get testcasess from the corpus + let scheduler = IndexesLenTimeMinimizerScheduler::new( + &edges_observer, + StdWeightedScheduler::with_schedule(&mut state, &edges_observer, Some(PowerSchedule::FAST)), + ); + + // A fuzzer with feedbacks and a corpus scheduler + let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); + + // The wrapped harness function, calling out to the LLVM-style harness + let mut harness = |input: &BytesInput| { + let target = input.target_bytes(); + let buf = target.as_slice(); + libfuzzer_test_one_input(buf); + ExitKind::Ok + }; + + let mut tracing_harness = harness; + let ctx_hook = CtxHook::new(); + // Create the executor for an in-process function with one observer for edge coverage and one for the execution time + let mut executor = HookableInProcessExecutor::with_timeout_generic( + tuple_list!(ctx_hook), + &mut harness, + tuple_list!(edges_observer, time_observer), + &mut fuzzer, + &mut state, + &mut mgr, + timeout, + )?; + + // Setup a tracing stage in which we log comparisons + let tracing = TracingStage::new( + InProcessExecutor::with_timeout( + &mut tracing_harness, + tuple_list!(cmplog_observer), + &mut fuzzer, + &mut state, + &mut mgr, + timeout * 10, + )?, + // Give it more time! + ); + + // The order of the stages matter! + let mut stages = tuple_list!(calibration, tracing, i2s, power); + + // Read tokens + if state.metadata_map().get::().is_none() { + let mut toks = Tokens::default(); + if let Some(tokenfile) = tokenfile { + toks.add_from_file(tokenfile)?; + } + #[cfg(any(target_os = "linux", target_vendor = "apple"))] + { + toks += autotokens()?; + } + + if !toks.is_empty() { + state.add_metadata(toks); + } + } + + // In case the corpus is empty (on first run), reset + if state.must_load_initial_inputs() { + state + .load_initial_inputs(&mut fuzzer, &mut executor, &mut mgr, &[seed_dir.clone()]) + .unwrap_or_else(|_| { + println!("Failed to load initial corpus at {:?}", &seed_dir); + process::exit(0); + }); + println!("We imported {} inputs from disk.", state.corpus().count()); + } + + // Remove target output (logs still survive) + #[cfg(unix)] + { + let null_fd = file_null.as_raw_fd(); + // dup2(null_fd, io::stdout().as_raw_fd())?; + if std::env::var("LIBAFL_FUZZBENCH_DEBUG").is_err() { + dup2(null_fd, io::stderr().as_raw_fd())?; + } + } + // reopen file to make sure we're at the end + log.replace(OpenOptions::new().append(true).create(true).open(logfile)?); + + fuzzer.fuzz_loop(&mut stages, &mut executor, &mut state, &mut mgr)?; + + // Never reached + Ok(()) +} diff --git a/fuzzers/fuzzbench_ctx/stub_rt.c b/fuzzers/fuzzbench_ctx/stub_rt.c new file mode 100644 index 0000000000..825d6780af --- /dev/null +++ b/fuzzers/fuzzbench_ctx/stub_rt.c @@ -0,0 +1,34 @@ +#include + +__attribute__((weak)) void __sanitizer_cov_trace_pc_guard_init(uint32_t *start, + uint32_t *stop) { +} + +__attribute__((weak)) void __sanitizer_cov_trace_pc_guard(uint32_t *guard) { +} + +__attribute__((weak)) void __cmplog_rtn_hook(uint8_t *ptr1, uint8_t *ptr2) { +} + +__attribute__((weak)) void __cmplog_rtn_gcc_stdstring_cstring( + uint8_t *stdstring, uint8_t *cstring) { +} + +__attribute__((weak)) void __cmplog_rtn_gcc_stdstring_stdstring( + uint8_t *stdstring1, uint8_t *stdstring2) { +} + +__attribute__((weak)) void __cmplog_rtn_llvm_stdstring_cstring( + uint8_t *stdstring, uint8_t *cstring) { +} + +__attribute__((weak)) void __cmplog_rtn_llvm_stdstring_stdstring( + uint8_t *stdstring1, uint8_t *stdstring2) { +} + +extern void libafl_main(void); + +int main(int argc, char **argv) { + libafl_main(); + return 0; +} diff --git a/fuzzers/fuzzbench_fork_qemu/Cargo.toml b/fuzzers/fuzzbench_fork_qemu/Cargo.toml index 705894e221..f2d74aa902 100644 --- a/fuzzers/fuzzbench_fork_qemu/Cargo.toml +++ b/fuzzers/fuzzbench_fork_qemu/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fuzzbench_fork_qemu" -version = "0.11.2" +version = "0.12.0" authors = ["Andrea Fioraldi ", "Dominik Maier "] edition = "2021" diff --git a/fuzzers/fuzzbench_fork_qemu/src/fuzzer.rs b/fuzzers/fuzzbench_fork_qemu/src/fuzzer.rs index 49bacd7d08..8af53417c8 100644 --- a/fuzzers/fuzzbench_fork_qemu/src/fuzzer.rs +++ b/fuzzers/fuzzbench_fork_qemu/src/fuzzer.rs @@ -26,7 +26,7 @@ use libafl::{ scheduled::havoc_mutations, token_mutations::I2SRandReplace, tokens_mutations, StdMOptMutator, StdScheduledMutator, Tokens, }, - observers::{ConstMapObserver, HitcountsMapObserver, TimeObserver}, + observers::{CanTrack, ConstMapObserver, HitcountsMapObserver, TimeObserver}, schedulers::{ powersched::PowerSchedule, IndexesLenTimeMinimizerScheduler, PowerQueueScheduler, }, @@ -34,28 +34,28 @@ use libafl::{ calibrate::CalibrationStage, power::StdPowerMutationalStage, ShadowTracingStage, StdMutationalStage, }, - state::{HasCorpus, HasMetadata, StdState}, - Error, + state::{HasCorpus, StdState}, + Error, HasMetadata, }; use libafl_bolts::{ - current_nanos, current_time, - os::dup2, + current_time, + os::{dup2, unix_signals::Signal}, rands::StdRand, shmem::{ShMemProvider, StdShMemProvider}, tuples::{tuple_list, Merge}, - AsMutSlice, AsSlice, + AsSlice, AsSliceMut, }; use libafl_qemu::{ cmplog::{CmpLogMap, CmpLogObserver, QemuCmpLogChildHelper}, - edges::{QemuEdgeCoverageChildHelper, EDGES_MAP_PTR, EDGES_MAP_SIZE}, + edges::{QemuEdgeCoverageChildHelper, EDGES_MAP_PTR, EDGES_MAP_SIZE_IN_USE}, elf::EasyElf, - emu::Emulator, filter_qemu_args, hooks::QemuHooks, - GuestReg, MmapPerms, QemuForkExecutor, Regs, + GuestReg, MmapPerms, Qemu, QemuExitReason, QemuExitReasonError, QemuForkExecutor, + QemuShutdownCause, Regs, }; #[cfg(unix)] -use nix::{self, unistd::dup}; +use nix::unistd::dup; /// The fuzzer main pub fn main() { @@ -148,33 +148,38 @@ fn fuzz( let args: Vec = env::args().collect(); let env: Vec<(String, String)> = env::vars().collect(); - let emu = Emulator::new(&args, &env)?; + let qemu = Qemu::init(&args, &env)?; let mut elf_buffer = Vec::new(); - let elf = EasyElf::from_file(emu.binary_path(), &mut elf_buffer)?; + let elf = EasyElf::from_file(qemu.binary_path(), &mut elf_buffer)?; let test_one_input_ptr = elf - .resolve_symbol("LLVMFuzzerTestOneInput", emu.load_addr()) + .resolve_symbol("LLVMFuzzerTestOneInput", qemu.load_addr()) .expect("Symbol LLVMFuzzerTestOneInput not found"); println!("LLVMFuzzerTestOneInput @ {test_one_input_ptr:#x}"); - emu.set_breakpoint(test_one_input_ptr); // LLVMFuzzerTestOneInput - unsafe { emu.run() }; + qemu.set_breakpoint(test_one_input_ptr); // LLVMFuzzerTestOneInput + unsafe { + match qemu.run() { + Ok(QemuExitReason::Breakpoint(_)) => {} + _ => panic!("Unexpected QEMU exit."), + } + } - println!("Break at {:#x}", emu.read_reg::<_, u64>(Regs::Rip).unwrap()); + println!("Break at {:#x}", qemu.read_reg::<_, u64>(Regs::Pc).unwrap()); - let stack_ptr: u64 = emu.read_reg(Regs::Rsp).unwrap(); + let stack_ptr: u64 = qemu.read_reg(Regs::Sp).unwrap(); let mut ret_addr = [0; 8]; - unsafe { emu.read_mem(stack_ptr, &mut ret_addr) }; + unsafe { qemu.read_mem(stack_ptr, &mut ret_addr) }; let ret_addr = u64::from_le_bytes(ret_addr); println!("Stack pointer = {stack_ptr:#x}"); println!("Return address = {ret_addr:#x}"); - emu.remove_breakpoint(test_one_input_ptr); // LLVMFuzzerTestOneInput - emu.set_breakpoint(ret_addr); // LLVMFuzzerTestOneInput ret addr + qemu.remove_breakpoint(test_one_input_ptr); // LLVMFuzzerTestOneInput + qemu.set_breakpoint(ret_addr); // LLVMFuzzerTestOneInput ret addr - let input_addr = emu.map_private(0, 4096, MmapPerms::ReadWrite).unwrap(); + let input_addr = qemu.map_private(0, 4096, MmapPerms::ReadWrite).unwrap(); println!("Placing input at {input_addr:#x}"); let log = RefCell::new( @@ -193,25 +198,22 @@ fn fuzz( let file_null = File::open("/dev/null")?; // 'While the stats are state, they are usually used in the broker - which is likely never restarted - let monitor = SimpleMonitor::with_user_monitor( - |s| { - #[cfg(unix)] - writeln!(&mut stdout_cpy, "{s}").unwrap(); - #[cfg(windows)] - println!("{s}"); - writeln!(log.borrow_mut(), "{:?} {s}", current_time()).unwrap(); - }, - true, - ); + let monitor = SimpleMonitor::with_user_monitor(|s| { + #[cfg(unix)] + writeln!(&mut stdout_cpy, "{s}").unwrap(); + #[cfg(windows)] + println!("{s}"); + writeln!(log.borrow_mut(), "{:?} {s}", current_time()).unwrap(); + }); let mut shmem_provider = StdShMemProvider::new()?; - let mut edges_shmem = shmem_provider.new_shmem(EDGES_MAP_SIZE).unwrap(); - let edges = edges_shmem.as_mut_slice(); + let mut edges_shmem = shmem_provider.new_shmem(EDGES_MAP_SIZE_IN_USE).unwrap(); + let edges = edges_shmem.as_slice_mut(); unsafe { EDGES_MAP_PTR = edges.as_mut_ptr() }; let mut cmp_shmem = shmem_provider.uninit_on_shmem::().unwrap(); - let cmplog = cmp_shmem.as_mut_slice(); + let cmplog = cmp_shmem.as_slice_mut(); // Beginning of a page should be properly aligned. #[allow(clippy::cast_ptr_alignment)] @@ -233,10 +235,11 @@ fn fuzz( // Create an observation channel using the coverage map let edges_observer = unsafe { - HitcountsMapObserver::new(ConstMapObserver::<_, EDGES_MAP_SIZE>::from_mut_ptr( + HitcountsMapObserver::new(ConstMapObserver::<_, EDGES_MAP_SIZE_IN_USE>::from_mut_ptr( "edges", edges.as_mut_ptr(), )) + .track_indices() }; // Create an observation channel to keep track of the execution time @@ -245,7 +248,7 @@ fn fuzz( // Create an observation channel using cmplog map let cmplog_observer = unsafe { CmpLogObserver::with_map_ptr("cmplog", cmplog_map_ptr, true) }; - let map_feedback = MaxMapFeedback::tracking(&edges_observer, true, false); + let map_feedback = MaxMapFeedback::new(&edges_observer); let calibration = CalibrationStage::new(&map_feedback); @@ -255,7 +258,7 @@ fn fuzz( // New maximization map feedback linked to the edges observer and the feedback state map_feedback, // Time feedback, this one does not need a feedback state - TimeFeedback::with_observer(&time_observer) + TimeFeedback::new(&time_observer) ); // A feedback to choose if an input is a solution or not @@ -265,7 +268,7 @@ fn fuzz( let mut state = state.unwrap_or_else(|| { StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryOnDiskCorpus::new(corpus_dir).unwrap(), // Corpus in which we store solutions (crashes in this example), @@ -294,11 +297,10 @@ fn fuzz( let power = StdPowerMutationalStage::new(mutator); // A minimization+queue policy to get testcasess from the corpus - let scheduler = IndexesLenTimeMinimizerScheduler::new(PowerQueueScheduler::new( - &mut state, + let scheduler = IndexesLenTimeMinimizerScheduler::new( &edges_observer, - PowerSchedule::FAST, - )); + PowerQueueScheduler::new(&mut state, &edges_observer, PowerSchedule::FAST), + ); // A fuzzer with feedbacks and a corpus scheduler let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); @@ -314,21 +316,28 @@ fn fuzz( } unsafe { - emu.write_mem(input_addr, buf); - - emu.write_reg(Regs::Rdi, input_addr).unwrap(); - emu.write_reg(Regs::Rsi, len as GuestReg).unwrap(); - emu.write_reg(Regs::Rip, test_one_input_ptr).unwrap(); - emu.write_reg(Regs::Rsp, stack_ptr).unwrap(); - - emu.run(); + qemu.write_mem(input_addr, buf); + + qemu.write_reg(Regs::Rdi, input_addr).unwrap(); + qemu.write_reg(Regs::Rsi, len as GuestReg).unwrap(); + qemu.write_reg(Regs::Rip, test_one_input_ptr).unwrap(); + qemu.write_reg(Regs::Rsp, stack_ptr).unwrap(); + + match qemu.run() { + Ok(QemuExitReason::Breakpoint(_)) => {} + Ok(QemuExitReason::End(QemuShutdownCause::HostSignal(Signal::SigInterrupt))) => { + process::exit(0) + } + Err(QemuExitReasonError::UnexpectedExit) => return ExitKind::Crash, + _ => panic!("Unexpected QEMU exit."), + } } ExitKind::Ok }; let mut hooks = QemuHooks::new( - emu.clone(), + qemu.clone(), tuple_list!( QemuEdgeCoverageChildHelper::default(), QemuCmpLogChildHelper::default(), diff --git a/fuzzers/fuzzbench_forkserver/Cargo.toml b/fuzzers/fuzzbench_forkserver/Cargo.toml index 9c30351062..5007c60107 100644 --- a/fuzzers/fuzzbench_forkserver/Cargo.toml +++ b/fuzzers/fuzzbench_forkserver/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fuzzbench_forkserver" -version = "0.11.2" +version = "0.12.0" authors = ["Andrea Fioraldi ", "Dominik Maier "] edition = "2021" diff --git a/fuzzers/fuzzbench_forkserver/src/main.rs b/fuzzers/fuzzbench_forkserver/src/main.rs index 0e4157cc09..ffa2b903f2 100644 --- a/fuzzers/fuzzbench_forkserver/src/main.rs +++ b/fuzzers/fuzzbench_forkserver/src/main.rs @@ -21,7 +21,9 @@ use libafl::{ scheduled::havoc_mutations, token_mutations::I2SRandReplace, tokens_mutations, StdMOptMutator, StdScheduledMutator, Tokens, }, - observers::{HitcountsMapObserver, StdCmpValuesObserver, StdMapObserver, TimeObserver}, + observers::{ + CanTrack, HitcountsMapObserver, StdCmpValuesObserver, StdMapObserver, TimeObserver, + }, schedulers::{ powersched::PowerSchedule, IndexesLenTimeMinimizerScheduler, StdWeightedScheduler, }, @@ -29,16 +31,16 @@ use libafl::{ calibrate::CalibrationStage, power::StdPowerMutationalStage, StdMutationalStage, TracingStage, }, - state::{HasCorpus, HasMetadata, StdState}, - Error, + state::{HasCorpus, StdState}, + Error, HasMetadata, }; use libafl_bolts::{ - current_nanos, current_time, + current_time, ownedref::OwnedRefMut, rands::StdRand, shmem::{ShMem, ShMemProvider, UnixShMemProvider}, tuples::{tuple_list, Merge}, - AsMutSlice, + AsSliceMut, }; use libafl_targets::cmps::AFLppCmpLogMap; use nix::sys::signal::Signal; @@ -222,7 +224,7 @@ fn fuzz( // a large initial map size that should be enough // to house all potential coverage maps for our targets // (we will eventually reduce the used size according to the actual map) - const MAP_SIZE: usize = 2_621_440; + const MAP_SIZE: usize = 65_536; let log = RefCell::new(OpenOptions::new().append(true).create(true).open(logfile)?); @@ -243,18 +245,19 @@ fn fuzz( let mut shmem = shmem_provider.new_shmem(MAP_SIZE).unwrap(); // let the forkserver know the shmid shmem.write_to_env("__AFL_SHM_ID").unwrap(); - let shmem_buf = shmem.as_mut_slice(); + let shmem_buf = shmem.as_slice_mut(); // To let know the AFL++ binary that we have a big map std::env::set_var("AFL_MAP_SIZE", format!("{}", MAP_SIZE)); // Create an observation channel using the hitcounts map of AFL++ - let edges_observer = - unsafe { HitcountsMapObserver::new(StdMapObserver::new("shared_mem", shmem_buf)) }; + let edges_observer = unsafe { + HitcountsMapObserver::new(StdMapObserver::new("shared_mem", shmem_buf)).track_indices() + }; // Create an observation channel to keep track of the execution time let time_observer = TimeObserver::new("time"); - let map_feedback = MaxMapFeedback::tracking(&edges_observer, true, false); + let map_feedback = MaxMapFeedback::new(&edges_observer); let calibration = CalibrationStage::new(&map_feedback); @@ -264,7 +267,7 @@ fn fuzz( // New maximization map feedback linked to the edges observer and the feedback state map_feedback, // Time feedback, this one does not need a feedback state - TimeFeedback::with_observer(&time_observer) + TimeFeedback::new(&time_observer) ); // A feedback to choose if an input is a solution or not @@ -273,7 +276,7 @@ fn fuzz( // create a State from scratch let mut state = StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryOnDiskCorpus::::new(corpus_dir).unwrap(), // Corpus in which we store solutions (crashes in this example), @@ -300,11 +303,14 @@ fn fuzz( let power = StdPowerMutationalStage::new(mutator); // A minimization+queue policy to get testcasess from the corpus - let scheduler = IndexesLenTimeMinimizerScheduler::new(StdWeightedScheduler::with_schedule( - &mut state, + let scheduler = IndexesLenTimeMinimizerScheduler::new( &edges_observer, - Some(PowerSchedule::EXPLORE), - )); + StdWeightedScheduler::with_schedule( + &mut state, + &edges_observer, + Some(PowerSchedule::EXPLORE), + ), + ); // A fuzzer with feedbacks and a corpus scheduler let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); diff --git a/fuzzers/fuzzbench_forkserver_cmplog/Cargo.toml b/fuzzers/fuzzbench_forkserver_cmplog/Cargo.toml index 03d2c313ef..1eeb90b94a 100644 --- a/fuzzers/fuzzbench_forkserver_cmplog/Cargo.toml +++ b/fuzzers/fuzzbench_forkserver_cmplog/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fuzzbench_forkserver_cmplog" -version = "0.11.2" +version = "0.12.0" authors = ["Andrea Fioraldi ", "Dominik Maier "] edition = "2021" diff --git a/fuzzers/fuzzbench_forkserver_cmplog/src/main.rs b/fuzzers/fuzzbench_forkserver_cmplog/src/main.rs index 9a3734615d..08676f9668 100644 --- a/fuzzers/fuzzbench_forkserver_cmplog/src/main.rs +++ b/fuzzers/fuzzbench_forkserver_cmplog/src/main.rs @@ -9,7 +9,7 @@ use std::{ use clap::{Arg, ArgAction, Command}; use libafl::{ - corpus::{Corpus, HasCurrentCorpusIdx, InMemoryOnDiskCorpus, OnDiskCorpus}, + corpus::{Corpus, InMemoryOnDiskCorpus, OnDiskCorpus}, events::SimpleEventManager, executors::forkserver::ForkserverExecutor, feedback_or, @@ -21,7 +21,7 @@ use libafl::{ scheduled::havoc_mutations, token_mutations::AFLppRedQueen, tokens_mutations, StdMOptMutator, Tokens, }, - observers::{HitcountsMapObserver, StdMapObserver, TimeObserver}, + observers::{CanTrack, HitcountsMapObserver, StdMapObserver, TimeObserver}, schedulers::{ powersched::PowerSchedule, IndexesLenTimeMinimizerScheduler, StdWeightedScheduler, }, @@ -29,16 +29,16 @@ use libafl::{ calibrate::CalibrationStage, mutational::MultiMutationalStage, power::StdPowerMutationalStage, ColorizationStage, IfStage, }, - state::{HasCorpus, HasMetadata, StdState}, - Error, + state::{HasCorpus, HasCurrentTestcase, StdState}, + Error, HasMetadata, }; use libafl_bolts::{ - current_nanos, current_time, + current_time, ownedref::OwnedRefMut, rands::StdRand, shmem::{ShMem, ShMemProvider, UnixShMemProvider}, - tuples::{tuple_list, Merge}, - AsMutSlice, + tuples::{tuple_list, Handler, Merge}, + AsSliceMut, }; use libafl_targets::{ cmps::{observers::AFLppCmpLogObserver, stages::AFLppCmplogTracingStage}, @@ -225,7 +225,7 @@ fn fuzz( // a large initial map size that should be enough // to house all potential coverage maps for our targets // (we will eventually reduce the used size according to the actual map) - const MAP_SIZE: usize = 2_621_440; + const MAP_SIZE: usize = 65_536; let log = RefCell::new(OpenOptions::new().append(true).create(true).open(logfile)?); @@ -246,18 +246,19 @@ fn fuzz( let mut shmem = shmem_provider.new_shmem(MAP_SIZE).unwrap(); // let the forkserver know the shmid shmem.write_to_env("__AFL_SHM_ID").unwrap(); - let shmem_buf = shmem.as_mut_slice(); + let shmem_buf = shmem.as_slice_mut(); // To let know the AFL++ binary that we have a big map std::env::set_var("AFL_MAP_SIZE", format!("{MAP_SIZE}")); // Create an observation channel using the hitcounts map of AFL++ - let edges_observer = - unsafe { HitcountsMapObserver::new(StdMapObserver::new("shared_mem", shmem_buf)) }; + let edges_observer = unsafe { + HitcountsMapObserver::new(StdMapObserver::new("shared_mem", shmem_buf)).track_indices() + }; // Create an observation channel to keep track of the execution time let time_observer = TimeObserver::new("time"); - let map_feedback = MaxMapFeedback::tracking(&edges_observer, true, false); + let map_feedback = MaxMapFeedback::new(&edges_observer); let calibration = CalibrationStage::new(&map_feedback); @@ -267,7 +268,7 @@ fn fuzz( // New maximization map feedback linked to the edges observer and the feedback state map_feedback, // Time feedback, this one does not need a feedback state - TimeFeedback::with_observer(&time_observer) + TimeFeedback::new(&time_observer) ); // A feedback to choose if an input is a solution or not @@ -276,7 +277,7 @@ fn fuzz( // create a State from scratch let mut state = StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryOnDiskCorpus::::new(corpus_dir).unwrap(), // Corpus in which we store solutions (crashes in this example), @@ -303,11 +304,14 @@ fn fuzz( let power = StdPowerMutationalStage::new(mutator); // A minimization+queue policy to get testcasess from the corpus - let scheduler = IndexesLenTimeMinimizerScheduler::new(StdWeightedScheduler::with_schedule( - &mut state, + let scheduler = IndexesLenTimeMinimizerScheduler::new( &edges_observer, - Some(PowerSchedule::EXPLORE), - )); + StdWeightedScheduler::with_schedule( + &mut state, + &edges_observer, + Some(PowerSchedule::EXPLORE), + ), + ); // A fuzzer with feedbacks and a corpus scheduler let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); @@ -351,6 +355,7 @@ fn fuzz( let cmpmap = unsafe { OwnedRefMut::from_shmem(&mut cmplog_shmem) }; let cmplog_observer = AFLppCmpLogObserver::new("cmplog", cmpmap, true); + let cmplog_ref = cmplog_observer.handle(); let cmplog_executor = ForkserverExecutor::builder() .program(exec) @@ -363,7 +368,7 @@ fn fuzz( .build(tuple_list!(cmplog_observer)) .unwrap(); - let tracing = AFLppCmplogTracingStage::with_cmplog_observer_name(cmplog_executor, "cmplog"); + let tracing = AFLppCmplogTracingStage::with_cmplog_observer(cmplog_executor, cmplog_ref); // Setup a randomic Input2State stage let rq = MultiMutationalStage::new(AFLppRedQueen::with_cmplog_options(true, true)); @@ -373,14 +378,8 @@ fn fuzz( state: &mut StdState<_, InMemoryOnDiskCorpus<_>, _, _>, _event_manager: &mut _| -> Result { - let Some(corpus_id) = state.current_corpus_idx()? else { - return Err(Error::illegal_state( - "state is not currently processing a corpus index", - )); - }; - - let corpus = state.corpus().get(corpus_id)?.borrow(); - let res = corpus.scheduled_count() == 1; // let's try on the 2nd trial + let testcase = state.current_testcase()?; + let res = testcase.scheduled_count() == 1; // let's try on the 2nd trial Ok(res) }; diff --git a/fuzzers/fuzzbench_qemu/Cargo.toml b/fuzzers/fuzzbench_qemu/Cargo.toml index c6e754e7bd..4a7b5ef638 100644 --- a/fuzzers/fuzzbench_qemu/Cargo.toml +++ b/fuzzers/fuzzbench_qemu/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fuzzbench_qemu" -version = "0.11.2" +version = "0.12.0" authors = ["Andrea Fioraldi ", "Dominik Maier "] edition = "2021" diff --git a/fuzzers/fuzzbench_qemu/src/fuzzer.rs b/fuzzers/fuzzbench_qemu/src/fuzzer.rs index b2bedb50cb..bd4430825d 100644 --- a/fuzzers/fuzzbench_qemu/src/fuzzer.rs +++ b/fuzzers/fuzzbench_qemu/src/fuzzer.rs @@ -25,7 +25,7 @@ use libafl::{ scheduled::havoc_mutations, token_mutations::I2SRandReplace, tokens_mutations, StdMOptMutator, StdScheduledMutator, Tokens, }, - observers::{HitcountsMapObserver, TimeObserver, VariableMapObserver}, + observers::{CanTrack, HitcountsMapObserver, TimeObserver, VariableMapObserver}, schedulers::{ powersched::PowerSchedule, IndexesLenTimeMinimizerScheduler, PowerQueueScheduler, }, @@ -33,12 +33,13 @@ use libafl::{ calibrate::CalibrationStage, power::StdPowerMutationalStage, ShadowTracingStage, StdMutationalStage, }, - state::{HasCorpus, HasMetadata, StdState}, - Error, + state::{HasCorpus, StdState}, + Error, HasMetadata, }; use libafl_bolts::{ - current_nanos, current_time, - os::dup2, + current_time, + os::{dup2, unix_signals::Signal}, + ownedref::OwnedMutSlice, rands::StdRand, shmem::{ShMemProvider, StdShMemProvider}, tuples::{tuple_list, Merge}, @@ -47,9 +48,9 @@ use libafl_bolts::{ use libafl_qemu::{ // asan::{init_with_asan, QemuAsanHelper}, cmplog::{CmpLogObserver, QemuCmpLogHelper}, - edges::edges_map_mut_slice, + edges::edges_map_mut_ptr, edges::QemuEdgeCoverageHelper, - edges::MAX_EDGES_NUM, + edges::{EDGES_MAP_SIZE_IN_USE, MAX_EDGES_FOUND}, elf::EasyElf, filter_qemu_args, hooks::QemuHooks, @@ -57,11 +58,15 @@ use libafl_qemu::{ GuestReg, //snapshot::QemuSnapshotHelper, MmapPerms, + Qemu, QemuExecutor, + QemuExitReason, + QemuExitReasonError, + QemuShutdownCause, Regs, }; #[cfg(unix)] -use nix::{self, unistd::dup}; +use nix::unistd::dup; pub const MAX_INPUT_SIZE: usize = 1048576; // 1MB @@ -172,34 +177,39 @@ fn fuzz( let args: Vec = env::args().collect(); let env: Vec<(String, String)> = env::vars().collect(); - let emu = Emulator::new(&args, &env).unwrap(); + let qemu = Qemu::init(&args, &env).unwrap(); // let (emu, asan) = init_with_asan(&mut args, &mut env).unwrap(); let mut elf_buffer = Vec::new(); - let elf = EasyElf::from_file(emu.binary_path(), &mut elf_buffer)?; + let elf = EasyElf::from_file(qemu.binary_path(), &mut elf_buffer)?; let test_one_input_ptr = elf - .resolve_symbol("LLVMFuzzerTestOneInput", emu.load_addr()) + .resolve_symbol("LLVMFuzzerTestOneInput", qemu.load_addr()) .expect("Symbol LLVMFuzzerTestOneInput not found"); println!("LLVMFuzzerTestOneInput @ {test_one_input_ptr:#x}"); - emu.set_breakpoint(test_one_input_ptr); // LLVMFuzzerTestOneInput - unsafe { emu.run() }; + qemu.set_breakpoint(test_one_input_ptr); // LLVMFuzzerTestOneInput + unsafe { + match qemu.run() { + Ok(QemuExitReason::Breakpoint(_)) => {} + _ => panic!("Unexpected QEMU exit."), + } + } - println!("Break at {:#x}", emu.read_reg::<_, u64>(Regs::Rip).unwrap()); + println!("Break at {:#x}", qemu.read_reg::<_, u64>(Regs::Pc).unwrap()); - let stack_ptr: u64 = emu.read_reg(Regs::Rsp).unwrap(); + let stack_ptr: u64 = qemu.read_reg(Regs::Sp).unwrap(); let mut ret_addr = [0; 8]; - unsafe { emu.read_mem(stack_ptr, &mut ret_addr) }; + unsafe { qemu.read_mem(stack_ptr, &mut ret_addr) }; let ret_addr = u64::from_le_bytes(ret_addr); println!("Stack pointer = {stack_ptr:#x}"); println!("Return address = {ret_addr:#x}"); - emu.remove_breakpoint(test_one_input_ptr); // LLVMFuzzerTestOneInput - emu.set_breakpoint(ret_addr); // LLVMFuzzerTestOneInput ret addr + qemu.remove_breakpoint(test_one_input_ptr); // LLVMFuzzerTestOneInput + qemu.set_breakpoint(ret_addr); // LLVMFuzzerTestOneInput ret addr - let input_addr = emu + let input_addr = qemu .map_private(0, MAX_INPUT_SIZE, MmapPerms::ReadWrite) .unwrap(); println!("Placing input at {input_addr:#x}"); @@ -248,9 +258,10 @@ fn fuzz( let edges_observer = unsafe { HitcountsMapObserver::new(VariableMapObserver::from_mut_slice( "edges", - edges_map_mut_slice(), - addr_of_mut!(MAX_EDGES_NUM), + OwnedMutSlice::from_raw_parts_mut(edges_map_mut_ptr(), EDGES_MAP_SIZE_IN_USE), + addr_of_mut!(MAX_EDGES_FOUND), )) + .track_indices() }; // Create an observation channel to keep track of the execution time @@ -259,7 +270,7 @@ fn fuzz( // Create an observation channel using cmplog map let cmplog_observer = CmpLogObserver::new("cmplog", true); - let map_feedback = MaxMapFeedback::tracking(&edges_observer, true, false); + let map_feedback = MaxMapFeedback::new(&edges_observer); let calibration = CalibrationStage::new(&map_feedback); @@ -269,7 +280,7 @@ fn fuzz( // New maximization map feedback linked to the edges observer and the feedback state map_feedback, // Time feedback, this one does not need a feedback state - TimeFeedback::with_observer(&time_observer) + TimeFeedback::new(&time_observer) ); // A feedback to choose if an input is a solution or not @@ -279,7 +290,7 @@ fn fuzz( let mut state = state.unwrap_or_else(|| { StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryOnDiskCorpus::new(corpus_dir).unwrap(), // Corpus in which we store solutions (crashes in this example), @@ -308,11 +319,10 @@ fn fuzz( let power = StdPowerMutationalStage::new(mutator); // A minimization+queue policy to get testcasess from the corpus - let scheduler = IndexesLenTimeMinimizerScheduler::new(PowerQueueScheduler::new( - &mut state, + let scheduler = IndexesLenTimeMinimizerScheduler::new( &edges_observer, - PowerSchedule::FAST, - )); + PowerQueueScheduler::new(&mut state, &edges_observer, PowerSchedule::FAST), + ); // A fuzzer with feedbacks and a corpus scheduler let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); @@ -328,21 +338,28 @@ fn fuzz( } unsafe { - emu.write_mem(input_addr, buf); - - emu.write_reg(Regs::Rdi, input_addr).unwrap(); - emu.write_reg(Regs::Rsi, len as GuestReg).unwrap(); - emu.write_reg(Regs::Rip, test_one_input_ptr).unwrap(); - emu.write_reg(Regs::Rsp, stack_ptr).unwrap(); - - emu.run(); + qemu.write_mem(input_addr, buf); + + qemu.write_reg(Regs::Rdi, input_addr).unwrap(); + qemu.write_reg(Regs::Rsi, len as GuestReg).unwrap(); + qemu.write_reg(Regs::Rip, test_one_input_ptr).unwrap(); + qemu.write_reg(Regs::Rsp, stack_ptr).unwrap(); + + match qemu.run() { + Ok(QemuExitReason::Breakpoint(_)) => {} + Ok(QemuExitReason::End(QemuShutdownCause::HostSignal(Signal::SigInterrupt))) => { + process::exit(0) + } + Err(QemuExitReasonError::UnexpectedExit) => return ExitKind::Crash, + _ => panic!("Unexpected QEMU exit."), + } } ExitKind::Ok }; let mut hooks = QemuHooks::new( - emu.clone(), + qemu.clone(), tuple_list!( QemuEdgeCoverageHelper::default(), QemuCmpLogHelper::default(), diff --git a/fuzzers/fuzzbench_text/Cargo.toml b/fuzzers/fuzzbench_text/Cargo.toml index 76b772e643..bde12220f5 100644 --- a/fuzzers/fuzzbench_text/Cargo.toml +++ b/fuzzers/fuzzbench_text/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fuzzbench_text" -version = "0.11.2" +version = "0.12.0" authors = ["Andrea Fioraldi ", "Dominik Maier "] edition = "2021" diff --git a/fuzzers/fuzzbench_text/Makefile.toml b/fuzzers/fuzzbench_text/Makefile.toml index cbf7f0bdfc..94faa860ce 100644 --- a/fuzzers/fuzzbench_text/Makefile.toml +++ b/fuzzers/fuzzbench_text/Makefile.toml @@ -83,7 +83,7 @@ rm -rf libafl_unix_shmem_server || true mkdir in || true echo a > in/a # Allow sigterm as exit code -timeout 31s ./${FUZZER_NAME} -o out -i in >fuzz_stdout.log || true +timeout 31s ./${FUZZER_NAME} -o out -i in | tee fuzz_stdout.log || true cat fuzz_stdout.log if grep -qa "objectives: 1" fuzz_stdout.log; then echo "Fuzzer is working" diff --git a/fuzzers/fuzzbench_text/src/lib.rs b/fuzzers/fuzzbench_text/src/lib.rs index 573ed1ff83..b8924015ca 100644 --- a/fuzzers/fuzzbench_text/src/lib.rs +++ b/fuzzers/fuzzbench_text/src/lib.rs @@ -34,7 +34,7 @@ use libafl::{ token_mutations::I2SRandReplace, tokens_mutations, StdMOptMutator, StdScheduledMutator, Tokens, }, - observers::{HitcountsMapObserver, TimeObserver}, + observers::{CanTrack, HitcountsMapObserver, TimeObserver}, schedulers::{ powersched::PowerSchedule, IndexesLenTimeMinimizerScheduler, StdWeightedScheduler, }, @@ -42,11 +42,11 @@ use libafl::{ calibrate::CalibrationStage, power::StdPowerMutationalStage, GeneralizationStage, StdMutationalStage, TracingStage, }, - state::{HasCorpus, HasMetadata, StdState}, - Error, + state::{HasCorpus, StdState}, + Error, HasMetadata, }; use libafl_bolts::{ - current_nanos, current_time, + current_time, os::dup2, rands::StdRand, shmem::{ShMemProvider, StdShMemProvider}, @@ -59,7 +59,7 @@ use libafl_targets::{ libfuzzer_initialize, libfuzzer_test_one_input, std_edges_map_observer, CmpLogObserver, }; #[cfg(unix)] -use nix::{self, unistd::dup}; +use nix::unistd::dup; /// The fuzzer main (as `no_mangle` C function) #[no_mangle] @@ -310,14 +310,15 @@ fn fuzz_binary( // Create an observation channel using the coverage map // We don't use the hitcounts (see the Cargo.toml, we use pcguard_edges) - let edges_observer = HitcountsMapObserver::new(unsafe { std_edges_map_observer("edges") }); + let edges_observer = + HitcountsMapObserver::new(unsafe { std_edges_map_observer("edges") }).track_indices(); // Create an observation channel to keep track of the execution time let time_observer = TimeObserver::new("time"); let cmplog_observer = CmpLogObserver::new("cmplog", true); - let map_feedback = MaxMapFeedback::tracking(&edges_observer, true, false); + let map_feedback = MaxMapFeedback::new(&edges_observer); let calibration = CalibrationStage::new(&map_feedback); @@ -327,7 +328,7 @@ fn fuzz_binary( // New maximization map feedback linked to the edges observer and the feedback state map_feedback, // Time feedback, this one does not need a feedback state - TimeFeedback::with_observer(&time_observer) + TimeFeedback::new(&time_observer) ); // A feedback to choose if an input is a solution or not let mut objective = CrashFeedback::new(); @@ -336,7 +337,7 @@ fn fuzz_binary( let mut state = state.unwrap_or_else(|| { StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryOnDiskCorpus::new(corpus_dir).unwrap(), // Corpus in which we store solutions (crashes in this example), @@ -374,11 +375,14 @@ fn fuzz_binary( let power = StdPowerMutationalStage::new(mutator); // A minimization+queue policy to get testcasess from the corpus - let scheduler = IndexesLenTimeMinimizerScheduler::new(StdWeightedScheduler::with_schedule( - &mut state, + let scheduler = IndexesLenTimeMinimizerScheduler::new( &edges_observer, - Some(PowerSchedule::EXPLORE), - )); + StdWeightedScheduler::with_schedule( + &mut state, + &edges_observer, + Some(PowerSchedule::EXPLORE), + ), + ); // A fuzzer with feedbacks and a corpus scheduler let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); @@ -516,7 +520,9 @@ fn fuzz_text( // Create an observation channel using the coverage map // We don't use the hitcounts (see the Cargo.toml, we use pcguard_edges) - let edges_observer = HitcountsMapObserver::new(unsafe { std_edges_map_observer("edges") }); + let edges_observer = HitcountsMapObserver::new(unsafe { std_edges_map_observer("edges") }) + .track_indices() + .track_novelties(); // Create an observation channel to keep track of the execution time let time_observer = TimeObserver::new("time"); @@ -524,7 +530,7 @@ fn fuzz_text( let cmplog_observer = CmpLogObserver::new("cmplog", true); // New maximization map feedback linked to the edges observer and the feedback state - let map_feedback = MaxMapFeedback::tracking(&edges_observer, true, true); + let map_feedback = MaxMapFeedback::new(&edges_observer); let calibration = CalibrationStage::new(&map_feedback); @@ -533,7 +539,7 @@ fn fuzz_text( let mut feedback = feedback_or!( map_feedback, // Time feedback, this one does not need a feedback state - TimeFeedback::with_observer(&time_observer) + TimeFeedback::new(&time_observer) ); // A feedback to choose if an input is a solution or not @@ -543,7 +549,7 @@ fn fuzz_text( let mut state = state.unwrap_or_else(|| { StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryOnDiskCorpus::new(corpus_dir).unwrap(), // Corpus in which we store solutions (crashes in this example), @@ -594,11 +600,14 @@ fn fuzz_text( let grimoire = StdMutationalStage::transforming(grimoire_mutator); // A minimization+queue policy to get testcasess from the corpus - let scheduler = IndexesLenTimeMinimizerScheduler::new(StdWeightedScheduler::with_schedule( - &mut state, + let scheduler = IndexesLenTimeMinimizerScheduler::new( &edges_observer, - Some(PowerSchedule::EXPLORE), - )); + StdWeightedScheduler::with_schedule( + &mut state, + &edges_observer, + Some(PowerSchedule::EXPLORE), + ), + ); // A fuzzer with feedbacks and a corpus scheduler let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); diff --git a/fuzzers/libafl_atheris/Cargo.toml b/fuzzers/libafl_atheris/Cargo.toml index 93d4229378..e3489eb925 100644 --- a/fuzzers/libafl_atheris/Cargo.toml +++ b/fuzzers/libafl_atheris/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "libafl_atheris" -version = "0.11.2" +version = "0.12.0" authors = ["Andrea Fioraldi ", "Dominik Maier "] edition = "2021" diff --git a/fuzzers/libafl_atheris/src/lib.rs b/fuzzers/libafl_atheris/src/lib.rs index 2645713e5d..2ca552d22d 100644 --- a/fuzzers/libafl_atheris/src/lib.rs +++ b/fuzzers/libafl_atheris/src/lib.rs @@ -25,15 +25,14 @@ use libafl::{ scheduled::{havoc_mutations, tokens_mutations, StdScheduledMutator}, token_mutations::{I2SRandReplace, Tokens}, }, - observers::{HitcountsMapObserver, StdMapObserver, TimeObserver}, + observers::{CanTrack, HitcountsMapObserver, StdMapObserver, TimeObserver}, schedulers::{IndexesLenTimeMinimizerScheduler, QueueScheduler}, stages::{StdMutationalStage, TracingStage}, - state::{HasCorpus, HasMetadata, StdState}, - Error, + state::{HasCorpus, StdState}, + Error, HasMetadata, }; use libafl_bolts::{ core_affinity::Cores, - current_nanos, rands::StdRand, shmem::{ShMemProvider, StdShMemProvider}, tuples::{tuple_list, Merge}, @@ -136,7 +135,8 @@ pub extern "C" fn LLVMFuzzerRunDriver( let edges_observer = HitcountsMapObserver::new(StdMapObserver::from_mut_slice( "edges", edges.into_iter().next().unwrap(), - )); + )) + .track_indices(); // Create an observation channel to keep track of the execution time let time_observer = TimeObserver::new("time"); @@ -148,9 +148,9 @@ pub extern "C" fn LLVMFuzzerRunDriver( // This one is composed by two Feedbacks in OR let mut feedback = feedback_or!( // New maximization map feedback linked to the edges observer and the feedback state - MaxMapFeedback::tracking(&edges_observer, true, false), + MaxMapFeedback::new(&edges_observer), // Time feedback, this one does not need a feedback state - TimeFeedback::with_observer(&time_observer) + TimeFeedback::new(&time_observer) ); // A feedback to choose if an input is a solution or not @@ -160,7 +160,7 @@ pub extern "C" fn LLVMFuzzerRunDriver( let mut state = state.unwrap_or_else(|| { StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryCorpus::new(), // Corpus in which we store solutions (crashes in this example), @@ -183,7 +183,8 @@ pub extern "C" fn LLVMFuzzerRunDriver( } // A minimization+queue policy to get testcasess from the corpus - let scheduler = IndexesLenTimeMinimizerScheduler::new(QueueScheduler::new()); + let scheduler = + IndexesLenTimeMinimizerScheduler::new(&edges_observer, QueueScheduler::new()); // A fuzzer with feedbacks and a corpus scheduler let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); diff --git a/fuzzers/libfuzzer_libmozjpeg/Cargo.toml b/fuzzers/libfuzzer_libmozjpeg/Cargo.toml index a17aae7cda..c07245e583 100644 --- a/fuzzers/libfuzzer_libmozjpeg/Cargo.toml +++ b/fuzzers/libfuzzer_libmozjpeg/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "libfuzzer_libmozjpeg" -version = "0.11.2" +version = "0.12.0" authors = ["Andrea Fioraldi ", "Dominik Maier "] edition = "2021" diff --git a/fuzzers/libfuzzer_libmozjpeg/Makefile.toml b/fuzzers/libfuzzer_libmozjpeg/Makefile.toml index 8d70fcf4bf..f96e02ef2c 100644 --- a/fuzzers/libfuzzer_libmozjpeg/Makefile.toml +++ b/fuzzers/libfuzzer_libmozjpeg/Makefile.toml @@ -99,7 +99,7 @@ windows_alias = "unsupported" script_runner = "@shell" script=''' rm -rf libafl_unix_shmem_server || true -(timeout 31s ./${FUZZER_NAME} >fuzz_stdout.log 2>/dev/null || true) & +(timeout 31s ./${FUZZER_NAME} | tee fuzz_stdout.log 2>/dev/null || true) & sleep 0.2 timeout 30s ./${FUZZER_NAME} >/dev/null 2>/dev/null || true if grep -qa "corpus: 30" fuzz_stdout.log; then diff --git a/fuzzers/libfuzzer_libmozjpeg/src/lib.rs b/fuzzers/libfuzzer_libmozjpeg/src/lib.rs index 458aab5cbd..99db317f11 100644 --- a/fuzzers/libfuzzer_libmozjpeg/src/lib.rs +++ b/fuzzers/libfuzzer_libmozjpeg/src/lib.rs @@ -22,11 +22,10 @@ use libafl::{ observers::StdMapObserver, schedulers::RandScheduler, stages::mutational::StdMutationalStage, - state::{HasCorpus, HasMetadata, StdState}, - Error, + state::{HasCorpus, StdState}, + Error, HasMetadata, }; use libafl_bolts::{ - current_nanos, rands::StdRand, tuples::{tuple_list, Merge}, AsSlice, @@ -100,7 +99,7 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re let mut state = state.unwrap_or_else(|| { StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryCorpus::new(), // Corpus in which we store solutions (crashes in this example), diff --git a/fuzzers/libfuzzer_libpng/Cargo.toml b/fuzzers/libfuzzer_libpng/Cargo.toml index c5aba18c9c..d21558c18b 100644 --- a/fuzzers/libfuzzer_libpng/Cargo.toml +++ b/fuzzers/libfuzzer_libpng/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "libfuzzer_libpng" -version = "0.11.2" +version = "0.12.0" authors = ["Andrea Fioraldi ", "Dominik Maier "] edition = "2021" diff --git a/fuzzers/libfuzzer_libpng/Makefile.toml b/fuzzers/libfuzzer_libpng/Makefile.toml index 94430ed162..fb1dc6cfd2 100644 --- a/fuzzers/libfuzzer_libpng/Makefile.toml +++ b/fuzzers/libfuzzer_libpng/Makefile.toml @@ -161,7 +161,7 @@ windows_alias = "unsupported" script_runner = "@shell" script=''' rm -rf libafl_unix_shmem_server || true -(timeout 31s ./${FUZZER_NAME} >fuzz_stdout.log 2>/dev/null || true) & +(timeout 31s ./${FUZZER_NAME} | tee fuzz_stdout.log 2>/dev/null || true) & sleep 0.2 timeout 30s ./${FUZZER_NAME} >/dev/null 2>/dev/null || true if grep -qa "corpus: 30" fuzz_stdout.log; then @@ -177,7 +177,7 @@ dependencies = [ "fuzzer" ] script_runner = "@shell" script=''' rm -rf libafl_unix_shmem_server || true -(timeout 31s ./${FUZZER_NAME} >fuzz_stdout.log 2>/dev/null || true) & +(timeout 31s ./${FUZZER_NAME} | tee fuzz_stdout.log 2>/dev/null || true) & sleep 0.2 timeout 30s ./${FUZZER_NAME} >/dev/null 2>/dev/null || true ''' diff --git a/fuzzers/libfuzzer_libpng/src/lib.rs b/fuzzers/libfuzzer_libpng/src/lib.rs index e1e5cb00a6..f1fff436c6 100644 --- a/fuzzers/libfuzzer_libpng/src/lib.rs +++ b/fuzzers/libfuzzer_libpng/src/lib.rs @@ -1,9 +1,5 @@ //! A libfuzzer-like fuzzer with llmp-multithreading support and restarts //! The example harness is built for libpng. -use mimalloc::MiMalloc; -#[global_allocator] -static GLOBAL: MiMalloc = MiMalloc; - use core::time::Duration; #[cfg(feature = "crash")] use std::ptr; @@ -22,21 +18,24 @@ use libafl::{ scheduled::{havoc_mutations, tokens_mutations, StdScheduledMutator}, token_mutations::Tokens, }, - observers::{HitcountsMapObserver, StdMapObserver, TimeObserver}, + observers::{CanTrack, HitcountsMapObserver, StdMapObserver, TimeObserver}, schedulers::{ powersched::PowerSchedule, IndexesLenTimeMinimizerScheduler, StdWeightedScheduler, }, stages::{calibrate::CalibrationStage, power::StdPowerMutationalStage}, - state::{HasCorpus, HasMetadata, StdState}, - Error, + state::{HasCorpus, StdState}, + Error, HasMetadata, }; use libafl_bolts::{ - current_nanos, rands::StdRand, tuples::{tuple_list, Merge}, AsSlice, }; -use libafl_targets::{libfuzzer_initialize, libfuzzer_test_one_input, EDGES_MAP, MAX_EDGES_NUM}; +use libafl_targets::{libfuzzer_initialize, libfuzzer_test_one_input, EDGES_MAP, MAX_EDGES_FOUND}; +use mimalloc::MiMalloc; + +#[global_allocator] +static GLOBAL: MiMalloc = MiMalloc; /// The main fn, `no_mangle` as it is a C main #[cfg(not(test))] @@ -83,14 +82,15 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re HitcountsMapObserver::new(StdMapObserver::from_mut_ptr( "edges", EDGES_MAP.as_mut_ptr(), - MAX_EDGES_NUM, + MAX_EDGES_FOUND, )) + .track_indices() }; // Create an observation channel to keep track of the execution time let time_observer = TimeObserver::new("time"); - let map_feedback = MaxMapFeedback::tracking(&edges_observer, true, false); + let map_feedback = MaxMapFeedback::new(&edges_observer); let calibration = CalibrationStage::new(&map_feedback); @@ -100,7 +100,7 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re // New maximization map feedback linked to the edges observer and the feedback state map_feedback, // Time feedback, this one does not need a feedback state - TimeFeedback::with_observer(&time_observer) + TimeFeedback::new(&time_observer) ); // A feedback to choose if an input is a solution or not @@ -110,7 +110,7 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re let mut state = state.unwrap_or_else(|| { StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryCorpus::new(), // Corpus in which we store solutions (crashes in this example), @@ -147,11 +147,10 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re let mut stages = tuple_list!(calibration, power); // A minimization+queue policy to get testcasess from the corpus - let scheduler = IndexesLenTimeMinimizerScheduler::new(StdWeightedScheduler::with_schedule( - &mut state, + let scheduler = IndexesLenTimeMinimizerScheduler::new( &edges_observer, - Some(PowerSchedule::FAST), - )); + StdWeightedScheduler::with_schedule(&mut state, &edges_observer, Some(PowerSchedule::FAST)), + ); // A fuzzer with feedbacks and a corpus scheduler let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); diff --git a/fuzzers/libfuzzer_libpng_accounting/Cargo.toml b/fuzzers/libfuzzer_libpng_accounting/Cargo.toml index f9b82634ad..9e0d8cd800 100644 --- a/fuzzers/libfuzzer_libpng_accounting/Cargo.toml +++ b/fuzzers/libfuzzer_libpng_accounting/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "libfuzzer_libpng_accounting" -version = "0.11.2" +version = "0.12.0" authors = ["Andrea Fioraldi ", "Dominik Maier "] edition = "2021" diff --git a/fuzzers/libfuzzer_libpng_accounting/Makefile.toml b/fuzzers/libfuzzer_libpng_accounting/Makefile.toml index b37c152b71..13aaf5e3a3 100644 --- a/fuzzers/libfuzzer_libpng_accounting/Makefile.toml +++ b/fuzzers/libfuzzer_libpng_accounting/Makefile.toml @@ -98,7 +98,7 @@ windows_alias = "unsupported" script_runner = "@shell" script=''' rm -rf libafl_unix_shmem_server || true -timeout 31s ./${FUZZER_NAME} --cores 0 --input ./corpus >fuzz_stdout.log 2>/dev/null || true +timeout 31s ./${FUZZER_NAME} --cores 0 --input ./corpus | tee fuzz_stdout.log 2>/dev/null || true if grep -qa "corpus: 30" fuzz_stdout.log; then echo "Fuzzer is working" else @@ -112,7 +112,7 @@ dependencies = [ "fuzzer" ] script_runner = "@shell" script=''' rm -rf libafl_unix_shmem_server || true -timeout 31s ./${FUZZER_NAME} --cores 0 --input ./corpus >fuzz_stdout.log 2>/dev/null || true +timeout 31s ./${FUZZER_NAME} --cores 0 --input ./corpus | tee fuzz_stdout.log 2>/dev/null || true ''' dependencies = [ "fuzzer" ] diff --git a/fuzzers/libfuzzer_libpng_accounting/src/lib.rs b/fuzzers/libfuzzer_libpng_accounting/src/lib.rs index beb6fc0d45..c52cfd68a8 100644 --- a/fuzzers/libfuzzer_libpng_accounting/src/lib.rs +++ b/fuzzers/libfuzzer_libpng_accounting/src/lib.rs @@ -2,10 +2,6 @@ //! The example harness is built for libpng. //! In this example, you will see the use of the `launcher` feature. //! The `launcher` will spawn new processes for each cpu core. -use mimalloc::MiMalloc; -#[global_allocator] -static GLOBAL: MiMalloc = MiMalloc; - use core::time::Duration; use std::{env, net::SocketAddr, path::PathBuf}; @@ -23,23 +19,27 @@ use libafl::{ scheduled::{havoc_mutations, tokens_mutations, StdScheduledMutator}, token_mutations::Tokens, }, - observers::{HitcountsMapObserver, StdMapObserver, TimeObserver}, + observers::{CanTrack, HitcountsMapObserver, StdMapObserver, TimeObserver}, schedulers::{CoverageAccountingScheduler, QueueScheduler}, stages::mutational::StdMutationalStage, - state::{HasCorpus, HasMetadata, StdState}, - Error, + state::{HasCorpus, StdState}, + Error, HasMetadata, }; use libafl_bolts::{ core_affinity::Cores, - current_nanos, rands::StdRand, shmem::{ShMemProvider, StdShMemProvider}, tuples::{tuple_list, Merge}, AsSlice, }; use libafl_targets::{ - libfuzzer_initialize, libfuzzer_test_one_input, ACCOUNTING_MEMOP_MAP, EDGES_MAP, MAX_EDGES_NUM, + libfuzzer_initialize, libfuzzer_test_one_input, ACCOUNTING_MEMOP_MAP, EDGES_MAP, + MAX_EDGES_FOUND, }; +use mimalloc::MiMalloc; + +#[global_allocator] +static GLOBAL: MiMalloc = MiMalloc; /// Parse a millis string to a [`Duration`]. Used for arg parsing. fn timeout_from_millis_str(time: &str) -> Result { @@ -55,11 +55,11 @@ fn timeout_from_millis_str(time: &str) -> Result { )] struct Opt { #[arg( - short, - long, - value_parser = Cores::from_cmdline, - help = "Spawn a client in each of the provided cores. Broker runs in the 0th core. 'all' to select all available cores. 'none' to run a client without binding to any core. eg: '1,2-4,6' selects the cores 1,2,3,4,6.", - name = "CORES" + short, + long, + value_parser = Cores::from_cmdline, + help = "Spawn a client in each of the provided cores. Broker runs in the 0th core. 'all' to select all available cores. 'none' to run a client without binding to any core. eg: '1,2-4,6' selects the cores 1,2,3,4,6.", + name = "CORES" )] cores: Cores, @@ -75,7 +75,13 @@ struct Opt { #[arg(short = 'a', long, help = "Specify a remote broker", name = "REMOTE")] remote_broker_addr: Option, - #[arg(short, long, help = "Set an initial corpus directory", name = "INPUT")] + #[arg( + short, + long, + help = "Set an initial corpus directory", + name = "INPUT", + required = true + )] input: Vec, #[arg( @@ -88,12 +94,12 @@ struct Opt { output: PathBuf, #[arg( - value_parser = timeout_from_millis_str, - short, - long, - help = "Set the execution timeout in milliseconds, default is 10000", - name = "TIMEOUT", - default_value = "10000" + value_parser = timeout_from_millis_str, + short, + long, + help = "Set the execution timeout in milliseconds, default is 10000", + name = "TIMEOUT", + default_value = "10000" )] timeout: Duration, /* @@ -133,8 +139,9 @@ pub extern "C" fn libafl_main() { let mut run_client = |state: Option<_>, mut restarting_mgr, _core_id| { // Create an observation channel using the coverage map let edges_observer = HitcountsMapObserver::new(unsafe { - StdMapObserver::from_mut_ptr("edges", EDGES_MAP.as_mut_ptr(), MAX_EDGES_NUM) - }); + StdMapObserver::from_mut_ptr("edges", EDGES_MAP.as_mut_ptr(), MAX_EDGES_FOUND) + }) + .track_indices(); // Create an observation channel to keep track of the execution time let time_observer = TimeObserver::new("time"); @@ -143,9 +150,9 @@ pub extern "C" fn libafl_main() { // This one is composed by two Feedbacks in OR let mut feedback = feedback_or!( // New maximization map feedback linked to the edges observer and the feedback state - MaxMapFeedback::tracking(&edges_observer, true, false), + MaxMapFeedback::new(&edges_observer), // Time feedback, this one does not need a feedback state - TimeFeedback::with_observer(&time_observer) + TimeFeedback::new(&time_observer) ); // A feedback to choose if an input is a solution or not @@ -155,7 +162,7 @@ pub extern "C" fn libafl_main() { let mut state = state.unwrap_or_else(|| { StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryCorpus::new(), // Corpus in which we store solutions (crashes in this example), @@ -188,10 +195,12 @@ pub extern "C" fn libafl_main() { let mut stages = tuple_list!(StdMutationalStage::new(mutator)); // A minimization+queue policy to get testcasess from the corpus - let scheduler = - CoverageAccountingScheduler::new(&mut state, QueueScheduler::new(), unsafe { - &ACCOUNTING_MEMOP_MAP - }); + let scheduler = CoverageAccountingScheduler::new( + &edges_observer, + &mut state, + QueueScheduler::new(), + unsafe { &ACCOUNTING_MEMOP_MAP }, + ); // A fuzzer with feedbacks and a corpus scheduler let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); diff --git a/fuzzers/libfuzzer_libpng_aflpp_ui/Makefile.toml b/fuzzers/libfuzzer_libpng_aflpp_ui/Makefile.toml index 2606491ecb..3624a0b321 100644 --- a/fuzzers/libfuzzer_libpng_aflpp_ui/Makefile.toml +++ b/fuzzers/libfuzzer_libpng_aflpp_ui/Makefile.toml @@ -171,7 +171,7 @@ dependencies = [ "fuzzer" ] script_runner = "@shell" script=''' rm -rf libafl_unix_shmem_server || true -(timeout --foreground 11s ./${FUZZER_NAME} >fuzz_stdout.log 2>/dev/null || true) & +(timeout --foreground 11s ./${FUZZER_NAME} | tee fuzz_stdout.log 2>/dev/null || true) & sleep 0.2 timeout --foreground 10s ./${FUZZER_NAME} >/dev/null 2>/dev/null || true ''' diff --git a/fuzzers/libfuzzer_libpng_aflpp_ui/src/lib.rs b/fuzzers/libfuzzer_libpng_aflpp_ui/src/lib.rs index a28d3ce6f2..340e71b1fb 100644 --- a/fuzzers/libfuzzer_libpng_aflpp_ui/src/lib.rs +++ b/fuzzers/libfuzzer_libpng_aflpp_ui/src/lib.rs @@ -1,9 +1,5 @@ //! A libfuzzer-like fuzzer with llmp-multithreading support and restarts //! The example harness is built for libpng. -use mimalloc::MiMalloc; -#[global_allocator] -static GLOBAL: MiMalloc = MiMalloc; - use core::time::Duration; #[cfg(feature = "crash")] use std::ptr; @@ -22,21 +18,24 @@ use libafl::{ scheduled::{havoc_mutations, tokens_mutations, StdScheduledMutator}, token_mutations::Tokens, }, - observers::{HitcountsMapObserver, StdMapObserver, TimeObserver}, + observers::{CanTrack, HitcountsMapObserver, StdMapObserver, TimeObserver}, schedulers::{ powersched::PowerSchedule, IndexesLenTimeMinimizerScheduler, StdWeightedScheduler, }, stages::{calibrate::CalibrationStage, power::StdPowerMutationalStage, stats::AflStatsStage}, - state::{HasCorpus, HasMetadata, StdState}, - Error, + state::{HasCorpus, StdState}, + Error, HasMetadata, }; use libafl_bolts::{ - current_nanos, rands::StdRand, tuples::{tuple_list, Merge}, AsSlice, }; -use libafl_targets::{libfuzzer_initialize, libfuzzer_test_one_input, EDGES_MAP, MAX_EDGES_NUM}; +use libafl_targets::{libfuzzer_initialize, libfuzzer_test_one_input, EDGES_MAP, MAX_EDGES_FOUND}; +use mimalloc::MiMalloc; + +#[global_allocator] +static GLOBAL: MiMalloc = MiMalloc; /// The main fn, `no_mangle` as it is a C main #[cfg(not(test))] @@ -97,14 +96,15 @@ fn fuzz( HitcountsMapObserver::new(StdMapObserver::from_mut_ptr( "edges", EDGES_MAP.as_mut_ptr(), - MAX_EDGES_NUM, + MAX_EDGES_FOUND, )) + .track_indices() }; // Create an observation channel to keep track of the execution time let time_observer = TimeObserver::new("time"); - let map_feedback = MaxMapFeedback::tracking(&edges_observer, true, false); + let map_feedback = MaxMapFeedback::new(&edges_observer); let calibration = CalibrationStage::new(&map_feedback); @@ -114,7 +114,7 @@ fn fuzz( // New maximization map feedback linked to the edges observer and the feedback state map_feedback, // Time feedback, this one does not need a feedback state - TimeFeedback::with_observer(&time_observer) + TimeFeedback::new(&time_observer) ); // A feedback to choose if an input is a solution or not @@ -124,7 +124,7 @@ fn fuzz( let mut state = state.unwrap_or_else(|| { StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryOnDiskCorpus::new(corpus_dir).unwrap(), // Corpus in which we store solutions (crashes in this example), @@ -163,11 +163,10 @@ fn fuzz( let mut stages = tuple_list!(calibration, power, aflstats); // A minimization+queue policy to get testcasess from the corpus - let scheduler = IndexesLenTimeMinimizerScheduler::new(StdWeightedScheduler::with_schedule( - &mut state, + let scheduler = IndexesLenTimeMinimizerScheduler::new( &edges_observer, - Some(PowerSchedule::FAST), - )); + StdWeightedScheduler::with_schedule(&mut state, &edges_observer, Some(PowerSchedule::FAST)), + ); // A fuzzer with feedbacks and a corpus scheduler let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); diff --git a/fuzzers/libfuzzer_libpng_centralized/Cargo.toml b/fuzzers/libfuzzer_libpng_centralized/Cargo.toml index bb20666004..74766d032a 100644 --- a/fuzzers/libfuzzer_libpng_centralized/Cargo.toml +++ b/fuzzers/libfuzzer_libpng_centralized/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "libfuzzer_libpng_launcher_centralized" -version = "0.11.2" +version = "0.12.0" authors = ["Andrea Fioraldi ", "Dominik Maier "] edition = "2021" diff --git a/fuzzers/libfuzzer_libpng_centralized/Makefile.toml b/fuzzers/libfuzzer_libpng_centralized/Makefile.toml index fe8768420b..a1e30b393d 100644 --- a/fuzzers/libfuzzer_libpng_centralized/Makefile.toml +++ b/fuzzers/libfuzzer_libpng_centralized/Makefile.toml @@ -84,7 +84,7 @@ windows_alias = "unsupported" [tasks.run_unix] script_runner = "@shell" script=''' -./${FUZZER_NAME} --cores 0 --input ./corpus +./${FUZZER_NAME} --cores 0-1 --input ./corpus ''' dependencies = [ "fuzzer" ] @@ -98,7 +98,7 @@ windows_alias = "unsupported" script_runner = "@shell" script=''' rm -rf libafl_unix_shmem_server || true -timeout 31s ./${FUZZER_NAME} --cores 0 --input ./corpus 2>/dev/null >fuzz_stdout.log || true +timeout 31s ./${FUZZER_NAME} --cores 0-1 --input ./corpus 2>/dev/null | tee fuzz_stdout.log || true if grep -qa "corpus: 30" fuzz_stdout.log; then echo "Fuzzer is working" else @@ -112,7 +112,7 @@ dependencies = [ "fuzzer" ] script_runner = "@shell" script=''' rm -rf libafl_unix_shmem_server || true -timeout 31s ./${FUZZER_NAME} --cores 0 --input ./corpus 2>/dev/null >fuzz_stdout.log || true +timeout 31s ./${FUZZER_NAME} --cores 0 --input ./corpus 2>/dev/null | tee fuzz_stdout.log || true ''' dependencies = [ "fuzzer" ] diff --git a/fuzzers/libfuzzer_libpng_centralized/src/lib.rs b/fuzzers/libfuzzer_libpng_centralized/src/lib.rs index d799e81196..169f2e12b0 100644 --- a/fuzzers/libfuzzer_libpng_centralized/src/lib.rs +++ b/fuzzers/libfuzzer_libpng_centralized/src/lib.rs @@ -2,17 +2,13 @@ //! The example harness is built for libpng. //! In this example, you will see the use of the `launcher` feature. //! The `launcher` will spawn new processes for each cpu core. -use mimalloc::MiMalloc; -#[global_allocator] -static GLOBAL: MiMalloc = MiMalloc; - use core::time::Duration; use std::{env, net::SocketAddr, path::PathBuf}; use clap::{self, Parser}; use libafl::{ corpus::{Corpus, InMemoryCorpus, OnDiskCorpus}, - events::{launcher::CentralizedLauncher, EventConfig}, + events::{centralized::CentralizedEventManager, launcher::CentralizedLauncher, EventConfig}, executors::{inprocess::InProcessExecutor, ExitKind}, feedback_or, feedback_or_fast, feedbacks::{CrashFeedback, MaxMapFeedback, TimeFeedback, TimeoutFeedback}, @@ -23,21 +19,24 @@ use libafl::{ scheduled::{havoc_mutations, tokens_mutations, StdScheduledMutator}, token_mutations::Tokens, }, - observers::{HitcountsMapObserver, TimeObserver}, + observers::{CanTrack, HitcountsMapObserver, TimeObserver}, schedulers::{IndexesLenTimeMinimizerScheduler, QueueScheduler}, stages::mutational::StdMutationalStage, - state::{HasCorpus, HasMetadata, StdState}, - Error, + state::{HasCorpus, StdState}, + Error, HasMetadata, }; use libafl_bolts::{ core_affinity::{CoreId, Cores}, - current_nanos, rands::StdRand, shmem::{ShMemProvider, StdShMemProvider}, tuples::{tuple_list, Merge}, AsSlice, }; use libafl_targets::{libfuzzer_initialize, libfuzzer_test_one_input, std_edges_map_observer}; +use mimalloc::MiMalloc; + +#[global_allocator] +static GLOBAL: MiMalloc = MiMalloc; /// Parse a millis string to a [`Duration`]. Used for arg parsing. fn timeout_from_millis_str(time: &str) -> Result { @@ -53,11 +52,11 @@ fn timeout_from_millis_str(time: &str) -> Result { )] struct Opt { #[arg( - short, - long, - value_parser = Cores::from_cmdline, - help = "Spawn a client in each of the provided cores. Broker runs in the 0th core. 'all' to select all available cores. 'none' to run a client without binding to any core. eg: '1,2-4,6' selects the cores 1,2,3,4,6.", - name = "CORES" + short, + long, + value_parser = Cores::from_cmdline, + help = "Spawn a client in each of the provided cores. Broker runs in the 0th core. 'all' to select all available cores. 'none' to run a client without binding to any core. eg: '1,2-4,6' selects the cores 1,2,3,4,6.", + name = "CORES" )] cores: Cores, @@ -73,7 +72,13 @@ struct Opt { #[arg(short = 'a', long, help = "Specify a remote broker", name = "REMOTE")] remote_broker_addr: Option, - #[arg(short, long, help = "Set an initial corpus directory", name = "INPUT")] + #[arg( + short, + long, + help = "Set an initial corpus directory", + name = "INPUT", + required = true + )] input: Vec, #[arg( @@ -86,12 +91,12 @@ struct Opt { output: PathBuf, #[arg( - value_parser = timeout_from_millis_str, - short, - long, - help = "Set the exeucution timeout in milliseconds, default is 10000", - name = "TIMEOUT", - default_value = "10000" + value_parser = timeout_from_millis_str, + short, + long, + help = "Set the exeucution timeout in milliseconds, default is 10000", + name = "TIMEOUT", + default_value = "10000" )] timeout: Duration, /* @@ -130,9 +135,12 @@ pub extern "C" fn libafl_main() { let monitor = MultiMonitor::new(|s| println!("{s}")); - let mut run_client = |state: Option<_>, mut mgr, _core_id: CoreId| { + let mut run_client = |state: Option<_>, + mut mgr: CentralizedEventManager<_, _>, + _core_id: CoreId| { // Create an observation channel using the coverage map - let edges_observer = HitcountsMapObserver::new(unsafe { std_edges_map_observer("edges") }); + let edges_observer = + HitcountsMapObserver::new(unsafe { std_edges_map_observer("edges") }).track_indices(); // Create an observation channel to keep track of the execution time let time_observer = TimeObserver::new("time"); @@ -141,9 +149,9 @@ pub extern "C" fn libafl_main() { // This one is composed by two Feedbacks in OR let mut feedback = feedback_or!( // New maximization map feedback linked to the edges observer and the feedback state - MaxMapFeedback::tracking(&edges_observer, true, false), + MaxMapFeedback::new(&edges_observer), // Time feedback, this one does not need a feedback state - TimeFeedback::with_observer(&time_observer) + TimeFeedback::new(&time_observer) ); // A feedback to choose if an input is a solution or not @@ -153,7 +161,7 @@ pub extern "C" fn libafl_main() { let mut state = state.unwrap_or_else(|| { StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryCorpus::new(), // Corpus in which we store solutions (crashes in this example), @@ -186,7 +194,8 @@ pub extern "C" fn libafl_main() { let mut stages = tuple_list!(StdMutationalStage::new(mutator)); // A minimization+queue policy to get testcasess from the corpus - let scheduler = IndexesLenTimeMinimizerScheduler::new(QueueScheduler::new()); + let scheduler = + IndexesLenTimeMinimizerScheduler::new(&edges_observer, QueueScheduler::new()); // A fuzzer with feedbacks and a corpus scheduler let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); @@ -201,7 +210,7 @@ pub extern "C" fn libafl_main() { // Create the executor for an in-process function with one observer for edge coverage and one for the execution time #[cfg(target_os = "linux")] - let mut executor = InProcessExecutor::batched_timeouts( + let mut executor = InProcessExecutor::batched_timeout( &mut harness, tuple_list!(edges_observer, time_observer), &mut fuzzer, @@ -234,16 +243,23 @@ pub extern "C" fn libafl_main() { .unwrap_or_else(|_| panic!("Failed to load initial corpus at {:?}", &opt.input)); println!("We imported {} inputs from disk.", state.corpus().count()); } - - fuzzer.fuzz_loop(&mut stages, &mut executor, &mut state, &mut mgr)?; + if !mgr.is_main() { + fuzzer.fuzz_loop(&mut stages, &mut executor, &mut state, &mut mgr)?; + } else { + let mut empty_stages = tuple_list!(); + fuzzer.fuzz_loop(&mut empty_stages, &mut executor, &mut state, &mut mgr)?; + } Ok(()) }; + let mut main_run_client = run_client.clone(); // clone it just for borrow checker + match CentralizedLauncher::builder() .shmem_provider(shmem_provider) .configuration(EventConfig::from_name("default")) .monitor(monitor) .run_client(&mut run_client) + .main_run_client(&mut main_run_client) .cores(&cores) .broker_port(broker_port) .remote_broker_addr(opt.remote_broker_addr) diff --git a/fuzzers/libfuzzer_libpng_cmin/Cargo.toml b/fuzzers/libfuzzer_libpng_cmin/Cargo.toml index eb26923805..f4435c1d8c 100644 --- a/fuzzers/libfuzzer_libpng_cmin/Cargo.toml +++ b/fuzzers/libfuzzer_libpng_cmin/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "libfuzzer_libpng_cmin" -version = "0.11.2" +version = "0.12.0" authors = ["Andrea Fioraldi ", "Dominik Maier ", "Addison Crump "] edition = "2021" diff --git a/fuzzers/libfuzzer_libpng_cmin/Makefile.toml b/fuzzers/libfuzzer_libpng_cmin/Makefile.toml index 1df326c8f0..29d66bea15 100644 --- a/fuzzers/libfuzzer_libpng_cmin/Makefile.toml +++ b/fuzzers/libfuzzer_libpng_cmin/Makefile.toml @@ -161,7 +161,7 @@ windows_alias = "unsupported" script_runner = "@shell" script=''' rm -rf libafl_unix_shmem_server || true -timeout 31s ./${FUZZER_NAME} >fuzz_stdout.log & +timeout 31s ./${FUZZER_NAME} | tee fuzz_stdout.log & sleep 0.2 timeout 30s ./${FUZZER_NAME} >/dev/null 2>/dev/null || true if grep -qa "corpus: 30" fuzz_stdout.log; then @@ -177,7 +177,7 @@ dependencies = [ "fuzzer" ] script_runner = "@shell" script=''' rm -rf libafl_unix_shmem_server || true -timeout 31s ./${FUZZER_NAME} >fuzz_stdout.log & +timeout 31s ./${FUZZER_NAME} | tee fuzz_stdout.log & sleep 0.2 timeout 30s ./${FUZZER_NAME} >/dev/null 2>/dev/null || true ''' diff --git a/fuzzers/libfuzzer_libpng_cmin/src/lib.rs b/fuzzers/libfuzzer_libpng_cmin/src/lib.rs index 59d9a449b9..e3b660bfe7 100644 --- a/fuzzers/libfuzzer_libpng_cmin/src/lib.rs +++ b/fuzzers/libfuzzer_libpng_cmin/src/lib.rs @@ -1,9 +1,5 @@ //! A libfuzzer-like fuzzer with llmp-multithreading support and restarts //! The example harness is built for libpng. -use mimalloc::MiMalloc; -#[global_allocator] -static GLOBAL: MiMalloc = MiMalloc; - use core::time::Duration; #[cfg(feature = "crash")] use std::ptr; @@ -25,21 +21,24 @@ use libafl::{ scheduled::{havoc_mutations, tokens_mutations, StdScheduledMutator}, token_mutations::Tokens, }, - observers::{HitcountsMapObserver, TimeObserver}, + observers::{CanTrack, HitcountsMapObserver, TimeObserver}, schedulers::{ powersched::PowerSchedule, IndexesLenTimeMinimizerScheduler, StdWeightedScheduler, }, stages::{calibrate::CalibrationStage, power::StdPowerMutationalStage}, - state::{HasCorpus, HasMetadata, StdState}, - Error, + state::{HasCorpus, StdState}, + Error, HasMetadata, }; use libafl_bolts::{ - current_nanos, rands::StdRand, tuples::{tuple_list, Merge}, AsSlice, }; use libafl_targets::{libfuzzer_initialize, libfuzzer_test_one_input, std_edges_map_observer}; +use mimalloc::MiMalloc; + +#[global_allocator] +static GLOBAL: MiMalloc = MiMalloc; /// The main fn, `no_mangle` as it is a C main #[cfg(not(test))] @@ -82,14 +81,15 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re }; // Create an observation channel using the coverage map - let edges_observer = HitcountsMapObserver::new(unsafe { std_edges_map_observer("edges") }); + let edges_observer = + HitcountsMapObserver::new(unsafe { std_edges_map_observer("edges") }).track_indices(); let minimizer = StdCorpusMinimizer::new(&edges_observer); // Create an observation channel to keep track of the execution time let time_observer = TimeObserver::new("time"); - let map_feedback = MaxMapFeedback::tracking(&edges_observer, true, false); + let map_feedback = MaxMapFeedback::new(&edges_observer); let calibration = CalibrationStage::new(&map_feedback); @@ -99,7 +99,7 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re // New maximization map feedback linked to the edges observer and the feedback state map_feedback, // Time feedback, this one does not need a feedback state - TimeFeedback::with_observer(&time_observer) + TimeFeedback::new(&time_observer) ); // A feedback to choose if an input is a solution or not @@ -109,7 +109,7 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re let mut state = state.unwrap_or_else(|| { StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryCorpus::new(), // Corpus in which we store solutions (crashes in this example), @@ -146,11 +146,10 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re let mut stages = tuple_list!(calibration, power); // A minimization+queue policy to get testcasess from the corpus - let scheduler = IndexesLenTimeMinimizerScheduler::new(StdWeightedScheduler::with_schedule( - &mut state, + let scheduler = IndexesLenTimeMinimizerScheduler::new( &edges_observer, - Some(PowerSchedule::FAST), - )); + StdWeightedScheduler::with_schedule(&mut state, &edges_observer, Some(PowerSchedule::FAST)), + ); // A fuzzer with feedbacks and a corpus scheduler let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); diff --git a/fuzzers/libfuzzer_libpng_ctx/.gitignore b/fuzzers/libfuzzer_libpng_ctx/.gitignore deleted file mode 100644 index a977a2ca5b..0000000000 --- a/fuzzers/libfuzzer_libpng_ctx/.gitignore +++ /dev/null @@ -1 +0,0 @@ -libpng-* \ No newline at end of file diff --git a/fuzzers/libfuzzer_libpng_ctx/Cargo.toml b/fuzzers/libfuzzer_libpng_ctx/Cargo.toml deleted file mode 100644 index 53de79a405..0000000000 --- a/fuzzers/libfuzzer_libpng_ctx/Cargo.toml +++ /dev/null @@ -1,32 +0,0 @@ -[package] -name = "libfuzzer_libpng_ctx" -version = "0.11.2" -authors = ["Andrea Fioraldi ", "Dominik Maier "] -edition = "2021" - -[features] -default = ["std"] -std = [] - -[profile.release] -lto = true -codegen-units = 1 -opt-level = 3 -debug = true - -[build-dependencies] -cc = { version = "1.0", features = ["parallel"] } -which = "4.4" - -[dependencies] -libafl = { path = "../../libafl/", features = ["std", "derive", "llmp_compression", "introspection"] } -libafl_bolts = { path = "../../libafl_bolts/" } -libafl_targets = { path = "../../libafl_targets/", features = ["libfuzzer"] } -# TODO Include it only when building cc -libafl_cc = { path = "../../libafl_cc/" } -clap = { version = "4.0", features = ["derive"] } -mimalloc = { version = "*", default-features = false } - -[lib] -name = "libfuzzer_libpng" -crate-type = ["staticlib"] diff --git a/fuzzers/libfuzzer_libpng_ctx/Makefile.toml b/fuzzers/libfuzzer_libpng_ctx/Makefile.toml deleted file mode 100644 index 8d66dbc8e3..0000000000 --- a/fuzzers/libfuzzer_libpng_ctx/Makefile.toml +++ /dev/null @@ -1,134 +0,0 @@ -# Variables -[env] -FUZZER_NAME='fuzzer_libpng_ctx' -PROJECT_DIR = { script = ["pwd"] } -CARGO_TARGET_DIR = { value = "${PROJECT_DIR}/target", condition = { env_not_set = ["CARGO_TARGET_DIR"] } } -PROFILE = { value = "release", condition = {env_not_set = ["PROFILE"]} } -PROFILE_DIR = {value = "release", condition = {env_not_set = ["PROFILE_DIR"] }} -LIBAFL_CC = '${CARGO_TARGET_DIR}/${PROFILE_DIR}/libafl_cc' -LIBAFL_CXX = '${CARGO_TARGET_DIR}/${PROFILE}/libafl_cxx' -FUZZER = '${CARGO_TARGET_DIR}/${PROFILE_DIR}/${FUZZER_NAME}' - -[tasks.unsupported] -script_runner="@shell" -script=''' -echo "Cargo-make not integrated yet on this platform" -''' - -# libpng -[tasks.libpng] -linux_alias = "libpng_unix" -mac_alias = "libpng_unix" -windows_alias = "unsupported" - -[tasks.libpng_unix] -condition = { files_not_exist = ["./libpng-1.6.37"]} -script_runner="@shell" -script=''' -wget https://github.com/glennrp/libpng/archive/refs/tags/v1.6.37.tar.gz -tar -xvf v1.6.37.tar.gz -''' - -# Compilers -[tasks.cxx] -linux_alias = "cxx_unix" -mac_alias = "cxx_unix" -windows_alias = "unsupported" - -[tasks.cxx_unix] -command = "cargo" -args = ["build" , "--profile", "${PROFILE}"] - -[tasks.cc] -linux_alias = "cc_unix" -mac_alias = "cc_unix" -windows_alias = "unsupported" - -[tasks.cc_unix] -command = "cargo" -args = ["build" , "--profile", "${PROFILE}"] - -# Library -[tasks.lib] -linux_alias = "lib_unix" -mac_alias = "lib_unix" -windows_alias = "unsupported" - -[tasks.lib_unix] -script_runner="@shell" -script=''' -cd libpng-1.6.37 && ./configure --enable-shared=no --with-pic=yes --enable-hardware-optimizations=yes -cd "${PROJECT_DIR}" -make -C libpng-1.6.37 CC="${CARGO_TARGET_DIR}/${PROFILE_DIR}/libafl_cc" CXX="${CARGO_TARGET_DIR}/${PROFILE_DIR}/libafl_cxx" -''' -dependencies = [ "libpng", "cxx", "cc" ] - - -# Harness -[tasks.fuzzer] -linux_alias = "fuzzer_unix" -mac_alias = "fuzzer_unix" -windows_alias = "unsupported" - -[tasks.fuzzer_unix] -command = "${CARGO_TARGET_DIR}/${PROFILE_DIR}/libafl_cxx" -args = ["${PROJECT_DIR}/harness.cc", "${PROJECT_DIR}/libpng-1.6.37/.libs/libpng16.a", "-I", "${PROJECT_DIR}/libpng-1.6.37/", "-o", "${FUZZER_NAME}", "-lm", "-lz"] -dependencies = [ "lib", "cxx", "cc" ] - -# Run the fuzzer -[tasks.run] -linux_alias = "run_unix" -mac_alias = "run_unix" -windows_alias = "unsupported" - -[tasks.run_unix] -script_runner = "@shell" -script=''' -./${FUZZER_NAME} --cores 0 --input ./corpus -''' -dependencies = [ "fuzzer" ] - -# Test -[tasks.test] -linux_alias = "test_unix" -mac_alias = "test_mac" -windows_alias = "unsupported" - -[tasks.test_unix] -script_runner = "@shell" -script=''' -rm -rf libafl_unix_shmem_server || true -timeout 31s ./${FUZZER_NAME} --cores 0 --input ./corpus >fuzz_stdout.log 2>/dev/null || true -if grep -qa "corpus: 30" fuzz_stdout.log; then - echo "Fuzzer is working" -elif grep -qa "objectives: 1" fuzz_stdout.log; then - echo "Fuzzer finds timeout or crash" -else - echo "Fuzzer does not generate any testcases or any crashes" - exit 1 -fi -''' -dependencies = [ "fuzzer" ] - -[tasks.test_mac] -script_runner = "@shell" -script=''' -rm -rf libafl_unix_shmem_server || true -timeout 31s ./${FUZZER_NAME} --cores 0 --input ./corpus >fuzz_stdout.log 2>/dev/null || true -''' - -# Clean up -[tasks.clean] -linux_alias = "clean_unix" -mac_alias = "clean_unix" -windows_alias = "unsupported" - -[tasks.clean_unix] -# Disable default `clean` definition -clear = true -script_runner="@shell" -script=''' -rm -f ./${FUZZER_NAME} -make -C libpng-1.6.37 clean -cargo clean -''' diff --git a/fuzzers/libfuzzer_libpng_ctx/README.md b/fuzzers/libfuzzer_libpng_ctx/README.md deleted file mode 100644 index b9ffdecb35..0000000000 --- a/fuzzers/libfuzzer_libpng_ctx/README.md +++ /dev/null @@ -1,47 +0,0 @@ -# Libfuzzer for libpng, with launcher - -This folder contains an example fuzzer for libpng, using LLMP for fast multi-process fuzzing and crash detection. -To show off crash detection, we added a `ud2` instruction to the harness, edit harness.cc if you want a non-crashing example. -It has been tested on Linux. - -In contrast to the normal libfuzzer libpng example, this uses the `launcher` feature, that automatically spawns `n` child processes, and binds them to a free core. - -## Build - -To build this example, run - -```bash -cargo build --release -``` - -This will build the library with the fuzzer (src/lib.rs) with the libfuzzer compatibility layer and the SanitizerCoverage runtime functions for coverage feedback. -In addition, it will also build two C and C++ compiler wrappers (bin/libafl_c(libafl_c/xx).rs) that you must use to compile the target. - -Then download libpng, and unpack the archive: -```bash -wget https://github.com/glennrp/libpng/archive/refs/tags/v1.6.37.tar.gz -tar -xvf v1.6.37.tar.gz -``` - -Now compile libpng, using the libafl_cc compiler wrapper: - -```bash -cd libpng-1.6.37 -./configure -make CC=../target/release/libafl_cc CXX=../target/release/libafl_cxx -j `nproc` -``` - -You can find the static lib at `libpng-1.6.37/.libs/libpng16.a`. - -Now, we have to build the libfuzzer harness and link all together to create our fuzzer binary. - -``` -cd .. -./target/release/libafl_cxx ./harness.cc libpng-1.6.37/.libs/libpng16.a -I libpng-1.6.37/ -o fuzzer_libpng -lz -lm -``` - -Afterwards, the fuzzer will be ready to run. - -## Run - -Just run once, the launcher feature should do the rest. \ No newline at end of file diff --git a/fuzzers/libfuzzer_libpng_ctx/corpus/not_kitty.png b/fuzzers/libfuzzer_libpng_ctx/corpus/not_kitty.png deleted file mode 100644 index eff7c1707b..0000000000 Binary files a/fuzzers/libfuzzer_libpng_ctx/corpus/not_kitty.png and /dev/null differ diff --git a/fuzzers/libfuzzer_libpng_ctx/corpus/not_kitty_alpha.png b/fuzzers/libfuzzer_libpng_ctx/corpus/not_kitty_alpha.png deleted file mode 100644 index 2fb8da2c8f..0000000000 Binary files a/fuzzers/libfuzzer_libpng_ctx/corpus/not_kitty_alpha.png and /dev/null differ diff --git a/fuzzers/libfuzzer_libpng_ctx/corpus/not_kitty_gamma.png b/fuzzers/libfuzzer_libpng_ctx/corpus/not_kitty_gamma.png deleted file mode 100644 index 939d9d29a9..0000000000 Binary files a/fuzzers/libfuzzer_libpng_ctx/corpus/not_kitty_gamma.png and /dev/null differ diff --git a/fuzzers/libfuzzer_libpng_ctx/corpus/not_kitty_icc.png b/fuzzers/libfuzzer_libpng_ctx/corpus/not_kitty_icc.png deleted file mode 100644 index f0c7804d99..0000000000 Binary files a/fuzzers/libfuzzer_libpng_ctx/corpus/not_kitty_icc.png and /dev/null differ diff --git a/fuzzers/libfuzzer_libpng_ctx/harness.cc b/fuzzers/libfuzzer_libpng_ctx/harness.cc deleted file mode 100644 index e26e707e17..0000000000 --- a/fuzzers/libfuzzer_libpng_ctx/harness.cc +++ /dev/null @@ -1,188 +0,0 @@ -// libpng_read_fuzzer.cc -// Copyright 2017-2018 Glenn Randers-Pehrson -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that may -// be found in the LICENSE file https://cs.chromium.org/chromium/src/LICENSE - -// Last changed in libpng 1.6.35 [July 15, 2018] - -// The modifications in 2017 by Glenn Randers-Pehrson include -// 1. addition of a PNG_CLEANUP macro, -// 2. setting the option to ignore ADLER32 checksums, -// 3. adding "#include " which is needed on some platforms -// to provide memcpy(). -// 4. adding read_end_info() and creating an end_info structure. -// 5. adding calls to png_set_*() transforms commonly used by browsers. - -#include -#include -#include - -#include - -#define PNG_INTERNAL -#include "png.h" - -#define PNG_CLEANUP \ - if (png_handler.png_ptr) { \ - if (png_handler.row_ptr) \ - png_free(png_handler.png_ptr, png_handler.row_ptr); \ - if (png_handler.end_info_ptr) \ - png_destroy_read_struct(&png_handler.png_ptr, &png_handler.info_ptr, \ - &png_handler.end_info_ptr); \ - else if (png_handler.info_ptr) \ - png_destroy_read_struct(&png_handler.png_ptr, &png_handler.info_ptr, \ - nullptr); \ - else \ - png_destroy_read_struct(&png_handler.png_ptr, nullptr, nullptr); \ - png_handler.png_ptr = nullptr; \ - png_handler.row_ptr = nullptr; \ - png_handler.info_ptr = nullptr; \ - png_handler.end_info_ptr = nullptr; \ - } - -struct BufState { - const uint8_t *data; - size_t bytes_left; -}; - -struct PngObjectHandler { - png_infop info_ptr = nullptr; - png_structp png_ptr = nullptr; - png_infop end_info_ptr = nullptr; - png_voidp row_ptr = nullptr; - BufState *buf_state = nullptr; - - ~PngObjectHandler() { - if (row_ptr) { png_free(png_ptr, row_ptr); } - if (end_info_ptr) - png_destroy_read_struct(&png_ptr, &info_ptr, &end_info_ptr); - else if (info_ptr) - png_destroy_read_struct(&png_ptr, &info_ptr, nullptr); - else - png_destroy_read_struct(&png_ptr, nullptr, nullptr); - delete buf_state; - } -}; - -void user_read_data(png_structp png_ptr, png_bytep data, size_t length) { - BufState *buf_state = static_cast(png_get_io_ptr(png_ptr)); - if (length > buf_state->bytes_left) { png_error(png_ptr, "read error"); } - memcpy(data, buf_state->data, length); - buf_state->bytes_left -= length; - buf_state->data += length; -} - -static const int kPngHeaderSize = 8; - -// Entry point for LibFuzzer. -// Roughly follows the libpng book example: -// http://www.libpng.org/pub/png/book/chapter13.html -extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { - if (size < kPngHeaderSize) { return 0; } - - std::vector v(data, data + size); - if (png_sig_cmp(v.data(), 0, kPngHeaderSize)) { - // not a PNG. - return 0; - } - - PngObjectHandler png_handler; - png_handler.png_ptr = nullptr; - png_handler.row_ptr = nullptr; - png_handler.info_ptr = nullptr; - png_handler.end_info_ptr = nullptr; - - png_handler.png_ptr = - png_create_read_struct(PNG_LIBPNG_VER_STRING, nullptr, nullptr, nullptr); - if (!png_handler.png_ptr) { return 0; } - - png_handler.info_ptr = png_create_info_struct(png_handler.png_ptr); - if (!png_handler.info_ptr) { - PNG_CLEANUP - return 0; - } - - png_handler.end_info_ptr = png_create_info_struct(png_handler.png_ptr); - if (!png_handler.end_info_ptr) { - PNG_CLEANUP - return 0; - } - - png_set_crc_action(png_handler.png_ptr, PNG_CRC_QUIET_USE, PNG_CRC_QUIET_USE); -#ifdef PNG_IGNORE_ADLER32 - png_set_option(png_handler.png_ptr, PNG_IGNORE_ADLER32, PNG_OPTION_ON); -#endif - - // Setting up reading from buffer. - png_handler.buf_state = new BufState(); - png_handler.buf_state->data = data + kPngHeaderSize; - png_handler.buf_state->bytes_left = size - kPngHeaderSize; - png_set_read_fn(png_handler.png_ptr, png_handler.buf_state, user_read_data); - png_set_sig_bytes(png_handler.png_ptr, kPngHeaderSize); - - if (setjmp(png_jmpbuf(png_handler.png_ptr))) { - PNG_CLEANUP - return 0; - } - - // Reading. - png_read_info(png_handler.png_ptr, png_handler.info_ptr); - - // reset error handler to put png_deleter into scope. - if (setjmp(png_jmpbuf(png_handler.png_ptr))) { - PNG_CLEANUP - return 0; - } - - png_uint_32 width, height; - int bit_depth, color_type, interlace_type, compression_type; - int filter_type; - - if (!png_get_IHDR(png_handler.png_ptr, png_handler.info_ptr, &width, &height, - &bit_depth, &color_type, &interlace_type, &compression_type, - &filter_type)) { - PNG_CLEANUP - return 0; - } - - // This is going to be too slow. - if (width && height > 100000000 / width) { - PNG_CLEANUP -#ifdef HAS_DUMMY_CRASH - #ifdef __aarch64__ - asm volatile(".word 0xf7f0a000\n"); - #else - asm("ud2"); - #endif -#endif - return 0; - } - - // Set several transforms that browsers typically use: - png_set_gray_to_rgb(png_handler.png_ptr); - png_set_expand(png_handler.png_ptr); - png_set_packing(png_handler.png_ptr); - png_set_scale_16(png_handler.png_ptr); - png_set_tRNS_to_alpha(png_handler.png_ptr); - - int passes = png_set_interlace_handling(png_handler.png_ptr); - - png_read_update_info(png_handler.png_ptr, png_handler.info_ptr); - - png_handler.row_ptr = - png_malloc(png_handler.png_ptr, - png_get_rowbytes(png_handler.png_ptr, png_handler.info_ptr)); - - for (int pass = 0; pass < passes; ++pass) { - for (png_uint_32 y = 0; y < height; ++y) { - png_read_row(png_handler.png_ptr, - static_cast(png_handler.row_ptr), nullptr); - } - } - - png_read_end(png_handler.png_ptr, png_handler.end_info_ptr); - - PNG_CLEANUP - return 0; -} diff --git a/fuzzers/libfuzzer_libpng_ctx/src/lib.rs b/fuzzers/libfuzzer_libpng_ctx/src/lib.rs deleted file mode 100644 index 2e422fb71c..0000000000 --- a/fuzzers/libfuzzer_libpng_ctx/src/lib.rs +++ /dev/null @@ -1,243 +0,0 @@ -//! A libfuzzer-like fuzzer with llmp-multithreading support and restarts -//! The example harness is built for libpng. -//! In this example, you will see the use of the `launcher` feature. -//! The `launcher` will spawn new processes for each cpu core. -use mimalloc::MiMalloc; -#[global_allocator] -static GLOBAL: MiMalloc = MiMalloc; - -use core::time::Duration; -use std::{env, net::SocketAddr, path::PathBuf}; - -use clap::{self, Parser}; -use libafl::{ - corpus::{Corpus, InMemoryCorpus, OnDiskCorpus}, - events::{launcher::Launcher, EventConfig}, - executors::{inprocess::InProcessExecutor, ExitKind}, - feedback_or, feedback_or_fast, - feedbacks::{CrashFeedback, MaxMapFeedback, TimeFeedback, TimeoutFeedback}, - fuzzer::{Fuzzer, StdFuzzer}, - inputs::{BytesInput, HasTargetBytes}, - monitors::MultiMonitor, - mutators::{ - scheduled::{havoc_mutations, tokens_mutations, StdScheduledMutator}, - token_mutations::Tokens, - }, - observers::{HitcountsMapObserver, StdMapObserver, TimeObserver}, - schedulers::{IndexesLenTimeMinimizerScheduler, QueueScheduler}, - stages::mutational::StdMutationalStage, - state::{HasCorpus, HasMetadata, StdState}, - Error, -}; -use libafl_bolts::{ - core_affinity::Cores, - current_nanos, - rands::StdRand, - shmem::{ShMemProvider, StdShMemProvider}, - tuples::{tuple_list, Merge}, - AsSlice, -}; -use libafl_targets::{libfuzzer_initialize, libfuzzer_test_one_input, std_edges_map_observer}; - -fn timeout_from_millis_str(time: &str) -> Result { - Ok(Duration::from_millis(time.parse()?)) -} - -#[derive(Debug, Parser)] -#[command( - name = "libfuzzer_libpng_ctx", - about = "A clone of libfuzzer using LibAFL for a libpng harness", - author = "Andrea Fioraldi , Dominik Maier " -)] -struct Opt { - #[arg( - short, - long, - value_parser = Cores::from_cmdline, - help = "Spawn a client in each of the provided cores. Broker runs in the 0th core. 'all' to select all available cores. 'none' to run a client without binding to any core. eg: '1,2-4,6' selects the cores 1,2,3,4,6.", - name = "CORES" - )] - cores: Cores, - - #[arg( - short = 'p', - long, - help = "Choose the broker TCP port, default is 1337", - name = "PORT", - default_value = "1337" - )] - broker_port: u16, - - #[arg(short = 'a', long, help = "Specify a remote broker", name = "REMOTE")] - remote_broker_addr: Option, - - #[arg(short, long, help = "Set an initial corpus directory", name = "INPUT")] - input: Vec, - - #[arg( - short, - long, - help = "Set the output directory, default is ./out", - name = "OUTPUT", - default_value = "./out" - )] - output: PathBuf, - - #[arg( - short, - long, - value_parser = timeout_from_millis_str, - help = "Set the exeucution timeout in milliseconds, default is 10000", - name = "TIMEOUT", - default_value = "10000", - )] - timeout: Duration, - /* - // The tokens are hardcoded in this example. - #[arg( - - short = "x", - long, - help = "Feed the fuzzer with an user-specified list of tokens (often called \"dictionary\"", - name = "TOKENS", - multiple = true - )] - tokens: Vec,*/ -} - -/// The main fn, `no_mangle` as it is a C symbol -#[no_mangle] -pub extern "C" fn libafl_main() { - // Registry the metadata types used in this fuzzer - // Needed only on no_std - // unsafe { RegistryBuilder::register::(); } - let opt = Opt::parse(); - - let broker_port = opt.broker_port; - - let cores = opt.cores; - - println!( - "Workdir: {:?}", - env::current_dir().unwrap().to_string_lossy().to_string() - ); - - let shmem_provider = StdShMemProvider::new().expect("Failed to init shared memory"); - - let monitor = MultiMonitor::new(|s| println!("{s}")); - - let mut run_client = |state: Option<_>, mut restarting_mgr, _core_id| { - // Create an observation channel using the coverage map - let edges_observer = unsafe { HitcountsMapObserver::new(std_edges_map_observer("edges")) }; - - // Create an observation channel to keep track of the execution time - let time_observer = TimeObserver::new("time"); - - // Feedback to rate the interestingness of an input - // This one is composed by two Feedbacks in OR - let mut feedback = feedback_or!( - // New maximization map feedback linked to the edges observer and the feedback state - MaxMapFeedback::tracking(&edges_observer, true, false), - // Time feedback, this one does not need a feedback state - TimeFeedback::with_observer(&time_observer) - ); - - // A feedback to choose if an input is a solution or not - let mut objective = feedback_or_fast!(CrashFeedback::new(), TimeoutFeedback::new()); - - // If not restarting, create a State from scratch - let mut state = state.unwrap_or_else(|| { - StdState::new( - // RNG - StdRand::with_seed(current_nanos()), - // Corpus that will be evolved, we keep it in memory for performance - InMemoryCorpus::new(), - // Corpus in which we store solutions (crashes in this example), - // on disk so the user can get them after stopping the fuzzer - OnDiskCorpus::new(opt.output.clone()).unwrap(), - // States of the feedbacks. - // The feedbacks can report the data that should persist in the State. - &mut feedback, - // Same for objective feedbacks - &mut objective, - ) - .unwrap() - }); - - println!("We're a client, let's fuzz :)"); - - // Create a PNG dictionary if not existing - if state.metadata_map().get::().is_none() { - state.add_metadata(Tokens::from([ - vec![137, 80, 78, 71, 13, 10, 26, 10], // PNG header - "IHDR".as_bytes().to_vec(), - "IDAT".as_bytes().to_vec(), - "PLTE".as_bytes().to_vec(), - "IEND".as_bytes().to_vec(), - ])); - } - - // Setup a basic mutator with a mutational stage - let mutator = StdScheduledMutator::new(havoc_mutations().merge(tokens_mutations())); - let mut stages = tuple_list!(StdMutationalStage::new(mutator)); - - // A minimization+queue policy to get testcasess from the corpus - let scheduler = IndexesLenTimeMinimizerScheduler::new(QueueScheduler::new()); - - // A fuzzer with feedbacks and a corpus scheduler - let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); - - // The wrapped harness function, calling out to the LLVM-style harness - let mut harness = |input: &BytesInput| { - let target = input.target_bytes(); - let buf = target.as_slice(); - libfuzzer_test_one_input(buf); - ExitKind::Ok - }; - - // Create the executor for an in-process function with one observer for edge coverage and one for the execution time - let mut executor = InProcessExecutor::with_timeout( - &mut harness, - tuple_list!(edges_observer, time_observer), - &mut fuzzer, - &mut state, - &mut restarting_mgr, - opt.timeout, - )?; - - // The actual target run starts here. - // Call LLVMFUzzerInitialize() if present. - let args: Vec = env::args().collect(); - if libfuzzer_initialize(&args) == -1 { - println!("Warning: LLVMFuzzerInitialize failed with -1"); - } - - // In case the corpus is empty (on first run), reset - if state.must_load_initial_inputs() { - state - .load_initial_inputs(&mut fuzzer, &mut executor, &mut restarting_mgr, &opt.input) - .unwrap_or_else(|_| panic!("Failed to load initial corpus at {:?}", &opt.input)); - println!("We imported {} inputs from disk.", state.corpus().count()); - } - - fuzzer.fuzz_loop(&mut stages, &mut executor, &mut state, &mut restarting_mgr)?; - Ok(()) - }; - - match Launcher::builder() - .shmem_provider(shmem_provider) - .configuration(EventConfig::from_name("default")) - .monitor(monitor) - .run_client(&mut run_client) - .cores(&cores) - .broker_port(broker_port) - .remote_broker_addr(opt.remote_broker_addr) - .stdout_file(Some("/dev/null")) - .build() - .launch() - { - Ok(()) => (), - Err(Error::ShuttingDown) => println!("Fuzzing stopped by user. Good bye."), - Err(err) => panic!("Failed to run launcher: {err:?}"), - } -} diff --git a/fuzzers/libfuzzer_libpng_launcher/Cargo.toml b/fuzzers/libfuzzer_libpng_launcher/Cargo.toml index ded6b8436c..78669692b8 100644 --- a/fuzzers/libfuzzer_libpng_launcher/Cargo.toml +++ b/fuzzers/libfuzzer_libpng_launcher/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "libfuzzer_libpng_launcher" -version = "0.11.2" +version = "0.12.0" authors = ["Andrea Fioraldi ", "Dominik Maier "] edition = "2021" diff --git a/fuzzers/libfuzzer_libpng_launcher/Makefile.toml b/fuzzers/libfuzzer_libpng_launcher/Makefile.toml index 5b30335d80..86f63f1613 100644 --- a/fuzzers/libfuzzer_libpng_launcher/Makefile.toml +++ b/fuzzers/libfuzzer_libpng_launcher/Makefile.toml @@ -99,7 +99,7 @@ windows_alias = "unsupported" script_runner = "@shell" script=''' rm -rf libafl_unix_shmem_server || true -timeout 31s ./${FUZZER_NAME}.coverage --broker-port 21337 --cores 0 --input ./corpus 2>/dev/null >fuzz_stdout.log || true +timeout 31s ./${FUZZER_NAME}.coverage --broker-port 21337 --cores 0 --input ./corpus 2>/dev/null | tee fuzz_stdout.log || true if grep -qa "corpus: 30" fuzz_stdout.log; then echo "Fuzzer is working" else @@ -113,7 +113,7 @@ dependencies = [ "fuzzer" ] script_runner = "@shell" script=''' rm -rf libafl_unix_shmem_server || true -timeout 31s ./${FUZZER_NAME} --cores 0 --input ./corpus 2>/dev/null >fuzz_stdout.log || true +timeout 31s ./${FUZZER_NAME} --cores 0 --input ./corpus 2>/dev/null | tee fuzz_stdout.log || true ''' dependencies = [ "fuzzer" ] diff --git a/fuzzers/libfuzzer_libpng_launcher/src/lib.rs b/fuzzers/libfuzzer_libpng_launcher/src/lib.rs index 3b452bf681..e9def09841 100644 --- a/fuzzers/libfuzzer_libpng_launcher/src/lib.rs +++ b/fuzzers/libfuzzer_libpng_launcher/src/lib.rs @@ -2,10 +2,6 @@ //! The example harness is built for libpng. //! In this example, you will see the use of the `launcher` feature. //! The `launcher` will spawn new processes for each cpu core. -use mimalloc::MiMalloc; -#[global_allocator] -static GLOBAL: MiMalloc = MiMalloc; - use core::time::Duration; use std::{env, net::SocketAddr, path::PathBuf}; @@ -23,21 +19,24 @@ use libafl::{ scheduled::{havoc_mutations, tokens_mutations, StdScheduledMutator}, token_mutations::Tokens, }, - observers::{HitcountsMapObserver, TimeObserver}, + observers::{CanTrack, HitcountsMapObserver, TimeObserver}, schedulers::{IndexesLenTimeMinimizerScheduler, QueueScheduler}, stages::mutational::StdMutationalStage, - state::{HasCorpus, HasMetadata, StdState}, - Error, + state::{HasCorpus, StdState}, + Error, HasMetadata, }; use libafl_bolts::{ core_affinity::Cores, - current_nanos, rands::StdRand, shmem::{ShMemProvider, StdShMemProvider}, tuples::{tuple_list, Merge}, AsSlice, }; use libafl_targets::{libfuzzer_initialize, libfuzzer_test_one_input, std_edges_map_observer}; +use mimalloc::MiMalloc; + +#[global_allocator] +static GLOBAL: MiMalloc = MiMalloc; /// Parse a millis string to a [`Duration`]. Used for arg parsing. fn timeout_from_millis_str(time: &str) -> Result { @@ -53,11 +52,11 @@ fn timeout_from_millis_str(time: &str) -> Result { )] struct Opt { #[arg( - short, - long, - value_parser = Cores::from_cmdline, - help = "Spawn a client in each of the provided cores. Broker runs in the 0th core. 'all' to select all available cores. 'none' to run a client without binding to any core. eg: '1,2-4,6' selects the cores 1,2,3,4,6.", - name = "CORES" + short, + long, + value_parser = Cores::from_cmdline, + help = "Spawn a client in each of the provided cores. Broker runs in the 0th core. 'all' to select all available cores. 'none' to run a client without binding to any core. eg: '1,2-4,6' selects the cores 1,2,3,4,6.", + name = "CORES" )] cores: Cores, @@ -73,7 +72,13 @@ struct Opt { #[arg(short = 'a', long, help = "Specify a remote broker", name = "REMOTE")] remote_broker_addr: Option, - #[arg(short, long, help = "Set an initial corpus directory", name = "INPUT")] + #[arg( + short, + long, + help = "Set an initial corpus directory", + name = "INPUT", + required = true + )] input: Vec, #[arg( @@ -86,12 +91,12 @@ struct Opt { output: PathBuf, #[arg( - value_parser = timeout_from_millis_str, - short, - long, - help = "Set the exeucution timeout in milliseconds, default is 10000", - name = "TIMEOUT", - default_value = "10000" + value_parser = timeout_from_millis_str, + short, + long, + help = "Set the exeucution timeout in milliseconds, default is 10000", + name = "TIMEOUT", + default_value = "10000" )] timeout: Duration, /* @@ -133,7 +138,8 @@ pub extern "C" fn libafl_main() { let mut run_client = |state: Option<_>, mut restarting_mgr, _core_id| { // Create an observation channel using the coverage map - let edges_observer = HitcountsMapObserver::new(unsafe { std_edges_map_observer("edges") }); + let edges_observer = + HitcountsMapObserver::new(unsafe { std_edges_map_observer("edges") }).track_indices(); // Create an observation channel to keep track of the execution time let time_observer = TimeObserver::new("time"); @@ -142,9 +148,9 @@ pub extern "C" fn libafl_main() { // This one is composed by two Feedbacks in OR let mut feedback = feedback_or!( // New maximization map feedback linked to the edges observer and the feedback state - MaxMapFeedback::tracking(&edges_observer, true, false), + MaxMapFeedback::new(&edges_observer), // Time feedback, this one does not need a feedback state - TimeFeedback::with_observer(&time_observer) + TimeFeedback::new(&time_observer) ); // A feedback to choose if an input is a solution or not @@ -154,7 +160,7 @@ pub extern "C" fn libafl_main() { let mut state = state.unwrap_or_else(|| { StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryCorpus::new(), // Corpus in which we store solutions (crashes in this example), @@ -187,7 +193,8 @@ pub extern "C" fn libafl_main() { let mut stages = tuple_list!(StdMutationalStage::new(mutator)); // A minimization+queue policy to get testcasess from the corpus - let scheduler = IndexesLenTimeMinimizerScheduler::new(QueueScheduler::new()); + let scheduler = + IndexesLenTimeMinimizerScheduler::new(&edges_observer, QueueScheduler::new()); // A fuzzer with feedbacks and a corpus scheduler let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); @@ -202,7 +209,7 @@ pub extern "C" fn libafl_main() { // Create the executor for an in-process function with one observer for edge coverage and one for the execution time #[cfg(target_os = "linux")] - let mut executor = InProcessExecutor::batched_timeouts( + let mut executor = InProcessExecutor::batched_timeout( &mut harness, tuple_list!(edges_observer, time_observer), &mut fuzzer, diff --git a/fuzzers/libfuzzer_libpng_norestart/Makefile.toml b/fuzzers/libfuzzer_libpng_norestart/Makefile.toml index 88cac0c2e1..a696ff7f42 100644 --- a/fuzzers/libfuzzer_libpng_norestart/Makefile.toml +++ b/fuzzers/libfuzzer_libpng_norestart/Makefile.toml @@ -104,7 +104,7 @@ rm -rf libafl_unix_shmem_server || true rm -rf corpus/ || true mkdir corpus/ || true cp seeds/* corpus/ || true -timeout 31s ./${FUZZER_NAME} --cores 0 --input ./corpus 2>/dev/null >fuzz_stdout.log || true +timeout 31s ./${FUZZER_NAME} --cores 0 --input ./corpus 2>/dev/null | tee fuzz_stdout.log || true if grep -qa "corpus: 30" fuzz_stdout.log; then echo "Fuzzer is working" else diff --git a/fuzzers/libfuzzer_libpng_norestart/src/lib.rs b/fuzzers/libfuzzer_libpng_norestart/src/lib.rs index a9de4ae99b..f21f4baa64 100644 --- a/fuzzers/libfuzzer_libpng_norestart/src/lib.rs +++ b/fuzzers/libfuzzer_libpng_norestart/src/lib.rs @@ -2,17 +2,17 @@ //! The example harness is built for libpng. //! In this example, you will see the use of the `launcher` feature. //! The `launcher` will spawn new processes for each cpu core. -use mimalloc::MiMalloc; -#[global_allocator] -static GLOBAL: MiMalloc = MiMalloc; use core::time::Duration; use std::{env, net::SocketAddr, path::PathBuf}; -use clap::{self, Parser}; +use clap::Parser; use libafl::{ corpus::{Corpus, InMemoryOnDiskCorpus, OnDiskCorpus}, - events::{launcher::Launcher, EventConfig, EventRestarter, LlmpRestartingEventManager}, + events::{ + launcher::Launcher, llmp::LlmpShouldSaveState, EventConfig, EventRestarter, + LlmpRestartingEventManager, + }, executors::{inprocess::InProcessExecutor, ExitKind}, feedback_or, feedback_or_fast, feedbacks::{CrashFeedback, MaxMapFeedback, TimeFeedback, TimeoutFeedback}, @@ -23,21 +23,24 @@ use libafl::{ scheduled::{havoc_mutations, tokens_mutations, StdScheduledMutator}, token_mutations::Tokens, }, - observers::{HitcountsMapObserver, TimeObserver}, + observers::{CanTrack, HitcountsMapObserver, TimeObserver}, schedulers::{IndexesLenTimeMinimizerScheduler, QueueScheduler}, stages::mutational::StdMutationalStage, - state::{HasCorpus, HasMetadata, StdState}, - Error, + state::{HasCorpus, StdState}, + Error, HasMetadata, }; use libafl_bolts::{ core_affinity::Cores, - current_nanos, rands::StdRand, shmem::{ShMemProvider, StdShMemProvider}, tuples::{tuple_list, Merge}, AsSlice, }; use libafl_targets::{libfuzzer_initialize, libfuzzer_test_one_input, std_edges_map_observer}; +use mimalloc::MiMalloc; + +#[global_allocator] +static GLOBAL: MiMalloc = MiMalloc; /// Parse a millis string to a [`Duration`]. Used for arg parsing. fn timeout_from_millis_str(time: &str) -> Result { @@ -53,11 +56,11 @@ fn timeout_from_millis_str(time: &str) -> Result { )] struct Opt { #[arg( - short, - long, - value_parser = Cores::from_cmdline, - help = "Spawn a client in each of the provided cores. Broker runs in the 0th core. 'all' to select all available cores. 'none' to run a client without binding to any core. eg: '1,2-4,6' selects the cores 1,2,3,4,6.", - name = "CORES" + short, + long, + value_parser = Cores::from_cmdline, + help = "Spawn a client in each of the provided cores. Broker runs in the 0th core. 'all' to select all available cores. 'none' to run a client without binding to any core. eg: '1,2-4,6' selects the cores 1,2,3,4,6.", + name = "CORES" )] cores: Cores, @@ -73,7 +76,13 @@ struct Opt { #[arg(short = 'a', long, help = "Specify a remote broker", name = "REMOTE")] remote_broker_addr: Option, - #[arg(short, long, help = "Set an the corpus directories", name = "INPUT")] + #[arg( + short, + long, + help = "Set an the corpus directories", + name = "INPUT", + required = true + )] input: Vec, #[arg( @@ -86,12 +95,12 @@ struct Opt { output: PathBuf, #[arg( - value_parser = timeout_from_millis_str, - short, - long, - help = "Set the exeucution timeout in milliseconds, default is 10000", - name = "TIMEOUT", - default_value = "10000" + value_parser = timeout_from_millis_str, + short, + long, + help = "Set the exeucution timeout in milliseconds, default is 10000", + name = "TIMEOUT", + default_value = "10000" )] timeout: Duration, @@ -151,10 +160,11 @@ pub extern "C" fn libafl_main() { ); let mut run_client = |state: Option<_>, - mut restarting_mgr: LlmpRestartingEventManager<_, _>, - _core_id| { + mut restarting_mgr: LlmpRestartingEventManager<_, _, _>, + core_id| { // Create an observation channel using the coverage map - let edges_observer = HitcountsMapObserver::new(unsafe { std_edges_map_observer("edges") }); + let edges_observer = + HitcountsMapObserver::new(unsafe { std_edges_map_observer("edges") }).track_indices(); // Create an observation channel to keep track of the execution time let time_observer = TimeObserver::new("time"); @@ -163,9 +173,9 @@ pub extern "C" fn libafl_main() { // This one is composed by two Feedbacks in OR let mut feedback = feedback_or!( // New maximization map feedback linked to the edges observer and the feedback state - MaxMapFeedback::tracking(&edges_observer, true, false), + MaxMapFeedback::new(&edges_observer), // Time feedback, this one does not need a feedback state - TimeFeedback::with_observer(&time_observer) + TimeFeedback::new(&time_observer) ); // A feedback to choose if an input is a solution or not @@ -175,7 +185,7 @@ pub extern "C" fn libafl_main() { let mut state = state.unwrap_or_else(|| { StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryOnDiskCorpus::new(&opt.input[0]).unwrap(), // Corpus in which we store solutions (crashes in this example), @@ -208,7 +218,8 @@ pub extern "C" fn libafl_main() { let mut stages = tuple_list!(StdMutationalStage::new(mutator)); // A minimization+queue policy to get testcasess from the corpus - let scheduler = IndexesLenTimeMinimizerScheduler::new(QueueScheduler::new()); + let scheduler = + IndexesLenTimeMinimizerScheduler::new(&edges_observer, QueueScheduler::new()); // A fuzzer with feedbacks and a corpus scheduler let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); @@ -240,7 +251,14 @@ pub extern "C" fn libafl_main() { // In case the corpus is empty (on first run), reset if state.must_load_initial_inputs() { state - .load_initial_inputs(&mut fuzzer, &mut executor, &mut restarting_mgr, &opt.input) + .load_initial_inputs_multicore( + &mut fuzzer, + &mut executor, + &mut restarting_mgr, + &opt.input, + &core_id, + &cores, + ) .unwrap_or_else(|_| panic!("Failed to load initial corpus at {:?}", &opt.input)); println!("We imported {} inputs from disk.", state.corpus().count()); } @@ -266,7 +284,11 @@ pub extern "C" fn libafl_main() { .broker_port(broker_port) .remote_broker_addr(opt.remote_broker_addr) .stdout_file(Some("/dev/null")) - .serialize_state(!opt.reload_corpus) + .serialize_state(if opt.reload_corpus { + LlmpShouldSaveState::OOMSafeNever + } else { + LlmpShouldSaveState::OOMSafeOnRestart + }) .build() .launch() { diff --git a/fuzzers/libfuzzer_libpng_tcp_manager/Cargo.toml b/fuzzers/libfuzzer_libpng_tcp_manager/Cargo.toml index a8bfb2ea67..054f102f63 100644 --- a/fuzzers/libfuzzer_libpng_tcp_manager/Cargo.toml +++ b/fuzzers/libfuzzer_libpng_tcp_manager/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "libfuzzer_libpng_tcp_manager" -version = "0.11.2" +version = "0.12.0" authors = ["Andrea Fioraldi ", "Dominik Maier "] edition = "2021" diff --git a/fuzzers/libfuzzer_libpng_tcp_manager/Makefile.toml b/fuzzers/libfuzzer_libpng_tcp_manager/Makefile.toml index 94430ed162..fb1dc6cfd2 100644 --- a/fuzzers/libfuzzer_libpng_tcp_manager/Makefile.toml +++ b/fuzzers/libfuzzer_libpng_tcp_manager/Makefile.toml @@ -161,7 +161,7 @@ windows_alias = "unsupported" script_runner = "@shell" script=''' rm -rf libafl_unix_shmem_server || true -(timeout 31s ./${FUZZER_NAME} >fuzz_stdout.log 2>/dev/null || true) & +(timeout 31s ./${FUZZER_NAME} | tee fuzz_stdout.log 2>/dev/null || true) & sleep 0.2 timeout 30s ./${FUZZER_NAME} >/dev/null 2>/dev/null || true if grep -qa "corpus: 30" fuzz_stdout.log; then @@ -177,7 +177,7 @@ dependencies = [ "fuzzer" ] script_runner = "@shell" script=''' rm -rf libafl_unix_shmem_server || true -(timeout 31s ./${FUZZER_NAME} >fuzz_stdout.log 2>/dev/null || true) & +(timeout 31s ./${FUZZER_NAME} | tee fuzz_stdout.log 2>/dev/null || true) & sleep 0.2 timeout 30s ./${FUZZER_NAME} >/dev/null 2>/dev/null || true ''' diff --git a/fuzzers/libfuzzer_libpng_tcp_manager/src/lib.rs b/fuzzers/libfuzzer_libpng_tcp_manager/src/lib.rs index 72ca166bd5..60dab533fa 100644 --- a/fuzzers/libfuzzer_libpng_tcp_manager/src/lib.rs +++ b/fuzzers/libfuzzer_libpng_tcp_manager/src/lib.rs @@ -1,9 +1,5 @@ //! A libfuzzer-like fuzzer with llmp-multithreading support and restarts //! The example harness is built for libpng. -use mimalloc::MiMalloc; -#[global_allocator] -static GLOBAL: MiMalloc = MiMalloc; - use core::time::Duration; #[cfg(feature = "crash")] use std::ptr; @@ -22,21 +18,24 @@ use libafl::{ scheduled::{havoc_mutations, tokens_mutations, StdScheduledMutator}, token_mutations::Tokens, }, - observers::{HitcountsMapObserver, StdMapObserver, TimeObserver}, + observers::{CanTrack, HitcountsMapObserver, StdMapObserver, TimeObserver}, schedulers::{ powersched::PowerSchedule, IndexesLenTimeMinimizerScheduler, StdWeightedScheduler, }, stages::{calibrate::CalibrationStage, power::StdPowerMutationalStage}, - state::{HasCorpus, HasMetadata, StdState}, - Error, + state::{HasCorpus, StdState}, + Error, HasMetadata, }; use libafl_bolts::{ - current_nanos, rands::StdRand, tuples::{tuple_list, Merge}, AsSlice, }; -use libafl_targets::{libfuzzer_initialize, libfuzzer_test_one_input, EDGES_MAP, MAX_EDGES_NUM}; +use libafl_targets::{libfuzzer_initialize, libfuzzer_test_one_input, EDGES_MAP, MAX_EDGES_FOUND}; +use mimalloc::MiMalloc; + +#[global_allocator] +static GLOBAL: MiMalloc = MiMalloc; /// The main fn, `no_mangle` as it is a C main #[no_mangle] @@ -81,14 +80,15 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re HitcountsMapObserver::new(StdMapObserver::from_mut_ptr( "edges", EDGES_MAP.as_mut_ptr(), - MAX_EDGES_NUM, + MAX_EDGES_FOUND, )) + .track_indices() }; // Create an observation channel to keep track of the execution time let time_observer = TimeObserver::new("time"); - let map_feedback = MaxMapFeedback::tracking(&edges_observer, true, false); + let map_feedback = MaxMapFeedback::new(&edges_observer); let calibration = CalibrationStage::new(&map_feedback); @@ -98,7 +98,7 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re // New maximization map feedback linked to the edges observer and the feedback state map_feedback, // Time feedback, this one does not need a feedback state - TimeFeedback::with_observer(&time_observer) + TimeFeedback::new(&time_observer) ); // A feedback to choose if an input is a solution or not @@ -108,7 +108,7 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re let mut state = state.unwrap_or_else(|| { StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryCorpus::new(), // Corpus in which we store solutions (crashes in this example), @@ -145,11 +145,10 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re let mut stages = tuple_list!(calibration, power); // A minimization+queue policy to get testcasess from the corpus - let scheduler = IndexesLenTimeMinimizerScheduler::new(StdWeightedScheduler::with_schedule( - &mut state, + let scheduler = IndexesLenTimeMinimizerScheduler::new( &edges_observer, - Some(PowerSchedule::FAST), - )); + StdWeightedScheduler::with_schedule(&mut state, &edges_observer, Some(PowerSchedule::FAST)), + ); // A fuzzer with feedbacks and a corpus scheduler let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); diff --git a/fuzzers/libfuzzer_reachability/.gitignore b/fuzzers/libfuzzer_reachability/.gitignore deleted file mode 100644 index a977a2ca5b..0000000000 --- a/fuzzers/libfuzzer_reachability/.gitignore +++ /dev/null @@ -1 +0,0 @@ -libpng-* \ No newline at end of file diff --git a/fuzzers/libfuzzer_reachability/README.md b/fuzzers/libfuzzer_reachability/README.md deleted file mode 100644 index 77510a1b80..0000000000 --- a/fuzzers/libfuzzer_reachability/README.md +++ /dev/null @@ -1,84 +0,0 @@ -# Libfuzzer for libpng - -This folder contains an example fuzzer for libpng, using LLMP for fast multi-process fuzzing and crash detection. - -In contrast to other fuzzer examples, this setup uses `fuzz_loop_for`, to occasionally respawn the fuzzer executor. -While this costs performance, it can be useful for targets with memory leaks or other instabilities. -If your target is really instable, however, consider exchanging the `InProcessExecutor` for a `ForkserverExecutor` instead. - -To show off crash detection, we added a `ud2` instruction to the harness, edit harness.cc if you want a non-crashing example. -It has been tested on Linux. - -## Build - -To build this example, run - -```bash -cargo build --release -clang -c weak.c -o weak.o -``` - -This will build the library with the fuzzer (src/lib.rs) with the libfuzzer compatibility layer and the SanitizerCoverage runtime functions for coverage feedback. -In addition, it will also build two C and C++ compiler wrappers (bin/libafl_c(libafl_c/xx).rs) that you must use to compile the target. - -The compiler wrappers, `libafl_cc` and `libafl_cxx`, will end up in `./target/release/` (or `./target/debug`, in case you did not build with the `--release` flag). - -Then download libpng, and unpack the archive: -```bash -wget https://github.com/glennrp/libpng/archive/refs/tags/v1.6.37.tar.gz -tar -xvf v1.6.37.tar.gz -``` -Run `patch libpng-1.6.37/png.c diff.patch` before compiling the libpng -Now compile libpng, using the libafl_cc compiler wrapper: - -```bash -cd libpng-1.6.37 -./configure -LIBAFL_WEAK=../weak.o make CC="$(pwd)/../target/release/libafl_cc" CXX="$(pwd)/../target/release/libafl_cxx" -j `nproc` -``` - -You can find the static lib at `libpng-1.6.37/.libs/libpng16.a`. - -Now, we have to build the libfuzzer harness and link all together to create our fuzzer binary. - -``` -cd .. -LIBAFL_WEAK=./weak.o ./target/release/libafl_cxx ./harness.cc libpng-1.6.37/.libs/libpng16.a -I libpng-1.6.37/ -o fuzzer_libpng -lz -lm -``` - -Afterward, the fuzzer will be ready to run. -Note that, unless you use the `launcher`, you will have to run the binary multiple times to actually start the fuzz process, see `Run` in the following. -This allows you to run multiple different builds of the same fuzzer alongside, for example, with and without ASAN (`-fsanitize=address`) or with different mutators. - -This example also shows you how to use a user-defined variable from LibAFL. -`diff.patch` adds an array `__libafl_target_list` to `png.c`. In order to read from this variable from LibAFL, you need to weakly define __libafl_target_list as in `weak.c`. -For building, you have to set `LIBAFL_WEAK` to point to the compiled `weak.o`, so that the compiler can find this `weak.o` file and link successfully. - -## Run - -The first time you run the binary, the broker will open a tcp port (currently on port `1337`), waiting for fuzzer clients to connect. This port is local and only used for the initial handshake. All further communication happens via shared map, to be independent of the kernel. Currently you must run the clients from the libfuzzer_libpng directory for them to be able to access the PNG corpus. - -``` -./fuzzer_libpng - -[libafl/src/bolts/llmp.rs:407] "We're the broker" = "We\'re the broker" -Doing broker things. Run this tool again to start fuzzing in a client. -``` - -And after running the above again in a separate terminal: - -``` -[libafl/src/bolts/llmp.rs:1464] "New connection" = "New connection" -[libafl/src/bolts/llmp.rs:1464] addr = 127.0.0.1:33500 -[libafl/src/bolts/llmp.rs:1464] stream.peer_addr().unwrap() = 127.0.0.1:33500 -[LOG Debug]: Loaded 4 initial testcases. -[New Testcase #2] clients: 3, corpus: 6, objectives: 0, executions: 5, exec/sec: 0 -< fuzzing stats > -``` - -As this example uses in-process fuzzing, we added a Restarting Event Manager (`setup_restarting_mgr`). -This means each client will start itself again to listen for crashes and timeouts. -By restarting the actual fuzzer, it can recover from these exit conditions. - -In any real-world scenario, you should use `taskset` to pin each client to an empty CPU core, the lib does not pick an empty core automatically (yet). - diff --git a/fuzzers/libfuzzer_reachability/corpus/not_kitty.png b/fuzzers/libfuzzer_reachability/corpus/not_kitty.png deleted file mode 100644 index eff7c1707b..0000000000 Binary files a/fuzzers/libfuzzer_reachability/corpus/not_kitty.png and /dev/null differ diff --git a/fuzzers/libfuzzer_reachability/corpus/not_kitty_alpha.png b/fuzzers/libfuzzer_reachability/corpus/not_kitty_alpha.png deleted file mode 100644 index 2fb8da2c8f..0000000000 Binary files a/fuzzers/libfuzzer_reachability/corpus/not_kitty_alpha.png and /dev/null differ diff --git a/fuzzers/libfuzzer_reachability/corpus/not_kitty_gamma.png b/fuzzers/libfuzzer_reachability/corpus/not_kitty_gamma.png deleted file mode 100644 index 939d9d29a9..0000000000 Binary files a/fuzzers/libfuzzer_reachability/corpus/not_kitty_gamma.png and /dev/null differ diff --git a/fuzzers/libfuzzer_reachability/corpus/not_kitty_icc.png b/fuzzers/libfuzzer_reachability/corpus/not_kitty_icc.png deleted file mode 100644 index f0c7804d99..0000000000 Binary files a/fuzzers/libfuzzer_reachability/corpus/not_kitty_icc.png and /dev/null differ diff --git a/fuzzers/libfuzzer_reachability/diff.patch b/fuzzers/libfuzzer_reachability/diff.patch deleted file mode 100644 index 8c0ca68b83..0000000000 --- a/fuzzers/libfuzzer_reachability/diff.patch +++ /dev/null @@ -1,9 +0,0 @@ -15a16,19 -> #define TARGET_SIZE 4 -> -> size_t __lafl_dummy_list[TARGET_SIZE] = {0}; -> size_t *__libafl_target_list = __lafl_dummy_list; -2562a2567 -> __lafl_dummy_list[0] = 1; -2584a2590 -> __lafl_dummy_list[1] = 1; diff --git a/fuzzers/libfuzzer_reachability/harness.cc b/fuzzers/libfuzzer_reachability/harness.cc deleted file mode 100644 index e26e707e17..0000000000 --- a/fuzzers/libfuzzer_reachability/harness.cc +++ /dev/null @@ -1,188 +0,0 @@ -// libpng_read_fuzzer.cc -// Copyright 2017-2018 Glenn Randers-Pehrson -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that may -// be found in the LICENSE file https://cs.chromium.org/chromium/src/LICENSE - -// Last changed in libpng 1.6.35 [July 15, 2018] - -// The modifications in 2017 by Glenn Randers-Pehrson include -// 1. addition of a PNG_CLEANUP macro, -// 2. setting the option to ignore ADLER32 checksums, -// 3. adding "#include " which is needed on some platforms -// to provide memcpy(). -// 4. adding read_end_info() and creating an end_info structure. -// 5. adding calls to png_set_*() transforms commonly used by browsers. - -#include -#include -#include - -#include - -#define PNG_INTERNAL -#include "png.h" - -#define PNG_CLEANUP \ - if (png_handler.png_ptr) { \ - if (png_handler.row_ptr) \ - png_free(png_handler.png_ptr, png_handler.row_ptr); \ - if (png_handler.end_info_ptr) \ - png_destroy_read_struct(&png_handler.png_ptr, &png_handler.info_ptr, \ - &png_handler.end_info_ptr); \ - else if (png_handler.info_ptr) \ - png_destroy_read_struct(&png_handler.png_ptr, &png_handler.info_ptr, \ - nullptr); \ - else \ - png_destroy_read_struct(&png_handler.png_ptr, nullptr, nullptr); \ - png_handler.png_ptr = nullptr; \ - png_handler.row_ptr = nullptr; \ - png_handler.info_ptr = nullptr; \ - png_handler.end_info_ptr = nullptr; \ - } - -struct BufState { - const uint8_t *data; - size_t bytes_left; -}; - -struct PngObjectHandler { - png_infop info_ptr = nullptr; - png_structp png_ptr = nullptr; - png_infop end_info_ptr = nullptr; - png_voidp row_ptr = nullptr; - BufState *buf_state = nullptr; - - ~PngObjectHandler() { - if (row_ptr) { png_free(png_ptr, row_ptr); } - if (end_info_ptr) - png_destroy_read_struct(&png_ptr, &info_ptr, &end_info_ptr); - else if (info_ptr) - png_destroy_read_struct(&png_ptr, &info_ptr, nullptr); - else - png_destroy_read_struct(&png_ptr, nullptr, nullptr); - delete buf_state; - } -}; - -void user_read_data(png_structp png_ptr, png_bytep data, size_t length) { - BufState *buf_state = static_cast(png_get_io_ptr(png_ptr)); - if (length > buf_state->bytes_left) { png_error(png_ptr, "read error"); } - memcpy(data, buf_state->data, length); - buf_state->bytes_left -= length; - buf_state->data += length; -} - -static const int kPngHeaderSize = 8; - -// Entry point for LibFuzzer. -// Roughly follows the libpng book example: -// http://www.libpng.org/pub/png/book/chapter13.html -extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { - if (size < kPngHeaderSize) { return 0; } - - std::vector v(data, data + size); - if (png_sig_cmp(v.data(), 0, kPngHeaderSize)) { - // not a PNG. - return 0; - } - - PngObjectHandler png_handler; - png_handler.png_ptr = nullptr; - png_handler.row_ptr = nullptr; - png_handler.info_ptr = nullptr; - png_handler.end_info_ptr = nullptr; - - png_handler.png_ptr = - png_create_read_struct(PNG_LIBPNG_VER_STRING, nullptr, nullptr, nullptr); - if (!png_handler.png_ptr) { return 0; } - - png_handler.info_ptr = png_create_info_struct(png_handler.png_ptr); - if (!png_handler.info_ptr) { - PNG_CLEANUP - return 0; - } - - png_handler.end_info_ptr = png_create_info_struct(png_handler.png_ptr); - if (!png_handler.end_info_ptr) { - PNG_CLEANUP - return 0; - } - - png_set_crc_action(png_handler.png_ptr, PNG_CRC_QUIET_USE, PNG_CRC_QUIET_USE); -#ifdef PNG_IGNORE_ADLER32 - png_set_option(png_handler.png_ptr, PNG_IGNORE_ADLER32, PNG_OPTION_ON); -#endif - - // Setting up reading from buffer. - png_handler.buf_state = new BufState(); - png_handler.buf_state->data = data + kPngHeaderSize; - png_handler.buf_state->bytes_left = size - kPngHeaderSize; - png_set_read_fn(png_handler.png_ptr, png_handler.buf_state, user_read_data); - png_set_sig_bytes(png_handler.png_ptr, kPngHeaderSize); - - if (setjmp(png_jmpbuf(png_handler.png_ptr))) { - PNG_CLEANUP - return 0; - } - - // Reading. - png_read_info(png_handler.png_ptr, png_handler.info_ptr); - - // reset error handler to put png_deleter into scope. - if (setjmp(png_jmpbuf(png_handler.png_ptr))) { - PNG_CLEANUP - return 0; - } - - png_uint_32 width, height; - int bit_depth, color_type, interlace_type, compression_type; - int filter_type; - - if (!png_get_IHDR(png_handler.png_ptr, png_handler.info_ptr, &width, &height, - &bit_depth, &color_type, &interlace_type, &compression_type, - &filter_type)) { - PNG_CLEANUP - return 0; - } - - // This is going to be too slow. - if (width && height > 100000000 / width) { - PNG_CLEANUP -#ifdef HAS_DUMMY_CRASH - #ifdef __aarch64__ - asm volatile(".word 0xf7f0a000\n"); - #else - asm("ud2"); - #endif -#endif - return 0; - } - - // Set several transforms that browsers typically use: - png_set_gray_to_rgb(png_handler.png_ptr); - png_set_expand(png_handler.png_ptr); - png_set_packing(png_handler.png_ptr); - png_set_scale_16(png_handler.png_ptr); - png_set_tRNS_to_alpha(png_handler.png_ptr); - - int passes = png_set_interlace_handling(png_handler.png_ptr); - - png_read_update_info(png_handler.png_ptr, png_handler.info_ptr); - - png_handler.row_ptr = - png_malloc(png_handler.png_ptr, - png_get_rowbytes(png_handler.png_ptr, png_handler.info_ptr)); - - for (int pass = 0; pass < passes; ++pass) { - for (png_uint_32 y = 0; y < height; ++y) { - png_read_row(png_handler.png_ptr, - static_cast(png_handler.row_ptr), nullptr); - } - } - - png_read_end(png_handler.png_ptr, png_handler.end_info_ptr); - - PNG_CLEANUP - return 0; -} diff --git a/fuzzers/libfuzzer_reachability/src/bin/libafl_cc.rs b/fuzzers/libfuzzer_reachability/src/bin/libafl_cc.rs deleted file mode 100644 index 36b2acbdf8..0000000000 --- a/fuzzers/libfuzzer_reachability/src/bin/libafl_cc.rs +++ /dev/null @@ -1,38 +0,0 @@ -use std::env; - -use libafl_cc::{ClangWrapper, CompilerWrapper, ToolWrapper}; - -pub fn main() { - let args: Vec = env::args().collect(); - if args.len() > 1 { - let mut dir = env::current_exe().unwrap(); - let weak = env::var("LIBAFL_WEAK").unwrap(); - let wrapper_name = dir.file_name().unwrap().to_str().unwrap(); - - let is_cpp = match wrapper_name[wrapper_name.len()-2..].to_lowercase().as_str() { - "cc" => false, - "++" | "pp" | "xx" => true, - _ => panic!("Could not figure out if c or c++ wrapper was called. Expected {dir:?} to end with c or cxx"), - }; - - dir.pop(); - - let mut cc = ClangWrapper::new(); - if let Some(code) = cc - .cpp(is_cpp) - // silence the compiler wrapper output, needed for some configure scripts. - .silence(true) - .parse_args(&args) - .expect("Failed to parse the command line") - .add_link_arg(weak) - .link_staticlib(&dir, "libfuzzer_libpng") - .add_arg("-fsanitize-coverage=trace-pc-guard") - .run() - .expect("Failed to run the wrapped compiler") - { - std::process::exit(code); - } - } else { - panic!("LibAFL CC: No Arguments given"); - } -} diff --git a/fuzzers/libfuzzer_reachability/src/bin/libafl_cxx.rs b/fuzzers/libfuzzer_reachability/src/bin/libafl_cxx.rs deleted file mode 100644 index dabd22971a..0000000000 --- a/fuzzers/libfuzzer_reachability/src/bin/libafl_cxx.rs +++ /dev/null @@ -1,5 +0,0 @@ -pub mod libafl_cc; - -fn main() { - libafl_cc::main(); -} diff --git a/fuzzers/libfuzzer_reachability/src/lib.rs b/fuzzers/libfuzzer_reachability/src/lib.rs deleted file mode 100644 index 1093888e3b..0000000000 --- a/fuzzers/libfuzzer_reachability/src/lib.rs +++ /dev/null @@ -1,164 +0,0 @@ -//! A libfuzzer-like fuzzer with llmp-multithreading support and restarts -//! The example harness is built for libpng. -use mimalloc::MiMalloc; -#[global_allocator] -static GLOBAL: MiMalloc = MiMalloc; - -use std::{env, path::PathBuf}; - -use libafl::{ - corpus::{Corpus, InMemoryCorpus, OnDiskCorpus}, - events::{setup_restarting_mgr_std, EventConfig, EventRestarter}, - executors::{inprocess::InProcessExecutor, ExitKind}, - feedbacks::{MaxMapFeedback, ReachabilityFeedback}, - fuzzer::{Fuzzer, StdFuzzer}, - inputs::{BytesInput, HasTargetBytes}, - monitors::SimpleMonitor, - mutators::scheduled::{havoc_mutations, StdScheduledMutator}, - observers::{HitcountsMapObserver, StdMapObserver}, - schedulers::RandScheduler, - stages::mutational::StdMutationalStage, - state::{HasCorpus, StdState}, - Error, -}; -use libafl_bolts::{current_nanos, rands::StdRand, tuples::tuple_list, AsSlice}; -use libafl_targets::{libfuzzer_initialize, libfuzzer_test_one_input, std_edges_map_observer}; - -const TARGET_SIZE: usize = 4; - -extern "C" { - static __libafl_target_list: *mut usize; -} - -/// The main fn, `no_mangle` as it is a C symbol -#[no_mangle] -pub extern "C" fn libafl_main() { - // Registry the metadata types used in this fuzzer - // Needed only on no_std - // unsafe { RegistryBuilder::register::(); } - - println!( - "Workdir: {:?}", - env::current_dir().unwrap().to_string_lossy().to_string() - ); - fuzz( - &[PathBuf::from("./corpus")], - PathBuf::from("./crashes"), - 1337, - ) - .expect("An error occurred while fuzzing"); -} - -/// The actual fuzzer -fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Result<(), Error> { - // 'While the stats are state, they are usually used in the broker - which is likely never restarted - let monitor = SimpleMonitor::new(|s| println!("{s}")); - - // The restarting state will spawn the same process again as child, then restarted it each time it crashes. - let (state, mut restarting_mgr) = - match setup_restarting_mgr_std(monitor, broker_port, EventConfig::AlwaysUnique) { - Ok(res) => res, - Err(err) => match err { - Error::ShuttingDown => { - return Ok(()); - } - _ => { - panic!("Failed to setup the restarter: {err}"); - } - }, - }; - - // Create an observation channel using the coverage map - let edges_observer = HitcountsMapObserver::new(unsafe { std_edges_map_observer("edges") }); - - let reachability_observer = - unsafe { StdMapObserver::from_mut_ptr("png.c", __libafl_target_list, TARGET_SIZE) }; - - // Feedback to rate the interestingness of an input - let mut feedback = MaxMapFeedback::new(&edges_observer); - - // A feedback to choose if an input is a solution or not - let mut objective = ReachabilityFeedback::new(&reachability_observer); - - // If not restarting, create a State from scratch - let mut state = state.unwrap_or_else(|| { - StdState::new( - // RNG - StdRand::with_seed(current_nanos()), - // Corpus that will be evolved, we keep it in memory for performance - InMemoryCorpus::new(), - // Corpus in which we store solutions (crashes in this example), - // on disk so the user can get them after stopping the fuzzer - OnDiskCorpus::new(objective_dir).unwrap(), - // States of the feedbacks. - // The feedbacks can report the data that should persist in the State. - &mut feedback, - // Same for objective feedbacks - &mut objective, - ) - .unwrap() - }); - - println!("We're a client, let's fuzz :)"); - - // Setup a basic mutator with a mutational stage - let mutator = StdScheduledMutator::new(havoc_mutations()); - let mut stages = tuple_list!(StdMutationalStage::new(mutator)); - - // A random policy to get testcasess from the corpus - let scheduler = RandScheduler::new(); - - // A fuzzer with feedbacks and a corpus scheduler - let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); - - // The wrapped harness function, calling out to the LLVM-style harness - let mut harness = |input: &BytesInput| { - let target = input.target_bytes(); - let buf = target.as_slice(); - libfuzzer_test_one_input(buf); - ExitKind::Ok - }; - - // Create the executor for an in-process function with one observer for edge coverage and one for the execution time - let mut executor = InProcessExecutor::new( - &mut harness, - tuple_list!(edges_observer, reachability_observer), - &mut fuzzer, - &mut state, - &mut restarting_mgr, - )?; - - // The actual target run starts here. - // Call LLVMFUzzerInitialize() if present. - let args: Vec = env::args().collect(); - if libfuzzer_initialize(&args) == -1 { - println!("Warning: LLVMFuzzerInitialize failed with -1"); - } - - // In case the corpus is empty (on first run), reset - if state.must_load_initial_inputs() { - state - .load_initial_inputs(&mut fuzzer, &mut executor, &mut restarting_mgr, corpus_dirs) - .unwrap_or_else(|_| panic!("Failed to load initial corpus at {:?}", corpus_dirs)); - println!("We imported {} inputs from disk.", state.corpus().count()); - } - - // This fuzzer restarts after 1 mio `fuzz_one` executions. - // Each fuzz_one will internally do many executions of the target. - // If your target is very instable, setting a low count here may help. - // However, you will lose a lot of performance that way. - let iters = 1_000_000; - fuzzer.fuzz_loop_for( - &mut stages, - &mut executor, - &mut state, - &mut restarting_mgr, - iters, - )?; - - // It's important, that we store the state before restarting! - // Else, the parent will not respawn a new child and quit. - restarting_mgr.on_restart(&mut state)?; - - Ok(()) -} diff --git a/fuzzers/libfuzzer_reachability/weak.c b/fuzzers/libfuzzer_reachability/weak.c deleted file mode 100644 index 4228811157..0000000000 --- a/fuzzers/libfuzzer_reachability/weak.c +++ /dev/null @@ -1,2 +0,0 @@ -#include -__attribute__((weak, visibility("default"))) size_t *__libafl_target_list; diff --git a/fuzzers/libfuzzer_stb_image/Cargo.toml b/fuzzers/libfuzzer_stb_image/Cargo.toml index 1476c1e086..162af6439a 100644 --- a/fuzzers/libfuzzer_stb_image/Cargo.toml +++ b/fuzzers/libfuzzer_stb_image/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "libfuzzer_stb_image" -version = "0.11.2" +version = "0.12.0" authors = ["Andrea Fioraldi ", "Dominik Maier "] edition = "2021" build = "build.rs" diff --git a/fuzzers/libfuzzer_stb_image/Makefile.toml b/fuzzers/libfuzzer_stb_image/Makefile.toml index 5c88c2a497..9c17671641 100644 --- a/fuzzers/libfuzzer_stb_image/Makefile.toml +++ b/fuzzers/libfuzzer_stb_image/Makefile.toml @@ -62,7 +62,7 @@ windows_alias = "test_windows" script_runner = "@shell" script=''' rm -rf libafl_unix_shmem_server || true -(timeout 31s ./${FUZZER_NAME} >fuzz_stdout.log 2>/dev/null || true) & +(timeout 31s ./${FUZZER_NAME} | tee fuzz_stdout.log 2>/dev/null || true) & sleep 0.2 timeout 30s ./${FUZZER_NAME} >/dev/null 2>/dev/null || true if grep -qa "corpus: 30" fuzz_stdout.log; then @@ -78,7 +78,7 @@ dependencies = [ "fuzzer" ] script_runner = "@shell" script=''' rm -rf libafl_unix_shmem_server || true -(timeout 31s ./${FUZZER_NAME} >fuzz_stdout.log 2>/dev/null || true) & +(timeout 31s ./${FUZZER_NAME} | tee fuzz_stdout.log 2>/dev/null || true) & sleep 0.2 timeout 30s ./${FUZZER_NAME} >/dev/null 2>/dev/null || true ''' diff --git a/fuzzers/libfuzzer_stb_image/src/main.rs b/fuzzers/libfuzzer_stb_image/src/main.rs index 913eeaf7ad..c59fa70cf7 100644 --- a/fuzzers/libfuzzer_stb_image/src/main.rs +++ b/fuzzers/libfuzzer_stb_image/src/main.rs @@ -19,13 +19,13 @@ use libafl::{ scheduled::{havoc_mutations, StdScheduledMutator}, token_mutations::I2SRandReplace, }, - observers::TimeObserver, + observers::{CanTrack, TimeObserver}, schedulers::{IndexesLenTimeMinimizerScheduler, QueueScheduler}, stages::{ShadowTracingStage, StdMutationalStage}, state::{HasCorpus, StdState}, Error, }; -use libafl_bolts::{current_nanos, rands::StdRand, tuples::tuple_list, AsSlice}; +use libafl_bolts::{rands::StdRand, tuples::tuple_list, AsSlice}; use libafl_targets::{ libfuzzer_initialize, libfuzzer_test_one_input, std_edges_map_observer, CmpLogObserver, }; @@ -68,7 +68,7 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re // Create an observation channel using the coverage map // We don't use the hitcounts (see the Cargo.toml, we use pcguard_edges) - let edges_observer = unsafe { std_edges_map_observer("edges") }; + let edges_observer = unsafe { std_edges_map_observer("edges").track_indices() }; // Create an observation channel to keep track of the execution time let time_observer = TimeObserver::new("time"); @@ -79,9 +79,9 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re // This one is composed by two Feedbacks in OR let mut feedback = feedback_or!( // New maximization map feedback linked to the edges observer and the feedback state - MaxMapFeedback::tracking(&edges_observer, true, false), + MaxMapFeedback::new(&edges_observer), // Time feedback, this one does not need a feedback state - TimeFeedback::with_observer(&time_observer) + TimeFeedback::new(&time_observer) ); // A feedback to choose if an input is a solution or not @@ -91,7 +91,7 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re let mut state = state.unwrap_or_else(|| { StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryCorpus::new(), // Corpus in which we store solutions (crashes in this example), @@ -109,7 +109,7 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re println!("We're a client, let's fuzz :)"); // A minimization+queue policy to get testcasess from the corpus - let scheduler = IndexesLenTimeMinimizerScheduler::new(QueueScheduler::new()); + let scheduler = IndexesLenTimeMinimizerScheduler::new(&edges_observer, QueueScheduler::new()); // A fuzzer with feedbacks and a corpus scheduler let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); diff --git a/fuzzers/libfuzzer_stb_image_concolic/fuzzer/Cargo.toml b/fuzzers/libfuzzer_stb_image_concolic/fuzzer/Cargo.toml index 60e056abcd..6801768195 100644 --- a/fuzzers/libfuzzer_stb_image_concolic/fuzzer/Cargo.toml +++ b/fuzzers/libfuzzer_stb_image_concolic/fuzzer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "libfuzzer_stb_image_concolic" -version = "0.11.2" +version = "0.12.0" authors = ["Andrea Fioraldi ", "Dominik Maier ", "Julius Hohnerlein"] edition = "2021" build = "build.rs" diff --git a/fuzzers/libfuzzer_stb_image_concolic/fuzzer/src/main.rs b/fuzzers/libfuzzer_stb_image_concolic/fuzzer/src/main.rs index 840278baf6..26d5af417e 100644 --- a/fuzzers/libfuzzer_stb_image_concolic/fuzzer/src/main.rs +++ b/fuzzers/libfuzzer_stb_image_concolic/fuzzer/src/main.rs @@ -1,11 +1,7 @@ //! A libfuzzer-like fuzzer with llmp-multithreading support and restarts //! The example harness is built for `stb_image`. -use mimalloc::MiMalloc; -#[global_allocator] -static GLOBAL: MiMalloc = MiMalloc; - use std::{ - env, + env, fs, path::PathBuf, process::{Child, Command, Stdio}, time::Duration, @@ -32,7 +28,7 @@ use libafl::{ serialization_format::{DEFAULT_ENV_NAME, DEFAULT_SIZE}, ConcolicObserver, }, - TimeObserver, + CanTrack, TimeObserver, }, schedulers::{IndexesLenTimeMinimizerScheduler, QueueScheduler}, stages::{ @@ -46,12 +42,16 @@ use libafl_bolts::{ current_nanos, rands::StdRand, shmem::{ShMem, ShMemProvider, StdShMemProvider}, - tuples::tuple_list, - AsMutSlice, AsSlice, Named, + tuples::{tuple_list, Handler}, + AsSlice, AsSliceMut, }; use libafl_targets::{ libfuzzer_initialize, libfuzzer_test_one_input, std_edges_map_observer, CmpLogObserver, }; +use mimalloc::MiMalloc; + +#[global_allocator] +static GLOBAL: MiMalloc = MiMalloc; #[derive(Debug, Parser)] struct Opt { @@ -60,7 +60,6 @@ struct Opt { concolic: bool, } -use std::fs; pub fn main() { // Registry the metadata types used in this fuzzer // Needed only on no_std @@ -107,7 +106,7 @@ fn fuzz( // Create an observation channel using the coverage map // We don't use the hitcounts (see the Cargo.toml, we use pcguard_edges) - let edges_observer = unsafe { std_edges_map_observer("edges") }; + let edges_observer = unsafe { std_edges_map_observer("edges").track_indices() }; // Create an observation channel to keep track of the execution time let time_observer = TimeObserver::new("time"); @@ -118,9 +117,9 @@ fn fuzz( // This one is composed by two Feedbacks in OR let mut feedback = feedback_or!( // New maximization map feedback linked to the edges observer and the feedback state - MaxMapFeedback::tracking(&edges_observer, true, false), + MaxMapFeedback::new(&edges_observer), // Time feedback, this one does not need a feedback state - TimeFeedback::with_observer(&time_observer) + TimeFeedback::new(&time_observer) ); // A feedback to choose if an input is a solution or not @@ -148,7 +147,7 @@ fn fuzz( println!("We're a client, let's fuzz :)"); // A minimization+queue policy to get testcasess from the corpus - let scheduler = IndexesLenTimeMinimizerScheduler::new(QueueScheduler::new()); + let scheduler = IndexesLenTimeMinimizerScheduler::new(&edges_observer, QueueScheduler::new()); // A fuzzer with feedbacks and a corpus scheduler let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); @@ -207,10 +206,8 @@ fn fuzz( concolic_shmem.write_to_env(DEFAULT_ENV_NAME).unwrap(); // The concolic observer observers the concolic shared memory map. - let concolic_observer = - ConcolicObserver::new("concolic".to_string(), concolic_shmem.as_mut_slice()); - - let concolic_observer_name = concolic_observer.name().to_string(); + let concolic_observer = ConcolicObserver::new("concolic", concolic_shmem.as_slice_mut()); + let concolic_ref = concolic_observer.handle(); // The order of the stages matter! let mut stages = tuple_list!( @@ -219,7 +216,7 @@ fn fuzz( TracingStage::new( MyCommandConfigurator.into_executor(tuple_list!(concolic_observer)) ), - concolic_observer_name, + concolic_ref, ), // Use the concolic trace for z3-based solving SimpleConcolicMutationalStage::default(), @@ -240,8 +237,8 @@ fn fuzz( #[derive(Default, Debug)] pub struct MyCommandConfigurator; -impl CommandConfigurator for MyCommandConfigurator { - fn spawn_child(&mut self, input: &I) -> Result { +impl CommandConfigurator for MyCommandConfigurator { + fn spawn_child(&mut self, input: &BytesInput) -> Result { input.to_file("cur_input")?; Ok(Command::new("./target_symcc.out") diff --git a/fuzzers/libfuzzer_stb_image_concolic/runtime/Cargo.toml b/fuzzers/libfuzzer_stb_image_concolic/runtime/Cargo.toml index 9407e764b2..83e24ea161 100644 --- a/fuzzers/libfuzzer_stb_image_concolic/runtime/Cargo.toml +++ b/fuzzers/libfuzzer_stb_image_concolic/runtime/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "example_runtime" -version = "0.11.2" +version = "0.12.0" edition = "2021" authors = ["Julius Hohnerlein "] diff --git a/fuzzers/libfuzzer_stb_image_sugar/Cargo.toml b/fuzzers/libfuzzer_stb_image_sugar/Cargo.toml index 815a8a1045..fd15998cf2 100644 --- a/fuzzers/libfuzzer_stb_image_sugar/Cargo.toml +++ b/fuzzers/libfuzzer_stb_image_sugar/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "libfuzzer_stb_image_sugar" -version = "0.11.2" +version = "0.12.0" authors = ["Andrea Fioraldi ", "Dominik Maier "] edition = "2021" build = "build.rs" diff --git a/fuzzers/libfuzzer_stb_image_sugar/Makefile.toml b/fuzzers/libfuzzer_stb_image_sugar/Makefile.toml index 5ade8fafa2..357a3865be 100644 --- a/fuzzers/libfuzzer_stb_image_sugar/Makefile.toml +++ b/fuzzers/libfuzzer_stb_image_sugar/Makefile.toml @@ -60,7 +60,7 @@ windows_alias = "test_windows" script_runner = "@shell" script=''' rm -rf libafl_unix_shmem_server || true -timeout 31s ./${FUZZER_NAME} 2>/dev/null >fuzz_stdout.log || true +timeout 31s ./${FUZZER_NAME} 2>/dev/null | tee fuzz_stdout.log || true echo "The test is skipped. See https://github.com/AFLplusplus/LibAFL/issues/1176" ''' dependencies = [ "fuzzer" ] diff --git a/fuzzers/libfuzzer_windows_asan/Makefile.toml b/fuzzers/libfuzzer_windows_asan/Makefile.toml index 43c0e3bcb1..16112f488d 100644 --- a/fuzzers/libfuzzer_windows_asan/Makefile.toml +++ b/fuzzers/libfuzzer_windows_asan/Makefile.toml @@ -81,6 +81,11 @@ windows_alias = "test_windows" # TODO [tasks.test_windows] script_runner = "@shell" script=''' +start "" "${FUZZER_NAME}.exe" +start "" "${FUZZER_NAME}.exe" +#ping is for timeout +ping -n 10 127.0.0.1>NUL && taskkill /im ${FUZZER_NAME}.exe /F +>nul 2>nul dir /a-d "crashes\*" && (echo Files exist) || (exit /b 1337) ''' dependencies = [ "fuzzer" ] diff --git a/fuzzers/libfuzzer_windows_asan/harness.cpp b/fuzzers/libfuzzer_windows_asan/harness.cpp index 4c4e3b0f72..46004dca58 100644 --- a/fuzzers/libfuzzer_windows_asan/harness.cpp +++ b/fuzzers/libfuzzer_windows_asan/harness.cpp @@ -11,6 +11,6 @@ void asan_crash() { extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { // abort(); - asan_crash(); + if (size == 10) { asan_crash(); } return 0; } diff --git a/fuzzers/libfuzzer_windows_asan/src/bin/libafl_cc.rs b/fuzzers/libfuzzer_windows_asan/src/bin/libafl_cc.rs index 53610b57cb..82aa485d3f 100644 --- a/fuzzers/libfuzzer_windows_asan/src/bin/libafl_cc.rs +++ b/fuzzers/libfuzzer_windows_asan/src/bin/libafl_cc.rs @@ -30,6 +30,7 @@ pub fn main() { .expect("Failed to parse the command line") .link_staticlib(&dir, "libfuzzer_windows_asan") .add_arg("-lOleAut32.lib") + .add_arg("-lntdll.lib") .add_arg("-fsanitize-coverage=trace-pc-guard") .add_arg("-fsanitize=address") .run() diff --git a/fuzzers/libfuzzer_windows_asan/src/lib.rs b/fuzzers/libfuzzer_windows_asan/src/lib.rs index 2f0eeab1da..0d1401a8d5 100644 --- a/fuzzers/libfuzzer_windows_asan/src/lib.rs +++ b/fuzzers/libfuzzer_windows_asan/src/lib.rs @@ -11,7 +11,7 @@ use libafl::{ inputs::{BytesInput, HasTargetBytes}, monitors::MultiMonitor, mutators::scheduled::{havoc_mutations, tokens_mutations, StdScheduledMutator}, - observers::{HitcountsMapObserver, TimeObserver}, + observers::{CanTrack, HitcountsMapObserver, TimeObserver}, schedulers::{ powersched::PowerSchedule, IndexesLenTimeMinimizerScheduler, StdWeightedScheduler, }, @@ -20,7 +20,6 @@ use libafl::{ Error, }; use libafl_bolts::{ - current_nanos, rands::StdRand, tuples::{tuple_list, Merge}, AsSlice, @@ -61,12 +60,13 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re }; // Create an observation channel using the coverage map - let edges_observer = unsafe { HitcountsMapObserver::new(std_edges_map_observer("edges")) }; + let edges_observer = + unsafe { HitcountsMapObserver::new(std_edges_map_observer("edges")).track_indices() }; // Create an observation channel to keep track of the execution time let time_observer = TimeObserver::new("time"); - let map_feedback = MaxMapFeedback::tracking(&edges_observer, true, false); + let map_feedback = MaxMapFeedback::new(&edges_observer); let calibration = CalibrationStage::new(&map_feedback); @@ -76,7 +76,7 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re // New maximization map feedback linked to the edges observer and the feedback state map_feedback, // Time feedback, this one does not need a feedback state - TimeFeedback::with_observer(&time_observer) + TimeFeedback::new(&time_observer) ); // A feedback to choose if an input is a solution or not @@ -86,7 +86,7 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re let mut state = state.unwrap_or_else(|| { StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryCorpus::new(), // Corpus in which we store solutions (crashes in this example), @@ -112,11 +112,10 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re let mut stages = tuple_list!(calibration, power); // A minimization+queue policy to get testcasess from the corpus - let scheduler = IndexesLenTimeMinimizerScheduler::new(StdWeightedScheduler::with_schedule( - &mut state, + let scheduler = IndexesLenTimeMinimizerScheduler::new( &edges_observer, - Some(PowerSchedule::FAST), - )); + StdWeightedScheduler::with_schedule(&mut state, &edges_observer, Some(PowerSchedule::FAST)), + ); // A fuzzer with feedbacks and a corpus scheduler let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); diff --git a/fuzzers/nautilus_sync/Cargo.toml b/fuzzers/nautilus_sync/Cargo.toml index be123bfd8e..2daebcf1c4 100644 --- a/fuzzers/nautilus_sync/Cargo.toml +++ b/fuzzers/nautilus_sync/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "nautilus_sync" -version = "0.11.2" +version = "0.12.0" authors = ["Andrea Fioraldi ", "Dominik Maier "] edition = "2021" diff --git a/fuzzers/nautilus_sync/Makefile.toml b/fuzzers/nautilus_sync/Makefile.toml index df846ec169..c741b0d41e 100644 --- a/fuzzers/nautilus_sync/Makefile.toml +++ b/fuzzers/nautilus_sync/Makefile.toml @@ -106,7 +106,7 @@ windows_alias = "unsupported" script_runner = "@shell" script=''' rm -rf libafl_unix_shmem_server || true -timeout 31s ./${FUZZER_NAME} --cores 0 >fuzz_stdout.log 2>/dev/null || true +timeout 31s ./${FUZZER_NAME} --cores 0 | tee fuzz_stdout.log 2>/dev/null || true if grep -qa "corpus: 8" fuzz_stdout.log; then echo "Fuzzer is working" else @@ -120,7 +120,7 @@ dependencies = [ "fuzzer" ] script_runner = "@shell" script=''' rm -rf libafl_unix_shmem_server || true -timeout 31s ./${FUZZER_NAME} --cores 0 >fuzz_stdout.log 2>/dev/null || true +timeout 31s ./${FUZZER_NAME} --cores 0 | tee fuzz_stdout.log 2>/dev/null || true ''' dependencies = [ "fuzzer" ] diff --git a/fuzzers/nautilus_sync/src/lib.rs b/fuzzers/nautilus_sync/src/lib.rs index e4f8c95c3c..6e37d7118a 100644 --- a/fuzzers/nautilus_sync/src/lib.rs +++ b/fuzzers/nautilus_sync/src/lib.rs @@ -6,7 +6,7 @@ static GLOBAL: MiMalloc = MiMalloc; use std::ptr::write_volatile; use std::{env, net::SocketAddr, path::PathBuf, time::Duration}; -use clap::{self, Parser}; +use clap::Parser; use libafl::{ corpus::{InMemoryCorpus, OnDiskCorpus}, events::{launcher::Launcher, llmp::LlmpEventConverter, EventConfig}, @@ -23,12 +23,11 @@ use libafl::{ none_input_converter, schedulers::QueueScheduler, stages::{mutational::StdMutationalStage, sync::SyncFromBrokerStage}, - state::{HasMetadata, StdState}, - Error, + state::StdState, + Error, HasMetadata, }; use libafl_bolts::{ core_affinity::Cores, - current_nanos, rands::StdRand, shmem::{ShMemProvider, StdShMemProvider}, tuples::tuple_list, @@ -130,6 +129,9 @@ pub extern "C" fn libafl_main() { .unwrap() }); + // to disconnect the event coverter from the broker later + // call detach_from_broker( port) + let mut run_client = |state: Option<_>, mut mgr, _core_id| { let mut bytes = vec![]; @@ -156,7 +158,7 @@ pub extern "C" fn libafl_main() { let mut state = state.unwrap_or_else(|| { StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryCorpus::new(), // Corpus in which we store solutions (crashes in this example), diff --git a/fuzzers/nyx_libxml2_parallel/Cargo.toml b/fuzzers/nyx_libxml2_parallel/Cargo.toml index 28a152feab..cf2ebf64df 100644 --- a/fuzzers/nyx_libxml2_parallel/Cargo.toml +++ b/fuzzers/nyx_libxml2_parallel/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "nyx_libxml2_parallel" -version = "0.11.2" +version = "0.12.0" edition = "2021" default-run = "nyx_libxml2_parallel" diff --git a/fuzzers/nyx_libxml2_parallel/src/main.rs b/fuzzers/nyx_libxml2_parallel/src/main.rs index f4bd9b8a0f..fb836c26f6 100644 --- a/fuzzers/nyx_libxml2_parallel/src/main.rs +++ b/fuzzers/nyx_libxml2_parallel/src/main.rs @@ -15,11 +15,11 @@ use libafl::{ }; use libafl_bolts::{ core_affinity::{CoreId, Cores}, - rands::{RandomSeed, StdRand}, + rands::StdRand, shmem::{ShMemProvider, StdShMemProvider}, tuples::tuple_list, }; -use libafl_nyx::{executor::NyxExecutor, helper::NyxHelper}; +use libafl_nyx::{executor::NyxExecutor, helper::NyxHelper, settings::NyxSettings}; fn main() { let shmem_provider = StdShMemProvider::new().expect("Failed to init shared memory"); @@ -32,21 +32,15 @@ fn main() { // region: fuzzer start function let mut run_client = |state: Option<_>, mut restarting_mgr, core_id: CoreId| { - // nyx shared dir, created by nyx-fuzz/packer/packer/nyx_packer.py - let share_dir = Path::new("/tmp/nyx_libxml2/"); - let cpu_id = core_id.0.try_into().unwrap(); - let parallel_mode = true; // nyx stuff - let mut helper = NyxHelper::new( - share_dir, - cpu_id, - true, - parallel_mode, - Some(parent_cpu_id.0.try_into().unwrap()), - ) - .unwrap(); - let observer = - unsafe { StdMapObserver::from_mut_ptr("trace", helper.trace_bits, helper.map_size) }; + let settings = NyxSettings::builder() + .cpu_id(core_id.0) + .parent_cpu_id(Some(parent_cpu_id.0)) + .build(); + let helper = NyxHelper::new("/tmp/nyx_libxml2/", settings).unwrap(); + let observer = unsafe { + StdMapObserver::from_mut_ptr("trace", helper.bitmap_buffer, helper.bitmap_size) + }; let input = BytesInput::new(b"22".to_vec()); let rand = StdRand::new(); @@ -60,7 +54,7 @@ fn main() { let mut feedback = MaxMapFeedback::new(&observer); let mut objective = CrashFeedback::new(); let scheduler = RandScheduler::new(); - let mut executor = NyxExecutor::new(&mut helper, tuple_list!(observer)).unwrap(); + let mut executor = NyxExecutor::new(helper, tuple_list!(observer)); // If not restarting, create a State from scratch let mut state = state.unwrap_or_else(|| { diff --git a/fuzzers/nyx_libxml2_standalone/Cargo.toml b/fuzzers/nyx_libxml2_standalone/Cargo.toml index 52dce042ea..0c72633140 100644 --- a/fuzzers/nyx_libxml2_standalone/Cargo.toml +++ b/fuzzers/nyx_libxml2_standalone/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "nyx_libxml2_standalone" -version = "0.11.2" +version = "0.12.0" edition = "2021" default-run = "nyx_libxml2_standalone" diff --git a/fuzzers/nyx_libxml2_standalone/setup_libxml2.sh b/fuzzers/nyx_libxml2_standalone/setup_libxml2.sh index 4475265e61..e942831e02 100755 --- a/fuzzers/nyx_libxml2_standalone/setup_libxml2.sh +++ b/fuzzers/nyx_libxml2_standalone/setup_libxml2.sh @@ -35,3 +35,5 @@ python3 "../../libafl_nyx/packer/packer/nyx_packer.py" \ -file "/tmp/input" \ --fast_reload_mode \ --purge || exit + +python3 ../../libafl_nyx/packer/packer/nyx_config_gen.py /tmp/nyx_libxml2/ Kernel || exit diff --git a/fuzzers/nyx_libxml2_standalone/src/main.rs b/fuzzers/nyx_libxml2_standalone/src/main.rs index 3ce5ebc938..b2ad16dbc3 100644 --- a/fuzzers/nyx_libxml2_standalone/src/main.rs +++ b/fuzzers/nyx_libxml2_standalone/src/main.rs @@ -1,4 +1,4 @@ -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use libafl::{ corpus::{CachedOnDiskCorpus, Corpus, OnDiskCorpus, Testcase}, @@ -13,21 +13,15 @@ use libafl::{ state::StdState, Fuzzer, StdFuzzer, }; -use libafl_bolts::{ - rands::{RandomSeed, StdRand}, - tuples::tuple_list, -}; -use libafl_nyx::{executor::NyxExecutor, helper::NyxHelper}; +use libafl_bolts::{rands::StdRand, tuples::tuple_list}; +use libafl_nyx::{executor::NyxExecutor, helper::NyxHelper, settings::NyxSettings}; fn main() { - let share_dir = Path::new("/tmp/nyx_libxml2/"); - let cpu_id = 0; - let parallel_mode = false; - // nyx stuff - let mut helper = NyxHelper::new(share_dir, cpu_id, true, parallel_mode, None).unwrap(); + let settings = NyxSettings::builder().cpu_id(0).parent_cpu_id(None).build(); + let helper = NyxHelper::new("/tmp/nyx_libxml2/", settings).unwrap(); let observer = - unsafe { StdMapObserver::from_mut_ptr("trace", helper.trace_bits, helper.map_size) }; + unsafe { StdMapObserver::from_mut_ptr("trace", helper.bitmap_buffer, helper.bitmap_size) }; let input = BytesInput::new(b"22".to_vec()); let rand = StdRand::new(); @@ -50,7 +44,7 @@ fn main() { let monitor = TuiMonitor::new(ui); let mut mgr = SimpleEventManager::new(monitor); - let mut executor = NyxExecutor::new(&mut helper, tuple_list!(observer)).unwrap(); + let mut executor = NyxExecutor::new(helper, tuple_list!(observer)); let mutator = StdScheduledMutator::new(havoc_mutations()); let mut stages = tuple_list!(StdMutationalStage::new(mutator)); diff --git a/fuzzers/push_stage_harness/src/main.rs b/fuzzers/push_stage_harness/src/main.rs index 201a3a860c..27c78c26ef 100644 --- a/fuzzers/push_stage_harness/src/main.rs +++ b/fuzzers/push_stage_harness/src/main.rs @@ -31,10 +31,11 @@ fn signals_set(idx: usize) { unsafe { SIGNALS[idx] = 1 }; } -#[allow(clippy::similar_names)] +#[allow(clippy::similar_names, clippy::manual_assert)] pub fn main() { // Create an observation channel using the signals map - let observer = unsafe { StdMapObserver::new("signals", &mut SIGNALS) }; + let observer = + unsafe { StdMapObserver::from_mut_ptr("signals", SIGNALS.as_mut_ptr(), SIGNALS.len()) }; // Feedback to rate the interestingness of an input let mut feedback = MaxMapFeedback::new(&observer); @@ -90,8 +91,6 @@ pub fn main() { let exit_kind = Rc::new(Cell::new(None)); - let stage_idx = 0; - let observers = tuple_list!(observer); let shared_state = PushStageSharedState::new(fuzzer, state, observers, mgr); @@ -101,7 +100,6 @@ pub fn main() { mutator, Rc::new(RefCell::new(Some(shared_state))), exit_kind.clone(), - stage_idx, ); // Loop, the input, getting a new entry from the push stage each iteration. diff --git a/fuzzers/python_qemu/README.md b/fuzzers/python_qemu/README.md new file mode 100644 index 0000000000..7b2f08143a --- /dev/null +++ b/fuzzers/python_qemu/README.md @@ -0,0 +1,17 @@ +# Python LibAFL QEMU + +## Build + +First, install python bindings (check `LibAFL/bindings/pylibafl`) and use the virtual environment. + +Then, create the `in` folder and put some input inside +```bash +$ mkdir in +$ echo aaaaa > in/input +``` + +## Run + +```bash +$ python fuzzer.py +``` diff --git a/fuzzers/python_qemu/fuzzer.py b/fuzzers/python_qemu/fuzzer.py index fd0245a497..71fc023580 100644 --- a/fuzzers/python_qemu/fuzzer.py +++ b/fuzzers/python_qemu/fuzzer.py @@ -6,7 +6,7 @@ MAX_SIZE = 0x100 BINARY_PATH = './a.out' -emu = qemu.Emulator(['qemu-x86_64', BINARY_PATH], []) +emu = qemu.Qemu(['qemu-x86_64', BINARY_PATH], []) elf = lief.parse(BINARY_PATH) test_one_input = elf.get_function_address("LLVMFuzzerTestOneInput") diff --git a/fuzzers/qemu_cmin/Cargo.toml b/fuzzers/qemu_cmin/Cargo.toml index 69700d998c..4155c684c8 100644 --- a/fuzzers/qemu_cmin/Cargo.toml +++ b/fuzzers/qemu_cmin/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "qemu_cmin" -version = "0.11.2" +version = "0.12.0" authors = ["Andrea Fioraldi ", "Dominik Maier ", "WorksButNotTested"] edition = "2021" diff --git a/fuzzers/qemu_cmin/Makefile.toml b/fuzzers/qemu_cmin/Makefile.toml index a28fafea8a..c43f51af6b 100644 --- a/fuzzers/qemu_cmin/Makefile.toml +++ b/fuzzers/qemu_cmin/Makefile.toml @@ -8,6 +8,7 @@ TARGET_DIR = "${CARGO_MAKE_CRATE_TARGET_DIRECTORY}/x86_64" LIBPNG_ARCH = "x86_64" LIBPNG_OPTIMIZATIONS = "yes" FEATURE = "x86_64" +LIBAFL_QEMU_CLONE_DIR = "${CARGO_MAKE_CRATE_TARGET_DIRECTORY}/qemu-libafl-bridge" #LIBAFL_DEBUG_OUTPUT = "1" #CUSTOM_QEMU_DIR= "~/qemu-libafl-bridge" @@ -19,6 +20,7 @@ TARGET_DIR = "${CARGO_MAKE_CRATE_TARGET_DIRECTORY}/arm" LIBPNG_ARCH = "arm" LIBPNG_OPTIMIZATIONS = "yes" FEATURE = "arm" +LIBAFL_QEMU_CLONE_DIR = "${CARGO_MAKE_CRATE_TARGET_DIRECTORY}/qemu-libafl-bridge" [env.aarch64] CROSS_CC = "aarch64-linux-gnu-gcc" @@ -28,6 +30,7 @@ TARGET_DIR = "${CARGO_MAKE_CRATE_TARGET_DIRECTORY}/aarch64" LIBPNG_ARCH = "aarch64" LIBPNG_OPTIMIZATIONS = "yes" FEATURE = "aarch64" +LIBAFL_QEMU_CLONE_DIR = "${CARGO_MAKE_CRATE_TARGET_DIRECTORY}/qemu-libafl-bridge" [env.x86_64] CROSS_CC = "x86_64-linux-gnu-gcc" @@ -37,6 +40,7 @@ TARGET_DIR = "${CARGO_MAKE_CRATE_TARGET_DIRECTORY}/x86_64" LIBPNG_ARCH = "x86_64" LIBPNG_OPTIMIZATIONS = "yes" FEATURE = "x86_64" +LIBAFL_QEMU_CLONE_DIR = "${CARGO_MAKE_CRATE_TARGET_DIRECTORY}/qemu-libafl-bridge" [env.i386] CROSS_CC = "x86_64-linux-gnu-gcc" @@ -46,6 +50,7 @@ TARGET_DIR = "${CARGO_MAKE_CRATE_TARGET_DIRECTORY}/i386" LIBPNG_ARCH = "i386" LIBPNG_OPTIMIZATIONS = "yes" FEATURE = "i386" +LIBAFL_QEMU_CLONE_DIR = "${CARGO_MAKE_CRATE_TARGET_DIRECTORY}/qemu-libafl-bridge" [env.mips] CROSS_CC = "mipsel-linux-gnu-gcc" @@ -55,6 +60,7 @@ TARGET_DIR = "${CARGO_MAKE_CRATE_TARGET_DIRECTORY}/mips" LIBPNG_ARCH = "mips" LIBPNG_OPTIMIZATIONS = "yes" FEATURE = "mips" +LIBAFL_QEMU_CLONE_DIR = "${CARGO_MAKE_CRATE_TARGET_DIRECTORY}/qemu-libafl-bridge" [env.ppc] CROSS_CC = "powerpc-linux-gnu-gcc" @@ -64,6 +70,7 @@ TARGET_DIR = "${CARGO_MAKE_CRATE_TARGET_DIRECTORY}/ppc" LIBPNG_ARCH = "ppc" LIBPNG_OPTIMIZATIONS = "no" FEATURE = "ppc" +LIBAFL_QEMU_CLONE_DIR = "${CARGO_MAKE_CRATE_TARGET_DIRECTORY}/qemu-libafl-bridge" [tasks.unsupported] script_runner="@shell" @@ -242,6 +249,20 @@ mac_alias = "unsupported" windows_alias = "unsupported" [tasks.test_unix] +dependencies = [ "lightweight" ] +# Tidy up after we've run our tests so we don't hog all the disk space +command = "cargo" +args = [ + "make", + "clean", +] + +[tasks.test_full] +linux_alias = "test_unix_full" +mac_alias = "unsupported" +windows_alias = "unsupported" + +[tasks.test_unix_full] dependencies = [ "all" ] # Tidy up after we've run our tests so we don't hog all the disk space command = "cargo" @@ -321,3 +342,9 @@ dependencies = [ "mips", "ppc" ] + +[tasks.lightweight] +dependencies = [ + "arm", + "x86_64", +] diff --git a/fuzzers/qemu_cmin/src/fuzzer.rs b/fuzzers/qemu_cmin/src/fuzzer.rs index 6a7e5e5133..18f4cbc7bf 100644 --- a/fuzzers/qemu_cmin/src/fuzzer.rs +++ b/fuzzers/qemu_cmin/src/fuzzer.rs @@ -20,18 +20,17 @@ use libafl::{ }; use libafl_bolts::{ core_affinity::Cores, - current_nanos, + os::unix_signals::Signal, rands::StdRand, shmem::{ShMemProvider, StdShMemProvider}, tuples::tuple_list, - AsMutSlice, AsSlice, + AsSlice, AsSliceMut, }; use libafl_qemu::{ - edges::{QemuEdgeCoverageChildHelper, EDGES_MAP_PTR, EDGES_MAP_SIZE}, + edges::{QemuEdgeCoverageChildHelper, EDGES_MAP_PTR, EDGES_MAP_SIZE_IN_USE}, elf::EasyElf, - emu::Emulator, - ArchExtras, CallingConvention, GuestAddr, GuestReg, MmapPerms, QemuForkExecutor, QemuHooks, - Regs, + ArchExtras, CallingConvention, GuestAddr, GuestReg, MmapPerms, Qemu, QemuExitReason, + QemuExitReasonError, QemuForkExecutor, QemuHooks, QemuShutdownCause, Regs, }; #[derive(Default)] @@ -63,10 +62,10 @@ impl From for Str { #[derive(Parser, Debug)] #[clap(author, version, about, long_about = None)] #[command( - name = format!("qemu_cmin-{}",env!("CPU_TARGET")), - version = Version::default(), - about, - long_about = "Tool for generating minimizing corpus using QEMU instrumentation" +name = format ! ("qemu_cmin-{}", env ! ("CPU_TARGET")), +version = Version::default(), +about, +long_about = "Tool for generating minimizing corpus using QEMU instrumentation" )] pub struct FuzzerOptions { #[arg(long, help = "Output directory")] @@ -113,40 +112,37 @@ pub fn fuzz() -> Result<(), Error> { env::remove_var("LD_LIBRARY_PATH"); let env: Vec<(String, String)> = env::vars().collect(); - let emu = Emulator::new(&options.args, &env).unwrap(); + let qemu = Qemu::init(&options.args, &env).unwrap(); let mut elf_buffer = Vec::new(); - let elf = EasyElf::from_file(emu.binary_path(), &mut elf_buffer).unwrap(); + let elf = EasyElf::from_file(qemu.binary_path(), &mut elf_buffer).unwrap(); let test_one_input_ptr = elf - .resolve_symbol("LLVMFuzzerTestOneInput", emu.load_addr()) + .resolve_symbol("LLVMFuzzerTestOneInput", qemu.load_addr()) .expect("Symbol LLVMFuzzerTestOneInput not found"); log::debug!("LLVMFuzzerTestOneInput @ {test_one_input_ptr:#x}"); - emu.entry_break(test_one_input_ptr); + qemu.entry_break(test_one_input_ptr); - let pc: GuestReg = emu.read_reg(Regs::Pc).unwrap(); + let pc: GuestReg = qemu.read_reg(Regs::Pc).unwrap(); log::debug!("Break at {pc:#x}"); - let ret_addr: GuestAddr = emu.read_return_address().unwrap(); + let ret_addr: GuestAddr = qemu.read_return_address().unwrap(); log::debug!("Return address = {ret_addr:#x}"); - emu.set_breakpoint(ret_addr); + qemu.set_breakpoint(ret_addr); - let input_addr = emu + let input_addr = qemu .map_private(0, MAX_INPUT_SIZE, MmapPerms::ReadWrite) .unwrap(); log::debug!("Placing input at {input_addr:#x}"); - let stack_ptr: GuestAddr = emu.read_reg(Regs::Sp).unwrap(); + let stack_ptr: GuestAddr = qemu.read_reg(Regs::Sp).unwrap(); let mut shmem_provider = StdShMemProvider::new().expect("Failed to init shared memory"); - let monitor = SimpleMonitor::with_user_monitor( - |s| { - println!("{s}"); - }, - true, - ); + let monitor = SimpleMonitor::with_user_monitor(|s| { + println!("{s}"); + }); let (state, mut mgr) = match SimpleRestartingEventManager::launch(monitor, &mut shmem_provider) { Ok(res) => res, @@ -160,25 +156,25 @@ pub fn fuzz() -> Result<(), Error> { }, }; - let mut edges_shmem = shmem_provider.new_shmem(EDGES_MAP_SIZE).unwrap(); - let edges = edges_shmem.as_mut_slice(); + let mut edges_shmem = shmem_provider.new_shmem(EDGES_MAP_SIZE_IN_USE).unwrap(); + let edges = edges_shmem.as_slice_mut(); unsafe { EDGES_MAP_PTR = edges.as_mut_ptr() }; let edges_observer = unsafe { - HitcountsMapObserver::new(ConstMapObserver::<_, EDGES_MAP_SIZE>::from_mut_ptr( + HitcountsMapObserver::new(ConstMapObserver::<_, EDGES_MAP_SIZE_IN_USE>::from_mut_ptr( "edges", edges.as_mut_ptr(), )) }; - let mut feedback = MaxMapFeedback::tracking(&edges_observer, true, false); + let mut feedback = MaxMapFeedback::new(&edges_observer); #[allow(clippy::let_unit_value)] let mut objective = (); let mut state = state.unwrap_or_else(|| { StdState::new( - StdRand::with_seed(current_nanos()), + StdRand::new(), InMemoryOnDiskCorpus::new(PathBuf::from(options.output)).unwrap(), NopCorpus::new(), &mut feedback, @@ -201,24 +197,29 @@ pub fn fuzz() -> Result<(), Error> { let len = len as GuestReg; unsafe { - emu.write_mem(input_addr, buf); - emu.write_reg(Regs::Pc, test_one_input_ptr).unwrap(); - emu.write_reg(Regs::Sp, stack_ptr).unwrap(); - emu.write_return_address(ret_addr).unwrap(); - emu.write_function_argument(CallingConvention::Cdecl, 0, input_addr) + qemu.write_mem(input_addr, buf); + qemu.write_reg(Regs::Pc, test_one_input_ptr).unwrap(); + qemu.write_reg(Regs::Sp, stack_ptr).unwrap(); + qemu.write_return_address(ret_addr).unwrap(); + qemu.write_function_argument(CallingConvention::Cdecl, 0, input_addr) .unwrap(); - emu.write_function_argument(CallingConvention::Cdecl, 1, len) + qemu.write_function_argument(CallingConvention::Cdecl, 1, len) .unwrap(); - emu.run(); + + match qemu.run() { + Ok(QemuExitReason::Breakpoint(_)) => {} + Ok(QemuExitReason::End(QemuShutdownCause::HostSignal(Signal::SigInterrupt))) => { + process::exit(0) + } + Err(QemuExitReasonError::UnexpectedExit) => return ExitKind::Crash, + _ => panic!("Unexpected QEMU exit."), + } } ExitKind::Ok }; - let mut hooks = QemuHooks::new( - emu.clone(), - tuple_list!(QemuEdgeCoverageChildHelper::default(),), - ); + let mut hooks = QemuHooks::new(qemu, tuple_list!(QemuEdgeCoverageChildHelper::default(),)); let mut executor = QemuForkExecutor::new( &mut hooks, diff --git a/fuzzers/qemu_coverage/Cargo.toml b/fuzzers/qemu_coverage/Cargo.toml index d127210f51..a02d70b9bc 100644 --- a/fuzzers/qemu_coverage/Cargo.toml +++ b/fuzzers/qemu_coverage/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "qemu_coverage" -version = "0.11.2" +version = "0.12.0" authors = ["Andrea Fioraldi ", "Dominik Maier ", "WorksButNotTested"] edition = "2021" diff --git a/fuzzers/qemu_coverage/Makefile.toml b/fuzzers/qemu_coverage/Makefile.toml index d8972bc283..3ec4818066 100644 --- a/fuzzers/qemu_coverage/Makefile.toml +++ b/fuzzers/qemu_coverage/Makefile.toml @@ -8,6 +8,7 @@ TARGET_DIR = "${CARGO_MAKE_CRATE_TARGET_DIRECTORY}/x86_64" LIBPNG_ARCH = "x86_64" LIBPNG_OPTIMIZATIONS = "yes" FEATURE = "x86_64" +LIBAFL_QEMU_CLONE_DIR = "${CARGO_MAKE_CRATE_TARGET_DIRECTORY}/qemu-libafl-bridge" #LIBAFL_DEBUG_OUTPUT = "1" #CUSTOM_QEMU_DIR= "~/qemu-libafl-bridge" @@ -19,6 +20,7 @@ TARGET_DIR = "${CARGO_MAKE_CRATE_TARGET_DIRECTORY}/arm" LIBPNG_ARCH = "arm" LIBPNG_OPTIMIZATIONS = "yes" FEATURE = "arm" +LIBAFL_QEMU_CLONE_DIR = "${CARGO_MAKE_CRATE_TARGET_DIRECTORY}/qemu-libafl-bridge" [env.aarch64] CROSS_CC = "aarch64-linux-gnu-gcc" @@ -28,6 +30,7 @@ TARGET_DIR = "${CARGO_MAKE_CRATE_TARGET_DIRECTORY}/aarch64" LIBPNG_ARCH = "aarch64" LIBPNG_OPTIMIZATIONS = "yes" FEATURE = "aarch64" +LIBAFL_QEMU_CLONE_DIR = "${CARGO_MAKE_CRATE_TARGET_DIRECTORY}/qemu-libafl-bridge" [env.x86_64] CROSS_CC = "x86_64-linux-gnu-gcc" @@ -37,6 +40,7 @@ TARGET_DIR = "${CARGO_MAKE_CRATE_TARGET_DIRECTORY}/x86_64" LIBPNG_ARCH = "x86_64" LIBPNG_OPTIMIZATIONS = "yes" FEATURE = "x86_64" +LIBAFL_QEMU_CLONE_DIR = "${CARGO_MAKE_CRATE_TARGET_DIRECTORY}/qemu-libafl-bridge" [env.i386] CROSS_CC = "x86_64-linux-gnu-gcc" @@ -46,6 +50,7 @@ TARGET_DIR = "${CARGO_MAKE_CRATE_TARGET_DIRECTORY}/i386" LIBPNG_ARCH = "i386" LIBPNG_OPTIMIZATIONS = "yes" FEATURE = "i386" +LIBAFL_QEMU_CLONE_DIR = "${CARGO_MAKE_CRATE_TARGET_DIRECTORY}/qemu-libafl-bridge" [env.mips] CROSS_CC = "mipsel-linux-gnu-gcc" @@ -55,6 +60,7 @@ TARGET_DIR = "${CARGO_MAKE_CRATE_TARGET_DIRECTORY}/mips" LIBPNG_ARCH = "mips" LIBPNG_OPTIMIZATIONS = "yes" FEATURE = "mips" +LIBAFL_QEMU_CLONE_DIR = "${CARGO_MAKE_CRATE_TARGET_DIRECTORY}/qemu-libafl-bridge" [env.ppc] CROSS_CC = "powerpc-linux-gnu-gcc" @@ -64,6 +70,7 @@ TARGET_DIR = "${CARGO_MAKE_CRATE_TARGET_DIRECTORY}/ppc" LIBPNG_ARCH = "ppc" LIBPNG_OPTIMIZATIONS = "no" FEATURE = "ppc" +LIBAFL_QEMU_CLONE_DIR = "${CARGO_MAKE_CRATE_TARGET_DIRECTORY}/qemu-libafl-bridge" [tasks.unsupported] script_runner="@shell" @@ -241,6 +248,20 @@ mac_alias = "unsupported" windows_alias = "unsupported" [tasks.test_unix] +dependencies = [ "lightweight" ] +# Tidy up after we've run our tests so we don't hog all the disk space +command = "cargo" +args = [ + "make", + "clean", +] + +[tasks.test_full] +linux_alias = "test_unix_full" +mac_alias = "unsupported" +windows_alias = "unsupported" + +[tasks.test_unix_full] dependencies = [ "all" ] # Tidy up after we've run our tests so we don't hog all the disk space command = "cargo" @@ -320,3 +341,9 @@ dependencies = [ "mips", "ppc" ] + +[tasks.lightweight] +dependencies = [ + "arm", + "x86_64", +] diff --git a/fuzzers/qemu_coverage/src/fuzzer.rs b/fuzzers/qemu_coverage/src/fuzzer.rs index 334c5dcc72..e2edb5da56 100644 --- a/fuzzers/qemu_coverage/src/fuzzer.rs +++ b/fuzzers/qemu_coverage/src/fuzzer.rs @@ -8,27 +8,27 @@ use std::{env, fs::DirEntry, io, path::PathBuf, process}; use clap::{builder::Str, Parser}; use libafl::{ corpus::{Corpus, NopCorpus}, - events::{launcher::Launcher, EventConfig, EventRestarter}, + events::{launcher::Launcher, EventConfig, EventRestarter, LlmpRestartingEventManager}, executors::ExitKind, fuzzer::StdFuzzer, inputs::{BytesInput, HasTargetBytes}, monitors::MultiMonitor, - prelude::LlmpRestartingEventManager, schedulers::QueueScheduler, state::{HasCorpus, StdState}, Error, }; use libafl_bolts::{ core_affinity::Cores, - current_nanos, + os::unix_signals::Signal, rands::StdRand, shmem::{ShMemProvider, StdShMemProvider}, tuples::tuple_list, AsSlice, }; use libafl_qemu::{ - drcov::QemuDrCovHelper, elf::EasyElf, emu::Emulator, ArchExtras, CallingConvention, GuestAddr, - GuestReg, MmapPerms, QemuExecutor, QemuHooks, QemuInstrumentationAddressRangeFilter, Regs, + drcov::QemuDrCovHelper, elf::EasyElf, ArchExtras, CallingConvention, GuestAddr, GuestReg, + MmapPerms, Qemu, QemuExecutor, QemuExitReason, QemuHooks, + QemuInstrumentationAddressRangeFilter, QemuShutdownCause, Regs, }; use rangemap::RangeMap; @@ -119,51 +119,59 @@ pub fn fuzz() { env::remove_var("LD_LIBRARY_PATH"); let env: Vec<(String, String)> = env::vars().collect(); - let emu = Emulator::new(&options.args, &env).unwrap(); + let qemu = Qemu::init(&options.args, &env).unwrap(); let mut elf_buffer = Vec::new(); - let elf = EasyElf::from_file(emu.binary_path(), &mut elf_buffer).unwrap(); + let elf = EasyElf::from_file(qemu.binary_path(), &mut elf_buffer).unwrap(); let test_one_input_ptr = elf - .resolve_symbol("LLVMFuzzerTestOneInput", emu.load_addr()) + .resolve_symbol("LLVMFuzzerTestOneInput", qemu.load_addr()) .expect("Symbol LLVMFuzzerTestOneInput not found"); log::debug!("LLVMFuzzerTestOneInput @ {test_one_input_ptr:#x}"); - emu.entry_break(test_one_input_ptr); + qemu.entry_break(test_one_input_ptr); - for m in emu.mappings() { + for m in qemu.mappings() { log::debug!( "Mapping: 0x{:016x}-0x{:016x}, {}", m.start(), m.end(), - m.path().unwrap_or("") + m.path().unwrap_or(&"".to_string()) ); } - let pc: GuestReg = emu.read_reg(Regs::Pc).unwrap(); + let pc: GuestReg = qemu.read_reg(Regs::Pc).unwrap(); log::debug!("Break at {pc:#x}"); - let ret_addr: GuestAddr = emu.read_return_address().unwrap(); + let ret_addr: GuestAddr = qemu.read_return_address().unwrap(); log::debug!("Return address = {ret_addr:#x}"); - emu.set_breakpoint(ret_addr); + qemu.set_breakpoint(ret_addr); - let input_addr = emu + let input_addr = qemu .map_private(0, MAX_INPUT_SIZE, MmapPerms::ReadWrite) .unwrap(); log::debug!("Placing input at {input_addr:#x}"); - let stack_ptr: GuestAddr = emu.read_reg(Regs::Sp).unwrap(); + let stack_ptr: GuestAddr = qemu.read_reg(Regs::Sp).unwrap(); let reset = |buf: &[u8], len: GuestReg| -> Result<(), String> { unsafe { - emu.write_mem(input_addr, buf); - emu.write_reg(Regs::Pc, test_one_input_ptr)?; - emu.write_reg(Regs::Sp, stack_ptr)?; - emu.write_return_address(ret_addr)?; - emu.write_function_argument(CallingConvention::Cdecl, 0, input_addr)?; - emu.write_function_argument(CallingConvention::Cdecl, 1, len)?; - emu.run(); + qemu.write_mem(input_addr, buf); + qemu.write_reg(Regs::Pc, test_one_input_ptr)?; + qemu.write_reg(Regs::Sp, stack_ptr)?; + qemu.write_return_address(ret_addr)?; + qemu.write_function_argument(CallingConvention::Cdecl, 0, input_addr)?; + qemu.write_function_argument(CallingConvention::Cdecl, 1, len)?; + + match qemu.run() { + Ok(QemuExitReason::Breakpoint(_)) => {} + Ok(QemuExitReason::End(QemuShutdownCause::HostSignal(Signal::SigInterrupt))) => { + process::exit(0) + } + _ => panic!("Unexpected QEMU exit."), + } + Ok(()) } }; @@ -181,101 +189,102 @@ pub fn fuzz() { ExitKind::Ok }; - let mut run_client = |state: Option<_>, mut mgr: LlmpRestartingEventManager<_, _>, core_id| { - let core_idx = options - .cores - .position(core_id) - .expect("Failed to get core index"); - let files = corpus_files - .iter() - .skip(files_per_core * core_idx) - .take(files_per_core) - .map(|x| x.path()) - .collect::>(); - - if files.is_empty() { - mgr.send_exiting()?; - Err(Error::ShuttingDown)? - } - - #[allow(clippy::let_unit_value)] - let mut feedback = (); - - #[allow(clippy::let_unit_value)] - let mut objective = (); - - let mut state = state.unwrap_or_else(|| { - StdState::new( - StdRand::with_seed(current_nanos()), - NopCorpus::new(), - NopCorpus::new(), - &mut feedback, - &mut objective, - ) - .unwrap() - }); - - let scheduler = QueueScheduler::new(); - let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); - - let rangemap = emu - .mappings() - .filter_map(|m| { - m.path() - .map(|p| ((m.start() as usize)..(m.end() as usize), p.to_string())) - .filter(|(_, p)| !p.is_empty()) - }) - .enumerate() - .fold( - RangeMap::::new(), - |mut rm, (i, (r, p))| { - rm.insert(r, (i as u16, p)); - rm - }, + let mut run_client = + |state: Option<_>, mut mgr: LlmpRestartingEventManager<_, _, _>, core_id| { + let core_idx = options + .cores + .position(core_id) + .expect("Failed to get core index"); + let files = corpus_files + .iter() + .skip(files_per_core * core_idx) + .take(files_per_core) + .map(|x| x.path()) + .collect::>(); + + if files.is_empty() { + mgr.send_exiting()?; + Err(Error::ShuttingDown)? + } + + #[allow(clippy::let_unit_value)] + let mut feedback = (); + + #[allow(clippy::let_unit_value)] + let mut objective = (); + + let mut state = state.unwrap_or_else(|| { + StdState::new( + StdRand::new(), + NopCorpus::new(), + NopCorpus::new(), + &mut feedback, + &mut objective, + ) + .unwrap() + }); + + let scheduler = QueueScheduler::new(); + let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); + + let rangemap = qemu + .mappings() + .filter_map(|m| { + m.path() + .map(|p| ((m.start() as usize)..(m.end() as usize), p.to_string())) + .filter(|(_, p)| !p.is_empty()) + }) + .enumerate() + .fold( + RangeMap::::new(), + |mut rm, (i, (r, p))| { + rm.insert(r, (i as u16, p)); + rm + }, + ); + + let mut coverage = PathBuf::from(&options.coverage); + let coverage_name = coverage.file_stem().unwrap().to_str().unwrap(); + let coverage_extension = coverage.extension().unwrap_or_default().to_str().unwrap(); + let core = core_id.0; + coverage.set_file_name(format!("{coverage_name}-{core:03}.{coverage_extension}")); + + let mut hooks = QemuHooks::new( + qemu, + tuple_list!(QemuDrCovHelper::new( + QemuInstrumentationAddressRangeFilter::None, + rangemap, + coverage, + false, + )), ); - let mut coverage = PathBuf::from(&options.coverage); - let coverage_name = coverage.file_stem().unwrap().to_str().unwrap(); - let coverage_extension = coverage.extension().unwrap_or_default().to_str().unwrap(); - let core = core_id.0; - coverage.set_file_name(format!("{coverage_name}-{core:03}.{coverage_extension}")); - - let mut hooks = QemuHooks::new( - emu.clone(), - tuple_list!(QemuDrCovHelper::new( - QemuInstrumentationAddressRangeFilter::None, - rangemap, - PathBuf::from(coverage), - false, - )), - ); + let mut executor = QemuExecutor::new( + &mut hooks, + &mut harness, + (), + &mut fuzzer, + &mut state, + &mut mgr, + options.timeout, + ) + .expect("Failed to create QemuExecutor"); - let mut executor = QemuExecutor::new( - &mut hooks, - &mut harness, - (), - &mut fuzzer, - &mut state, - &mut mgr, - options.timeout, - ) - .expect("Failed to create QemuExecutor"); - - if state.must_load_initial_inputs() { - state - .load_initial_inputs_by_filenames(&mut fuzzer, &mut executor, &mut mgr, &files) - .unwrap_or_else(|_| { - println!("Failed to load initial corpus at {:?}", &corpus_dir); - process::exit(0); - }); - log::debug!("We imported {} inputs from disk.", state.corpus().count()); - } + if state.must_load_initial_inputs() { + state + .load_initial_inputs_by_filenames(&mut fuzzer, &mut executor, &mut mgr, &files) + .unwrap_or_else(|_| { + println!("Failed to load initial corpus at {:?}", &corpus_dir); + process::exit(0); + }); + log::debug!("We imported {} inputs from disk.", state.corpus().count()); + } - log::debug!("Processed {} inputs from disk.", files.len()); + log::debug!("Processed {} inputs from disk.", files.len()); - mgr.send_exiting()?; - Err(Error::ShuttingDown)? - }; + mgr.send_exiting()?; + Err(Error::ShuttingDown)? + }; match Launcher::builder() .shmem_provider(StdShMemProvider::new().expect("Failed to init shared memory")) diff --git a/fuzzers/qemu_launcher/Cargo.toml b/fuzzers/qemu_launcher/Cargo.toml index 00bb1c2941..30c23a3026 100644 --- a/fuzzers/qemu_launcher/Cargo.toml +++ b/fuzzers/qemu_launcher/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "qemu_launcher" -version = "0.11.2" +version = "0.12.0" authors = ["Andrea Fioraldi ", "Dominik Maier "] edition = "2021" @@ -38,7 +38,7 @@ vergen = { version = "8.2.1", features = ["build", "cargo", "git", "gitcl", "rus [dependencies] clap = { version = "4.3.0", features = ["derive", "string"]} libafl = { path = "../../libafl/" } -libafl_bolts = { path = "../../libafl_bolts/" } +libafl_bolts = { path = "../../libafl_bolts/", features = ["errors_backtrace"] } libafl_qemu = { path = "../../libafl_qemu/", features = ["usermode"] } log = {version = "0.4.20" } nix = { version = "0.27", features = ["fs"] } diff --git a/fuzzers/qemu_launcher/Makefile.toml b/fuzzers/qemu_launcher/Makefile.toml index b64103f258..6f508eaf03 100644 --- a/fuzzers/qemu_launcher/Makefile.toml +++ b/fuzzers/qemu_launcher/Makefile.toml @@ -216,8 +216,7 @@ ${CROSS_CXX} \ -I"${TARGET_DIR}/build-zlib/zlib/include" \ -L"${TARGET_DIR}/build-zlib/zlib/lib" \ -o"${TARGET_DIR}/libpng-harness-${CARGO_MAKE_PROFILE}" \ - -lm \ - -static + -lm ''' dependencies = [ "libpng" ] @@ -280,6 +279,42 @@ args = [ ] dependencies = [ "harness", "fuzzer" ] +[tasks.asan] +linux_alias = "asan_unix" +mac_alias = "unsupported" +windows_alias = "unsupported" + +[tasks.asan_unix] +command = "${TARGET_DIR}/${PROFILE_DIR}/qemu_launcher-${CARGO_MAKE_PROFILE}" +args = [ + "--input", "./corpus", + "--output", "${TARGET_DIR}/output/", + "--log", "${TARGET_DIR}/output/log.txt", + "--cores", "0", + "--asan-cores", "0", + "--", + "${TARGET_DIR}/libpng-harness-${CARGO_MAKE_PROFILE}", +] +dependencies = [ "harness", "fuzzer" ] + +[tasks.asan_guest] +linux_alias = "asan_guest_unix" +mac_alias = "unsupported" +windows_alias = "unsupported" + +[tasks.asan_guest_unix] +command = "${TARGET_DIR}/${PROFILE_DIR}/qemu_launcher-${CARGO_MAKE_PROFILE}" +args = [ + "--input", "./corpus", + "--output", "${TARGET_DIR}/output/", + "--log", "${TARGET_DIR}/output/log.txt", + "--cores", "0", + "--asan-guest-cores", "0", + "--", + "${TARGET_DIR}/libpng-harness-${CARGO_MAKE_PROFILE}", +] +dependencies = [ "harness", "fuzzer" ] + [tasks.test] linux_alias = "test_unix" mac_alias = "unsupported" @@ -297,7 +332,7 @@ timeout 10s "$(find ${TARGET_DIR} -name 'qemu_launcher')" -o out -i in -j ../inj if [ -z "$(grep -Ei "found.*injection" fuzz.log)" ]; then echo "Fuzzer does not generate any testcases or any crashes" echo "Logs:" - tail fuzz.log + cat fuzz.log exit 1 else echo "Fuzzer is working" diff --git a/fuzzers/qemu_launcher/README.md b/fuzzers/qemu_launcher/README.md index 10111d6471..bc9c9b767d 100644 --- a/fuzzers/qemu_launcher/README.md +++ b/fuzzers/qemu_launcher/README.md @@ -26,7 +26,8 @@ sudo apt install \ gcc-mipsel-linux-gnu \ g++-mipsel-linux-gnu \ gcc-powerpc-linux-gnu \ - g++-powerpc-linux-gnu + g++-powerpc-linux-gnu \ + libsqlite3-dev ``` ## Run diff --git a/fuzzers/qemu_launcher/src/client.rs b/fuzzers/qemu_launcher/src/client.rs index 39e2827dae..483e4a5b4e 100644 --- a/fuzzers/qemu_launcher/src/client.rs +++ b/fuzzers/qemu_launcher/src/client.rs @@ -11,11 +11,12 @@ use libafl_bolts::{core_affinity::CoreId, rands::StdRand, tuples::tuple_list}; #[cfg(feature = "injections")] use libafl_qemu::injections::QemuInjectionHelper; use libafl_qemu::{ - asan::{init_with_asan, QemuAsanHelper}, + asan::{init_qemu_with_asan, QemuAsanHelper}, + asan_guest::{init_qemu_with_asan_guest, QemuAsanGuestHelper}, cmplog::QemuCmpLogHelper, edges::QemuEdgeCoverageHelper, elf::EasyElf, - ArchExtras, Emulator, GuestAddr, QemuInstrumentationAddressRangeFilter, + ArchExtras, GuestAddr, Qemu, QemuInstrumentationAddressRangeFilter, }; use crate::{ @@ -53,21 +54,18 @@ impl<'a> Client<'a> { .collect::>() } - fn start_pc(emu: &Emulator) -> Result { + fn start_pc(qemu: &Qemu) -> Result { let mut elf_buffer = Vec::new(); - let elf = EasyElf::from_file(emu.binary_path(), &mut elf_buffer)?; + let elf = EasyElf::from_file(qemu.binary_path(), &mut elf_buffer)?; let start_pc = elf - .resolve_symbol("LLVMFuzzerTestOneInput", emu.load_addr()) + .resolve_symbol("LLVMFuzzerTestOneInput", qemu.load_addr()) .ok_or_else(|| Error::empty_optional("Symbol LLVMFuzzerTestOneInput not found"))?; Ok(start_pc) } #[allow(clippy::similar_names)] // elf != self - fn coverage_filter( - &self, - emu: &Emulator, - ) -> Result { + fn coverage_filter(&self, qemu: &Qemu) -> Result { /* Conversion is required on 32-bit targets, but not on 64-bit ones */ if let Some(includes) = &self.options.include { #[cfg_attr(target_pointer_width = "64", allow(clippy::useless_conversion))] @@ -91,9 +89,9 @@ impl<'a> Client<'a> { Ok(QemuInstrumentationAddressRangeFilter::DenyList(rules)) } else { let mut elf_buffer = Vec::new(); - let elf = EasyElf::from_file(emu.binary_path(), &mut elf_buffer)?; + let elf = EasyElf::from_file(qemu.binary_path(), &mut elf_buffer)?; let range = elf - .get_section(".text", emu.load_addr()) + .get_section(".text", qemu.load_addr()) .ok_or_else(|| Error::key_not_found("Failed to find .text section"))?; Ok(QemuInstrumentationAddressRangeFilter::AllowList(vec![ range, @@ -113,16 +111,26 @@ impl<'a> Client<'a> { let mut env = self.env(); log::debug!("ENV: {:#?}", env); - let (emu, mut asan) = { - if self.options.is_asan_core(core_id) { - let (emu, asan) = init_with_asan(&mut args, &mut env)?; - (emu, Some(asan)) + let is_asan = self.options.is_asan_core(core_id); + let is_asan_guest = self.options.is_asan_guest_core(core_id); + + if is_asan && is_asan_guest { + Err(Error::empty_optional("Multiple ASAN modes configured"))?; + } + + let (qemu, mut asan, mut asan_lib) = { + if is_asan { + let (emu, asan) = init_qemu_with_asan(&mut args, &mut env)?; + (emu, Some(asan), None) + } else if is_asan_guest { + let (emu, asan_lib) = init_qemu_with_asan_guest(&mut args, &mut env)?; + (emu, None, Some(asan_lib)) } else { - (Emulator::new(&args, &env)?, None) + (Qemu::init(&args, &env)?, None, None) } }; - let start_pc = Self::start_pc(&emu)?; + let start_pc = Self::start_pc(&qemu)?; log::debug!("start_pc @ {start_pc:#x}"); #[cfg(not(feature = "injections"))] @@ -146,22 +154,21 @@ impl<'a> Client<'a> { let extra_tokens = injection_helper.as_ref().map(|h| h.tokens.clone()); - emu.entry_break(start_pc); + qemu.entry_break(start_pc); - let ret_addr: GuestAddr = emu + let ret_addr: GuestAddr = qemu .read_return_address() .map_err(|e| Error::unknown(format!("Failed to read return address: {e:}")))?; log::debug!("ret_addr = {ret_addr:#x}"); - emu.set_breakpoint(ret_addr); + qemu.set_breakpoint(ret_addr); - let is_asan = self.options.is_asan_core(core_id); let is_cmplog = self.options.is_cmplog_core(core_id); - let edge_coverage_helper = QemuEdgeCoverageHelper::new(self.coverage_filter(&emu)?); + let edge_coverage_helper = QemuEdgeCoverageHelper::new(self.coverage_filter(&qemu)?); let instance = Instance::builder() .options(self.options) - .emu(&emu) + .qemu(&qemu) .mgr(mgr) .core_id(core_id) .extra_tokens(extra_tokens); @@ -187,6 +194,27 @@ impl<'a> Client<'a> { state, ) } + } else if is_asan_guest && is_cmplog { + if let Some(injection_helper) = injection_helper { + instance.build().run( + tuple_list!( + edge_coverage_helper, + QemuCmpLogHelper::default(), + QemuAsanGuestHelper::default(&qemu, asan_lib.take().unwrap()), + injection_helper + ), + state, + ) + } else { + instance.build().run( + tuple_list!( + edge_coverage_helper, + QemuCmpLogHelper::default(), + QemuAsanGuestHelper::default(&qemu, asan_lib.take().unwrap()), + ), + state, + ) + } } else if is_asan { if let Some(injection_helper) = injection_helper { instance.build().run( @@ -206,6 +234,12 @@ impl<'a> Client<'a> { state, ) } + } else if is_asan_guest { + let helpers = tuple_list!( + edge_coverage_helper, + QemuAsanGuestHelper::default(&qemu, asan_lib.take().unwrap()) + ); + instance.build().run(helpers, state) } else if is_cmplog { if let Some(injection_helper) = injection_helper { instance.build().run( diff --git a/fuzzers/qemu_launcher/src/harness.rs b/fuzzers/qemu_launcher/src/harness.rs index 43b2aabb33..c8272b4a41 100644 --- a/fuzzers/qemu_launcher/src/harness.rs +++ b/fuzzers/qemu_launcher/src/harness.rs @@ -4,10 +4,10 @@ use libafl::{ Error, }; use libafl_bolts::AsSlice; -use libafl_qemu::{ArchExtras, CallingConvention, Emulator, GuestAddr, GuestReg, MmapPerms, Regs}; +use libafl_qemu::{ArchExtras, CallingConvention, GuestAddr, GuestReg, MmapPerms, Qemu, Regs}; pub struct Harness<'a> { - emu: &'a Emulator, + qemu: &'a Qemu, input_addr: GuestAddr, pc: GuestAddr, stack_ptr: GuestAddr, @@ -17,25 +17,25 @@ pub struct Harness<'a> { pub const MAX_INPUT_SIZE: usize = 1_048_576; // 1MB impl<'a> Harness<'a> { - pub fn new(emu: &Emulator) -> Result { - let input_addr = emu + pub fn new(qemu: &Qemu) -> Result { + let input_addr = qemu .map_private(0, MAX_INPUT_SIZE, MmapPerms::ReadWrite) .map_err(|e| Error::unknown(format!("Failed to map input buffer: {e:}")))?; - let pc: GuestReg = emu + let pc: GuestReg = qemu .read_reg(Regs::Pc) .map_err(|e| Error::unknown(format!("Failed to read PC: {e:}")))?; - let stack_ptr: GuestAddr = emu + let stack_ptr: GuestAddr = qemu .read_reg(Regs::Sp) .map_err(|e| Error::unknown(format!("Failed to read stack pointer: {e:}")))?; - let ret_addr: GuestAddr = emu + let ret_addr: GuestAddr = qemu .read_return_address() .map_err(|e| Error::unknown(format!("Failed to read return address: {e:}")))?; Ok(Harness { - emu, + qemu, input_addr, pc, stack_ptr, @@ -58,29 +58,29 @@ impl<'a> Harness<'a> { } let len = len as GuestReg; - unsafe { self.emu.write_mem(self.input_addr, buf) }; + unsafe { self.qemu.write_mem(self.input_addr, buf) }; - self.emu + self.qemu .write_reg(Regs::Pc, self.pc) .map_err(|e| Error::unknown(format!("Failed to write PC: {e:}")))?; - self.emu + self.qemu .write_reg(Regs::Sp, self.stack_ptr) .map_err(|e| Error::unknown(format!("Failed to write SP: {e:}")))?; - self.emu + self.qemu .write_return_address(self.ret_addr) .map_err(|e| Error::unknown(format!("Failed to write return address: {e:}")))?; - self.emu + self.qemu .write_function_argument(CallingConvention::Cdecl, 0, self.input_addr) .map_err(|e| Error::unknown(format!("Failed to write argument 0: {e:}")))?; - self.emu + self.qemu .write_function_argument(CallingConvention::Cdecl, 1, len) .map_err(|e| Error::unknown(format!("Failed to write argument 1: {e:}")))?; unsafe { - let _ = self.emu.run(); + let _ = self.qemu.run(); }; Ok(()) } diff --git a/fuzzers/qemu_launcher/src/instance.rs b/fuzzers/qemu_launcher/src/instance.rs index 81ee9b086b..519eaa6b89 100644 --- a/fuzzers/qemu_launcher/src/instance.rs +++ b/fuzzers/qemu_launcher/src/instance.rs @@ -1,4 +1,4 @@ -use core::ptr::addr_of_mut; +use core::{fmt::Debug, ptr::addr_of_mut}; use std::{marker::PhantomData, process}; #[cfg(feature = "simplemgr")] @@ -18,7 +18,7 @@ use libafl::{ scheduled::havoc_mutations, token_mutations::I2SRandReplace, tokens_mutations, StdMOptMutator, StdScheduledMutator, Tokens, }, - observers::{HitcountsMapObserver, TimeObserver, VariableMapObserver}, + observers::{CanTrack, HitcountsMapObserver, TimeObserver, VariableMapObserver}, schedulers::{ powersched::PowerSchedule, IndexesLenTimeMinimizerScheduler, PowerQueueScheduler, }, @@ -26,22 +26,22 @@ use libafl::{ calibrate::CalibrationStage, power::StdPowerMutationalStage, ShadowTracingStage, StagesTuple, StdMutationalStage, }, - state::{HasCorpus, HasMetadata, StdState, UsesState}, - Error, + state::{HasCorpus, StdState, UsesState}, + Error, HasMetadata, }; #[cfg(not(feature = "simplemgr"))] use libafl_bolts::shmem::StdShMemProvider; use libafl_bolts::{ core_affinity::CoreId, - current_nanos, + ownedref::OwnedMutSlice, rands::StdRand, tuples::{tuple_list, Merge}, }; use libafl_qemu::{ cmplog::CmpLogObserver, - edges::{edges_map_mut_slice, MAX_EDGES_NUM}, - helper::QemuHelperTuple, - Emulator, QemuExecutor, QemuHooks, + edges::{edges_map_mut_ptr, EDGES_MAP_SIZE_IN_USE, MAX_EDGES_FOUND}, + helpers::QemuHelperTuple, + Qemu, QemuExecutor, QemuHooks, }; use typed_builder::TypedBuilder; @@ -54,12 +54,12 @@ pub type ClientState = pub type ClientMgr = SimpleEventManager; #[cfg(not(feature = "simplemgr"))] pub type ClientMgr = - MonitorTypedEventManager, M>; + MonitorTypedEventManager, M>; #[derive(TypedBuilder)] pub struct Instance<'a, M: Monitor> { options: &'a FuzzerOptions, - emu: &'a Emulator, + qemu: &'a Qemu, mgr: ClientMgr, core_id: CoreId, extra_tokens: Option>, @@ -70,23 +70,24 @@ pub struct Instance<'a, M: Monitor> { impl<'a, M: Monitor> Instance<'a, M> { pub fn run(&mut self, helpers: QT, state: Option) -> Result<(), Error> where - QT: QemuHelperTuple, + QT: QemuHelperTuple + Debug, { - let mut hooks = QemuHooks::new(self.emu.clone(), helpers); + let mut hooks = QemuHooks::new(*self.qemu, helpers); // Create an observation channel using the coverage map let edges_observer = unsafe { HitcountsMapObserver::new(VariableMapObserver::from_mut_slice( "edges", - edges_map_mut_slice(), - addr_of_mut!(MAX_EDGES_NUM), + OwnedMutSlice::from_raw_parts_mut(edges_map_mut_ptr(), EDGES_MAP_SIZE_IN_USE), + addr_of_mut!(MAX_EDGES_FOUND), )) + .track_indices() }; // Create an observation channel to keep track of the execution time let time_observer = TimeObserver::new("time"); - let map_feedback = MaxMapFeedback::tracking(&edges_observer, true, false); + let map_feedback = MaxMapFeedback::new(&edges_observer); let calibration = CalibrationStage::new(&map_feedback); @@ -96,7 +97,7 @@ impl<'a, M: Monitor> Instance<'a, M> { // New maximization map feedback linked to the edges observer and the feedback state map_feedback, // Time feedback, this one does not need a feedback state - TimeFeedback::with_observer(&time_observer) + TimeFeedback::new(&time_observer) ); // A feedback to choose if an input is a solution or not @@ -108,7 +109,7 @@ impl<'a, M: Monitor> Instance<'a, M> { None => { StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryOnDiskCorpus::no_meta(self.options.queue_dir(self.core_id))?, // Corpus in which we store solutions (crashes in this example), @@ -124,11 +125,10 @@ impl<'a, M: Monitor> Instance<'a, M> { }; // A minimization+queue policy to get testcasess from the corpus - let scheduler = IndexesLenTimeMinimizerScheduler::new(PowerQueueScheduler::new( - &mut state, + let scheduler = IndexesLenTimeMinimizerScheduler::new( &edges_observer, - PowerSchedule::FAST, - )); + PowerQueueScheduler::new(&mut state, &edges_observer, PowerSchedule::FAST), + ); let observers = tuple_list!(edges_observer, time_observer); @@ -147,7 +147,7 @@ impl<'a, M: Monitor> Instance<'a, M> { state.add_metadata(tokens); - let harness = Harness::new(self.emu)?; + let harness = Harness::new(self.qemu)?; let mut harness = |input: &BytesInput| harness.run(input); // A fuzzer with feedbacks and a corpus scheduler diff --git a/fuzzers/qemu_launcher/src/options.rs b/fuzzers/qemu_launcher/src/options.rs index 284d9baee8..20e9f598de 100644 --- a/fuzzers/qemu_launcher/src/options.rs +++ b/fuzzers/qemu_launcher/src/options.rs @@ -51,6 +51,9 @@ pub struct FuzzerOptions { #[arg(long, help = "Cpu cores to use for ASAN", value_parser = Cores::from_cmdline)] pub asan_cores: Option, + #[arg(long, help = "Cpu cores to use for ASAN", value_parser = Cores::from_cmdline)] + pub asan_guest_cores: Option, + #[arg(long, help = "Cpu cores to use for CmpLog", value_parser = Cores::from_cmdline)] pub cmplog_cores: Option, @@ -113,6 +116,12 @@ impl FuzzerOptions { .map_or(false, |c| c.contains(core_id)) } + pub fn is_asan_guest_core(&self, core_id: CoreId) -> bool { + self.asan_guest_cores + .as_ref() + .map_or(false, |c| c.contains(core_id)) + } + pub fn is_cmplog_core(&self, core_id: CoreId) -> bool { self.cmplog_cores .as_ref() diff --git a/fuzzers/qemu_systemmode/Cargo.toml b/fuzzers/qemu_systemmode/Cargo.toml index 519a978b55..1517816928 100644 --- a/fuzzers/qemu_systemmode/Cargo.toml +++ b/fuzzers/qemu_systemmode/Cargo.toml @@ -1,13 +1,19 @@ [package] name = "qemu_systemmode" -version = "0.11.2" +version = "0.12.0" authors = ["Andrea Fioraldi ", "Dominik Maier "] edition = "2021" [features] -default = ["std"] +default = ["std", "classic"] std = [] +classic = [] # The classic way to interact with LibAFL QEMU, with direct calls to QEMU's functions +breakpoint = [] # Uses the command system, with breakpoints +sync_exit = [] # Uses the command system, with sync exit. + +shared = ["libafl_qemu/shared"] + [profile.release] incremental = true debug = true @@ -18,4 +24,8 @@ codegen-units = 1 libafl = { path = "../../libafl/" } libafl_bolts = { path = "../../libafl_bolts/" } libafl_qemu = { path = "../../libafl_qemu/", features = ["arm", "systemmode"] } +libafl_qemu_sys = { path = "../../libafl_qemu/libafl_qemu_sys", features = ["arm", "systemmode"] } env_logger = "*" + +[build-dependencies] +libafl_qemu_build = { path = "../../libafl_qemu/libafl_qemu_build" } diff --git a/fuzzers/qemu_systemmode/Makefile.toml b/fuzzers/qemu_systemmode/Makefile.toml new file mode 100644 index 0000000000..2afe843b23 --- /dev/null +++ b/fuzzers/qemu_systemmode/Makefile.toml @@ -0,0 +1,203 @@ +env_scripts = [ +''' +#!@duckscript +profile = get_env PROFILE + +if eq ${profile} "dev" + set_env PROFILE_DIR debug +else + set_env PROFILE_DIR ${profile} +end +''', +''' +#!@duckscript +runs_on_ci = get_env RUN_ON_CI + +if ${runs_on_ci} + cargo_target_dir = get_env CARGO_MAKE_CRATE_TARGET_DIRECTORY + set_env TARGET_DIR ${cargo_target_dir} + set_env KERNEL ${cargo_target_dir}/example.elf +end +''' +] + +[env] +PROFILE = { value = "release", condition = { env_not_set = ["PROFILE"] } } +TARGET_DIR = "${CARGO_MAKE_CRATE_TARGET_DIRECTORY}/${FEATURE}" +LIBAFL_QEMU_CLONE_DIR = "${CARGO_MAKE_CRATE_TARGET_DIRECTORY}/qemu-libafl-bridge" +KERNEL = "${TARGET_DIR}/example.elf" + +[tasks.target_dir] +condition = { files_not_exist = [ "${TARGET_DIR}" ] } +script_runner="@shell" +script=''' +mkdir -p ${TARGET_DIR} +''' + +[tasks.image] +dependencies = ["target_dir"] +condition = { files_not_exist = [ "${TARGET_DIR}/dummy.qcow2" ] } +script_runner="@shell" +script=''' +qemu-img create -f qcow2 ${TARGET_DIR}/dummy.qcow2 32M +''' + +[tasks.target] +dependencies = ["target_dir"] +condition = { env_set = [ "TARGET_DEFINE" ] } +command = "arm-none-eabi-gcc" +args = [ + "-ggdb", + "-ffreestanding", + "-nostartfiles", + "-lgcc", + "-T", "${CARGO_MAKE_WORKING_DIRECTORY}/example/mps2_m3.ld", + "-mcpu=cortex-m3", + "${CARGO_MAKE_WORKING_DIRECTORY}/example/main.c", + "${CARGO_MAKE_WORKING_DIRECTORY}/example/startup.c", + "-D", "${TARGET_DEFINE}", + "-I", "${TARGET_DIR}/${PROFILE_DIR}/include", + "-o", "${TARGET_DIR}/example.elf", +] + +[tasks.build_fuzzer] +condition = { env_set = [ "FEATURE" ] } +command = "cargo" +args = [ + "build", + "--profile", + "${PROFILE}", + "--no-default-features", + "--features", "std,${FEATURE}", + "--target-dir", "${TARGET_DIR}", +] +dependencies = ["image"] + +[tasks.run_fuzzer] +command = "${TARGET_DIR}/${PROFILE_DIR}/qemu_systemmode" +args = [ + "-icount", "shift=auto,align=off,sleep=off", + "-machine", "mps2-an385", + "-monitor", "null", + "-kernel", "${TARGET_DIR}/example.elf", + "-serial", "null", + "-nographic", + "-snapshot", + "-drive", "if=none,format=qcow2,file=${TARGET_DIR}/dummy.qcow2", + "-S", +] +dependencies = ["target"] + +[tasks.test_fuzzer] +condition = { env_set = [ "FEATURE" ] } +script_runner="@shell" +script=''' +TMP_DIR=$(mktemp -d) + +cargo make build_$FEATURE +timeout 15s cargo make ${FEATURE} | tee $TMP_DIR/fuzz.log 2>&1 || true + +if [ -z "$(grep 'Objective' $TMP_DIR/fuzz.log)" ]; then + echo "qemu_systemmode ${FEATURE}: Fuzzer did not find the objective in $TMP_DIR/fuzz.log" + exit 1 +else + echo "qemu_systemmode ${FEATURE}: Objective found." +fi +''' + +[tasks.build_classic] +command = "cargo" +args = [ + "make", + "-e", "FEATURE=classic", + "-e", "TARGET_DEFINE=TARGET_CLASSIC", + "build_fuzzer", +] + +[tasks.test_classic] +command = "cargo" +args = [ + "make", + "-e", "FEATURE=classic", + "test_fuzzer", +] + +[tasks.build_breakpoint] +command = "cargo" +args = [ + "make", + "-e", "FEATURE=breakpoint", + "-e", "TARGET_DEFINE=TARGET_BREAKPOINT", + "build_fuzzer", +] + +[tasks.test_breakpoint] +command = "cargo" +args = [ + "make", + "-e", "FEATURE=breakpoint", + "test_fuzzer", +] + +[tasks.build_sync_exit] +command = "cargo" +args = [ + "make", + "-e", "FEATURE=sync_exit", + "-e", "TARGET_DEFINE=TARGET_SYNC_EXIT", + "build_fuzzer", +] + +[tasks.test_sync_exit] +command = "cargo" +args = [ + "make", + "-e", "FEATURE=sync_exit", + "test_fuzzer", +] + +[tasks.classic] +command = "cargo" +args = [ + "make", + "-e", "FEATURE=classic", + "-e", "TARGET_DEFINE=TARGET_CLASSIC", + "run_fuzzer", +] + +[tasks.breakpoint] +command = "cargo" +args = [ + "make", + "-e", "FEATURE=breakpoint", + "-e", "TARGET_DEFINE=TARGET_BREAKPOINT", + "run_fuzzer", +] + +[tasks.sync_exit] +command = "cargo" +args = [ + "make", + "-e", "FEATURE=sync_exit", + "-e", "TARGET_DEFINE=TARGET_SYNC_EXIT", + "run_fuzzer", +] + +[tasks.test] +clear = true +run_task = { name = ["test_classic", "test_breakpoint", "test_sync_exit"] } + +[tasks.build] +clear = true +run_task = { name = ["build_classic", "build_breakpoint", "build_sync_exit"] } + +[tasks.run] +alias="classic" + +[tasks.clean] +clear = true +script_runner="@shell" +script=''' +rm -rf ${CARGO_MAKE_CRATE_TARGET_DIRECTORY} +cargo clean +''' diff --git a/fuzzers/qemu_systemmode/README.md b/fuzzers/qemu_systemmode/README.md index 14098dc09c..12ad9af951 100644 --- a/fuzzers/qemu_systemmode/README.md +++ b/fuzzers/qemu_systemmode/README.md @@ -2,25 +2,44 @@ This folder contains an example fuzzer for the qemu systemmode, using LLMP for fast multi-process fuzzing and crash detection. -## Build +It comes in three flavours (can be set through features): + +-`classic`: The low-level way to interact with QEMU. +-`breakpoint`: Interaction with QEMU using the command system, leveraging breakpoints. +-`sync_exit`: Interaction with QEMU using the command system, leveraging sync exits. -To build this example, run +## Prerequisite +You will need to have `qemu-img` and `arm-none-eabi-gcc` installed. + +On Ubuntu and Debian, you will need to run ```bash -cargo build --release -cd example; sh build.sh; cd .. +sudo apt update +sudo apt -y install qemu-utils gcc-arm-none-eabi ``` -This will build the the fuzzer (src/fuzzer.rs) and a small example binary based on FreeRTOS, which can run under a qemu emulation target. +## Build + +```bash +cargo make build +``` ## Run -Since the instrumentation is based on snapshtos QEMU needs a virtual drive (even if it is unused...). -Create on and then run the fuzzer: ```bash -# create an image -qemu-img create -f qcow2 dummy.qcow2 32M -# run the fuzzer -KERNEL=./example/example.elf target/release/qemu_systemmode -icount shift=auto,align=off,sleep=off -machine mps2-an385 -monitor null -kernel ./example/example.elf -serial null -nographic -snapshot -drive if=none,format=qcow2,file=dummy.qcow2 -S +cargo make run ``` -Currently the ``KERNEL`` variable is needed because the fuzzer does not parse QEMUs arguments to find the binary. \ No newline at end of file + +It is also possible to run the fuzzer with the other features: + +```bash +cargo make +``` + +With feature being `classic`, `breakpoint` or `sync_exit`. + +This will build the desired fuzzer (src/fuzzer_.rs) and a small example binary based on FreeRTOS, which can run under a qemu emulation target. +Since the instrumentation is based on snapshots, QEMU needs a virtual drive (even if it is unused...). +Thus, the makefile creates a dummy QCOW2 image `dummy.qcow2` (can be found in the `target directory`). +Currently, the ``KERNEL`` variable is needed because the fuzzer does not parse QEMUs arguments to find the binary. +It is automatically set in the build script. \ No newline at end of file diff --git a/fuzzers/qemu_systemmode/build.rs b/fuzzers/qemu_systemmode/build.rs new file mode 100644 index 0000000000..9f7f8443ac --- /dev/null +++ b/fuzzers/qemu_systemmode/build.rs @@ -0,0 +1,19 @@ +use libafl_qemu_build::build_libafl_qemu; + +#[macro_export] +macro_rules! assert_unique_feature { + () => {}; + ($first:tt $(,$rest:tt)*) => { + $( + #[cfg(all(not(any(doc, feature = "clippy")), feature = $first, feature = $rest))] + compile_error!(concat!("features \"", $first, "\" and \"", $rest, "\" cannot be used together")); + )* + assert_unique_feature!($($rest),*); + } +} + +fn main() { + assert_unique_feature!("classic", "breakpoint", "sync_exit"); + + build_libafl_qemu(); +} diff --git a/fuzzers/qemu_systemmode/example/build.sh b/fuzzers/qemu_systemmode/example/build.sh deleted file mode 100755 index ceb387b19a..0000000000 --- a/fuzzers/qemu_systemmode/example/build.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -arm-none-eabi-gcc -ggdb -ffreestanding -nostartfiles -lgcc -T mps2_m3.ld -mcpu=cortex-m3 main.c startup.c -o example.elf \ No newline at end of file diff --git a/fuzzers/qemu_systemmode/example/main.c b/fuzzers/qemu_systemmode/example/main.c index 13777fb35a..40944fab37 100644 --- a/fuzzers/qemu_systemmode/example/main.c +++ b/fuzzers/qemu_systemmode/example/main.c @@ -1,8 +1,15 @@ +#ifdef TARGET_SYNC_EXIT + #include "libafl_qemu.h" +#endif + int __attribute__((noinline)) BREAKPOINT() { for (;;) {} } int LLVMFuzzerTestOneInput(unsigned int *Data, unsigned int Size) { +#ifdef TARGET_SYNC_EXIT + LIBAFL_QEMU_START_PHYS((unsigned int)Data, Size); +#endif if (Data[3] == 0) { while (1) {} } // cause a timeout @@ -19,6 +26,9 @@ int LLVMFuzzerTestOneInput(unsigned int *Data, unsigned int Size) { } } } +#ifdef TARGET_SYNC_EXIT + LIBAFL_QEMU_END(LIBAFL_QEMU_END_OK); +#endif return BREAKPOINT(); } unsigned int FUZZ_INPUT[] = { diff --git a/fuzzers/qemu_systemmode/src/fuzzer_breakpoint.rs b/fuzzers/qemu_systemmode/src/fuzzer_breakpoint.rs new file mode 100644 index 0000000000..09dc99a269 --- /dev/null +++ b/fuzzers/qemu_systemmode/src/fuzzer_breakpoint.rs @@ -0,0 +1,275 @@ +//! A fuzzer using qemu in systemmode for binary-only coverage of kernels +//! +use core::{ptr::addr_of_mut, time::Duration}; +use std::{env, path::PathBuf, process}; + +use libafl::{ + corpus::{Corpus, InMemoryCorpus, OnDiskCorpus}, + events::{launcher::Launcher, EventConfig, CTRL_C_EXIT}, + executors::ExitKind, + feedback_or, feedback_or_fast, + feedbacks::{CrashFeedback, MaxMapFeedback, TimeFeedback, TimeoutFeedback}, + fuzzer::{Fuzzer, StdFuzzer}, + inputs::BytesInput, + monitors::MultiMonitor, + mutators::scheduled::{havoc_mutations, StdScheduledMutator}, + observers::{CanTrack, HitcountsMapObserver, TimeObserver, VariableMapObserver}, + schedulers::{IndexesLenTimeMinimizerScheduler, QueueScheduler}, + stages::{CalibrationStage, StdMutationalStage}, + state::{HasCorpus, StdState}, + Error, +}; +use libafl_bolts::{ + core_affinity::Cores, + current_nanos, + ownedref::OwnedMutSlice, + rands::StdRand, + shmem::{ShMemProvider, StdShMemProvider}, + tuples::tuple_list, +}; +use libafl_qemu::{ + breakpoint::Breakpoint, + command::{Command, EmulatorMemoryChunk, EndCommand, StartCommand}, + edges::{edges_map_mut_ptr, QemuEdgeCoverageHelper, EDGES_MAP_SIZE_IN_USE, MAX_EDGES_FOUND}, + elf::EasyElf, + emu::Emulator, + executor::{stateful::StatefulQemuExecutor, QemuExecutorState}, + EmuExitReasonError, FastSnapshotManager, GuestPhysAddr, GuestReg, HandlerError, HandlerResult, + QemuHooks, StdEmuExitHandler, +}; + +// use libafl_qemu::QemuSnapshotBuilder; // for normal qemu snapshot + +pub static mut MAX_INPUT_SIZE: usize = 50; + +pub fn fuzz() { + env_logger::init(); + + if let Ok(s) = env::var("FUZZ_SIZE") { + str::parse::(&s).expect("FUZZ_SIZE was not a number"); + }; + // Hardcoded parameters + let timeout = Duration::from_secs(3); + let broker_port = 1337; + let cores = Cores::from_cmdline("1").unwrap(); + let corpus_dirs = [PathBuf::from("./corpus")]; + let objective_dir = PathBuf::from("./crashes"); + + let mut elf_buffer = Vec::new(); + let elf = EasyElf::from_file( + env::var("KERNEL").expect("KERNEL env not set"), + &mut elf_buffer, + ) + .unwrap(); + + let input_addr = elf + .resolve_symbol( + &env::var("FUZZ_INPUT").unwrap_or_else(|_| "FUZZ_INPUT".to_owned()), + 0, + ) + .expect("Symbol or env FUZZ_INPUT not found") as GuestPhysAddr; + println!("FUZZ_INPUT @ {input_addr:#x}"); + + let main_addr = elf + .resolve_symbol("main", 0) + .expect("Symbol main not found"); + println!("main address = {main_addr:#x}"); + + let breakpoint = elf + .resolve_symbol( + &env::var("BREAKPOINT").unwrap_or_else(|_| "BREAKPOINT".to_owned()), + 0, + ) + .expect("Symbol or env BREAKPOINT not found"); + println!("Breakpoint address = {breakpoint:#x}"); + + let mut run_client = |state: Option<_>, mut mgr, _core_id| { + // Initialize QEMU + let args: Vec = env::args().collect(); + let env: Vec<(String, String)> = env::vars().collect(); + + // Choose Snapshot Builder + // let emu_snapshot_manager = QemuSnapshotBuilder::new(true); + let emu_snapshot_manager = FastSnapshotManager::new(false); + + // Choose Exit Handler + let emu_exit_handler = StdEmuExitHandler::new(emu_snapshot_manager); + + // Create emulator + let emu = Emulator::new(&args, &env, emu_exit_handler).unwrap(); + + // Set breakpoints of interest with corresponding commands. + emu.add_breakpoint( + Breakpoint::with_command( + main_addr, + Command::StartCommand(StartCommand::new(EmulatorMemoryChunk::phys( + input_addr, + unsafe { MAX_INPUT_SIZE } as GuestReg, + None, + ))), + true, + ), + true, + ); + emu.add_breakpoint( + Breakpoint::with_command( + breakpoint, + Command::EndCommand(EndCommand::new(Some(ExitKind::Ok))), + false, + ), + true, + ); + + let devices = emu.list_devices(); + println!("Devices = {:?}", devices); + + // The wrapped harness function, calling out to the LLVM-style harness + let mut harness = + |input: &BytesInput, qemu_executor_state: &mut QemuExecutorState<_, _>| unsafe { + match emu.run(input, qemu_executor_state) { + Ok(handler_result) => match handler_result { + HandlerResult::UnhandledExit(unhandled_exit) => { + panic!("Unhandled exit: {}", unhandled_exit) + } + HandlerResult::EndOfRun(exit_kind) => return exit_kind, + HandlerResult::Interrupted => { + std::process::exit(CTRL_C_EXIT); + } + }, + Err(handler_error) => match handler_error { + HandlerError::QemuExitReasonError(emu_exit_reason_error) => { + match emu_exit_reason_error { + EmuExitReasonError::UnknownKind => panic!("unknown kind"), + EmuExitReasonError::UnexpectedExit => return ExitKind::Crash, + _ => { + panic!("Emu Exit unhandled error: {:?}", emu_exit_reason_error) + } + } + } + _ => panic!("Unhandled error: {:?}", handler_error), + }, + } + }; + + // Create an observation channel using the coverage map + let edges_observer = unsafe { + HitcountsMapObserver::new(VariableMapObserver::from_mut_slice( + "edges", + OwnedMutSlice::from_raw_parts_mut(edges_map_mut_ptr(), EDGES_MAP_SIZE_IN_USE), + addr_of_mut!(MAX_EDGES_FOUND), + )) + .track_indices() + }; + + // Create an observation channel to keep track of the execution time + let time_observer = TimeObserver::new("time"); + + // Feedback to rate the interestingness of an input + // This one is composed by two Feedbacks in OR + let mut feedback = feedback_or!( + // New maximization map feedback linked to the edges observer and the feedback state + MaxMapFeedback::new(&edges_observer), + // Time feedback, this one does not need a feedback state + TimeFeedback::new(&time_observer) + ); + + // A feedback to choose if an input is a solution or not + let mut objective = feedback_or_fast!(CrashFeedback::new(), TimeoutFeedback::new()); + + // If not restarting, create a State from scratch + let mut state = state.unwrap_or_else(|| { + StdState::new( + // RNG + StdRand::with_seed(current_nanos()), + // Corpus that will be evolved, we keep it in memory for performance + InMemoryCorpus::new(), + // Corpus in which we store solutions (crashes in this example), + // on disk so the user can get them after stopping the fuzzer + OnDiskCorpus::new(objective_dir.clone()).unwrap(), + // States of the feedbacks. + // The feedbacks can report the data that should persist in the State. + &mut feedback, + // Same for objective feedbacks + &mut objective, + ) + .unwrap() + }); + + // A minimization+queue policy to get testcasess from the corpus + let scheduler = + IndexesLenTimeMinimizerScheduler::new(&edges_observer, QueueScheduler::new()); + + // A fuzzer with feedbacks and a corpus scheduler + let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); + + let mut hooks = QemuHooks::new( + emu.qemu().clone(), + tuple_list!(QemuEdgeCoverageHelper::default()), + ); + + // Setup an havoc mutator with a mutational stage + let mutator = StdScheduledMutator::new(havoc_mutations()); + let calibration_feedback = MaxMapFeedback::new(&edges_observer); + let mut stages = tuple_list!( + StdMutationalStage::new(mutator), + CalibrationStage::new(&calibration_feedback) + ); + + // Create a QEMU in-process executor + let mut executor = StatefulQemuExecutor::new( + &mut hooks, + &mut harness, + tuple_list!(edges_observer, time_observer), + &mut fuzzer, + &mut state, + &mut mgr, + timeout, + ) + .expect("Failed to create QemuExecutor"); + + // Instead of calling the timeout handler and restart the process, trigger a breakpoint ASAP + executor.break_on_timeout(); + + if state.must_load_initial_inputs() { + state + .load_initial_inputs(&mut fuzzer, &mut executor, &mut mgr, &corpus_dirs) + .unwrap_or_else(|_| { + println!("Failed to load initial corpus at {:?}", &corpus_dirs); + process::exit(0); + }); + println!("We imported {} inputs from disk.", state.corpus().count()); + } + + fuzzer + .fuzz_loop(&mut stages, &mut executor, &mut state, &mut mgr) + .unwrap(); + Ok(()) + }; + + // The shared memory allocator + let shmem_provider = StdShMemProvider::new().expect("Failed to init shared memory"); + + // The stats reporter for the broker + let monitor = MultiMonitor::new(|s| println!("{s}")); + + // let monitor = SimpleMonitor::new(|s| println!("{s}")); + // let mut mgr = SimpleEventManager::new(monitor); + // run_client(None, mgr, 0); + + // Build and run a Launcher + match Launcher::builder() + .shmem_provider(shmem_provider) + .broker_port(broker_port) + .configuration(EventConfig::from_build_id()) + .monitor(monitor) + .run_client(&mut run_client) + .cores(&cores) + // .stdout_file(Some("/dev/null")) + .build() + .launch() + { + Ok(()) => (), + Err(Error::ShuttingDown) => println!("Fuzzing stopped by user. Good bye."), + Err(err) => panic!("Failed to run launcher: {err:?}"), + } +} diff --git a/fuzzers/qemu_systemmode/src/fuzzer.rs b/fuzzers/qemu_systemmode/src/fuzzer_classic.rs similarity index 80% rename from fuzzers/qemu_systemmode/src/fuzzer.rs rename to fuzzers/qemu_systemmode/src/fuzzer_classic.rs index b4749ae831..362a3d73d0 100644 --- a/fuzzers/qemu_systemmode/src/fuzzer.rs +++ b/fuzzers/qemu_systemmode/src/fuzzer_classic.rs @@ -5,7 +5,7 @@ use std::{env, path::PathBuf, process}; use libafl::{ corpus::{Corpus, InMemoryCorpus, OnDiskCorpus}, - events::{launcher::Launcher, EventConfig}, + events::{launcher::Launcher, EventConfig, CTRL_C_EXIT}, executors::ExitKind, feedback_or, feedback_or_fast, feedbacks::{CrashFeedback, MaxMapFeedback, TimeFeedback, TimeoutFeedback}, @@ -13,7 +13,7 @@ use libafl::{ inputs::{BytesInput, HasTargetBytes}, monitors::MultiMonitor, mutators::scheduled::{havoc_mutations, StdScheduledMutator}, - observers::{HitcountsMapObserver, TimeObserver, VariableMapObserver}, + observers::{CanTrack, HitcountsMapObserver, TimeObserver, VariableMapObserver}, schedulers::{IndexesLenTimeMinimizerScheduler, QueueScheduler}, stages::StdMutationalStage, state::{HasCorpus, StdState}, @@ -22,17 +22,20 @@ use libafl::{ use libafl_bolts::{ core_affinity::Cores, current_nanos, + os::unix_signals::Signal, + ownedref::OwnedMutSlice, rands::StdRand, shmem::{ShMemProvider, StdShMemProvider}, tuples::tuple_list, AsSlice, }; use libafl_qemu::{ - edges::{edges_map_mut_slice, QemuEdgeCoverageHelper, MAX_EDGES_NUM}, + edges::{edges_map_mut_ptr, QemuEdgeCoverageHelper, EDGES_MAP_SIZE_IN_USE, MAX_EDGES_FOUND}, elf::EasyElf, - emu::Emulator, - GuestPhysAddr, QemuExecutor, QemuHooks, Regs, + emu::Qemu, + QemuExecutor, QemuExitReason, QemuExitReasonError, QemuHooks, QemuShutdownCause, Regs, }; +use libafl_qemu_sys::GuestPhysAddr; pub static mut MAX_INPUT_SIZE: usize = 50; @@ -83,17 +86,20 @@ pub fn fuzz() { // Initialize QEMU let args: Vec = env::args().collect(); let env: Vec<(String, String)> = env::vars().collect(); - let emu = Emulator::new(&args, &env).unwrap(); + let qemu = Qemu::init(&args, &env).unwrap(); - emu.set_breakpoint(main_addr); + qemu.set_breakpoint(main_addr); unsafe { - emu.run().unwrap(); + match qemu.run() { + Ok(QemuExitReason::Breakpoint(_)) => {} + _ => panic!("Unexpected QEMU exit."), + } } - emu.remove_breakpoint(main_addr); + qemu.remove_breakpoint(main_addr); - emu.set_breakpoint(breakpoint); // BREAKPOINT + qemu.set_breakpoint(breakpoint); // BREAKPOINT - let devices = emu.list_devices(); + let devices = qemu.list_devices(); println!("Devices = {devices:?}"); // let saved_cpu_states: Vec<_> = (0..emu.num_cpus()) @@ -102,7 +108,7 @@ pub fn fuzz() { // emu.save_snapshot("start", true); - let snap = emu.create_fast_snapshot(true); + let snap = qemu.create_fast_snapshot(true); // The wrapped harness function, calling out to the LLVM-style harness let mut harness = |input: &BytesInput| { @@ -115,13 +121,20 @@ pub fn fuzz() { // len = MAX_INPUT_SIZE; } - emu.write_phys_mem(input_addr, buf); + qemu.write_phys_mem(input_addr, buf); - emu.run().unwrap(); + match qemu.run() { + Ok(QemuExitReason::Breakpoint(_)) => {} + Ok(QemuExitReason::End(QemuShutdownCause::HostSignal( + Signal::SigInterrupt, + ))) => process::exit(CTRL_C_EXIT), + Err(QemuExitReasonError::UnexpectedExit) => return ExitKind::Crash, + _ => panic!("Unexpected QEMU exit."), + } // If the execution stops at any point other then the designated breakpoint (e.g. a breakpoint on a panic method) we consider it a crash - let mut pcs = (0..emu.num_cpus()) - .map(|i| emu.cpu_from_index(i)) + let mut pcs = (0..qemu.num_cpus()) + .map(|i| qemu.cpu_from_index(i)) .map(|cpu| -> Result { cpu.read_reg(Regs::Pc) }); let ret = match pcs .find(|pc| (breakpoint..breakpoint + 5).contains(pc.as_ref().unwrap_or(&0))) @@ -139,7 +152,7 @@ pub fn fuzz() { // emu.load_snapshot("start", true); // OPTION 3: restore a fast devices+mem snapshot - emu.restore_fast_snapshot(snap); + qemu.restore_fast_snapshot(snap); ret } @@ -149,9 +162,10 @@ pub fn fuzz() { let edges_observer = unsafe { HitcountsMapObserver::new(VariableMapObserver::from_mut_slice( "edges", - edges_map_mut_slice(), - addr_of_mut!(MAX_EDGES_NUM), + OwnedMutSlice::from_raw_parts_mut(edges_map_mut_ptr(), EDGES_MAP_SIZE_IN_USE), + addr_of_mut!(MAX_EDGES_FOUND), )) + .track_indices() }; // Create an observation channel to keep track of the execution time @@ -161,9 +175,9 @@ pub fn fuzz() { // This one is composed by two Feedbacks in OR let mut feedback = feedback_or!( // New maximization map feedback linked to the edges observer and the feedback state - MaxMapFeedback::tracking(&edges_observer, true, true), + MaxMapFeedback::new(&edges_observer), // Time feedback, this one does not need a feedback state - TimeFeedback::with_observer(&time_observer) + TimeFeedback::new(&time_observer) ); // A feedback to choose if an input is a solution or not @@ -189,12 +203,14 @@ pub fn fuzz() { }); // A minimization+queue policy to get testcasess from the corpus - let scheduler = IndexesLenTimeMinimizerScheduler::new(QueueScheduler::new()); + let scheduler = + IndexesLenTimeMinimizerScheduler::new(&edges_observer, QueueScheduler::new()); // A fuzzer with feedbacks and a corpus scheduler let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); - let mut hooks = QemuHooks::new(emu.clone(), tuple_list!(QemuEdgeCoverageHelper::default())); + let mut hooks = + QemuHooks::new(qemu.clone(), tuple_list!(QemuEdgeCoverageHelper::default())); // Create a QEMU in-process executor let mut executor = QemuExecutor::new( diff --git a/fuzzers/qemu_systemmode/src/fuzzer_sync_exit.rs b/fuzzers/qemu_systemmode/src/fuzzer_sync_exit.rs new file mode 100644 index 0000000000..c6d2dacfd2 --- /dev/null +++ b/fuzzers/qemu_systemmode/src/fuzzer_sync_exit.rs @@ -0,0 +1,216 @@ +//! A fuzzer using qemu in systemmode for binary-only coverage of kernels +//! +use core::{ptr::addr_of_mut, time::Duration}; +use std::{env, path::PathBuf, process}; + +use libafl::{ + corpus::{Corpus, InMemoryCorpus, OnDiskCorpus}, + events::{launcher::Launcher, EventConfig, CTRL_C_EXIT}, + executors::ExitKind, + feedback_or, feedback_or_fast, + feedbacks::{CrashFeedback, MaxMapFeedback, TimeFeedback, TimeoutFeedback}, + fuzzer::{Fuzzer, StdFuzzer}, + inputs::BytesInput, + monitors::MultiMonitor, + mutators::scheduled::{havoc_mutations, StdScheduledMutator}, + observers::{CanTrack, HitcountsMapObserver, TimeObserver, VariableMapObserver}, + schedulers::{IndexesLenTimeMinimizerScheduler, QueueScheduler}, + stages::{CalibrationStage, StdMutationalStage}, + state::{HasCorpus, StdState}, + Error, +}; +use libafl_bolts::{ + core_affinity::Cores, + current_nanos, + ownedref::OwnedMutSlice, + rands::StdRand, + shmem::{ShMemProvider, StdShMemProvider}, + tuples::tuple_list, +}; +use libafl_qemu::{ + edges::{edges_map_mut_ptr, QemuEdgeCoverageHelper, EDGES_MAP_SIZE_IN_USE, MAX_EDGES_FOUND}, + emu::Emulator, + executor::{stateful::StatefulQemuExecutor, QemuExecutorState}, + EmuExitReasonError, FastSnapshotManager, HandlerError, HandlerResult, QemuHooks, + StdEmuExitHandler, +}; + +// use libafl_qemu::QemuSnapshotBuilder; for normal qemu snapshot + +pub fn fuzz() { + env_logger::init(); + + if let Ok(s) = env::var("FUZZ_SIZE") { + str::parse::(&s).expect("FUZZ_SIZE was not a number"); + }; + // Hardcoded parameters + let timeout = Duration::from_secs(3); + let broker_port = 1337; + let cores = Cores::from_cmdline("1").unwrap(); + let corpus_dirs = [PathBuf::from("./corpus")]; + let objective_dir = PathBuf::from("./crashes"); + + let mut run_client = |state: Option<_>, mut mgr, _core_id| { + // Initialize QEMU + let args: Vec = env::args().collect(); + let env: Vec<(String, String)> = env::vars().collect(); + // let emu_snapshot_manager = QemuSnapshotBuilder::new(true); + let emu_snapshot_manager = FastSnapshotManager::new(false); // Create a snapshot manager (normal or fast for now). + let emu_exit_handler: StdEmuExitHandler = + StdEmuExitHandler::new(emu_snapshot_manager); // Create an exit handler: it is the entity taking the decision of what should be done when QEMU returns. + let emu = Emulator::new(&args, &env, emu_exit_handler).unwrap(); // Create the emulator + + let devices = emu.list_devices(); + println!("Devices = {:?}", devices); + + // The wrapped harness function, calling out to the LLVM-style harness + let mut harness = + |input: &BytesInput, qemu_executor_state: &mut QemuExecutorState<_, _>| unsafe { + match emu.run(input, qemu_executor_state) { + Ok(handler_result) => match handler_result { + HandlerResult::UnhandledExit(unhandled_exit) => { + panic!("Unhandled exit: {}", unhandled_exit) + } + HandlerResult::EndOfRun(exit_kind) => exit_kind, + HandlerResult::Interrupted => { + println!("Interrupted."); + std::process::exit(CTRL_C_EXIT); + } + }, + Err(handler_error) => match handler_error { + HandlerError::QemuExitReasonError(emu_exit_reason_error) => { + match emu_exit_reason_error { + EmuExitReasonError::UnknownKind => panic!("unknown kind"), + EmuExitReasonError::UnexpectedExit => ExitKind::Crash, + _ => { + panic!("Emu Exit unhandled error: {:?}", emu_exit_reason_error) + } + } + } + _ => panic!("Unhandled error: {:?}", handler_error), + }, + } + }; + + // Create an observation channel using the coverage map + let edges_observer = unsafe { + HitcountsMapObserver::new(VariableMapObserver::from_mut_slice( + "edges", + OwnedMutSlice::from_raw_parts_mut(edges_map_mut_ptr(), EDGES_MAP_SIZE_IN_USE), + addr_of_mut!(MAX_EDGES_FOUND), + )) + .track_indices() + }; + + // Create an observation channel to keep track of the execution time + let time_observer = TimeObserver::new("time"); + + // Feedback to rate the interestingness of an input + // This one is composed by two Feedbacks in OR + let mut feedback = feedback_or!( + // New maximization map feedback linked to the edges observer and the feedback state + MaxMapFeedback::new(&edges_observer), + // Time feedback, this one does not need a feedback state + TimeFeedback::new(&time_observer) + ); + + // A feedback to choose if an input is a solution or not + let mut objective = feedback_or_fast!(CrashFeedback::new(), TimeoutFeedback::new()); + + // If not restarting, create a State from scratch + let mut state = state.unwrap_or_else(|| { + StdState::new( + // RNG + StdRand::with_seed(current_nanos()), + // Corpus that will be evolved, we keep it in memory for performance + InMemoryCorpus::new(), + // Corpus in which we store solutions (crashes in this example), + // on disk so the user can get them after stopping the fuzzer + OnDiskCorpus::new(objective_dir.clone()).unwrap(), + // States of the feedbacks. + // The feedbacks can report the data that should persist in the State. + &mut feedback, + // Same for objective feedbacks + &mut objective, + ) + .unwrap() + }); + + // A minimization+queue policy to get testcasess from the corpus + let scheduler = + IndexesLenTimeMinimizerScheduler::new(&edges_observer, QueueScheduler::new()); + + // A fuzzer with feedbacks and a corpus scheduler + let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); + + let mut hooks = QemuHooks::new( + emu.qemu().clone(), + tuple_list!(QemuEdgeCoverageHelper::default()), + ); + + // Setup an havoc mutator with a mutational stage + let mutator = StdScheduledMutator::new(havoc_mutations()); + let calibration_feedback = MaxMapFeedback::new(&edges_observer); + let mut stages = tuple_list!( + StdMutationalStage::new(mutator), + CalibrationStage::new(&calibration_feedback) + ); + + // Create a QEMU in-process executor + let mut executor = StatefulQemuExecutor::new( + &mut hooks, + &mut harness, + tuple_list!(edges_observer, time_observer), + &mut fuzzer, + &mut state, + &mut mgr, + timeout, + ) + .expect("Failed to create QemuExecutor"); + + // Instead of calling the timeout handler and restart the process, trigger a breakpoint ASAP + executor.break_on_timeout(); + + if state.must_load_initial_inputs() { + state + .load_initial_inputs(&mut fuzzer, &mut executor, &mut mgr, &corpus_dirs) + .unwrap_or_else(|_| { + println!("Failed to load initial corpus at {:?}", &corpus_dirs); + process::exit(0); + }); + println!("We imported {} inputs from disk.", state.corpus().count()); + } + + fuzzer + .fuzz_loop(&mut stages, &mut executor, &mut state, &mut mgr) + .unwrap(); + Ok(()) + }; + + // The shared memory allocator + let shmem_provider = StdShMemProvider::new().expect("Failed to init shared memory"); + + // The stats reporter for the broker + let monitor = MultiMonitor::new(|s| println!("{s}")); + + // let monitor = SimpleMonitor::new(|s| println!("{s}")); + // let mut mgr = SimpleEventManager::new(monitor); + // run_client(None, mgr, 0); + + // Build and run a Launcher + match Launcher::builder() + .shmem_provider(shmem_provider) + .broker_port(broker_port) + .configuration(EventConfig::from_build_id()) + .monitor(monitor) + .run_client(&mut run_client) + .cores(&cores) + // .stdout_file(Some("/dev/null")) + .build() + .launch() + { + Ok(()) => (), + Err(Error::ShuttingDown) => println!("Fuzzing stopped by user. Good bye."), + Err(err) => panic!("Failed to run launcher: {err:?}"), + } +} diff --git a/fuzzers/qemu_systemmode/src/main.rs b/fuzzers/qemu_systemmode/src/main.rs index bc1e80f767..8ca48d6eb6 100644 --- a/fuzzers/qemu_systemmode/src/main.rs +++ b/fuzzers/qemu_systemmode/src/main.rs @@ -1,10 +1,23 @@ //! A libfuzzer-like fuzzer using qemu for binary-only coverage -#[cfg(target_os = "linux")] -mod fuzzer; +#[cfg(all(target_os = "linux", feature = "classic"))] +mod fuzzer_classic; + +#[cfg(all(target_os = "linux", feature = "breakpoint"))] +mod fuzzer_breakpoint; + +#[cfg(all(target_os = "linux", feature = "sync_exit"))] +mod fuzzer_sync_exit; #[cfg(target_os = "linux")] pub fn main() { - fuzzer::fuzz(); + #[cfg(feature = "classic")] + fuzzer_classic::fuzz(); + + #[cfg(feature = "breakpoint")] + fuzzer_breakpoint::fuzz(); + + #[cfg(feature = "sync_exit")] + fuzzer_sync_exit::fuzz(); } #[cfg(not(target_os = "linux"))] diff --git a/fuzzers/tinyinst_simple/Cargo.toml b/fuzzers/tinyinst_simple/Cargo.toml index 2ef281ad47..236ff162a6 100644 --- a/fuzzers/tinyinst_simple/Cargo.toml +++ b/fuzzers/tinyinst_simple/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tinyinst_simple" -version = "0.11.2" +version = "0.12.0" edition = "2021" [dependencies] diff --git a/fuzzers/tinyinst_simple/Makefile.toml b/fuzzers/tinyinst_simple/Makefile.toml index ed85b91758..7b6aefc1a0 100644 --- a/fuzzers/tinyinst_simple/Makefile.toml +++ b/fuzzers/tinyinst_simple/Makefile.toml @@ -1,6 +1,7 @@ [env] PROFILE = { value = "release", condition = {env_not_set = ["PROFILE"]} } -PROFILE_DIR = {value = "release", condition = {env_not_set = ["PROFILE_DIR"] }} +PROFILE_DIR = { value = "release", condition = {env_not_set = ["PROFILE_DIR"] }} +CARGO_TARGET_DIR = { value = "target", condition = { env_not_set = ["CARGO_TARGET_DIR"] } } [tasks.unsupported] script_runner="@shell" @@ -10,10 +11,15 @@ echo "Cargo-make not integrated yet on this" # Harness [tasks.harness] -linux_alias = "unsupported" +linux_alias = "harness_linux" mac_alias = "unsupported" windows_alias = "harness_windows" +[tasks.harness_linux] +script=''' +clang test/test.cpp -o test.exe +''' + [tasks.harness_windows] script=''' cl test\test.cpp -o test.exe @@ -21,10 +27,15 @@ cl test\test.cpp -o test.exe # Fuzzer [tasks.fuzzer] -linux_alias = "unsupported" +linux_alias = "fuzzer_linux" mac_alias = "unsupported" windows_alias = "fuzzer_windows" +[tasks.fuzzer_linux] +dependencies = ["harness"] +command = "cargo" +args = ["build", "--profile", "${PROFILE}"] + [tasks.fuzzer_windows] dependencies = ["harness"] command = "cargo" @@ -32,10 +43,15 @@ args = ["build", "--profile", "${PROFILE}"] # Run the fuzzer [tasks.run] -linux_alias = "unsupported" +linux_alias = "run_linux" mac_alias = "unsupported" windows_alias = "run_windows" +[tasks.run_linux] +dependencies = ["harness", "fuzzer"] +command = "cargo" +args = ["run", "--profile", "${PROFILE}"] + [tasks.run_windows] dependencies = ["harness", "fuzzer"] command = "cargo" @@ -44,10 +60,25 @@ args = ["run", "--profile", "${PROFILE}"] # Run the fuzzer [tasks.test] -linux_alias = "unsupported" +linux_alias = "test_linux" mac_alias = "unsupported" windows_alias = "test_windows" +[tasks.test_linux] +script_runner="@shell" +script=''' +cp ${CARGO_TARGET_DIR}/${PROFILE_DIR}/tinyinst_simple . +echo running tests +timeout 5s ./tinyinst_simple || true +# corpus_discovered folder exists and is not empty +if [ -d "corpus_discovered" ] && [ -n "$(ls -A corpus_discovered)" ]; then + echo "Fuzzer works!" +else + exit 1 +fi +''' +dependencies = ["harness", "fuzzer"] + [tasks.test_windows] script_runner = "@shell" script=''' @@ -57,4 +88,4 @@ start "" "tinyinst_simple.exe" ping -n 10 127.0.0.1>NUL && taskkill /im tinyinst_simple.exe /F >nul 2>nul dir /a-d "corpus_discovered\*" && (echo Files exist) || (exit /b 1337) ''' -dependencies = [ "harness", "fuzzer" ] \ No newline at end of file +dependencies = ["harness", "fuzzer"] \ No newline at end of file diff --git a/fuzzers/tinyinst_simple/README.md b/fuzzers/tinyinst_simple/README.md index d20943d3e6..9d507df6c2 100644 --- a/fuzzers/tinyinst_simple/README.md +++ b/fuzzers/tinyinst_simple/README.md @@ -1,11 +1,12 @@ # Tinyinst example -This is a fuzzer example to show how libafl_tinyinst works +This is a fuzzer example to show how libafl_tinyinst works. ## How to build -1. Build the harness with `cl test\test.cpp -o test.exe` -2. Build the fuzzer with `cargo build --release`. The fuzzer is `target\release\tinyinst_simple.exe` +1. Install cxxbridge-cmd with `cargo install cxxbridge-cmd` +2. Build the harness with `cl test\test.cpp -o test.exe` +3. Build the fuzzer with `cargo build --release`. The fuzzer is `target\release\tinyinst_simple.exe` ## Run with cargo-make -Or, you can simple run it using cargo-make -1. Open up developer powershell so that you have access to cl (Windows Default Compiler) -2. Run `cargo make run` to run the fuzzer \ No newline at end of file +Or, you can simply run it using cargo-make +1. If on Windows, open up a developer powershell so that you have access to cl (Windows Default Compiler) +2. Run `cargo make run` to run the fuzzer diff --git a/fuzzers/tinyinst_simple/src/main.rs b/fuzzers/tinyinst_simple/src/main.rs index 60bf2307b5..a0d7349296 100644 --- a/fuzzers/tinyinst_simple/src/main.rs +++ b/fuzzers/tinyinst_simple/src/main.rs @@ -13,22 +13,20 @@ use libafl::{ state::StdState, Fuzzer, StdFuzzer, }; -#[cfg(target_vendor = "apple")] +#[cfg(unix)] use libafl_bolts::shmem::UnixShMemProvider; #[cfg(windows)] use libafl_bolts::shmem::Win32ShMemProvider; use libafl_bolts::{ - rands::{RandomSeed, StdRand}, - shmem::ShMemProvider, - tuples::tuple_list, + ownedref::OwnedMutPtr, rands::StdRand, shmem::ShMemProvider, tuples::tuple_list, }; use libafl_tinyinst::executor::TinyInstExecutorBuilder; static mut COVERAGE: Vec = vec![]; -#[cfg(not(any(target_vendor = "apple", windows)))] +#[cfg(not(any(target_vendor = "apple", windows, target_os = "linux")))] fn main() {} -#[cfg(any(target_vendor = "apple", windows))] +#[cfg(any(target_vendor = "apple", windows, target_os = "linux"))] fn main() { // Tinyinst things let tinyinst_args = vec!["-instrument_module".to_string(), "test.exe".to_string()]; @@ -39,12 +37,13 @@ fn main() { // use file to pass testcases // let args = vec!["test.exe".to_string(), "-f".to_string(), "@@".to_string()]; - let observer = unsafe { ListObserver::new("cov", &mut COVERAGE) }; - let mut feedback = ListFeedback::with_observer(&observer); + let coverage = unsafe { OwnedMutPtr::Ptr(core::ptr::addr_of_mut!(COVERAGE)) }; + let observer = ListObserver::new("cov", coverage); + let mut feedback = ListFeedback::new(&observer); #[cfg(windows)] let mut shmem_provider = Win32ShMemProvider::new().unwrap(); - #[cfg(target_vendor = "apple")] + #[cfg(unix)] let mut shmem_provider = UnixShMemProvider::new().unwrap(); let input = BytesInput::new(b"bad".to_vec()); diff --git a/fuzzers/tinyinst_simple/test/test.cpp b/fuzzers/tinyinst_simple/test/test.cpp index 9363d9f294..720be9eeea 100644 --- a/fuzzers/tinyinst_simple/test/test.cpp +++ b/fuzzers/tinyinst_simple/test/test.cpp @@ -23,8 +23,9 @@ limitations under the License. #include // shared memory stuff - -#if defined(WIN32) || defined(_WIN32) || defined(__WIN32) +#if defined(__linux__) + #include +#elif defined(WIN32) || defined(_WIN32) || defined(__WIN32) #include #else #include @@ -36,7 +37,19 @@ unsigned char *shm_data; bool use_shared_memory; -#if defined(WIN32) || defined(_WIN32) || defined(__WIN32) +#if defined(__linux__) + +int setup_shmem(const char *name) { + // map shared memory to process address space + shm_data = (unsigned char *)shmat(atoi(name), NULL, 0); + if (shm_data == (void *)-1) { + perror("Error in shmat"); + return 0; + } + return 1; +} + +#elif defined(WIN32) || defined(_WIN32) || defined(__WIN32) int setup_shmem(const char *name) { HANDLE map_file; @@ -71,7 +84,7 @@ int setup_shmem(const char *name) { // get shared memory file descriptor (NOT a file) fd = shm_open(name, O_RDONLY, S_IRUSR | S_IWUSR); if (fd == -1) { - printf("Error in shm_open\n"); + perror("Error in shm_open"); return 0; } @@ -79,7 +92,7 @@ int setup_shmem(const char *name) { shm_data = (unsigned char *)mmap(NULL, SHM_SIZE, PROT_READ, MAP_SHARED, fd, 0); if (shm_data == MAP_FAILED) { - printf("Error in mmap\n"); + perror("Error in mmap"); return 0; } @@ -101,7 +114,12 @@ char *crash = NULL; // actual target function -void FUZZ_TARGET_MODIFIERS fuzz(char *name) { +// Use extern "C" to preserve the function name for instrumentation +#ifdef __cplusplus +extern "C" +#endif // __cplusplus + void FUZZ_TARGET_MODIFIERS + fuzz(char *name) { char *sample_bytes = NULL; uint32_t sample_size = 0; diff --git a/fuzzers/tutorial/Cargo.toml b/fuzzers/tutorial/Cargo.toml index d3db9b8be0..b248b71754 100644 --- a/fuzzers/tutorial/Cargo.toml +++ b/fuzzers/tutorial/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tutorial" -version = "0.11.2" +version = "0.12.0" authors = ["Andrea Fioraldi ", "Dominik Maier "] edition = "2021" diff --git a/fuzzers/tutorial/src/lib.rs b/fuzzers/tutorial/src/lib.rs index 1c7fe08fe7..b60b44eeed 100644 --- a/fuzzers/tutorial/src/lib.rs +++ b/fuzzers/tutorial/src/lib.rs @@ -15,13 +15,13 @@ use libafl::{ fuzzer::StdFuzzer, inputs::HasTargetBytes, monitors::MultiMonitor, - observers::{HitcountsMapObserver, TimeObserver}, + observers::{CanTrack, HitcountsMapObserver, TimeObserver}, schedulers::{powersched::PowerSchedule, PowerQueueScheduler}, stages::{calibrate::CalibrationStage, power::StdPowerMutationalStage}, state::{HasCorpus, StdState}, Error, Fuzzer, }; -use libafl_bolts::{current_nanos, rands::StdRand, tuples::tuple_list, AsSlice}; +use libafl_bolts::{rands::StdRand, tuples::tuple_list, AsSlice}; use libafl_targets::{libfuzzer_initialize, libfuzzer_test_one_input, std_edges_map_observer}; mod input; @@ -81,12 +81,13 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re }; // Create an observation channel using the coverage map - let edges_observer = HitcountsMapObserver::new(unsafe { std_edges_map_observer("edges") }); + let edges_observer = + HitcountsMapObserver::new(unsafe { std_edges_map_observer("edges") }).track_indices(); // Create an observation channel to keep track of the execution time let time_observer = TimeObserver::new("time"); - let map_feedback = MaxMapFeedback::tracking(&edges_observer, true, false); + let map_feedback = MaxMapFeedback::new(&edges_observer); let calibration = CalibrationStage::new(&map_feedback); @@ -96,7 +97,7 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re // New maximization map feedback linked to the edges observer and the feedback state map_feedback, // Time feedback, this one does not need a feedback state - TimeFeedback::with_observer(&time_observer), + TimeFeedback::new(&time_observer), PacketLenFeedback::new() ); @@ -107,7 +108,7 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re let mut state = state.unwrap_or_else(|| { StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance InMemoryCorpus::new(), // Corpus in which we store solutions (crashes in this example), @@ -132,11 +133,10 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re let mut stages = tuple_list!(calibration, power); // A minimization+queue policy to get testcasess from the corpus - let scheduler = PacketLenMinimizerScheduler::new(PowerQueueScheduler::new( - &mut state, + let scheduler = PacketLenMinimizerScheduler::new( &edges_observer, - PowerSchedule::FAST, - )); + PowerQueueScheduler::new(&mut state, &edges_observer, PowerSchedule::FAST), + ); // A fuzzer with feedbacks and a corpus scheduler let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); diff --git a/fuzzers/tutorial/src/metadata.rs b/fuzzers/tutorial/src/metadata.rs index 785a0e701c..2115cdd0b9 100644 --- a/fuzzers/tutorial/src/metadata.rs +++ b/fuzzers/tutorial/src/metadata.rs @@ -1,3 +1,5 @@ +use std::borrow::Cow; + use libafl::{ corpus::Testcase, events::EventFirer, @@ -5,8 +7,8 @@ use libafl::{ feedbacks::{Feedback, MapIndexesMetadata}, observers::ObserversTuple, schedulers::{MinimizerScheduler, TestcaseScore}, - state::{HasCorpus, HasMetadata, State}, - Error, + state::{HasCorpus, State}, + Error, HasMetadata, }; use libafl_bolts::{Named, SerdeAny}; use serde::{Deserialize, Serialize}; @@ -32,8 +34,8 @@ where } } -pub type PacketLenMinimizerScheduler = - MinimizerScheduler; +pub type PacketLenMinimizerScheduler = + MinimizerScheduler; #[derive(Serialize, Deserialize, Default, Clone, Debug)] pub struct PacketLenFeedback { @@ -62,9 +64,10 @@ where } #[inline] - fn append_metadata( + fn append_metadata( &mut self, _state: &mut S, + _manager: &mut EM, _observers: &OT, testcase: &mut Testcase, ) -> Result<(), Error> { @@ -77,8 +80,9 @@ where impl Named for PacketLenFeedback { #[inline] - fn name(&self) -> &str { - "PacketLenFeedback" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("PacketLenFeedback"); + &NAME } } diff --git a/fuzzers/tutorial/src/mutator.rs b/fuzzers/tutorial/src/mutator.rs index ce9d85ad51..46bc7a7248 100644 --- a/fuzzers/tutorial/src/mutator.rs +++ b/fuzzers/tutorial/src/mutator.rs @@ -1,3 +1,5 @@ +use std::borrow::Cow; + use lain::traits::Mutatable; use libafl::{ mutators::{MutationResult, Mutator}, @@ -19,12 +21,7 @@ impl Mutator for LainMutator where S: HasRand, { - fn mutate( - &mut self, - state: &mut S, - input: &mut PacketData, - _stage_idx: i32, - ) -> Result { + fn mutate(&mut self, state: &mut S, input: &mut PacketData) -> Result { // Lain uses its own instance of StdRand, but we want to keep it in sync with LibAFL's state. self.inner.rng_mut().set_seed(state.rand_mut().next()); input.mutate(&mut self.inner, None); @@ -33,8 +30,9 @@ where } impl Named for LainMutator { - fn name(&self) -> &str { - "LainMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("LainMutator"); + &NAME } } diff --git a/libafl/Cargo.toml b/libafl/Cargo.toml index a365de7483..2008ab070b 100644 --- a/libafl/Cargo.toml +++ b/libafl/Cargo.toml @@ -32,9 +32,6 @@ introspection = [] ## Collects stats about scalability scalability_introspection = [] -## Will build the `pyo3` bindings -python = ["pyo3", "concat-idents", "libafl_bolts/python"] - ## Expose `libafl::prelude` for access without additional using directives prelude = ["libafl_bolts/prelude"] @@ -48,7 +45,7 @@ errors_backtrace = ["libafl_bolts/errors_backtrace"] corpus_btreemap = [] ## Enables gzip compression in certain parts of the lib -gzip = ["libafl_bolts/gzip"] +gzip = ["libafl_bolts/gzip"] ## If set, will use the `fork()` syscall to spawn children, instead of launching a new command, if supported by the OS (has no effect on `Windows`). fork = ["libafl_bolts/derive"] @@ -122,7 +119,7 @@ llmp_bind_public = ["libafl_bolts/llmp_bind_public"] llmp_compression = ["libafl_bolts/llmp_compression"] ## Enables debug output for LLMP (also needs a `logger` installed) -llmp_debug = ["libafl_bolts/llmp_debug"] +llmp_debug = ["std", "libafl_bolts/llmp_debug"] ## Reduces the initial map size for llmp llmp_small_maps = ["libafl_bolts/llmp_small_maps"] # reduces initial map size for llmp @@ -136,7 +133,7 @@ agpl = ["nautilus"] nautilus = ["grammartec", "std", "serde_json/std"] [build-dependencies] -reqwest = { version = "0.11", features = ["blocking"], optional = true} +reqwest = { version = "0.11", features = ["blocking"], optional = true } rustversion = "1.0" zip = { version = "0.6", optional = true } @@ -146,20 +143,20 @@ serde_json = { version = "1.0", default-features = false, features = ["alloc"] } bytecount = "0.6.3" [dependencies] -libafl_bolts = { version = "0.11.2", path = "../libafl_bolts", default-features = false, features = ["alloc"] } -libafl_derive = { version = "0.11.2", path = "../libafl_derive", optional = true } +libafl_bolts = { version = "0.12.0", path = "../libafl_bolts", default-features = false, features = ["alloc"] } +libafl_derive = { version = "0.12.0", path = "../libafl_derive", optional = true } rustversion = "1.0" tuple_list = { version = "0.1.3" } -hashbrown = { version = "0.14", features = ["serde", "ahash"], default-features=false } # A faster hashmap, nostd compatible +hashbrown = { version = "0.14", features = ["serde", "ahash"], default-features = false } # A faster hashmap, nostd compatible num-traits = { version = "0.2", default-features = false } serde = { version = "1.0", default-features = false, features = ["alloc", "derive"] } # serialization lib postcard = { version = "1.0", features = ["alloc"], default-features = false } # no_std compatible serde serialization format -bincode = {version = "1.3", optional = true } +bincode = { version = "1.3", optional = true } c2rust-bitfields = { version = "0.18", features = ["no_std"] } -ahash = { version = "0.8", default-features=false } # The hash function already used in hashbrown +ahash = { version = "0.8", default-features = false } # The hash function already used in hashbrown meminterval = { version = "0.4", features = ["serde"] } -backtrace = {version = "0.3", optional = true} # Used to get the stacktrace in StacktraceObserver +backtrace = { version = "0.3", optional = true } # Used to get the stacktrace in StacktraceObserver typed-builder = { version = "0.16", optional = true } # Implement the builder pattern at compiletime serde_json = { version = "1.0", optional = true, default-features = false, features = ["alloc"] } @@ -170,7 +167,7 @@ libm = "0.2.2" ratatui = { version = "0.23.0", default-features = false, features = ['crossterm'], optional = true } # Commandline rendering, for TUI Monitor crossterm = { version = "0.27.0", optional = true } -prometheus-client = { version= "0.21", optional = true} # For the prometheus monitor +prometheus-client = { version = "0.21", optional = true } # For the prometheus monitor tide = { version = "0.16.0", optional = true } async-std = { version = "1.12.0", features = ["attributes"], optional = true } futures = { version = "0.3.24", optional = true } @@ -181,7 +178,6 @@ wait-timeout = { version = "0.2", optional = true } # used by CommandExecutor to z3 = { version = "0.12.0", features = ["static-link-z3"], optional = true } # for concolic mutation -pyo3 = { version = "0.18", optional = true, features = ["serde", "macros"] } concat-idents = { version = "1.1.3", optional = true } libcasr = { version = "2.7", optional = true } @@ -190,6 +186,9 @@ bitvec = { version = "1.0", optional = true, features = ["serde"] } # used for s arrayvec = { version = "0.7.4", optional = true, default-features = false } # used for fixed-len collects +const_format = "0.2.32" # used for providing helpful compiler output +const_panic = "0.2.8" # similarly, for formatting const panic output + # optional-dev deps (change when target.'cfg(accessible(::std))'.test-dependencies will be stable) serial_test = { version = "2", optional = true, default-features = false, features = ["logging"] } diff --git a/libafl/src/common/mod.rs b/libafl/src/common/mod.rs new file mode 100644 index 0000000000..53a85a35cf --- /dev/null +++ b/libafl/src/common/mod.rs @@ -0,0 +1,149 @@ +//! This module defines trait shared across different `LibAFL` modules + +use alloc::boxed::Box; +use core::any::type_name; + +use libafl_bolts::{ + serdeany::{NamedSerdeAnyMap, SerdeAny, SerdeAnyMap}, + Error, +}; +/// Trait for elements offering metadata +pub trait HasMetadata { + /// A map, storing all metadata + fn metadata_map(&self) -> &SerdeAnyMap; + /// A map, storing all metadata (mutable) + fn metadata_map_mut(&mut self) -> &mut SerdeAnyMap; + + /// Add a metadata to the metadata map + #[inline] + fn add_metadata(&mut self, meta: M) + where + M: SerdeAny, + { + self.metadata_map_mut().insert(meta); + } + + /// Gets metadata, or inserts it using the given construction function `default` + fn metadata_or_insert_with(&mut self, default: impl FnOnce() -> M) -> &mut M + where + M: SerdeAny, + { + self.metadata_map_mut().get_or_insert_with::(default) + } + + /// Remove a metadata from the metadata map + #[inline] + fn remove_metadata(&mut self) -> Option> + where + M: SerdeAny, + { + self.metadata_map_mut().remove::() + } + + /// Check for a metadata + /// + /// # Note + /// For performance reasons, you likely want to use [`Self::metadata_or_insert_with`] instead + #[inline] + fn has_metadata(&self) -> bool + where + M: SerdeAny, + { + self.metadata_map().get::().is_some() + } + + /// To get metadata + #[inline] + fn metadata(&self) -> Result<&M, Error> + where + M: SerdeAny, + { + self.metadata_map() + .get::() + .ok_or_else(|| Error::key_not_found(format!("{} not found", type_name::()))) + } + + /// To get mutable metadata + #[inline] + fn metadata_mut(&mut self) -> Result<&mut M, Error> + where + M: SerdeAny, + { + self.metadata_map_mut() + .get_mut::() + .ok_or_else(|| Error::key_not_found(format!("{} not found", type_name::()))) + } +} + +/// Trait for elements offering named metadata +pub trait HasNamedMetadata { + /// A map, storing all metadata + fn named_metadata_map(&self) -> &NamedSerdeAnyMap; + /// A map, storing all metadata (mutable) + fn named_metadata_map_mut(&mut self) -> &mut NamedSerdeAnyMap; + + /// Add a metadata to the metadata map + #[inline] + fn add_named_metadata(&mut self, name: &str, meta: M) + where + M: SerdeAny, + { + self.named_metadata_map_mut().insert(name, meta); + } + + /// Add a metadata to the metadata map + #[inline] + fn remove_named_metadata(&mut self, name: &str) -> Option> + where + M: SerdeAny, + { + self.named_metadata_map_mut().remove::(name) + } + + /// Gets metadata, or inserts it using the given construction function `default` + fn named_metadata_or_insert_with( + &mut self, + name: &str, + default: impl FnOnce() -> M, + ) -> &mut M + where + M: SerdeAny, + { + self.named_metadata_map_mut() + .get_or_insert_with::(name, default) + } + + /// Check for a metadata + /// + /// # Note + /// You likely want to use [`Self::named_metadata_or_insert_with`] for performance reasons. + #[inline] + fn has_named_metadata(&self, name: &str) -> bool + where + M: SerdeAny, + { + self.named_metadata_map().contains::(name) + } + + /// To get named metadata + #[inline] + fn named_metadata(&self, name: &str) -> Result<&M, Error> + where + M: SerdeAny, + { + self.named_metadata_map() + .get::(name) + .ok_or_else(|| Error::key_not_found(format!("{} not found", type_name::()))) + } + + /// To get mutable named metadata + #[inline] + fn named_metadata_mut(&mut self, name: &str) -> Result<&mut M, Error> + where + M: SerdeAny, + { + self.named_metadata_map_mut() + .get_mut::(name) + .ok_or_else(|| Error::key_not_found(format!("{} not found", type_name::()))) + } +} diff --git a/libafl/src/corpus/cached.rs b/libafl/src/corpus/cached.rs index 812e1bb36c..d585871c0e 100644 --- a/libafl/src/corpus/cached.rs +++ b/libafl/src/corpus/cached.rs @@ -37,22 +37,69 @@ where type Input = I; } +impl CachedOnDiskCorpus +where + I: Input, +{ + fn cache_testcase<'a>( + &'a self, + testcase: &'a RefCell>, + idx: CorpusId, + ) -> Result<(), Error> { + if testcase.borrow().input().is_none() { + self.load_input_into(&mut testcase.borrow_mut())?; + let mut borrowed_num = 0; + while self.cached_indexes.borrow().len() >= self.cache_max_len { + let removed = self.cached_indexes.borrow_mut().pop_front().unwrap(); + + if let Ok(mut borrowed) = self.inner.get_from_all(removed)?.try_borrow_mut() { + *borrowed.input_mut() = None; + } else { + self.cached_indexes.borrow_mut().push_back(removed); + borrowed_num += 1; + if self.cache_max_len == borrowed_num { + break; + } + } + } + self.cached_indexes.borrow_mut().push_back(idx); + } + Ok(()) + } +} impl Corpus for CachedOnDiskCorpus where I: Input, { - /// Returns the number of elements + /// Returns the number of all enabled entries #[inline] fn count(&self) -> usize { self.inner.count() } - /// Add an entry to the corpus and return its index + /// Returns the number of all disabled entries + fn count_disabled(&self) -> usize { + self.inner.count_disabled() + } + + /// Returns the number of elements including disabled entries + #[inline] + fn count_all(&self) -> usize { + self.inner.count_all() + } + + /// Add an enabled testcase to the corpus and return its index #[inline] fn add(&mut self, testcase: Testcase) -> Result { self.inner.add(testcase) } + /// Add a disabled testcase to the corpus and return its index + #[inline] + fn add_disabled(&mut self, testcase: Testcase) -> Result { + self.inner.add_disabled(testcase) + } + /// Replaces the testcase at the given idx #[inline] fn replace(&mut self, idx: CorpusId, testcase: Testcase) -> Result, Error> { @@ -68,27 +115,18 @@ where Ok(testcase) } - /// Get by id + /// Get by id; considers only enabled testcases #[inline] fn get(&self, idx: CorpusId) -> Result<&RefCell>, Error> { let testcase = { self.inner.get(idx)? }; - if testcase.borrow().input().is_none() { - self.load_input_into(&mut testcase.borrow_mut())?; - let mut borrowed_num = 0; - while self.cached_indexes.borrow().len() >= self.cache_max_len { - let removed = self.cached_indexes.borrow_mut().pop_front().unwrap(); - if let Ok(mut borrowed) = self.inner.get(removed)?.try_borrow_mut() { - *borrowed.input_mut() = None; - } else { - self.cached_indexes.borrow_mut().push_back(removed); - borrowed_num += 1; - if self.cache_max_len == borrowed_num { - break; - } - } - } - self.cached_indexes.borrow_mut().push_back(idx); - } + self.cache_testcase(testcase, idx)?; + Ok(testcase) + } + /// Get by id; considers both enabled and disabled testcases + #[inline] + fn get_from_all(&self, idx: CorpusId) -> Result<&RefCell>, Error> { + let testcase = { self.inner.get_from_all(idx)? }; + self.cache_testcase(testcase, idx)?; Ok(testcase) } @@ -124,10 +162,16 @@ where self.inner.last() } + /// Get the nth corpus id; considers only enabled testcases #[inline] fn nth(&self, nth: usize) -> CorpusId { self.inner.nth(nth) } + /// Get the nth corpus id; considers both enabled and disabled testcases + #[inline] + fn nth_from_all(&self, nth: usize) -> CorpusId { + self.inner.nth_from_all(nth) + } #[inline] fn load_input_into(&self, testcase: &mut Testcase) -> Result<(), Error> { @@ -248,47 +292,3 @@ where &self.inner } } - -/// ``CachedOnDiskCorpus`` Python bindings -#[cfg(feature = "python")] -#[allow(clippy::unnecessary_fallible_conversions)] -pub mod pybind { - use alloc::string::String; - use std::path::PathBuf; - - use pyo3::prelude::*; - use serde::{Deserialize, Serialize}; - - use crate::{ - corpus::{pybind::PythonCorpus, CachedOnDiskCorpus}, - inputs::BytesInput, - }; - - #[pyclass(unsendable, name = "CachedOnDiskCorpus")] - #[allow(clippy::unsafe_derive_deserialize)] - #[derive(Serialize, Deserialize, Debug, Clone)] - /// Python class for CachedOnDiskCorpus - pub struct PythonCachedOnDiskCorpus { - /// Rust wrapped CachedOnDiskCorpus object - pub inner: CachedOnDiskCorpus, - } - - #[pymethods] - impl PythonCachedOnDiskCorpus { - #[new] - fn new(path: String, cache_max_len: usize) -> Self { - Self { - inner: CachedOnDiskCorpus::new(PathBuf::from(path), cache_max_len).unwrap(), - } - } - - fn as_corpus(slf: Py) -> PythonCorpus { - PythonCorpus::new_cached_on_disk(slf) - } - } - /// Register the classes to the python module - pub fn register(_py: Python, m: &PyModule) -> PyResult<()> { - m.add_class::()?; - Ok(()) - } -} diff --git a/libafl/src/corpus/inmemory.rs b/libafl/src/corpus/inmemory.rs index 25781fd4b6..f0ce03cfc2 100644 --- a/libafl/src/corpus/inmemory.rs +++ b/libafl/src/corpus/inmemory.rs @@ -28,28 +28,21 @@ where pub next: Option, } -#[cfg(not(feature = "corpus_btreemap"))] -/// The map type in which testcases are stored (enable the feature `corpus_btreemap` to use a `BTreeMap` instead of `HashMap`) -pub type TestcaseStorageMap = hashbrown::HashMap>; - -#[cfg(feature = "corpus_btreemap")] /// The map type in which testcases are stored (disable the feature `corpus_btreemap` to use a `HashMap` instead of `BTreeMap`) -pub type TestcaseStorageMap = - alloc::collections::btree_map::BTreeMap>>; - -/// Storage map for the testcases (used in `Corpus` implementations) with an incremental index #[derive(Default, Serialize, Deserialize, Clone, Debug)] #[serde(bound = "I: serde::de::DeserializeOwned")] -pub struct TestcaseStorage +pub struct TestcaseStorageMap where I: Input, { - /// The map in which testcases are stored - pub map: TestcaseStorageMap, + #[cfg(not(feature = "corpus_btreemap"))] + /// A map of `CorpusId` to `TestcaseStorageItem` + pub map: hashbrown::HashMap>, + #[cfg(feature = "corpus_btreemap")] + /// A map of `CorpusId` to `Testcase`. + pub map: alloc::collections::btree_map::BTreeMap>>, /// The keys in order (use `Vec::binary_search`) pub keys: Vec, - /// The progressive idx - progressive_idx: usize, /// First inserted idx #[cfg(not(feature = "corpus_btreemap"))] first_idx: Option, @@ -58,14 +51,7 @@ where last_idx: Option, } -impl UsesInput for TestcaseStorage -where - I: Input, -{ - type Input = I; -} - -impl TestcaseStorage +impl TestcaseStorageMap where I: Input, { @@ -83,43 +69,6 @@ where } } - /// Insert a testcase assigning a `CorpusId` to it - #[cfg(not(feature = "corpus_btreemap"))] - pub fn insert(&mut self, testcase: RefCell>) -> CorpusId { - let idx = CorpusId::from(self.progressive_idx); - self.progressive_idx += 1; - let prev = if let Some(last_idx) = self.last_idx { - self.map.get_mut(&last_idx).unwrap().next = Some(idx); - Some(last_idx) - } else { - None - }; - if self.first_idx.is_none() { - self.first_idx = Some(idx); - } - self.last_idx = Some(idx); - self.insert_key(idx); - self.map.insert( - idx, - TestcaseStorageItem { - testcase, - prev, - next: None, - }, - ); - idx - } - - /// Insert a testcase assigning a `CorpusId` to it - #[cfg(feature = "corpus_btreemap")] - pub fn insert(&mut self, testcase: RefCell>) -> CorpusId { - let idx = CorpusId::from(self.progressive_idx); - self.progressive_idx += 1; - self.insert_key(idx); - self.map.insert(idx, testcase); - idx - } - /// Replace a testcase given a `CorpusId` #[cfg(not(feature = "corpus_btreemap"))] pub fn replace(&mut self, idx: CorpusId, testcase: Testcase) -> Option> { @@ -270,13 +219,13 @@ where self.map.iter().next_back().map(|x| *x.0) } - /// Create new `TestcaseStorage` - #[must_use] - pub fn new() -> Self { + fn new() -> Self { Self { - map: TestcaseStorageMap::default(), - keys: vec![], - progressive_idx: 0, + #[cfg(not(feature = "corpus_btreemap"))] + map: hashbrown::HashMap::default(), + #[cfg(feature = "corpus_btreemap")] + map: alloc::collections::BTreeMap::default(), + keys: Vec::default(), #[cfg(not(feature = "corpus_btreemap"))] first_idx: None, #[cfg(not(feature = "corpus_btreemap"))] @@ -284,6 +233,98 @@ where } } } +/// Storage map for the testcases (used in `Corpus` implementations) with an incremental index +#[derive(Default, Serialize, Deserialize, Clone, Debug)] +#[serde(bound = "I: serde::de::DeserializeOwned")] +pub struct TestcaseStorage +where + I: Input, +{ + /// The map in which enabled testcases are stored + pub enabled: TestcaseStorageMap, + /// The map in which disabled testcases are stored + pub disabled: TestcaseStorageMap, + /// The progressive idx for both maps + progressive_idx: usize, +} + +impl UsesInput for TestcaseStorage +where + I: Input, +{ + type Input = I; +} + +impl TestcaseStorage +where + I: Input, +{ + /// Insert a testcase assigning a `CorpusId` to it + pub fn insert(&mut self, testcase: RefCell>) -> CorpusId { + self._insert(testcase, false) + } + + /// Insert a testcase assigning a `CorpusId` to it + pub fn insert_disabled(&mut self, testcase: RefCell>) -> CorpusId { + self._insert(testcase, true) + } + /// Insert a testcase assigning a `CorpusId` to it + #[cfg(not(feature = "corpus_btreemap"))] + fn _insert(&mut self, testcase: RefCell>, is_disabled: bool) -> CorpusId { + let idx = CorpusId::from(self.progressive_idx); + self.progressive_idx += 1; + let corpus = if is_disabled { + &mut self.disabled + } else { + &mut self.enabled + }; + let prev = if let Some(last_idx) = corpus.last_idx { + corpus.map.get_mut(&last_idx).unwrap().next = Some(idx); + Some(last_idx) + } else { + None + }; + if corpus.first_idx.is_none() { + corpus.first_idx = Some(idx); + } + corpus.last_idx = Some(idx); + corpus.insert_key(idx); + corpus.map.insert( + idx, + TestcaseStorageItem { + testcase, + prev, + next: None, + }, + ); + idx + } + + /// Insert a testcase assigning a `CorpusId` to it + #[cfg(feature = "corpus_btreemap")] + fn _insert(&mut self, testcase: RefCell>, is_disabled: bool) -> CorpusId { + let idx = CorpusId::from(self.progressive_idx); + self.progressive_idx += 1; + let corpus = if is_disabled { + &mut self.disabled + } else { + &mut self.enabled + }; + corpus.insert_key(idx); + corpus.map.insert(idx, testcase); + idx + } + + /// Create new `TestcaseStorage` + #[must_use] + pub fn new() -> Self { + Self { + enabled: TestcaseStorageMap::new(), + disabled: TestcaseStorageMap::new(), + progressive_idx: 0, + } + } +} /// A corpus handling all in memory. #[derive(Default, Serialize, Deserialize, Clone, Debug)] @@ -307,22 +348,44 @@ impl Corpus for InMemoryCorpus where I: Input, { - /// Returns the number of elements + /// Returns the number of all enabled entries #[inline] fn count(&self) -> usize { - self.storage.map.len() + self.storage.enabled.map.len() } - /// Add an entry to the corpus and return its index + /// Returns the number of all disabled entries + fn count_disabled(&self) -> usize { + self.storage.disabled.map.len() + } + + /// Returns the number of elements including disabled entries + #[inline] + fn count_all(&self) -> usize { + self.storage + .enabled + .map + .len() + .saturating_add(self.storage.disabled.map.len()) + } + + /// Add an enabled testcase to the corpus and return its index #[inline] fn add(&mut self, testcase: Testcase) -> Result { Ok(self.storage.insert(RefCell::new(testcase))) } + /// Add a disabled testcase to the corpus and return its index + #[inline] + fn add_disabled(&mut self, testcase: Testcase) -> Result { + Ok(self.storage.insert_disabled(RefCell::new(testcase))) + } + /// Replaces the testcase at the given idx #[inline] fn replace(&mut self, idx: CorpusId, testcase: Testcase) -> Result, Error> { self.storage + .enabled .replace(idx, testcase) .ok_or_else(|| Error::key_not_found(format!("Index {idx} not found"))) } @@ -331,18 +394,29 @@ where #[inline] fn remove(&mut self, idx: CorpusId) -> Result, Error> { self.storage + .enabled .remove(idx) .map(|x| x.take()) .ok_or_else(|| Error::key_not_found(format!("Index {idx} not found"))) } - /// Get by id + /// Get by id; considers only enabled testcases #[inline] fn get(&self, idx: CorpusId) -> Result<&RefCell>, Error> { self.storage + .enabled .get(idx) .ok_or_else(|| Error::key_not_found(format!("Index {idx} not found"))) } + /// Get by id; considers both enabled and disabled testcases + #[inline] + fn get_from_all(&self, idx: CorpusId) -> Result<&RefCell>, Error> { + let mut testcase = self.storage.enabled.get(idx); + if testcase.is_none() { + testcase = self.storage.disabled.get(idx); + } + testcase.ok_or_else(|| Error::key_not_found(format!("Index {idx} not found"))) + } /// Current testcase scheduled #[inline] @@ -358,27 +432,38 @@ where #[inline] fn next(&self, idx: CorpusId) -> Option { - self.storage.next(idx) + self.storage.enabled.next(idx) } #[inline] fn prev(&self, idx: CorpusId) -> Option { - self.storage.prev(idx) + self.storage.enabled.prev(idx) } #[inline] fn first(&self) -> Option { - self.storage.first() + self.storage.enabled.first() } #[inline] fn last(&self) -> Option { - self.storage.last() + self.storage.enabled.last() } + /// Get the nth corpus id; considers only enabled testcases #[inline] fn nth(&self, nth: usize) -> CorpusId { - self.storage.keys[nth] + self.storage.enabled.keys[nth] + } + + /// Get the nth corpus id; considers both enabled and disabled testcases + #[inline] + fn nth_from_all(&self, nth: usize) -> CorpusId { + let enabled_count = self.count(); + if nth >= enabled_count { + return self.storage.disabled.keys[nth.saturating_sub(enabled_count)]; + } + self.storage.enabled.keys[nth] } #[inline] @@ -426,44 +511,3 @@ where } } } - -/// `InMemoryCorpus` Python bindings -#[cfg(feature = "python")] -#[allow(clippy::unnecessary_fallible_conversions)] -pub mod pybind { - use pyo3::prelude::*; - use serde::{Deserialize, Serialize}; - - use crate::{ - corpus::{pybind::PythonCorpus, InMemoryCorpus}, - inputs::BytesInput, - }; - - #[pyclass(unsendable, name = "InMemoryCorpus")] - #[allow(clippy::unsafe_derive_deserialize)] - #[derive(Serialize, Deserialize, Debug, Clone)] - /// Python class for InMemoryCorpus - pub struct PythonInMemoryCorpus { - /// Rust wrapped InMemoryCorpus object - pub inner: InMemoryCorpus, - } - - #[pymethods] - impl PythonInMemoryCorpus { - #[new] - fn new() -> Self { - Self { - inner: InMemoryCorpus::new(), - } - } - - fn as_corpus(slf: Py) -> PythonCorpus { - PythonCorpus::new_in_memory(slf) - } - } - /// Register the classes to the python module - pub fn register(_py: Python, m: &PyModule) -> PyResult<()> { - m.add_class::()?; - Ok(()) - } -} diff --git a/libafl/src/corpus/inmemory_ondisk.rs b/libafl/src/corpus/inmemory_ondisk.rs index ef6036e064..ccd3c49cc4 100644 --- a/libafl/src/corpus/inmemory_ondisk.rs +++ b/libafl/src/corpus/inmemory_ondisk.rs @@ -24,8 +24,7 @@ use super::{ use crate::{ corpus::{Corpus, CorpusId, InMemoryCorpus, Testcase}, inputs::{Input, UsesInput}, - state::HasMetadata, - Error, + Error, HasMetadata, }; /// The [`Testcase`] metadata that'll be stored to disk @@ -65,13 +64,24 @@ impl Corpus for InMemoryOnDiskCorpus where I: Input, { - /// Returns the number of elements + /// Returns the number of all enabled entries #[inline] fn count(&self) -> usize { self.inner.count() } - /// Add an entry to the corpus and return its index + /// Returns the number of all disabled entries + fn count_disabled(&self) -> usize { + self.inner.count_disabled() + } + + /// Returns the number of elements including disabled entries + #[inline] + fn count_all(&self) -> usize { + self.inner.count_all() + } + + /// Add an enabled testcase to the corpus and return its index #[inline] fn add(&mut self, testcase: Testcase) -> Result { let idx = self.inner.add(testcase)?; @@ -81,6 +91,16 @@ where Ok(idx) } + /// Add a disabled testcase to the corpus and return its index + #[inline] + fn add_disabled(&mut self, testcase: Testcase) -> Result { + let idx = self.inner.add_disabled(testcase)?; + let testcase = &mut self.get_from_all(idx).unwrap().borrow_mut(); + self.save_testcase(testcase, idx)?; + *testcase.input_mut() = None; + Ok(idx) + } + /// Replaces the testcase at the given idx #[inline] fn replace(&mut self, idx: CorpusId, testcase: Testcase) -> Result, Error> { @@ -100,12 +120,18 @@ where Ok(entry) } - /// Get by id + /// Get by id; considers only enabled testcases #[inline] fn get(&self, idx: CorpusId) -> Result<&RefCell>, Error> { self.inner.get(idx) } + /// Get by id; considers both enabled and disabled testcases + #[inline] + fn get_from_all(&self, idx: CorpusId) -> Result<&RefCell>, Error> { + self.inner.get_from_all(idx) + } + /// Current testcase scheduled #[inline] fn current(&self) -> &Option { @@ -138,10 +164,16 @@ where self.inner.last() } + /// Get the nth corpus id; considers only enabled testcases #[inline] fn nth(&self, nth: usize) -> CorpusId { self.inner.nth(nth) } + /// Get the nth corpus id; considers both enabled and disabled testcases + #[inline] + fn nth_from_all(&self, nth: usize) -> CorpusId { + self.inner.nth_from_all(nth) + } fn load_input_into(&self, testcase: &mut Testcase) -> Result<(), Error> { if testcase.input_mut().is_none() { @@ -437,47 +469,3 @@ where &self.dir_path } } - -#[cfg(feature = "python")] -#[allow(clippy::unnecessary_fallible_conversions)] -/// `InMemoryOnDiskCorpus` Python bindings -pub mod pybind { - use alloc::string::String; - use std::path::PathBuf; - - use pyo3::prelude::*; - use serde::{Deserialize, Serialize}; - - use crate::{ - corpus::{pybind::PythonCorpus, InMemoryOnDiskCorpus}, - inputs::BytesInput, - }; - - #[pyclass(unsendable, name = "InMemoryOnDiskCorpus")] - #[allow(clippy::unsafe_derive_deserialize)] - #[derive(Serialize, Deserialize, Debug, Clone)] - /// Python class for InMemoryOnDiskCorpus - pub struct PythonInMemoryOnDiskCorpus { - /// Rust wrapped InMemoryOnDiskCorpus object - pub inner: InMemoryOnDiskCorpus, - } - - #[pymethods] - impl PythonInMemoryOnDiskCorpus { - #[new] - fn new(path: String) -> Self { - Self { - inner: InMemoryOnDiskCorpus::new(PathBuf::from(path)).unwrap(), - } - } - - fn as_corpus(slf: Py) -> PythonCorpus { - PythonCorpus::new_in_memory_on_disk(slf) - } - } - /// Register the classes to the python module - pub fn register(_py: Python, m: &PyModule) -> PyResult<()> { - m.add_class::()?; - Ok(()) - } -} diff --git a/libafl/src/corpus/minimizer.rs b/libafl/src/corpus/minimizer.rs index 8b4d602e57..9b9c2c4856 100644 --- a/libafl/src/corpus/minimizer.rs +++ b/libafl/src/corpus/minimizer.rs @@ -1,14 +1,15 @@ //! Whole corpus minimizers, for reducing the number of samples/the total size/the average runtime //! of your corpus. -use alloc::{ - string::{String, ToString}, - vec::Vec, -}; +use alloc::{borrow::Cow, string::ToString, vec::Vec}; use core::{hash::Hash, marker::PhantomData}; use hashbrown::{HashMap, HashSet}; -use libafl_bolts::{current_time, tuples::MatchName, AsIter, Named}; +use libafl_bolts::{ + current_time, + tuples::{Handle, Handler}, + AsIter, Named, +}; use num_traits::ToPrimitive; use z3::{ast::Bool, Config, Context, Optimize}; @@ -19,8 +20,8 @@ use crate::{ monitors::{AggregatorOps, UserStats, UserStatsValue}, observers::{MapObserver, ObserversTuple}, schedulers::{LenTimeMulTestcaseScore, RemovableScheduler, Scheduler, TestcaseScore}, - state::{HasCorpus, HasExecutions, HasMetadata, UsesState}, - Error, HasScheduler, + state::{HasCorpus, HasExecutions, UsesState}, + Error, HasMetadata, HasScheduler, }; /// `CorpusMinimizers` minimize corpora according to internal logic. See various implementations for @@ -49,43 +50,37 @@ where /// /// Algorithm based on WMOPT: #[derive(Debug)] -pub struct MapCorpusMinimizer -where - E: UsesState, - E::State: HasCorpus + HasMetadata, - TS: TestcaseScore, -{ - obs_name: String, +pub struct MapCorpusMinimizer { + obs_ref: Handle, phantom: PhantomData<(E, O, T, TS)>, } /// Standard corpus minimizer, which weights inputs by length and time. -pub type StdCorpusMinimizer = - MapCorpusMinimizer::State>>; +pub type StdCorpusMinimizer = + MapCorpusMinimizer::State>>; -impl MapCorpusMinimizer +impl MapCorpusMinimizer where E: UsesState, E::State: HasCorpus + HasMetadata, TS: TestcaseScore, + C: Named, { /// Constructs a new `MapCorpusMinimizer` from a provided observer. This observer will be used /// in the future to get observed maps from an executed input. - pub fn new(obs: &O) -> Self - where - O: Named, - { + pub fn new(obs: &C) -> Self { Self { - obs_name: obs.name().to_string(), + obs_ref: obs.handle(), phantom: PhantomData, } } } -impl CorpusMinimizer for MapCorpusMinimizer +impl CorpusMinimizer for MapCorpusMinimizer where E: UsesState, for<'a> O: MapObserver + AsIter<'a, Item = T>, + C: AsRef, E::State: HasMetadata + HasCorpus + HasExecutions, T: Copy + Hash + Eq, TS: TestcaseScore, @@ -149,7 +144,7 @@ where manager.fire( state, Event::UpdateUserStats { - name: "minimisation exec pass".to_string(), + name: Cow::from("minimisation exec pass"), value: UserStats::new(UserStatsValue::Ratio(curr, total), AggregatorOps::None), phantom: PhantomData, }, @@ -165,14 +160,12 @@ where )?; let seed_expr = Bool::fresh_const(&ctx, "seed"); - let obs: &O = executor - .observers() - .match_name::(&self.obs_name) - .expect("Observer must be present."); + let observers = executor.observers(); + let obs = observers[&self.obs_ref].as_ref(); // Store coverage, mapping coverage map indices to hit counts (if present) and the // associated seeds for the map indices with those hit counts. - for (i, e) in obs.as_iter().copied().enumerate() { + for (i, e) in obs.as_iter().map(|x| *x).enumerate() { if e != obs.initial() { cov_map .entry(i) diff --git a/libafl/src/corpus/mod.rs b/libafl/src/corpus/mod.rs index 0cfcc39c27..021b592e27 100644 --- a/libafl/src/corpus/mod.rs +++ b/libafl/src/corpus/mod.rs @@ -63,29 +63,49 @@ impl From for usize { } } -/// Utility macro to call `Corpus::random_id` +/// Utility macro to call `Corpus::random_id`; fetches only enabled testcases #[macro_export] macro_rules! random_corpus_id { ($corpus:expr, $rand:expr) => {{ - let cnt = $corpus.count() as u64; - let nth = $rand.below(cnt) as usize; + let cnt = $corpus.count(); + let nth = $rand.below(cnt); $corpus.nth(nth) }}; } +/// Utility macro to call `Corpus::random_id`; fetches both enabled and disabled testcases +/// Note: use `Corpus::get_from_all` as disabled entries are inaccessible from `Corpus::get` +#[macro_export] +macro_rules! random_corpus_id_with_disabled { + ($corpus:expr, $rand:expr) => {{ + let cnt = $corpus.count_all(); + let nth = $rand.below(cnt); + $corpus.nth_from_all(nth) + }}; +} + /// Corpus with all current [`Testcase`]s, or solutions pub trait Corpus: UsesInput + Serialize + for<'de> Deserialize<'de> { - /// Returns the number of elements + /// Returns the number of all enabled entries fn count(&self) -> usize; + /// Returns the number of all disabled entries + fn count_disabled(&self) -> usize; + + /// Returns the number of elements including disabled entries + fn count_all(&self) -> usize; + /// Returns true, if no elements are in this corpus yet fn is_empty(&self) -> bool { self.count() == 0 } - /// Add an entry to the corpus and return its index + /// Add an enabled testcase to the corpus and return its index fn add(&mut self, testcase: Testcase) -> Result; + /// Add a disabled testcase to the corpus and return its index + fn add_disabled(&mut self, testcase: Testcase) -> Result; + /// Replaces the [`Testcase`] at the given idx, returning the existing. fn replace( &mut self, @@ -96,9 +116,12 @@ pub trait Corpus: UsesInput + Serialize + for<'de> Deserialize<'de> { /// Removes an entry from the corpus, returning it if it was present. fn remove(&mut self, id: CorpusId) -> Result, Error>; - /// Get by id + /// Get by id; considers only enabled testcases fn get(&self, id: CorpusId) -> Result<&RefCell>, Error>; + /// Get by id; considers both enabled and disabled testcases + fn get_from_all(&self, id: CorpusId) -> Result<&RefCell>, Error>; + /// Current testcase scheduled fn current(&self) -> &Option; @@ -126,13 +149,16 @@ pub trait Corpus: UsesInput + Serialize + for<'de> Deserialize<'de> { } } - /// Get the nth corpus id + /// Get the nth corpus id; considers only enabled testcases fn nth(&self, nth: usize) -> CorpusId { self.ids() .nth(nth) .expect("Failed to get the {nth} CorpusId") } + /// Get the nth corpus id; considers both enabled and disabled testcases + fn nth_from_all(&self, nth: usize) -> CorpusId; + /// Method to load the input for this [`Testcase`] from persistent storage, /// if necessary, and if was not already loaded (`== Some(input)`). /// After this call, `testcase.input()` must always return `Some(input)`. @@ -200,247 +226,3 @@ where } } } - -/// `Corpus` Python bindings -#[cfg(feature = "python")] -#[allow(missing_docs)] -pub mod pybind { - use std::cell::RefCell; - - use pyo3::prelude::*; - use serde::{Deserialize, Serialize}; - - use crate::{ - corpus::{ - cached::pybind::PythonCachedOnDiskCorpus, inmemory::pybind::PythonInMemoryCorpus, - inmemory_ondisk::pybind::PythonInMemoryOnDiskCorpus, - ondisk::pybind::PythonOnDiskCorpus, testcase::pybind::PythonTestcaseWrapper, Corpus, - CorpusId, HasTestcase, Testcase, - }, - inputs::{BytesInput, UsesInput}, - Error, - }; - - #[derive(Serialize, Deserialize, Debug, Clone)] - enum PythonCorpusWrapper { - InMemory(Py), - CachedOnDisk(Py), - OnDisk(Py), - InMemoryOnDisk(Py), - } - - /// Corpus Trait binding - #[pyclass(unsendable, name = "Corpus")] - #[allow(clippy::unsafe_derive_deserialize)] - #[derive(Serialize, Deserialize, Debug, Clone)] - pub struct PythonCorpus { - wrapper: PythonCorpusWrapper, - } - - macro_rules! unwrap_me { - ($wrapper:expr, $name:ident, $body:block) => { - libafl_bolts::unwrap_me_body!( - $wrapper, - $name, - $body, - PythonCorpusWrapper, - { - InMemory, - InMemoryOnDisk, - CachedOnDisk, - OnDisk - } - ) - }; - } - - macro_rules! unwrap_me_mut { - ($wrapper:expr, $name:ident, $body:block) => { - libafl_bolts::unwrap_me_mut_body!( - $wrapper, - $name, - $body, - PythonCorpusWrapper, - { - InMemory, - InMemoryOnDisk, - CachedOnDisk, - OnDisk - } - ) - }; - } - - #[pymethods] - impl PythonCorpus { - #[staticmethod] - #[must_use] - pub fn new_in_memory(py_in_memory_corpus: Py) -> Self { - Self { - wrapper: PythonCorpusWrapper::InMemory(py_in_memory_corpus), - } - } - - #[staticmethod] - #[must_use] - pub fn new_cached_on_disk(py_cached_on_disk_corpus: Py) -> Self { - Self { - wrapper: PythonCorpusWrapper::CachedOnDisk(py_cached_on_disk_corpus), - } - } - - #[staticmethod] - #[must_use] - pub fn new_on_disk(py_on_disk_corpus: Py) -> Self { - Self { - wrapper: PythonCorpusWrapper::OnDisk(py_on_disk_corpus), - } - } - - #[staticmethod] - #[must_use] - pub fn new_in_memory_on_disk( - py_in_memory_on_disk_corpus: Py, - ) -> Self { - Self { - wrapper: PythonCorpusWrapper::InMemoryOnDisk(py_in_memory_on_disk_corpus), - } - } - - #[pyo3(name = "count")] - fn pycount(&self) -> usize { - self.count() - } - - #[pyo3(name = "current")] - fn pycurrent(&self) -> Option { - self.current().map(|x| x.0) - } - - #[pyo3(name = "get")] - fn pyget(&self, idx: usize) -> PythonTestcaseWrapper { - let t: &mut Testcase = unwrap_me!(self.wrapper, c, { - c.get(CorpusId::from(idx)) - .map(|v| unsafe { v.as_ptr().as_mut().unwrap() }) - .expect("PythonCorpus::get failed") - }); - PythonTestcaseWrapper::wrap(t) - } - } - - impl UsesInput for PythonCorpus { - type Input = BytesInput; - } - - impl Corpus for PythonCorpus { - #[inline] - fn count(&self) -> usize { - unwrap_me!(self.wrapper, c, { c.count() }) - } - - #[inline] - fn add(&mut self, testcase: Testcase) -> Result { - unwrap_me_mut!(self.wrapper, c, { c.add(testcase) }) - } - - #[inline] - fn replace( - &mut self, - idx: CorpusId, - testcase: Testcase, - ) -> Result, Error> { - unwrap_me_mut!(self.wrapper, c, { c.replace(idx, testcase) }) - } - - #[inline] - fn remove(&mut self, idx: CorpusId) -> Result, Error> { - unwrap_me_mut!(self.wrapper, c, { c.remove(idx) }) - } - - #[inline] - fn get(&self, idx: CorpusId) -> Result<&RefCell>, Error> { - let ptr = unwrap_me!(self.wrapper, c, { - c.get(idx) - .map(core::ptr::from_ref::>>) - })?; - Ok(unsafe { ptr.as_ref().unwrap() }) - } - - #[inline] - fn current(&self) -> &Option { - let ptr = unwrap_me!(self.wrapper, c, { - core::ptr::from_ref::>(c.current()) - }); - unsafe { ptr.as_ref().unwrap() } - } - - #[inline] - fn current_mut(&mut self) -> &mut Option { - let ptr = unwrap_me_mut!(self.wrapper, c, { - core::ptr::from_mut::>(c.current_mut()) - }); - unsafe { ptr.as_mut().unwrap() } - } - - fn next(&self, idx: CorpusId) -> Option { - unwrap_me!(self.wrapper, c, { c.next(idx) }) - } - - fn prev(&self, idx: CorpusId) -> Option { - unwrap_me!(self.wrapper, c, { c.prev(idx) }) - } - - fn first(&self) -> Option { - unwrap_me!(self.wrapper, c, { c.first() }) - } - - fn last(&self) -> Option { - unwrap_me!(self.wrapper, c, { c.last() }) - } - - fn load_input_into(&self, testcase: &mut Testcase) -> Result<(), Error> { - unwrap_me!(self.wrapper, c, { c.load_input_into(testcase) }) - } - - fn store_input_from(&self, testcase: &Testcase) -> Result<(), Error> { - unwrap_me!(self.wrapper, c, { c.store_input_from(testcase) }) - } - - /*fn ids<'a>(&'a self) -> CorpusIdIterator<'a, Self> { - CorpusIdIterator { - corpus: self, - cur: self.first(), - cur_back: self.last(), - } - } - - fn random_id(&self, next_random: u64) -> CorpusId { - let nth = (next_random as usize) % self.count(); - self.ids() - .nth(nth) - .expect("Failed to get a random CorpusId") - }*/ - } - - impl HasTestcase for PythonCorpus { - fn testcase( - &self, - id: CorpusId, - ) -> Result::Input>>, Error> { - Ok(self.get(id)?.borrow()) - } - - fn testcase_mut( - &self, - id: CorpusId, - ) -> Result::Input>>, Error> { - Ok(self.get(id)?.borrow_mut()) - } - } - - /// Register the classes to the python module - pub fn register(_py: Python, m: &PyModule) -> PyResult<()> { - m.add_class::()?; - Ok(()) - } -} diff --git a/libafl/src/corpus/nop.rs b/libafl/src/corpus/nop.rs index 091e2c29cb..ea6211a23d 100644 --- a/libafl/src/corpus/nop.rs +++ b/libafl/src/corpus/nop.rs @@ -28,18 +28,35 @@ impl Corpus for NopCorpus where I: Input, { - /// Returns the number of elements + /// Returns the number of all enabled entries #[inline] fn count(&self) -> usize { 0 } - /// Add an entry to the corpus and return its index + /// Returns the number of all disabled entries + fn count_disabled(&self) -> usize { + 0 + } + + /// Returns the number of all entries + #[inline] + fn count_all(&self) -> usize { + 0 + } + + /// Add an enabled testcase to the corpus and return its index #[inline] fn add(&mut self, _testcase: Testcase) -> Result { Err(Error::unsupported("Unsupported by NopCorpus")) } + /// Add a disabled testcase to the corpus and return its index + #[inline] + fn add_disabled(&mut self, _testcase: Testcase) -> Result { + Err(Error::unsupported("Unsupported by NopCorpus")) + } + /// Replaces the testcase at the given idx #[inline] fn replace(&mut self, _idx: CorpusId, _testcase: Testcase) -> Result, Error> { @@ -52,12 +69,18 @@ where Err(Error::unsupported("Unsupported by NopCorpus")) } - /// Get by id + /// Get by id; considers only enabled testcases #[inline] fn get(&self, _idx: CorpusId) -> Result<&RefCell>, Error> { Err(Error::unsupported("Unsupported by NopCorpus")) } + /// Get by id; considers both enabled and disabled testcases + #[inline] + fn get_from_all(&self, _idx: CorpusId) -> Result<&RefCell>, Error> { + Err(Error::unsupported("Unsupported by NopCorpus")) + } + /// Current testcase scheduled #[inline] fn current(&self) -> &Option { @@ -90,11 +113,18 @@ where None } + /// Get the nth corpus id; considers only enabled testcases #[inline] fn nth(&self, _nth: usize) -> CorpusId { CorpusId::from(0_usize) } + /// Get the nth corpus id; considers both enabled and disabled testcases + #[inline] + fn nth_from_all(&self, _nth: usize) -> CorpusId { + CorpusId::from(0_usize) + } + #[inline] fn load_input_into(&self, _testcase: &mut Testcase) -> Result<(), Error> { Err(Error::unsupported("Unsupported by NopCorpus")) diff --git a/libafl/src/corpus/ondisk.rs b/libafl/src/corpus/ondisk.rs index b7beeef1e4..926b7269c7 100644 --- a/libafl/src/corpus/ondisk.rs +++ b/libafl/src/corpus/ondisk.rs @@ -42,7 +42,7 @@ pub struct OnDiskMetadata<'a> { /// The exec time for this [`Testcase`] pub exec_time: &'a Option, /// The amount of executions for this [`Testcase`] - pub executions: &'a usize, + pub executions: &'a u64, } /// A corpus able to store [`Testcase`]s to disk, and load them from disk, when they are being used. @@ -71,18 +71,35 @@ impl Corpus for OnDiskCorpus where I: Input, { - /// Returns the number of elements + /// Returns the number of all enabled entries #[inline] fn count(&self) -> usize { self.inner.count() } - /// Add an entry to the corpus and return its index + /// Returns the number of all disabled entries + fn count_disabled(&self) -> usize { + self.inner.count_disabled() + } + + /// Returns the number of all entries + #[inline] + fn count_all(&self) -> usize { + self.inner.count_all() + } + + /// Add an enabled testcase to the corpus and return its index #[inline] fn add(&mut self, testcase: Testcase) -> Result { self.inner.add(testcase) } + /// Add a disabled testcase to the corpus and return its index + #[inline] + fn add_disabled(&mut self, testcase: Testcase) -> Result { + self.inner.add_disabled(testcase) + } + /// Replaces the testcase at the given idx #[inline] fn replace(&mut self, idx: CorpusId, testcase: Testcase) -> Result, Error> { @@ -95,12 +112,18 @@ where self.inner.remove(idx) } - /// Get by id + /// Get by id; will check the disabled corpus if not available in the enabled #[inline] fn get(&self, idx: CorpusId) -> Result<&RefCell>, Error> { self.inner.get(idx) } + /// Get by id; considers both enabled and disabled testcases + #[inline] + fn get_from_all(&self, idx: CorpusId) -> Result<&RefCell>, Error> { + self.inner.get_from_all(idx) + } + /// Current testcase scheduled #[inline] fn current(&self) -> &Option { @@ -133,10 +156,16 @@ where self.inner.last() } + /// Get the nth corpus id; considers only enabled testcases #[inline] fn nth(&self, nth: usize) -> CorpusId { self.inner.nth(nth) } + /// Get the nth corpus id; considers both enabled and disabled testcases + #[inline] + fn nth_from_all(&self, nth: usize) -> CorpusId { + self.inner.nth_from_all(nth) + } #[inline] fn load_input_into(&self, testcase: &mut Testcase) -> Result<(), Error> { @@ -260,47 +289,3 @@ where &self.dir_path } } - -#[cfg(feature = "python")] -#[allow(clippy::unnecessary_fallible_conversions)] -/// `OnDiskCorpus` Python bindings -pub mod pybind { - use alloc::string::String; - use std::path::PathBuf; - - use pyo3::prelude::*; - use serde::{Deserialize, Serialize}; - - use crate::{ - corpus::{pybind::PythonCorpus, OnDiskCorpus}, - inputs::BytesInput, - }; - - #[pyclass(unsendable, name = "OnDiskCorpus")] - #[allow(clippy::unsafe_derive_deserialize)] - #[derive(Serialize, Deserialize, Debug, Clone)] - /// Python class for OnDiskCorpus - pub struct PythonOnDiskCorpus { - /// Rust wrapped OnDiskCorpus object - pub inner: OnDiskCorpus, - } - - #[pymethods] - impl PythonOnDiskCorpus { - #[new] - fn new(path: String) -> Self { - Self { - inner: OnDiskCorpus::new(PathBuf::from(path)).unwrap(), - } - } - - fn as_corpus(slf: Py) -> PythonCorpus { - PythonCorpus::new_on_disk(slf) - } - } - /// Register the classes to the python module - pub fn register(_py: Python, m: &PyModule) -> PyResult<()> { - m.add_class::()?; - Ok(()) - } -} diff --git a/libafl/src/corpus/testcase.rs b/libafl/src/corpus/testcase.rs index 63f9eb0e56..2296d4f323 100644 --- a/libafl/src/corpus/testcase.rs +++ b/libafl/src/corpus/testcase.rs @@ -4,8 +4,6 @@ use alloc::string::String; use core::{ cell::{Ref, RefMut}, - default::Default, - option::Option, time::Duration, }; #[cfg(feature = "std")] @@ -18,8 +16,7 @@ use super::Corpus; use crate::{ corpus::CorpusId, inputs::{Input, UsesInput}, - state::HasMetadata, - Error, + Error, HasMetadata, }; /// Shorthand to receive a [`Ref`] or [`RefMut`] to a stored [`Testcase`], by [`CorpusId`]. @@ -61,11 +58,15 @@ where /// Cached len of the input, if any cached_len: Option, /// Number of executions done at discovery time - executions: usize, - /// Number of fuzzing iterations of this particular input updated in perform_mutational + executions: u64, + /// Number of fuzzing iterations of this particular input updated in `perform_mutational` scheduled_count: usize, /// Parent [`CorpusId`], if known parent_id: Option, + /// If the testcase is "disabled" + disabled: bool, + /// has found crash (or timeout) or not + objectives_found: usize, } impl HasMetadata for Testcase @@ -176,13 +177,13 @@ where /// Get the executions #[inline] - pub fn executions(&self) -> &usize { + pub fn executions(&self) -> &u64 { &self.executions } /// Get the executions (mutable) #[inline] - pub fn executions_mut(&mut self) -> &mut usize { + pub fn executions_mut(&mut self) -> &mut u64 { &mut self.executions } @@ -198,6 +199,18 @@ where self.scheduled_count = scheduled_count; } + /// Get `disabled` + #[inline] + pub fn disabled(&mut self) -> bool { + self.disabled + } + + /// Set the testcase as disabled + #[inline] + pub fn set_disabled(&mut self, disabled: bool) { + self.disabled = disabled; + } + /// Create a new Testcase instance given an input #[inline] pub fn new(mut input: I) -> Self { @@ -215,6 +228,8 @@ where executions: 0, scheduled_count: 0, parent_id: None, + disabled: false, + objectives_found: 0, } } @@ -235,6 +250,8 @@ where executions: 0, scheduled_count: 0, parent_id: Some(parent_id), + disabled: false, + objectives_found: 0, } } @@ -255,12 +272,14 @@ where executions: 0, scheduled_count: 0, parent_id: None, + disabled: false, + objectives_found: 0, } } /// Create a new Testcase instance given an [`Input`] and the number of executions #[inline] - pub fn with_executions(mut input: I, executions: usize) -> Self { + pub fn with_executions(mut input: I, executions: u64) -> Self { input.wrapped_as_testcase(); Self { input: Some(input), @@ -275,6 +294,8 @@ where executions, scheduled_count: 0, parent_id: None, + disabled: false, + objectives_found: 0, } } @@ -293,6 +314,16 @@ where pub fn set_parent_id_optional(&mut self, parent_id: Option) { self.parent_id = parent_id; } + + /// Gets how many objectives were found by mutating this testcase + pub fn objectives_found(&self) -> usize { + self.objectives_found + } + + /// Adds one objectives to the `objectives_found` counter. Mostly called from crash handler or executor. + pub fn found_objective(&mut self) { + self.objectives_found = self.objectives_found.saturating_add(1); + } } impl Default for Testcase @@ -315,6 +346,8 @@ where file_path: None, #[cfg(feature = "std")] metadata_path: None, + disabled: false, + objectives_found: 0, } } } @@ -368,15 +401,15 @@ where allow(clippy::unsafe_derive_deserialize) )] // for SerdeAny pub struct SchedulerTestcaseMetadata { - /// Number of bits set in bitmap, updated in calibrate_case + /// Number of bits set in bitmap, updated in `calibrate_case` bitmap_size: u64, /// Number of queue cycles behind handicap: u64, - /// Path depth, initialized in on_add + /// Path depth, initialized in `on_add` depth: u64, - /// Offset in n_fuzz + /// Offset in `n_fuzz` n_fuzz_entry: usize, - /// Cycles used to calibrate this (not really needed if it were not for on_replace and on_remove) + /// Cycles used to calibrate this (not really needed if it were not for `on_replace` and `on_remove`) cycle_and_time: (Duration, usize), } @@ -487,91 +520,3 @@ where } } } - -#[cfg(feature = "python")] -#[allow(missing_docs)] -/// `Testcase` Python bindings -pub mod pybind { - use alloc::{boxed::Box, vec::Vec}; - - use libafl_bolts::ownedref::OwnedMutPtr; - use pyo3::{prelude::*, types::PyDict}; - - use super::{HasMetadata, Testcase}; - use crate::{inputs::BytesInput, pybind::PythonMetadata}; - - /// `PythonTestcase` with fixed generics - pub type PythonTestcase = Testcase; - - #[pyclass(unsendable, name = "Testcase")] - #[derive(Debug)] - /// Python class for Testcase - pub struct PythonTestcaseWrapper { - /// Rust wrapped Testcase object - pub inner: OwnedMutPtr, - } - - impl PythonTestcaseWrapper { - pub fn wrap(r: &mut PythonTestcase) -> Self { - Self { - inner: OwnedMutPtr::Ptr(r), - } - } - - #[must_use] - pub fn unwrap(&self) -> &PythonTestcase { - self.inner.as_ref() - } - - pub fn unwrap_mut(&mut self) -> &mut PythonTestcase { - self.inner.as_mut() - } - } - - #[pymethods] - impl PythonTestcaseWrapper { - #[new] - fn new(input: Vec) -> Self { - Self { - inner: OwnedMutPtr::Owned(Box::new(PythonTestcase::new(BytesInput::new(input)))), - } - } - - #[getter] - fn exec_time_ms(&self) -> Option { - self.inner.as_ref().exec_time().map(|t| t.as_millis()) - } - - #[getter] - fn executions(&self) -> usize { - *self.inner.as_ref().executions() - } - - #[getter] - fn parent_id(&self) -> Option { - self.inner.as_ref().parent_id().map(|x| x.0) - } - - #[getter] - fn scheduled_count(&self) -> usize { - self.inner.as_ref().scheduled_count() - } - - fn metadata(&mut self) -> PyObject { - let meta = self.inner.as_mut().metadata_map_mut(); - if !meta.contains::() { - Python::with_gil(|py| { - let dict: Py = PyDict::new(py).into(); - meta.insert(PythonMetadata::new(dict.to_object(py))); - }); - } - meta.get::().unwrap().map.clone() - } - } - - /// Register the classes to the python module - pub fn register(_py: Python, m: &PyModule) -> PyResult<()> { - m.add_class::()?; - Ok(()) - } -} diff --git a/libafl/src/events/centralized.rs b/libafl/src/events/centralized.rs index 515e991144..ba4d1ac05a 100644 --- a/libafl/src/events/centralized.rs +++ b/libafl/src/events/centralized.rs @@ -1,10 +1,17 @@ -//! A wrapper manager to implement a main-secondary architecture with point-to-point channels +//! Centralized event manager is a special event manager that will be used to achieve a more efficient message passing architecture. + +// Some technical details.. +// A very standard multi-process fuzzing using centralized event manager will consist of 4 components +// 1. The "fuzzer clients", the fuzzer that will do the "normal" fuzzing +// 2. The "centralized broker, the broker that gathers all the testcases from all the fuzzer clients +// 3. The "main evaluator", the evaluator node that will evaluate all the testcases pass by the centralized event manager to see if the testcases are worth propagating +// 4. The "main broker", the gathers the stats from the fuzzer clients and broadcast the newly found testcases from the main evaluator. use alloc::{boxed::Box, string::String, vec::Vec}; use core::{marker::PhantomData, num::NonZeroUsize, time::Duration}; #[cfg(feature = "adaptive_serialization")] -use libafl_bolts::current_time; +use libafl_bolts::tuples::{Handle, Handler}; #[cfg(feature = "llmp_compression")] use libafl_bolts::{ compress::GzipCompressor, @@ -17,22 +24,24 @@ use libafl_bolts::{ }; use serde::{Deserialize, Serialize}; -use super::{CustomBufEventResult, HasCustomBufHandlers, ProgressReporter}; #[cfg(feature = "llmp_compression")] use crate::events::llmp::COMPRESS_THRESHOLD; +#[cfg(feature = "adaptive_serialization")] +use crate::observers::TimeObserver; #[cfg(feature = "scalability_introspection")] use crate::state::HasScalabilityMonitor; use crate::{ events::{ - llmp::EventStatsCollector, BrokerEventResult, Event, EventConfig, EventFirer, EventManager, - EventManagerId, EventProcessor, EventRestarter, HasEventManagerId, LogSeverity, + AdaptiveSerializer, BrokerEventResult, CustomBufEventResult, Event, EventConfig, + EventFirer, EventManager, EventManagerId, EventProcessor, EventRestarter, + HasCustomBufHandlers, HasEventManagerId, LogSeverity, ProgressReporter, }, executors::{Executor, HasObservers}, fuzzer::{EvaluatorObservers, ExecutionProcessor}, inputs::{Input, UsesInput}, observers::ObserversTuple, - state::{HasExecutions, HasLastReportTime, HasMetadata, UsesState}, - Error, + state::{HasExecutions, HasLastReportTime, UsesState}, + Error, HasMetadata, }; const _LLMP_TAG_TO_MAIN: Tag = Tag(0x3453453); @@ -86,19 +95,10 @@ where /// /// The port must not be bound yet to have a broker. #[cfg(feature = "std")] - pub fn on_port( - shmem_provider: SP, - port: u16, - client_timeout: Option, - ) -> Result { + pub fn on_port(shmem_provider: SP, port: u16) -> Result { Ok(Self { // TODO switch to false after solving the bug - llmp: LlmpBroker::with_keep_pages_attach_to_tcp( - shmem_provider, - port, - true, - client_timeout, - )?, + llmp: LlmpBroker::with_keep_pages_attach_to_tcp(shmem_provider, port, true)?, #[cfg(feature = "llmp_compression")] compressor: GzipCompressor::new(COMPRESS_THRESHOLD), phantom: PhantomData, @@ -225,6 +225,8 @@ where client: LlmpClient, #[cfg(feature = "llmp_compression")] compressor: GzipCompressor, + #[cfg(feature = "adaptive_serialization")] + time_ref: Handle, is_main: bool, } @@ -237,9 +239,9 @@ where } #[cfg(feature = "adaptive_serialization")] -impl EventStatsCollector for CentralizedEventManager +impl AdaptiveSerializer for CentralizedEventManager where - EM: EventStatsCollector + UsesState, + EM: AdaptiveSerializer + UsesState, SP: ShMemProvider + 'static, { fn serialization_time(&self) -> Duration { @@ -267,19 +269,23 @@ where fn should_serialize_cnt_mut(&mut self) -> &mut usize { self.inner.should_serialize_cnt_mut() } + + fn time_ref(&self) -> &Handle { + &self.time_ref + } } #[cfg(not(feature = "adaptive_serialization"))] -impl EventStatsCollector for CentralizedEventManager +impl AdaptiveSerializer for CentralizedEventManager where - EM: EventStatsCollector + UsesState, + EM: AdaptiveSerializer + UsesState, SP: ShMemProvider + 'static, { } impl EventFirer for CentralizedEventManager where - EM: EventStatsCollector + EventFirer + HasEventManagerId, + EM: AdaptiveSerializer + EventFirer + HasEventManagerId, SP: ShMemProvider + 'static, { fn fire( @@ -289,7 +295,8 @@ where ) -> Result<(), Error> { if !self.is_main { // secondary node - let is_nt = match &mut event { + let mut is_tc = false; + let is_nt_or_heartbeat = match &mut event { Event::NewTestcase { input: _, client_config: _, @@ -301,14 +308,26 @@ where forward_id, } => { *forward_id = Some(ClientId(self.inner.mgr_id().0 as u32)); + is_tc = true; true } + Event::UpdateExecStats { + time: _, + executions: _, + phantom: _, + } => true, // send it but this guy won't be handled. the only purpose is to keep this client alive else the broker thinks it is dead and will dc it _ => false, }; - if is_nt { - return self.forward_to_main(&event); + + if is_nt_or_heartbeat { + self.forward_to_main(&event)?; + if is_tc { + // early return here because we only send it to centralized not main broker. + return Ok(()); + } } } + // now inner llmp manager will process it self.inner.fire(state, event) } @@ -326,7 +345,7 @@ where where OT: ObserversTuple + Serialize, { - self.inner.serialize_observers(observers) + Ok(Some(postcard::to_allocvec(observers)?)) } #[cfg(feature = "adaptive_serialization")] @@ -334,40 +353,13 @@ where where OT: ObserversTuple + Serialize, { - const SERIALIZE_TIME_FACTOR: u32 = 4; - const SERIALIZE_PERCENTAGE_TRESHOLD: usize = 80; - - let exec_time = observers - .match_name::("time") - .map(|o| o.last_runtime().unwrap_or(Duration::ZERO)) - .unwrap(); - - let mut must_ser = (self.serialization_time() + self.deserialization_time()) - * SERIALIZE_TIME_FACTOR - < exec_time; - if must_ser { - *self.should_serialize_cnt_mut() += 1; - } - - if self.serializations_cnt() > 32 { - must_ser = (self.should_serialize_cnt() * 100 / self.serializations_cnt()) - > SERIALIZE_PERCENTAGE_TRESHOLD; - } - - if self.inner.serialization_time() == Duration::ZERO - || must_ser - || self.serializations_cnt().trailing_zeros() >= 8 - { - let start = current_time(); - let ser = postcard::to_allocvec(observers)?; - *self.inner.serialization_time_mut() = current_time() - start; - - *self.serializations_cnt_mut() += 1; - Ok(Some(ser)) - } else { - *self.serializations_cnt_mut() += 1; - Ok(None) - } + const SERIALIZE_TIME_FACTOR: u32 = 4; // twice as much as the normal llmp em's value cuz it does this job twice. + const SERIALIZE_PERCENTAGE_THRESHOLD: usize = 80; + self.inner.serialize_observers_adaptive( + observers, + SERIALIZE_TIME_FACTOR, + SERIALIZE_PERCENTAGE_THRESHOLD, + ) } fn configuration(&self) -> EventConfig { @@ -401,7 +393,7 @@ where impl EventProcessor for CentralizedEventManager where - EM: EventStatsCollector + EventProcessor + EventFirer + HasEventManagerId, + EM: AdaptiveSerializer + EventProcessor + EventFirer + HasEventManagerId, E: HasObservers + Executor, for<'a> E::Observers: Deserialize<'a>, Z: EvaluatorObservers @@ -427,7 +419,7 @@ where impl EventManager for CentralizedEventManager where - EM: EventStatsCollector + EventManager, + EM: AdaptiveSerializer + EventManager, EM::State: HasExecutions + HasMetadata + HasLastReportTime, E: HasObservers + Executor, for<'a> E::Observers: Deserialize<'a>, @@ -446,7 +438,7 @@ where fn add_custom_buf_handler( &mut self, handler: Box< - dyn FnMut(&mut Self::State, &String, &[u8]) -> Result, + dyn FnMut(&mut Self::State, &str, &[u8]) -> Result, >, ) { self.inner.add_custom_buf_handler(handler); @@ -455,7 +447,7 @@ where impl ProgressReporter for CentralizedEventManager where - EM: EventStatsCollector + ProgressReporter + HasEventManagerId, + EM: AdaptiveSerializer + ProgressReporter + HasEventManagerId, EM::State: HasMetadata + HasExecutions + HasLastReportTime, SP: ShMemProvider + 'static, { @@ -477,6 +469,7 @@ where SP: ShMemProvider + 'static, { /// Creates a new [`CentralizedEventManager`]. + #[cfg(not(feature = "adaptive_serialization"))] pub fn new(inner: EM, client: LlmpClient, is_main: bool) -> Result { Ok(Self { inner, @@ -487,24 +480,66 @@ where }) } + /// Creates a new [`CentralizedEventManager`]. + #[cfg(feature = "adaptive_serialization")] + pub fn new( + inner: EM, + client: LlmpClient, + is_main: bool, + time_obs: &TimeObserver, + ) -> Result { + Ok(Self { + inner, + client, + #[cfg(feature = "llmp_compression")] + compressor: GzipCompressor::new(COMPRESS_THRESHOLD), + time_ref: time_obs.handle(), + is_main, + }) + } + /// Create a centralized event manager on a port /// /// If the port is not yet bound, it will act as a broker; otherwise, it /// will act as a client. - #[cfg(feature = "std")] + #[cfg(all(feature = "std", not(feature = "adaptive_serialization")))] pub fn on_port(inner: EM, shmem_provider: SP, port: u16, is_main: bool) -> Result { + let client = LlmpClient::create_attach_to_tcp(shmem_provider, port)?; Ok(Self { inner, - client: LlmpClient::create_attach_to_tcp(shmem_provider, port)?, + client, #[cfg(feature = "llmp_compression")] compressor: GzipCompressor::new(COMPRESS_THRESHOLD), is_main, }) } + /// Create a centralized event manager on a port + /// + /// If the port is not yet bound, it will act as a broker; otherwise, it + /// will act as a client. + #[cfg(all(feature = "std", feature = "adaptive_serialization"))] + pub fn on_port( + inner: EM, + shmem_provider: SP, + port: u16, + is_main: bool, + time_obs: &TimeObserver, + ) -> Result { + let client = LlmpClient::create_attach_to_tcp(shmem_provider, port)?; + Ok(Self { + inner, + client, + #[cfg(feature = "llmp_compression")] + compressor: GzipCompressor::new(COMPRESS_THRESHOLD), + time_ref: time_obs.handle(), + is_main, + }) + } + /// If a client respawns, it may reuse the existing connection, previously /// stored by [`LlmpClient::to_env()`]. - #[cfg(feature = "std")] + #[cfg(all(feature = "std", not(feature = "adaptive_serialization")))] pub fn existing_client_from_env( inner: EM, shmem_provider: SP, @@ -520,27 +555,67 @@ where }) } - /// Describe the client event manager's LLMP parts in a restorable fashion - pub fn describe(&self) -> Result { - self.client.describe() + /// If a client respawns, it may reuse the existing connection, previously + /// stored by [`LlmpClient::to_env()`]. + #[cfg(all(feature = "std", feature = "adaptive_serialization"))] + pub fn existing_client_from_env( + inner: EM, + shmem_provider: SP, + env_name: &str, + is_main: bool, + time_obs: &TimeObserver, + ) -> Result { + Ok(Self { + inner, + client: LlmpClient::on_existing_from_env(shmem_provider, env_name)?, + #[cfg(feature = "llmp_compression")] + compressor: GzipCompressor::new(COMPRESS_THRESHOLD), + time_ref: time_obs.handle(), + is_main, + }) } /// Create an existing client from description + #[cfg(not(feature = "adaptive_serialization"))] + pub fn existing_client_from_description( + inner: EM, + shmem_provider: SP, + description: &LlmpClientDescription, + is_main: bool, + ) -> Result { + Ok(Self { + inner, + client: LlmpClient::existing_client_from_description(shmem_provider, description)?, + #[cfg(feature = "llmp_compression")] + compressor: GzipCompressor::new(COMPRESS_THRESHOLD), + is_main, + }) + } + + /// Create an existing client from description + #[cfg(feature = "adaptive_serialization")] pub fn existing_client_from_description( inner: EM, shmem_provider: SP, description: &LlmpClientDescription, is_main: bool, + time_obs: &TimeObserver, ) -> Result { Ok(Self { inner, client: LlmpClient::existing_client_from_description(shmem_provider, description)?, #[cfg(feature = "llmp_compression")] compressor: GzipCompressor::new(COMPRESS_THRESHOLD), + time_ref: time_obs.handle(), is_main, }) } + /// Describe the client event manager's LLMP parts in a restorable fashion + pub fn describe(&self) -> Result { + self.client.describe() + } + /// Write the config for a client [`EventManager`] to env vars, a new /// client can reattach using [`CentralizedEventManager::existing_client_from_env()`]. #[cfg(feature = "std")] @@ -556,7 +631,7 @@ where impl CentralizedEventManager where - EM: UsesState + EventFirer + EventStatsCollector + HasEventManagerId, + EM: UsesState + EventFirer + AdaptiveSerializer + HasEventManagerId, SP: ShMemProvider + 'static, { #[cfg(feature = "llmp_compression")] @@ -671,7 +746,7 @@ where { state.scalability_monitor_mut().testcase_with_observers += 1; } - fuzzer.process_execution( + fuzzer.execute_and_process( state, self, input.clone(), @@ -692,6 +767,7 @@ where false, )? }; + if let Some(item) = res.1 { if res.1.is_some() { self.inner.fire( diff --git a/libafl/src/events/hooks/mod.rs b/libafl/src/events/hooks/mod.rs new file mode 100644 index 0000000000..b10c0c8f0e --- /dev/null +++ b/libafl/src/events/hooks/mod.rs @@ -0,0 +1,84 @@ +//! Hooks for event managers, especifically these are used to hook before and and `handle_in_client`. +//! This will allow user to define pre/post-processing code when the event manager receives any message from +//! other clients +use libafl_bolts::ClientId; + +use crate::{events::Event, state::State, Error}; + +/// The hooks that are run before and after the event manager calls `handle_in_client` +pub trait EventManagerHook +where + S: State, +{ + /// The hook that runs before `handle_in_client` + /// Return false if you want to cancel the subsequent event handling + fn pre_exec( + &mut self, + state: &mut S, + client_id: ClientId, + event: &Event, + ) -> Result; + /// The hook that runs after `handle_in_client` + /// Return false if you want to cancel the subsequent event handling + fn post_exec(&mut self, state: &mut S, client_id: ClientId) -> Result; +} + +/// The tuples contains hooks to be executed for `handle_in_client` +pub trait EventManagerHooksTuple +where + S: State, +{ + /// The hook that runs before `handle_in_client` + fn pre_exec_all( + &mut self, + state: &mut S, + client_id: ClientId, + event: &Event, + ) -> Result; + /// The hook that runs after `handle_in_client` + fn post_exec_all(&mut self, state: &mut S, client_id: ClientId) -> Result; +} + +impl EventManagerHooksTuple for () +where + S: State, +{ + /// The hook that runs before `handle_in_client` + fn pre_exec_all( + &mut self, + _state: &mut S, + _client_id: ClientId, + _event: &Event, + ) -> Result { + Ok(true) + } + /// The hook that runs after `handle_in_client` + fn post_exec_all(&mut self, _state: &mut S, _client_id: ClientId) -> Result { + Ok(true) + } +} + +impl EventManagerHooksTuple for (Head, Tail) +where + Head: EventManagerHook, + Tail: EventManagerHooksTuple, + S: State, +{ + /// The hook that runs before `handle_in_client` + fn pre_exec_all( + &mut self, + state: &mut S, + client_id: ClientId, + event: &Event, + ) -> Result { + let first = self.0.pre_exec(state, client_id, event)?; + let second = self.1.pre_exec_all(state, client_id, event)?; + Ok(first & second) + } + /// The hook that runs after `handle_in_client` + fn post_exec_all(&mut self, state: &mut S, client_id: ClientId) -> Result { + let first = self.0.post_exec(state, client_id)?; + let second = self.1.post_exec_all(state, client_id)?; + Ok(first & second) + } +} diff --git a/libafl/src/events/launcher.rs b/libafl/src/events/launcher.rs index 75f250e9fb..b377364d98 100644 --- a/libafl/src/events/launcher.rs +++ b/libafl/src/events/launcher.rs @@ -15,6 +15,8 @@ use alloc::string::ToString; #[cfg(feature = "std")] use core::marker::PhantomData; +#[cfg(all(unix, feature = "std", feature = "fork"))] +use core::time::Duration; use core::{ fmt::{self, Debug, Formatter}, num::NonZeroUsize, @@ -23,29 +25,37 @@ use core::{ use std::net::SocketAddr; #[cfg(all(feature = "std", any(windows, not(feature = "fork"))))] use std::process::Stdio; -#[cfg(all(unix, feature = "std", feature = "fork"))] -use std::{fs::File, os::unix::io::AsRawFd, time::Duration}; +#[cfg(all(unix, feature = "std"))] +use std::{fs::File, os::unix::io::AsRawFd}; +#[cfg(all(unix, feature = "std"))] +use libafl_bolts::os::dup2; #[cfg(all(feature = "std", any(windows, not(feature = "fork"))))] use libafl_bolts::os::startable_self; +#[cfg(feature = "adaptive_serialization")] +use libafl_bolts::tuples::{Handle, Handler}; #[cfg(all(unix, feature = "std", feature = "fork"))] use libafl_bolts::{ core_affinity::get_core_ids, - os::{dup2, fork, ForkResult}, + os::{fork, ForkResult}, }; use libafl_bolts::{ core_affinity::{CoreId, Cores}, shmem::ShMemProvider, + tuples::tuple_list, }; #[cfg(feature = "std")] use typed_builder::TypedBuilder; +use super::hooks::EventManagerHooksTuple; #[cfg(all(unix, feature = "std", feature = "fork"))] use crate::events::{CentralizedEventManager, CentralizedLlmpEventBroker}; +#[cfg(feature = "adaptive_serialization")] +use crate::observers::TimeObserver; #[cfg(feature = "std")] use crate::{ events::{ - llmp::{LlmpRestartingEventManager, ManagerKind, RestartingMgr}, + llmp::{LlmpRestartingEventManager, LlmpShouldSaveState, ManagerKind, RestartingMgr}, EventConfig, }, monitors::Monitor, @@ -70,15 +80,16 @@ const LIBAFL_DEBUG_OUTPUT: &str = "LIBAFL_DEBUG_OUTPUT"; clippy::ignored_unit_patterns )] #[derive(TypedBuilder)] -pub struct Launcher<'a, CF, MT, S, SP> +pub struct Launcher<'a, CF, EMH, MT, S, SP> where - CF: FnOnce(Option, LlmpRestartingEventManager, CoreId) -> Result<(), Error>, + CF: FnOnce(Option, LlmpRestartingEventManager, CoreId) -> Result<(), Error>, + EMH: EventManagerHooksTuple, S::Input: 'a, MT: Monitor, SP: ShMemProvider + 'static, S: State + 'a, { - /// The ShmemProvider to use + /// The `ShmemProvider` to use shmem_provider: SP, /// The monitor instance to use monitor: MT, @@ -93,17 +104,19 @@ where /// The list of cores to run on cores: &'a Cores, /// A file name to write all client output to + #[cfg(all(unix, feature = "std"))] #[builder(default = None)] stdout_file: Option<&'a str>, - /// The actual, opened, stdout_file - so that we keep it open until the end + /// The actual, opened, `stdout_file` - so that we keep it open until the end #[cfg(all(unix, feature = "std", feature = "fork"))] #[builder(setter(skip), default = None)] opened_stdout_file: Option, /// A file name to write all client stderr output to. If not specified, output is sent to /// `stdout_file`. + #[cfg(all(unix, feature = "std"))] #[builder(default = None)] stderr_file: Option<&'a str>, - /// The actual, opened, stdout_file - so that we keep it open until the end + /// The actual, opened, `stdout_file` - so that we keep it open until the end #[cfg(all(unix, feature = "std", feature = "fork"))] #[builder(setter(skip), default = None)] opened_stderr_file: Option, @@ -111,6 +124,8 @@ where /// clusters. #[builder(default = None)] remote_broker_addr: Option, + #[cfg(feature = "adaptive_serialization")] + time_ref: Handle, /// If this launcher should spawn a new `broker` on `[Self::broker_port]` (default). /// The reason you may not want this is, if you already have a [`Launcher`] /// with a different configuration (for the same target) running on this machine. @@ -118,44 +133,74 @@ where #[builder(default = true)] spawn_broker: bool, /// Tell the manager to serialize or not the state on restart - #[builder(default = true)] - serialize_state: bool, + #[builder(default = LlmpShouldSaveState::OnRestart)] + serialize_state: LlmpShouldSaveState, #[builder(setter(skip), default = PhantomData)] - phantom_data: PhantomData<(&'a S, &'a SP)>, + phantom_data: PhantomData<(&'a S, &'a SP, EMH)>, } -impl Debug for Launcher<'_, CF, MT, S, SP> +impl Debug for Launcher<'_, CF, EMH, MT, S, SP> where - CF: FnOnce(Option, LlmpRestartingEventManager, CoreId) -> Result<(), Error>, + CF: FnOnce(Option, LlmpRestartingEventManager, CoreId) -> Result<(), Error>, + EMH: EventManagerHooksTuple, MT: Monitor + Clone, SP: ShMemProvider + 'static, S: State, { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - f.debug_struct("Launcher") + let mut dbg_struct = f.debug_struct("Launcher"); + dbg_struct .field("configuration", &self.configuration) .field("broker_port", &self.broker_port) .field("core", &self.cores) .field("spawn_broker", &self.spawn_broker) - .field("remote_broker_addr", &self.remote_broker_addr) - .field("stdout_file", &self.stdout_file) - .field("stderr_file", &self.stderr_file) - .finish_non_exhaustive() + .field("remote_broker_addr", &self.remote_broker_addr); + #[cfg(all(unix, feature = "std"))] + { + dbg_struct + .field("stdout_file", &self.stdout_file) + .field("stderr_file", &self.stderr_file); + } + + dbg_struct.finish_non_exhaustive() } } -#[cfg(feature = "std")] -impl<'a, CF, MT, S, SP> Launcher<'a, CF, MT, S, SP> +impl<'a, CF, MT, S, SP> Launcher<'a, CF, (), MT, S, SP> where - CF: FnOnce(Option, LlmpRestartingEventManager, CoreId) -> Result<(), Error>, + CF: FnOnce(Option, LlmpRestartingEventManager<(), S, SP>, CoreId) -> Result<(), Error>, MT: Monitor + Clone, S: State + HasExecutions, SP: ShMemProvider + 'static, { /// Launch the broker and the clients and fuzz #[cfg(all(unix, feature = "std", feature = "fork"))] - #[allow(clippy::similar_names)] pub fn launch(&mut self) -> Result<(), Error> { + Self::launch_with_hooks(self, tuple_list!()) + } + + /// Launch the broker and the clients and fuzz + #[cfg(all(feature = "std", any(windows, not(feature = "fork"))))] + #[allow(unused_mut, clippy::match_wild_err_arm)] + pub fn launch(&mut self) -> Result<(), Error> { + Self::launch_with_hooks(self, tuple_list!()) + } +} + +#[cfg(feature = "std")] +impl<'a, CF, EMH, MT, S, SP> Launcher<'a, CF, EMH, MT, S, SP> +where + CF: FnOnce(Option, LlmpRestartingEventManager, CoreId) -> Result<(), Error>, + EMH: EventManagerHooksTuple + Clone + Copy, + MT: Monitor + Clone, + S: State + HasExecutions, + SP: ShMemProvider + 'static, +{ + /// Launch the broker and the clients and fuzz with a user-supplied hook + #[cfg(all(unix, feature = "std", feature = "fork"))] + #[allow(clippy::similar_names)] + #[allow(clippy::too_many_lines)] + pub fn launch_with_hooks(&mut self, hooks: EMH) -> Result<(), Error> { if self.cores.ids.is_empty() { return Err(Error::illegal_argument( "No cores to spawn on given, cannot launch anything.", @@ -206,7 +251,7 @@ where self.shmem_provider.post_fork(true)?; #[cfg(feature = "std")] - std::thread::sleep(std::time::Duration::from_millis(index * 10)); + std::thread::sleep(Duration::from_millis(index * 10)); #[cfg(feature = "std")] if !debug_output { @@ -221,7 +266,7 @@ where } // Fuzzer client. keeps retrying the connection to broker till the broker starts - let (state, mgr) = RestartingMgr::::builder() + let builder = RestartingMgr::::builder() .shmem_provider(self.shmem_provider.clone()) .broker_port(self.broker_port) .kind(ManagerKind::Client { @@ -229,8 +274,10 @@ where }) .configuration(self.configuration) .serialize_state(self.serialize_state) - .build() - .launch()?; + .hooks(hooks); + #[cfg(feature = "adaptive_serialization")] + let builder = builder.time_ref(self.time_ref.clone()); + let (state, mgr) = builder.build().launch()?; return (self.run_client.take().unwrap())(state, mgr, *bind_to); } @@ -243,7 +290,7 @@ where log::info!("I am broker!!."); // TODO we don't want always a broker here, think about using different laucher process to spawn different configurations - RestartingMgr::::builder() + let builder = RestartingMgr::::builder() .shmem_provider(self.shmem_provider.clone()) .monitor(Some(self.monitor.clone())) .broker_port(self.broker_port) @@ -252,8 +299,12 @@ where .exit_cleanly_after(Some(NonZeroUsize::try_from(self.cores.ids.len()).unwrap())) .configuration(self.configuration) .serialize_state(self.serialize_state) - .build() - .launch()?; + .hooks(hooks); + + #[cfg(feature = "adaptive_serialization")] + let builder = builder.time_ref(self.time_ref.clone()); + + builder.build().launch()?; // Broker exited. kill all clients. for handle in &handles { @@ -282,7 +333,7 @@ where /// Launch the broker and the clients and fuzz #[cfg(all(feature = "std", any(windows, not(feature = "fork"))))] #[allow(unused_mut, clippy::match_wild_err_arm)] - pub fn launch(&mut self) -> Result<(), Error> { + pub fn launch_with_hooks(&mut self, hooks: EMH) -> Result<(), Error> { use libafl_bolts::core_affinity; let is_client = std::env::var(_AFL_LAUNCHER_CLIENT); @@ -290,12 +341,8 @@ where let mut handles = match is_client { Ok(core_conf) => { let core_id = core_conf.parse()?; - - // TODO: silence stdout and stderr for clients - // let debug_output = std::env::var(LIBAFL_DEBUG_OUTPUT).is_ok(); - // the actual client. do the fuzzing - let (state, mgr) = RestartingMgr::::builder() + let (state, mgr) = RestartingMgr::::builder() .shmem_provider(self.shmem_provider.clone()) .broker_port(self.broker_port) .kind(ManagerKind::Client { @@ -303,6 +350,7 @@ where }) .configuration(self.configuration) .serialize_state(self.serialize_state) + .hooks(hooks) .build() .launch()?; @@ -312,11 +360,6 @@ where // I am a broker // before going to the broker loop, spawn n clients - #[cfg(windows)] - if self.stdout_file.is_some() { - log::info!("Child process file stdio is not supported on Windows yet. Dumping to stdout instead..."); - } - let core_ids = core_affinity::get_core_ids().unwrap(); let num_cores = core_ids.len(); let mut handles = vec![]; @@ -324,22 +367,45 @@ where log::info!("spawning on cores: {:?}", self.cores); let debug_output = std::env::var("LIBAFL_DEBUG_OUTPUT").is_ok(); - + #[cfg(all(feature = "std", unix))] + { + // Set own stdout and stderr as set by the user + if !debug_output { + let opened_stdout_file = self + .stdout_file + .map(|filename| File::create(filename).unwrap()); + let opened_stderr_file = self + .stderr_file + .map(|filename| File::create(filename).unwrap()); + if let Some(file) = opened_stdout_file { + dup2(file.as_raw_fd(), libc::STDOUT_FILENO)?; + if let Some(stderr) = opened_stderr_file { + dup2(stderr.as_raw_fd(), libc::STDERR_FILENO)?; + } else { + dup2(file.as_raw_fd(), libc::STDERR_FILENO)?; + } + } + } + } //spawn clients for (id, _) in core_ids.iter().enumerate().take(num_cores) { if self.cores.ids.iter().any(|&x| x == id.into()) { - let stdio = if self.stdout_file.is_some() { - Stdio::inherit() - } else { - Stdio::null() - }; - + // Forward own stdio to child processes, if requested by user + let (mut stdout, mut stderr) = (Stdio::null(), Stdio::null()); + #[cfg(all(feature = "std", unix))] + { + if self.stdout_file.is_some() || self.stderr_file.is_some() { + stdout = Stdio::inherit(); + stderr = Stdio::inherit(); + }; + } std::env::set_var(_AFL_LAUNCHER_CLIENT, id.to_string()); let mut child = startable_self()?; let child = (if debug_output { &mut child } else { - child.stdout(stdio) + child.stdout(stdout); + child.stderr(stderr) }) .spawn()?; handles.push(child); @@ -363,7 +429,7 @@ where #[cfg(feature = "std")] log::info!("I am broker!!."); - RestartingMgr::::builder() + RestartingMgr::::builder() .shmem_provider(self.shmem_provider.clone()) .monitor(Some(self.monitor.clone())) .broker_port(self.broker_port) @@ -372,6 +438,7 @@ where .exit_cleanly_after(Some(NonZeroUsize::try_from(self.cores.ids.len()).unwrap())) .configuration(self.configuration) .serialize_state(self.serialize_state) + .hooks(hooks) .build() .launch()?; @@ -394,14 +461,20 @@ where } /// Provides a Launcher, which can be used to launch a fuzzing run on a specified list of cores with a single main and multiple secondary nodes +/// This is for centralized, the 4th argument of the closure should mean if this is the main node. #[cfg(all(unix, feature = "std", feature = "fork"))] #[derive(TypedBuilder)] #[allow(clippy::type_complexity, missing_debug_implementations)] -pub struct CentralizedLauncher<'a, CF, MT, S, SP> +pub struct CentralizedLauncher<'a, CF, MF, MT, S, SP> where CF: FnOnce( Option, - CentralizedEventManager, SP>, + CentralizedEventManager, SP>, // No hooks for centralized EM + CoreId, + ) -> Result<(), Error>, + MF: FnOnce( + Option, + CentralizedEventManager, SP>, // No hooks for centralized EM CoreId, ) -> Result<(), Error>, S::Input: 'a, @@ -409,7 +482,7 @@ where SP: ShMemProvider + 'static, S: State + 'a, { - /// The ShmemProvider to use + /// The `ShmemProvider` to use shmem_provider: SP, /// The monitor instance to use monitor: MT, @@ -418,18 +491,24 @@ where /// The 'main' function to run for each client forked. This probably shouldn't return #[builder(default, setter(strip_option))] run_client: Option, + /// The 'main' function to run for the main evaluator noed + #[builder(default, setter(strip_option))] + main_run_client: Option, /// The broker port to use (or to attach to, in case [`Self::spawn_broker`] is `false`) #[builder(default = 1337_u16)] broker_port: u16, /// The centralized broker port to use (or to attach to, in case [`Self::spawn_broker`] is `false`) #[builder(default = 1338_u16)] centralized_broker_port: u16, + /// The time observer by which to adaptively serialize + #[cfg(feature = "adaptive_serialization")] + time_obs: &'a TimeObserver, /// The list of cores to run on cores: &'a Cores, /// A file name to write all client output to #[builder(default = None)] stdout_file: Option<&'a str>, - /// The actual, opened, stdout_file - so that we keep it open until the end + /// The actual, opened, `stdout_file` - so that we keep it open until the end #[cfg(all(unix, feature = "std", feature = "fork"))] #[builder(setter(skip), default = None)] opened_stdout_file: Option, @@ -437,7 +516,7 @@ where /// `stdout_file`. #[builder(default = None)] stderr_file: Option<&'a str>, - /// The actual, opened, stdout_file - so that we keep it open until the end + /// The actual, opened, `stdout_file` - so that we keep it open until the end #[cfg(all(unix, feature = "std", feature = "fork"))] #[builder(setter(skip), default = None)] opened_stderr_file: Option, @@ -453,18 +532,23 @@ where #[builder(default = true)] spawn_broker: bool, /// Tell the manager to serialize or not the state on restart - #[builder(default = true)] - serialize_state: bool, + #[builder(default = LlmpShouldSaveState::OnRestart)] + serialize_state: LlmpShouldSaveState, #[builder(setter(skip), default = PhantomData)] phantom_data: PhantomData<(&'a S, &'a SP)>, } #[cfg(all(unix, feature = "std", feature = "fork"))] -impl Debug for CentralizedLauncher<'_, CF, MT, S, SP> +impl Debug for CentralizedLauncher<'_, CF, MF, MT, S, SP> where CF: FnOnce( Option, - CentralizedEventManager, SP>, + CentralizedEventManager, SP>, + CoreId, + ) -> Result<(), Error>, + MF: FnOnce( + Option, + CentralizedEventManager, SP>, // No hooks for centralized EM CoreId, ) -> Result<(), Error>, MT: Monitor + Clone, @@ -485,11 +569,16 @@ where } #[cfg(all(unix, feature = "std", feature = "fork"))] -impl<'a, CF, MT, S, SP> CentralizedLauncher<'a, CF, MT, S, SP> +impl<'a, CF, MF, MT, S, SP> CentralizedLauncher<'a, CF, MF, MT, S, SP> where CF: FnOnce( Option, - CentralizedEventManager, SP>, + CentralizedEventManager, SP>, + CoreId, + ) -> Result<(), Error>, + MF: FnOnce( + Option, + CentralizedEventManager, SP>, // No hooks for centralized EM CoreId, ) -> Result<(), Error>, MT: Monitor + Clone, @@ -498,7 +587,8 @@ where { #[allow(clippy::similar_names)] #[allow(clippy::too_many_lines)] - fn launch_internal(&mut self, client_timeout: Option) -> Result<(), Error> { + /// launch the broker and the client and fuzz + pub fn launch(&mut self) -> Result<(), Error> { if self.cores.ids.is_empty() { return Err(Error::illegal_argument( "No cores to spawn on given, cannot launch anything.", @@ -533,23 +623,24 @@ where self.shmem_provider.post_fork(false)?; handles.push(child.pid); #[cfg(feature = "std")] - log::info!("centralized broker spawned"); + log::info!("PID: {:#?} centralized broker spawned", std::process::id()); } ForkResult::Child => { log::info!("{:?} PostFork", unsafe { libc::getpid() }); + #[cfg(feature = "std")] + log::info!("PID: {:#?} I am centralized broker", std::process::id()); self.shmem_provider.post_fork(true)?; let mut broker: CentralizedLlmpEventBroker = CentralizedLlmpEventBroker::on_port( self.shmem_provider.clone(), self.centralized_broker_port, - client_timeout, )?; broker.broker_loop()?; } } - std::thread::sleep(std::time::Duration::from_millis(10)); + std::thread::sleep(Duration::from_millis(10)); // Spawn clients let mut index = 0_u64; @@ -568,7 +659,7 @@ where log::info!("{:?} PostFork", unsafe { libc::getpid() }); self.shmem_provider.post_fork(true)?; - std::thread::sleep(std::time::Duration::from_millis(index * 10)); + std::thread::sleep(Duration::from_millis(index * 10)); if !debug_output { if let Some(file) = &self.opened_stdout_file { @@ -582,7 +673,7 @@ where } // Fuzzer client. keeps retrying the connection to broker till the broker starts - let (state, mgr) = RestartingMgr::::builder() + let builder = RestartingMgr::<(), MT, S, SP>::builder() .shmem_provider(self.shmem_provider.clone()) .broker_port(self.broker_port) .kind(ManagerKind::Client { @@ -590,16 +681,30 @@ where }) .configuration(self.configuration) .serialize_state(self.serialize_state) - .build() - .launch()?; + .hooks(tuple_list!()); + #[cfg(feature = "adaptive_serialization")] + let builder = builder.time_ref(self.time_obs.handle()); + let (state, mgr) = builder.build().launch()?; + #[cfg(not(feature = "adaptive_serialization"))] + let c_mgr = CentralizedEventManager::on_port( + mgr, + self.shmem_provider.clone(), + self.centralized_broker_port, + index == 1, + )?; + #[cfg(feature = "adaptive_serialization")] let c_mgr = CentralizedEventManager::on_port( mgr, self.shmem_provider.clone(), self.centralized_broker_port, index == 1, + self.time_obs, )?; + if index == 1 { + return (self.main_run_client.take().unwrap())(state, c_mgr, *bind_to); + } return (self.run_client.take().unwrap())(state, c_mgr, *bind_to); } }; @@ -610,7 +715,7 @@ where log::info!("I am broker!!."); // TODO we don't want always a broker here, think about using different laucher process to spawn different configurations - RestartingMgr::::builder() + let builder = RestartingMgr::<(), MT, S, SP>::builder() .shmem_provider(self.shmem_provider.clone()) .monitor(Some(self.monitor.clone())) .broker_port(self.broker_port) @@ -619,8 +724,12 @@ where .exit_cleanly_after(Some(NonZeroUsize::try_from(self.cores.ids.len()).unwrap())) .configuration(self.configuration) .serialize_state(self.serialize_state) - .build() - .launch()?; + .hooks(tuple_list!()); + + #[cfg(feature = "adaptive_serialization")] + let builder = builder.time_ref(self.time_obs.handle()); + + builder.build().launch()?; // Broker exited. kill all clients. for handle in &handles { @@ -643,14 +752,4 @@ where Ok(()) } - - /// Launch the broker and the clients and fuzz - pub fn launch(&mut self) -> Result<(), Error> { - self.launch_internal(None) - } - - /// Launch the broker and the clients and fuzz with a given timeout for the clients - pub fn launch_with_client_timeout(&mut self, client_timeout: Duration) -> Result<(), Error> { - self.launch_internal(Some(client_timeout)) - } } diff --git a/libafl/src/events/llmp.rs b/libafl/src/events/llmp.rs index 5cc02ad97c..e350a47dbd 100644 --- a/libafl/src/events/llmp.rs +++ b/libafl/src/events/llmp.rs @@ -1,22 +1,18 @@ //! LLMP-backed event manager for scalable multi-processed fuzzing -use alloc::{ - boxed::Box, - string::{String, ToString}, - vec::Vec, -}; +#[cfg(feature = "std")] +use alloc::string::ToString; +use alloc::{boxed::Box, vec::Vec}; #[cfg(all(unix, not(miri), feature = "std"))] use core::ptr::addr_of_mut; #[cfg(feature = "std")] use core::sync::atomic::{compiler_fence, Ordering}; use core::{marker::PhantomData, num::NonZeroUsize, time::Duration}; #[cfg(feature = "std")] +use std::net::TcpStream; +#[cfg(feature = "std")] use std::net::{SocketAddr, ToSocketAddrs}; -#[cfg(feature = "std")] -use libafl_bolts::core_affinity::CoreId; -#[cfg(feature = "adaptive_serialization")] -use libafl_bolts::current_time; #[cfg(all(feature = "std", any(windows, not(feature = "fork"))))] use libafl_bolts::os::startable_self; #[cfg(all(unix, feature = "std", not(miri)))] @@ -29,21 +25,35 @@ use libafl_bolts::{ llmp::{LLMP_FLAG_COMPRESSED, LLMP_FLAG_INITIALIZED}, }; #[cfg(feature = "std")] +use libafl_bolts::{ + core_affinity::CoreId, + llmp::{recv_tcp_msg, send_tcp_msg, TcpRequest, TcpResponse}, + IP_LOCALHOST, +}; +#[cfg(feature = "adaptive_serialization")] +use libafl_bolts::{ + current_time, + tuples::{Handle, Handler}, +}; +#[cfg(feature = "std")] use libafl_bolts::{llmp::LlmpConnection, shmem::StdShMemProvider, staterestore::StateRestorer}; use libafl_bolts::{ llmp::{self, LlmpClient, LlmpClientDescription, Tag}, shmem::ShMemProvider, + tuples::tuple_list, ClientId, }; -#[cfg(feature = "std")] -use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; #[cfg(feature = "std")] use typed_builder::TypedBuilder; -use super::{CustomBufEventResult, CustomBufHandlerFn}; -#[cfg(all(unix, feature = "std"))] +use super::{hooks::EventManagerHooksTuple, CustomBufEventResult, CustomBufHandlerFn}; +#[cfg(any(feature = "std", feature = "adaptive_serialization"))] +use crate::events::AdaptiveSerializer; +#[cfg(all(unix, feature = "std", not(miri)))] use crate::events::EVENTMGR_SIGHANDLER_STATE; +#[cfg(feature = "adaptive_serialization")] +use crate::observers::TimeObserver; use crate::{ events::{ BrokerEventResult, Event, EventConfig, EventFirer, EventManager, EventManagerId, @@ -54,8 +64,8 @@ use crate::{ inputs::{Input, InputConverter, UsesInput}, monitors::Monitor, observers::ObserversTuple, - state::{HasExecutions, HasLastReportTime, HasMetadata, State, UsesState}, - Error, + state::{HasExecutions, HasLastReportTime, State, UsesState}, + Error, HasMetadata, }; /// Forward this to the client @@ -109,15 +119,10 @@ where /// /// The port must not be bound yet to have a broker. #[cfg(feature = "std")] - pub fn on_port( - shmem_provider: SP, - monitor: MT, - port: u16, - client_timeout: Option, - ) -> Result { + pub fn on_port(shmem_provider: SP, monitor: MT, port: u16) -> Result { Ok(Self { monitor, - llmp: llmp::LlmpBroker::create_attach_to_tcp(shmem_provider, port, client_timeout)?, + llmp: llmp::LlmpBroker::create_attach_to_tcp(shmem_provider, port)?, #[cfg(feature = "llmp_compression")] compressor: GzipCompressor::new(COMPRESS_THRESHOLD), phantom: PhantomData, @@ -208,7 +213,7 @@ where Ok(llmp::LlmpMsgHookResult::ForwardToClients) } } else { - monitor.display("Broker".into(), ClientId(0)); + monitor.display("Broker Heartbeat", ClientId(0)); Ok(llmp::LlmpMsgHookResult::Handled) } }, @@ -252,9 +257,10 @@ where if id == client_id { // do not update executions for forwarded messages, otherwise we loose the total order // as a forwarded msg with a lower executions may arrive after a stats msg with an higher executions - client.update_executions(*executions as u64, *time); + // this also means when you wrap this event manger with centralized EM, you will **NOT** get executions update with the new tc message + client.update_executions(*executions, *time); } - monitor.display(event.name().to_string(), id); + monitor.display(event.name(), id); Ok(BrokerEventResult::Forward) } Event::UpdateExecStats { @@ -265,8 +271,8 @@ where // TODO: The monitor buffer should be added on client add. monitor.client_stats_insert(client_id); let client = monitor.client_stats_mut_for(client_id); - client.update_executions(*executions as u64, *time); - monitor.display(event.name().to_string(), client_id); + client.update_executions(*executions, *time); + monitor.display(event.name(), client_id); Ok(BrokerEventResult::Handled) } Event::UpdateUserStats { @@ -278,7 +284,7 @@ where let client = monitor.client_stats_mut_for(client_id); client.update_user_stats(name.clone(), value.clone()); monitor.aggregate(name); - monitor.display(event.name().to_string(), client_id); + monitor.display(event.name(), client_id); Ok(BrokerEventResult::Handled) } #[cfg(feature = "introspection")] @@ -295,22 +301,27 @@ where let client = monitor.client_stats_mut_for(client_id); // Update the normal monitor for this client - client.update_executions(*executions as u64, *time); + client.update_executions(*executions, *time); // Update the performance monitor for this client client.update_introspection_monitor((**introspection_monitor).clone()); // Display the monitor via `.display` only on core #1 - monitor.display(event.name().to_string(), client_id); + monitor.display(event.name(), client_id); // Correctly handled the event Ok(BrokerEventResult::Handled) } - Event::Objective { objective_size } => { + Event::Objective { + objective_size, + executions, + time, + } => { monitor.client_stats_insert(client_id); let client = monitor.client_stats_mut_for(client_id); client.update_objective_size(*objective_size as u64); - monitor.display(event.name().to_string(), client_id); + client.update_executions(*executions, *time); + monitor.display(event.name(), client_id); Ok(BrokerEventResult::Handled) } Event::Log { @@ -329,39 +340,14 @@ where } } -/// Collected stats to decide if observers must be serialized or not -#[cfg(feature = "adaptive_serialization")] -pub trait EventStatsCollector { - /// Expose the collected observers serialization time - fn serialization_time(&self) -> Duration; - /// Expose the collected observers deserialization time - fn deserialization_time(&self) -> Duration; - /// How many times observers were serialized - fn serializations_cnt(&self) -> usize; - /// How many times shoukd have been serialized an observer - fn should_serialize_cnt(&self) -> usize; - - /// Expose the collected observers serialization time (mut) - fn serialization_time_mut(&mut self) -> &mut Duration; - /// Expose the collected observers deserialization time (mut) - fn deserialization_time_mut(&mut self) -> &mut Duration; - /// How many times observers were serialized (mut) - fn serializations_cnt_mut(&mut self) -> &mut usize; - /// How many times shoukd have been serialized an observer (mut) - fn should_serialize_cnt_mut(&mut self) -> &mut usize; -} - -/// Collected stats to decide if observers must be serialized or not -#[cfg(not(feature = "adaptive_serialization"))] -pub trait EventStatsCollector {} - /// An [`EventManager`] that forwards all events to other attached fuzzers on shared maps or via tcp, -/// using low-level message passing, [`libafl_bolts::llmp`]. -pub struct LlmpEventManager +/// using low-level message passing, [`llmp`]. +pub struct LlmpEventManager where S: State, SP: ShMemProvider + 'static, { + hooks: EMH, /// The LLMP client for inter process communication llmp: LlmpClient, /// The custom buf handler @@ -380,11 +366,13 @@ where serializations_cnt: usize, #[cfg(feature = "adaptive_serialization")] should_serialize_cnt: usize, + #[cfg(feature = "adaptive_serialization")] + time_ref: Handle, phantom: PhantomData, } #[cfg(feature = "adaptive_serialization")] -impl EventStatsCollector for LlmpEventManager +impl AdaptiveSerializer for LlmpEventManager where SP: ShMemProvider + 'static, S: State, @@ -414,9 +402,13 @@ where fn should_serialize_cnt_mut(&mut self) -> &mut usize { &mut self.should_serialize_cnt } + + fn time_ref(&self) -> &Handle { + &self.time_ref + } } -impl core::fmt::Debug for LlmpEventManager +impl core::fmt::Debug for LlmpEventManager where SP: ShMemProvider + 'static, S: State, @@ -434,7 +426,7 @@ where } } -impl Drop for LlmpEventManager +impl Drop for LlmpEventManager where SP: ShMemProvider + 'static, S: State, @@ -445,26 +437,20 @@ where } } -impl LlmpEventManager +impl LlmpEventManager<(), S, SP> where S: State, SP: ShMemProvider + 'static, { /// Create a manager from a raw LLMP client + #[cfg(not(feature = "adaptive_serialization"))] pub fn new(llmp: LlmpClient, configuration: EventConfig) -> Result { - Ok(Self { + Ok(LlmpEventManager { + hooks: tuple_list!(), llmp, #[cfg(feature = "llmp_compression")] compressor: GzipCompressor::new(COMPRESS_THRESHOLD), configuration, - #[cfg(feature = "adaptive_serialization")] - serialization_time: Duration::ZERO, - #[cfg(feature = "adaptive_serialization")] - deserialization_time: Duration::ZERO, - #[cfg(feature = "adaptive_serialization")] - serializations_cnt: 0, - #[cfg(feature = "adaptive_serialization")] - should_serialize_cnt: 0, phantom: PhantomData, custom_buf_handlers: vec![], }) @@ -474,85 +460,275 @@ where /// /// If the port is not yet bound, it will act as a broker; otherwise, it /// will act as a client. - #[cfg(feature = "std")] + #[cfg(all(feature = "std", not(feature = "adaptive_serialization")))] pub fn on_port( shmem_provider: SP, port: u16, configuration: EventConfig, + ) -> Result, Error> { + let llmp = LlmpClient::create_attach_to_tcp(shmem_provider, port)?; + Self::new(llmp, configuration) + } + + /// If a client respawns, it may reuse the existing connection, previously + /// stored by [`LlmpClient::to_env()`]. + #[cfg(all(feature = "std", not(feature = "adaptive_serialization")))] + pub fn existing_client_from_env( + shmem_provider: SP, + env_name: &str, + configuration: EventConfig, + ) -> Result, Error> { + let llmp = LlmpClient::on_existing_from_env(shmem_provider, env_name)?; + Self::new(llmp, configuration) + } + + /// Create an existing client from description + #[cfg(not(feature = "adaptive_serialization"))] + pub fn existing_client_from_description( + shmem_provider: SP, + description: &LlmpClientDescription, + configuration: EventConfig, + ) -> Result, Error> { + let llmp = LlmpClient::existing_client_from_description(shmem_provider, description)?; + Self::new(llmp, configuration) + } + + /// Create a manager from a raw LLMP client + #[cfg(feature = "adaptive_serialization")] + pub fn new( + llmp: LlmpClient, + configuration: EventConfig, + time_ref: Handle, ) -> Result { - Ok(Self { - llmp: LlmpClient::create_attach_to_tcp(shmem_provider, port)?, + Ok(LlmpEventManager { + hooks: tuple_list!(), + llmp, #[cfg(feature = "llmp_compression")] compressor: GzipCompressor::new(COMPRESS_THRESHOLD), configuration, - #[cfg(feature = "adaptive_serialization")] serialization_time: Duration::ZERO, - #[cfg(feature = "adaptive_serialization")] deserialization_time: Duration::ZERO, - #[cfg(feature = "adaptive_serialization")] serializations_cnt: 0, - #[cfg(feature = "adaptive_serialization")] should_serialize_cnt: 0, + time_ref, phantom: PhantomData, custom_buf_handlers: vec![], }) } + /// Create an LLMP event manager on a port + /// + /// If the port is not yet bound, it will act as a broker; otherwise, it + /// will act as a client. + #[cfg(all(feature = "std", feature = "adaptive_serialization"))] + pub fn on_port( + shmem_provider: SP, + port: u16, + configuration: EventConfig, + time_ref: Handle, + ) -> Result, Error> { + let llmp = LlmpClient::create_attach_to_tcp(shmem_provider, port)?; + Self::new(llmp, configuration, time_ref) + } + /// If a client respawns, it may reuse the existing connection, previously /// stored by [`LlmpClient::to_env()`]. - #[cfg(feature = "std")] + #[cfg(all(feature = "std", feature = "adaptive_serialization"))] pub fn existing_client_from_env( shmem_provider: SP, env_name: &str, configuration: EventConfig, + time_ref: Handle, + ) -> Result, Error> { + let llmp = LlmpClient::on_existing_from_env(shmem_provider, env_name)?; + Self::new(llmp, configuration, time_ref) + } + + /// Create an existing client from description + #[cfg(feature = "adaptive_serialization")] + pub fn existing_client_from_description( + shmem_provider: SP, + description: &LlmpClientDescription, + configuration: EventConfig, + time_ref: Handle, + ) -> Result, Error> { + let llmp = LlmpClient::existing_client_from_description(shmem_provider, description)?; + Self::new(llmp, configuration, time_ref) + } +} + +impl LlmpEventManager +where + S: State, + SP: ShMemProvider + 'static, +{ + /// Create a manager from a raw LLMP client with hooks + #[cfg(not(feature = "adaptive_serialization"))] + pub fn with_hooks( + llmp: LlmpClient, + configuration: EventConfig, + hooks: EMH, ) -> Result { Ok(Self { - llmp: LlmpClient::on_existing_from_env(shmem_provider, env_name)?, + hooks, + llmp, #[cfg(feature = "llmp_compression")] compressor: GzipCompressor::new(COMPRESS_THRESHOLD), configuration, - #[cfg(feature = "adaptive_serialization")] - serialization_time: Duration::ZERO, - #[cfg(feature = "adaptive_serialization")] - deserialization_time: Duration::ZERO, - #[cfg(feature = "adaptive_serialization")] - serializations_cnt: 0, - #[cfg(feature = "adaptive_serialization")] - should_serialize_cnt: 0, phantom: PhantomData, custom_buf_handlers: vec![], }) } - /// Describe the client event manager's LLMP parts in a restorable fashion - pub fn describe(&self) -> Result { - self.llmp.describe() + /// Create an LLMP event manager on a port with hook + /// + /// If the port is not yet bound, it will act as a broker; otherwise, it + /// will act as a client. + /// This will make a new connection to the broker so will return its new [`ClientId`], too + #[cfg(all(feature = "std", not(feature = "adaptive_serialization")))] + pub fn on_port_with_hooks( + shmem_provider: SP, + port: u16, + configuration: EventConfig, + hooks: EMH, + ) -> Result { + let llmp = LlmpClient::create_attach_to_tcp(shmem_provider, port)?; + Self::with_hooks(llmp, configuration, hooks) + } + + /// If a client respawns, it may reuse the existing connection, previously + /// stored by [`LlmpClient::to_env()`]. + /// create a event manager from env with hooks + #[cfg(all(feature = "std", not(feature = "adaptive_serialization")))] + pub fn existing_client_from_env_with_hooks( + shmem_provider: SP, + env_name: &str, + configuration: EventConfig, + hooks: EMH, + ) -> Result { + let llmp = LlmpClient::on_existing_from_env(shmem_provider, env_name)?; + Self::with_hooks(llmp, configuration, hooks) } /// Create an existing client from description - pub fn existing_client_from_description( + #[cfg(not(feature = "adaptive_serialization"))] + pub fn existing_client_from_description_with_hooks( shmem_provider: SP, description: &LlmpClientDescription, configuration: EventConfig, + hooks: EMH, + ) -> Result { + let llmp = LlmpClient::existing_client_from_description(shmem_provider, description)?; + Self::with_hooks(llmp, configuration, hooks) + } + + /// Create a manager from a raw LLMP client with hooks + #[cfg(feature = "adaptive_serialization")] + pub fn with_hooks( + llmp: LlmpClient, + configuration: EventConfig, + hooks: EMH, + time_ref: Handle, ) -> Result { Ok(Self { - llmp: LlmpClient::existing_client_from_description(shmem_provider, description)?, + hooks, + llmp, #[cfg(feature = "llmp_compression")] compressor: GzipCompressor::new(COMPRESS_THRESHOLD), configuration, - #[cfg(feature = "adaptive_serialization")] serialization_time: Duration::ZERO, - #[cfg(feature = "adaptive_serialization")] deserialization_time: Duration::ZERO, - #[cfg(feature = "adaptive_serialization")] serializations_cnt: 0, - #[cfg(feature = "adaptive_serialization")] should_serialize_cnt: 0, + time_ref, phantom: PhantomData, custom_buf_handlers: vec![], }) } + /// Create an LLMP event manager on a port with hook + /// + /// If the port is not yet bound, it will act as a broker; otherwise, it + /// will act as a client. + /// This will make a new connection to the broker so will return its new [`ClientId`], too + #[cfg(all(feature = "std", feature = "adaptive_serialization"))] + pub fn on_port_with_hooks( + shmem_provider: SP, + port: u16, + configuration: EventConfig, + hooks: EMH, + time_ref: Handle, + ) -> Result { + let llmp = LlmpClient::create_attach_to_tcp(shmem_provider, port)?; + Self::with_hooks(llmp, configuration, hooks, time_ref) + } + + /// If a client respawns, it may reuse the existing connection, previously + /// stored by [`LlmpClient::to_env()`]. + /// create a event manager from env with hooks + #[cfg(all(feature = "std", feature = "adaptive_serialization"))] + pub fn existing_client_from_env_with_hooks( + shmem_provider: SP, + env_name: &str, + configuration: EventConfig, + hooks: EMH, + time_ref: Handle, + ) -> Result { + let llmp = LlmpClient::on_existing_from_env(shmem_provider, env_name)?; + Self::with_hooks(llmp, configuration, hooks, time_ref) + } + + /// Create an existing client from description + #[cfg(feature = "adaptive_serialization")] + pub fn existing_client_from_description_with_hooks( + shmem_provider: SP, + description: &LlmpClientDescription, + configuration: EventConfig, + hooks: EMH, + time_ref: Handle, + ) -> Result { + let llmp = LlmpClient::existing_client_from_description(shmem_provider, description)?; + Self::with_hooks(llmp, configuration, hooks, time_ref) + } + + /// Calling this function will tell the llmp broker that this client is exiting + /// This should be called from the restarter not from the actual fuzzer client + /// This function serves the same roll as the `LlmpClient.send_exiting()` + /// However, from the the event restarter process it is forbidden to call `send_exiting()` + /// (You can call it and it compiles but you should never do so) + /// `send_exiting()` is exclusive to the fuzzer client. + #[cfg(feature = "std")] + pub fn detach_from_broker(&self, broker_port: u16) -> Result<(), Error> { + let client_id = self.llmp.sender().id(); + let Ok(mut stream) = TcpStream::connect((IP_LOCALHOST, broker_port)) else { + log::error!("Connection refused."); + return Ok(()); + }; + // The broker tells us hello we don't care we just tell it our client died + let TcpResponse::BrokerConnectHello { + broker_shmem_description: _, + hostname: _, + } = recv_tcp_msg(&mut stream)?.try_into()? + else { + return Err(Error::illegal_state( + "Received unexpected Broker Hello".to_string(), + )); + }; + let msg = TcpRequest::ClientQuit { client_id }; + // Send this mesasge off and we are leaving. + match send_tcp_msg(&mut stream, &msg) { + Ok(_) => (), + Err(e) => log::error!("Failed to send tcp message {:#?}", e), + } + log::info!("Asking he broker to be disconnected"); + Ok(()) + } + + /// Describe the client event manager's LLMP parts in a restorable fashion + pub fn describe(&self) -> Result { + self.llmp.describe() + } + /// Write the config for a client [`EventManager`] to env vars, a new /// client can reattach using [`LlmpEventManager::existing_client_from_env()`]. #[cfg(feature = "std")] @@ -561,8 +737,9 @@ where } } -impl LlmpEventManager +impl LlmpEventManager where + EMH: EventManagerHooksTuple, S: State + HasExecutions + HasMetadata, SP: ShMemProvider + 'static, { @@ -581,6 +758,9 @@ where for<'a> E::Observers: Deserialize<'a>, Z: ExecutionProcessor + EvaluatorObservers, { + if !self.hooks.pre_exec_all(state, client_id, &event)? { + return Ok(()); + } match event { Event::NewTestcase { input, @@ -609,7 +789,7 @@ where { state.scalability_monitor_mut().testcase_with_observers += 1; } - fuzzer.process_execution(state, self, input, &observers, &exit_kind, false)? + fuzzer.execute_and_process(state, self, input, &observers, &exit_kind, false)? } else { #[cfg(feature = "scalability_introspection")] { @@ -622,7 +802,6 @@ where if let Some(item) = res.1 { log::info!("Added received Testcase as item #{item}"); } - Ok(()) } Event::CustomBuf { tag, buf } => { for handler in &mut self.custom_buf_handlers { @@ -630,17 +809,20 @@ where break; } } - Ok(()) } - _ => Err(Error::unknown(format!( - "Received illegal message that message should not have arrived: {:?}.", - event.name() - ))), + _ => { + return Err(Error::unknown(format!( + "Received illegal message that message should not have arrived: {:?}.", + event.name() + ))); + } } + self.hooks.post_exec_all(state, client_id)?; + Ok(()) } } -impl LlmpEventManager { +impl LlmpEventManager { /// Send information that this client is exiting. /// The other side may free up all allocated memory. /// We are no longer allowed to send anything afterwards. @@ -649,7 +831,7 @@ impl LlmpEventManager { } } -impl UsesState for LlmpEventManager +impl UsesState for LlmpEventManager where S: State, SP: ShMemProvider, @@ -657,7 +839,7 @@ where type State = S; } -impl EventFirer for LlmpEventManager +impl EventFirer for LlmpEventManager where S: State, SP: ShMemProvider, @@ -711,39 +893,12 @@ where OT: ObserversTuple + Serialize, { const SERIALIZE_TIME_FACTOR: u32 = 2; - const SERIALIZE_PERCENTAGE_TRESHOLD: usize = 80; - - let exec_time = observers - .match_name::("time") - .map(|o| o.last_runtime().unwrap_or(Duration::ZERO)) - .unwrap(); - - let mut must_ser = (self.serialization_time() + self.deserialization_time()) - * SERIALIZE_TIME_FACTOR - < exec_time; - if must_ser { - *self.should_serialize_cnt_mut() += 1; - } - - if self.serializations_cnt() > 32 { - must_ser = (self.should_serialize_cnt() * 100 / self.serializations_cnt()) - > SERIALIZE_PERCENTAGE_TRESHOLD; - } - - if self.serialization_time() == Duration::ZERO - || must_ser - || self.serializations_cnt().trailing_zeros() >= 8 - { - let start = current_time(); - let ser = postcard::to_allocvec(observers)?; - *self.serialization_time_mut() = current_time() - start; - - *self.serializations_cnt_mut() += 1; - Ok(Some(ser)) - } else { - *self.serializations_cnt_mut() += 1; - Ok(None) - } + const SERIALIZE_PERCENTAGE_THRESHOLD: usize = 80; + self.serialize_observers_adaptive( + observers, + SERIALIZE_TIME_FACTOR, + SERIALIZE_PERCENTAGE_THRESHOLD, + ) } fn configuration(&self) -> EventConfig { @@ -751,7 +906,7 @@ where } } -impl EventRestarter for LlmpEventManager +impl EventRestarter for LlmpEventManager where S: State, SP: ShMemProvider, @@ -764,8 +919,9 @@ where } } -impl EventProcessor for LlmpEventManager +impl EventProcessor for LlmpEventManager where + EMH: EventManagerHooksTuple, S: State + HasExecutions + HasMetadata, SP: ShMemProvider, E: HasObservers + Executor, @@ -809,37 +965,38 @@ where } } -impl EventManager for LlmpEventManager +impl EventManager for LlmpEventManager where E: HasObservers + Executor, for<'a> E::Observers: Deserialize<'a>, + EMH: EventManagerHooksTuple, S: State + HasExecutions + HasMetadata + HasLastReportTime, SP: ShMemProvider, Z: EvaluatorObservers + ExecutionProcessor, { } -impl HasCustomBufHandlers for LlmpEventManager +impl HasCustomBufHandlers for LlmpEventManager where S: State, SP: ShMemProvider, { fn add_custom_buf_handler( &mut self, - handler: Box Result>, + handler: Box Result>, ) { self.custom_buf_handlers.push(handler); } } -impl ProgressReporter for LlmpEventManager +impl ProgressReporter for LlmpEventManager where S: State + HasExecutions + HasMetadata + HasLastReportTime, SP: ShMemProvider, { } -impl HasEventManagerId for LlmpEventManager +impl HasEventManagerId for LlmpEventManager where S: State, SP: ShMemProvider, @@ -850,25 +1007,60 @@ where } } +/// Specify if the State must be persistent over restarts +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum LlmpShouldSaveState { + /// Always save and restore the state on restart (not OOM resistant) + OnRestart, + /// Never save the state (not OOM resistant) + Never, + /// Best-effort save and restore the state on restart (OOM safe) + /// This adds additional runtime costs when processing events + OOMSafeOnRestart, + /// Never save the state (OOM safe) + /// This adds additional runtime costs when processing events + OOMSafeNever, +} + +impl LlmpShouldSaveState { + /// Check if the state must be saved `on_restart()` + #[must_use] + pub fn on_restart(&self) -> bool { + matches!( + self, + LlmpShouldSaveState::OnRestart | LlmpShouldSaveState::OOMSafeOnRestart + ) + } + + /// Check if the policy is OOM safe + #[must_use] + pub fn oom_safe(&self) -> bool { + matches!( + self, + LlmpShouldSaveState::OOMSafeOnRestart | LlmpShouldSaveState::OOMSafeNever + ) + } +} + /// A manager that can restart on the fly, storing states in-between (in `on_restart`) #[cfg(feature = "std")] #[derive(Debug)] -pub struct LlmpRestartingEventManager +pub struct LlmpRestartingEventManager where S: State, SP: ShMemProvider + 'static, //CE: CustomEvent, { /// The embedded LLMP event manager - llmp_mgr: LlmpEventManager, + llmp_mgr: LlmpEventManager, /// The staterestorer to serialize the state for the next runner staterestorer: StateRestorer, /// Decide if the state restorer must save the serialized state - save_state: bool, + save_state: LlmpShouldSaveState, } #[cfg(all(feature = "std", feature = "adaptive_serialization"))] -impl EventStatsCollector for LlmpRestartingEventManager +impl AdaptiveSerializer for LlmpRestartingEventManager where SP: ShMemProvider + 'static, S: State, @@ -898,10 +1090,14 @@ where fn should_serialize_cnt_mut(&mut self) -> &mut usize { self.llmp_mgr.should_serialize_cnt_mut() } + + fn time_ref(&self) -> &Handle { + &self.llmp_mgr.time_ref + } } #[cfg(all(feature = "std", not(feature = "adaptive_serialization")))] -impl EventStatsCollector for LlmpRestartingEventManager +impl AdaptiveSerializer for LlmpRestartingEventManager where SP: ShMemProvider + 'static, S: State, @@ -909,7 +1105,7 @@ where } #[cfg(feature = "std")] -impl UsesState for LlmpRestartingEventManager +impl UsesState for LlmpRestartingEventManager where S: State, SP: ShMemProvider + 'static, @@ -918,7 +1114,7 @@ where } #[cfg(feature = "std")] -impl ProgressReporter for LlmpRestartingEventManager +impl ProgressReporter for LlmpRestartingEventManager where S: State + HasExecutions + HasMetadata + HasLastReportTime, SP: ShMemProvider, @@ -926,7 +1122,7 @@ where } #[cfg(feature = "std")] -impl EventFirer for LlmpRestartingEventManager +impl EventFirer for LlmpRestartingEventManager where SP: ShMemProvider, S: State, @@ -938,7 +1134,9 @@ where event: Event<::Input>, ) -> Result<(), Error> { // Check if we are going to crash in the event, in which case we store our current state for the next runner - self.llmp_mgr.fire(state, event) + self.llmp_mgr.fire(state, event)?; + self.intermediate_save()?; + Ok(()) } fn serialize_observers(&mut self, observers: &OT) -> Result>, Error> @@ -954,7 +1152,7 @@ where } #[cfg(feature = "std")] -impl EventRestarter for LlmpRestartingEventManager +impl EventRestarter for LlmpRestartingEventManager where S: State + HasExecutions, SP: ShMemProvider, @@ -974,7 +1172,11 @@ where // First, reset the page to 0 so the next iteration can read read from the beginning of this page self.staterestorer.reset(); self.staterestorer.save(&( - if self.save_state { Some(state) } else { None }, + if self.save_state.on_restart() { + Some(state) + } else { + None + }, &self.llmp_mgr.describe()?, ))?; @@ -992,24 +1194,28 @@ where } #[cfg(feature = "std")] -impl EventProcessor for LlmpRestartingEventManager +impl EventProcessor for LlmpRestartingEventManager where - E: HasObservers + Executor, Z>, + E: HasObservers + Executor, Z>, for<'a> E::Observers: Deserialize<'a>, + EMH: EventManagerHooksTuple, S: State + HasExecutions + HasMetadata, SP: ShMemProvider + 'static, Z: EvaluatorObservers + ExecutionProcessor, //CE: CustomEvent, { fn process(&mut self, fuzzer: &mut Z, state: &mut S, executor: &mut E) -> Result { - self.llmp_mgr.process(fuzzer, state, executor) + let res = self.llmp_mgr.process(fuzzer, state, executor)?; + self.intermediate_save()?; + Ok(res) } } #[cfg(feature = "std")] -impl EventManager for LlmpRestartingEventManager +impl EventManager for LlmpRestartingEventManager where - E: HasObservers + Executor, Z>, + E: HasObservers + Executor, Z>, for<'a> E::Observers: Deserialize<'a>, + EMH: EventManagerHooksTuple, S: State + HasExecutions + HasMetadata + HasLastReportTime, SP: ShMemProvider + 'static, Z: EvaluatorObservers + ExecutionProcessor, //CE: CustomEvent, @@ -1017,7 +1223,7 @@ where } #[cfg(feature = "std")] -impl HasEventManagerId for LlmpRestartingEventManager +impl HasEventManagerId for LlmpRestartingEventManager where S: State, SP: ShMemProvider + 'static, @@ -1034,26 +1240,26 @@ const _ENV_FUZZER_RECEIVER: &str = "_AFL_ENV_FUZZER_RECEIVER"; const _ENV_FUZZER_BROKER_CLIENT_INITIAL: &str = "_AFL_ENV_FUZZER_BROKER_CLIENT"; #[cfg(feature = "std")] -impl LlmpRestartingEventManager +impl LlmpRestartingEventManager where S: State, SP: ShMemProvider + 'static, //CE: CustomEvent, { /// Create a new runner, the executed child doing the actual fuzzing. - pub fn new(llmp_mgr: LlmpEventManager, staterestorer: StateRestorer) -> Self { + pub fn new(llmp_mgr: LlmpEventManager, staterestorer: StateRestorer) -> Self { Self { llmp_mgr, staterestorer, - save_state: true, + save_state: LlmpShouldSaveState::OnRestart, } } /// Create a new runner specifying if it must save the serialized state on restart. pub fn with_save_state( - llmp_mgr: LlmpEventManager, + llmp_mgr: LlmpEventManager, staterestorer: StateRestorer, - save_state: bool, + save_state: LlmpShouldSaveState, ) -> Self { Self { llmp_mgr, @@ -1071,6 +1277,17 @@ where pub fn staterestorer_mut(&mut self) -> &mut StateRestorer { &mut self.staterestorer } + + /// Save LLMP state and empty state in staterestorer + pub fn intermediate_save(&mut self) -> Result<(), Error> { + // First, reset the page to 0 so the next iteration can read read from the beginning of this page + if self.save_state.oom_safe() { + self.staterestorer.reset(); + self.staterestorer + .save(&(None::, &self.llmp_mgr.describe()?))?; + } + Ok(()) + } } /// The kind of manager we're creating right now @@ -1091,13 +1308,19 @@ pub enum ManagerKind { /// Sets up a restarting fuzzer, using the [`StdShMemProvider`], and standard features. /// The restarting mgr is a combination of restarter and runner, that can be used on systems with and without `fork` support. /// The restarter will spawn a new process each time the child crashes or timeouts. -#[cfg(feature = "std")] +#[cfg(all(feature = "std", not(feature = "adaptive_serialization")))] #[allow(clippy::type_complexity)] pub fn setup_restarting_mgr_std( monitor: MT, broker_port: u16, configuration: EventConfig, -) -> Result<(Option, LlmpRestartingEventManager), Error> +) -> Result< + ( + Option, + LlmpRestartingEventManager<(), S, StdShMemProvider>, + ), + Error, +> where MT: Monitor + Clone, S: State + HasExecutions, @@ -1107,6 +1330,39 @@ where .monitor(Some(monitor)) .broker_port(broker_port) .configuration(configuration) + .hooks(tuple_list!()) + .build() + .launch() +} + +/// Sets up a restarting fuzzer, using the [`StdShMemProvider`], and standard features. +/// The restarting mgr is a combination of restarter and runner, that can be used on systems with and without `fork` support. +/// The restarter will spawn a new process each time the child crashes or timeouts. +#[cfg(all(feature = "std", feature = "adaptive_serialization"))] +#[allow(clippy::type_complexity)] +pub fn setup_restarting_mgr_std( + monitor: MT, + broker_port: u16, + configuration: EventConfig, + time_obs: &TimeObserver, +) -> Result< + ( + Option, + LlmpRestartingEventManager<(), S, StdShMemProvider>, + ), + Error, +> +where + MT: Monitor + Clone, + S: State + HasExecutions, +{ + RestartingMgr::builder() + .shmem_provider(StdShMemProvider::new()?) + .monitor(Some(monitor)) + .broker_port(broker_port) + .configuration(configuration) + .hooks(tuple_list!()) + .time_ref(time_obs.handle()) .build() .launch() } @@ -1117,9 +1373,10 @@ where #[cfg(feature = "std")] #[allow(clippy::default_trait_access, clippy::ignored_unit_patterns)] #[derive(TypedBuilder, Debug)] -pub struct RestartingMgr +pub struct RestartingMgr where - S: UsesInput + DeserializeOwned, + EMH: EventManagerHooksTuple, + S: State, SP: ShMemProvider + 'static, MT: Monitor, //CE: CustomEvent, @@ -1150,37 +1407,27 @@ where #[builder(default = None)] exit_cleanly_after: Option, /// Tell the manager to serialize or not the state on restart - #[builder(default = true)] - serialize_state: bool, + #[builder(default = LlmpShouldSaveState::OnRestart)] + serialize_state: LlmpShouldSaveState, + /// The hooks passed to event manager: + hooks: EMH, + #[cfg(feature = "adaptive_serialization")] + time_ref: Handle, #[builder(setter(skip), default = PhantomData)] - phantom_data: PhantomData, + phantom_data: PhantomData<(EMH, S)>, } #[cfg(feature = "std")] #[allow(clippy::type_complexity, clippy::too_many_lines)] -impl RestartingMgr +impl RestartingMgr where + EMH: EventManagerHooksTuple + Copy + Clone, SP: ShMemProvider, S: State + HasExecutions, MT: Monitor + Clone, { - /// Internal function, returns true when shuttdown is requested by a `SIGINT` signal - #[inline] - #[allow(clippy::unused_self)] - fn is_shutting_down() -> bool { - #[cfg(unix)] - unsafe { - core::ptr::read_volatile(core::ptr::addr_of!(EVENTMGR_SIGHANDLER_STATE.shutting_down)) - } - - #[cfg(windows)] - false - } - - fn launch_internal( - &mut self, - client_timeout: Option, - ) -> Result<(Option, LlmpRestartingEventManager), Error> { + /// Launch the broker and the clients and fuzz + pub fn launch(&mut self) -> Result<(Option, LlmpRestartingEventManager), Error> { // We start ourself as child process to actually fuzz let (staterestorer, new_shmem_provider, core_id) = if std::env::var(_ENV_FUZZER_SENDER) .is_err() @@ -1198,15 +1445,11 @@ where broker.broker_loop() }; - // We get here if we are on Unix, or we are a broker on Windows (or without forks). let (mgr, core_id) = match self.kind { ManagerKind::Any => { - let connection = LlmpConnection::on_port( - self.shmem_provider.clone(), - self.broker_port, - client_timeout, - )?; + let connection = + LlmpConnection::on_port(self.shmem_provider.clone(), self.broker_port)?; match connection { LlmpConnection::IsBroker { broker } => { let event_broker = LlmpEventBroker::::new( @@ -1224,7 +1467,19 @@ where return Err(Error::shutting_down()); } LlmpConnection::IsClient { client } => { - let mgr = LlmpEventManager::::new(client, self.configuration)?; + #[cfg(not(feature = "adaptive_serialization"))] + let mgr = LlmpEventManager::::with_hooks( + client, + self.configuration, + self.hooks, + )?; + #[cfg(feature = "adaptive_serialization")] + let mgr = LlmpEventManager::::with_hooks( + client, + self.configuration, + self.hooks, + self.time_ref.clone(), + )?; (mgr, None) } } @@ -1234,7 +1489,6 @@ where self.shmem_provider.clone(), self.monitor.take().unwrap(), self.broker_port, - client_timeout, )?; broker_things(event_broker, self.remote_broker_addr)?; @@ -1242,10 +1496,20 @@ where } ManagerKind::Client { cpu_core } => { // We are a client - let mgr = LlmpEventManager::::on_port( + #[cfg(not(feature = "adaptive_serialization"))] + let mgr = LlmpEventManager::::on_port_with_hooks( self.shmem_provider.clone(), self.broker_port, self.configuration, + self.hooks, + )?; + #[cfg(feature = "adaptive_serialization")] + let mgr = LlmpEventManager::::on_port_with_hooks( + self.shmem_provider.clone(), + self.broker_port, + self.configuration, + self.hooks, + self.time_ref.clone(), )?; (mgr, cpu_core) @@ -1272,15 +1536,6 @@ where // Store the information to a map. staterestorer.write_to_env(_ENV_FUZZER_SENDER)?; - // We setup signal handlers to clean up shmem segments used by state restorer - #[cfg(all(unix, not(miri)))] - if let Err(_e) = - unsafe { setup_signal_handler(addr_of_mut!(EVENTMGR_SIGHANDLER_STATE)) } - { - // We can live without a proper ctrl+c signal handler. Print and ignore. - log::error!("Failed to setup signal handlers: {_e}"); - } - let mut ctr: u64 = 0; // Client->parent loop loop { @@ -1293,7 +1548,7 @@ where match unsafe { fork() }? { ForkResult::Parent(handle) => { unsafe { - EVENTMGR_SIGHANDLER_STATE.set_exit_from_main(); + libc::signal(libc::SIGINT, libc::SIG_IGN); } self.shmem_provider.post_fork(false)?; handle.status() @@ -1305,39 +1560,61 @@ where } }; - #[cfg(all(unix, not(feature = "fork")))] + // If this guy wants to fork, then ignore sigit + #[cfg(any(windows, not(feature = "fork")))] unsafe { - EVENTMGR_SIGHANDLER_STATE.set_exit_from_main(); + #[cfg(windows)] + libafl_bolts::os::windows_exceptions::signal( + libafl_bolts::os::windows_exceptions::SIGINT, + libafl_bolts::os::windows_exceptions::sig_ign(), + ); + + #[cfg(unix)] + libc::signal(libc::SIGINT, libc::SIG_IGN); } // On Windows (or in any case without fork), we spawn ourself again #[cfg(any(windows, not(feature = "fork")))] let child_status = startable_self()?.status()?; - #[cfg(all(unix, not(feature = "fork")))] + #[cfg(any(windows, not(feature = "fork")))] let child_status = child_status.code().unwrap_or_default(); compiler_fence(Ordering::SeqCst); + if child_status == crate::events::CTRL_C_EXIT || staterestorer.wants_to_exit() { + // if ctrl-c is pressed, we end up in this branch + if let Err(err) = mgr.detach_from_broker(self.broker_port) { + log::error!("Failed to detach from broker: {err}"); + } + return Err(Error::shutting_down()); + } + #[allow(clippy::manual_assert)] - if !staterestorer.has_content() && self.serialize_state { + if !staterestorer.has_content() && !self.serialize_state.oom_safe() { + if let Err(err) = mgr.detach_from_broker(self.broker_port) { + log::error!("Failed to detach from broker: {err}"); + } #[cfg(unix)] - if child_status == 137 { - // Out of Memory, see https://tldp.org/LDP/abs/html/exitcodes.html - // and https://github.com/AFLplusplus/LibAFL/issues/32 for discussion. - panic!("Fuzzer-respawner: The fuzzed target crashed with an out of memory error! Fix your harness, or switch to another executor (for example, a forkserver)."); + if child_status == 9 { + panic!("Target received SIGKILL!. This could indicate the target crashed due to OOM, user sent SIGKILL, or the target was in an unrecoverable situation and could not save state to restart"); } - // Storing state in the last round did not work panic!("Fuzzer-respawner: Storing state in crashed fuzzer instance did not work, no point to spawn the next client! This can happen if the child calls `exit()`, in that case make sure it uses `abort()`, if it got killed unrecoverable (OOM), or if there is a bug in the fuzzer itself. (Child exited with: {child_status})"); } - if staterestorer.wants_to_exit() || Self::is_shutting_down() { - return Err(Error::shutting_down()); - } - ctr = ctr.wrapping_add(1); } } else { + // At this point we are the fuzzer *NOT* the restarter. + // We setup signal handlers to clean up shmem segments used by state restorer + #[cfg(all(unix, not(miri)))] + if let Err(_e) = + unsafe { setup_signal_handler(addr_of_mut!(EVENTMGR_SIGHANDLER_STATE)) } + { + // We can live without a proper ctrl+c signal handler. Print and ignore. + log::error!("Failed to setup signal handlers: {_e}"); + } + // We are the newly started fuzzing instance (i.e. on Windows), first, connect to our own restore map. // We get here *only on Windows*, if we were started by a restarting fuzzer. // A staterestorer and a receiver for single communication @@ -1356,14 +1633,25 @@ where // If we're restarting, deserialize the old state. let (state, mut mgr) = if let Some((state_opt, mgr_description)) = staterestorer.restore()? { + #[cfg(not(feature = "adaptive_serialization"))] + let llmp_mgr = LlmpEventManager::existing_client_from_description_with_hooks( + new_shmem_provider, + &mgr_description, + self.configuration, + self.hooks, + )?; + #[cfg(feature = "adaptive_serialization")] + let llmp_mgr = LlmpEventManager::existing_client_from_description_with_hooks( + new_shmem_provider, + &mgr_description, + self.configuration, + self.hooks, + self.time_ref.clone(), + )?; ( state_opt, LlmpRestartingEventManager::with_save_state( - LlmpEventManager::existing_client_from_description( - new_shmem_provider, - &mgr_description, - self.configuration, - )?, + llmp_mgr, staterestorer, self.serialize_state, ), @@ -1371,10 +1659,20 @@ where } else { log::info!("First run. Let's set it all up"); // Mgr to send and receive msgs from/to all other fuzzer instances - let mgr = LlmpEventManager::::existing_client_from_env( + #[cfg(not(feature = "adaptive_serialization"))] + let mgr = LlmpEventManager::::existing_client_from_env_with_hooks( + new_shmem_provider, + _ENV_FUZZER_BROKER_CLIENT_INITIAL, + self.configuration, + self.hooks, + )?; + #[cfg(feature = "adaptive_serialization")] + let mgr = LlmpEventManager::::existing_client_from_env_with_hooks( new_shmem_provider, _ENV_FUZZER_BROKER_CLIENT_INITIAL, self.configuration, + self.hooks, + self.time_ref.clone(), )?; ( @@ -1387,7 +1685,11 @@ where ) }; // We reset the staterestorer, the next staterestorer and receiver (after crash) will reuse the page from the initial message. - mgr.staterestorer.reset(); + if self.serialize_state.oom_safe() { + mgr.intermediate_save()?; + } else { + mgr.staterestorer.reset(); + } /* TODO: Not sure if this is needed // We commit an empty NO_RESTART message to this buf, against infinite loops, @@ -1397,23 +1699,10 @@ where Ok((state, mgr)) } - - /// Launch the restarting manager - pub fn launch(&mut self) -> Result<(Option, LlmpRestartingEventManager), Error> { - self.launch_internal(None) - } - - /// Launch the restarting manager with a custom client timeout - pub fn launch_with_client_timeout( - &mut self, - client_timeout: Duration, - ) -> Result<(Option, LlmpRestartingEventManager), Error> { - self.launch_internal(Some(client_timeout)) - } } /// A manager-like llmp client that converts between input types -pub struct LlmpEventConverter +pub struct LlmpEventConverter where S: UsesInput, SP: ShMemProvider + 'static, @@ -1431,7 +1720,7 @@ where phantom: PhantomData, } -impl core::fmt::Debug for LlmpEventConverter +impl core::fmt::Debug for LlmpEventConverter where SP: ShMemProvider + 'static, S: UsesInput, @@ -1453,9 +1742,9 @@ where } } -impl LlmpEventConverter +impl LlmpEventConverter where - S: UsesInput + HasExecutions, + S: UsesInput + HasExecutions + HasMetadata, SP: ShMemProvider + 'static, IC: InputConverter, ICB: InputConverter, @@ -1486,8 +1775,9 @@ where converter: Option, converter_back: Option, ) -> Result { + let llmp = LlmpClient::create_attach_to_tcp(shmem_provider, port)?; Ok(Self { - llmp: LlmpClient::create_attach_to_tcp(shmem_provider, port)?, + llmp, #[cfg(feature = "llmp_compression")] compressor: GzipCompressor::new(COMPRESS_THRESHOLD), converter, @@ -1546,7 +1836,7 @@ where executor: &mut E, state: &mut S, manager: &mut EM, - _client_id: ClientId, + client_id: ClientId, event: Event, ) -> Result<(), Error> where @@ -1566,7 +1856,7 @@ where executions: _, forward_id, } => { - log::info!("Received new Testcase to convert from {_client_id:?} (forward {forward_id:?}, forward {forward_id:?})"); + log::info!("Received new Testcase to convert from {client_id:?} (forward {forward_id:?}, forward {forward_id:?})"); let Some(converter) = self.converter_back.as_mut() else { return Ok(()); @@ -1579,6 +1869,7 @@ where converter.convert(input)?, false, )?; + if let Some(item) = res.1 { log::info!("Added received Testcase as item #{item}"); } @@ -1646,7 +1937,7 @@ where } } -impl UsesState for LlmpEventConverter +impl UsesState for LlmpEventConverter where S: State, SP: ShMemProvider, @@ -1657,7 +1948,7 @@ where type State = S; } -impl EventFirer for LlmpEventConverter +impl EventFirer for LlmpEventConverter where S: State, SP: ShMemProvider, @@ -1766,6 +2057,8 @@ where mod tests { use core::sync::atomic::{compiler_fence, Ordering}; + #[cfg(feature = "adaptive_serialization")] + use libafl_bolts::tuples::Handler; use libafl_bolts::{ llmp::{LlmpClient, LlmpSharedMap}, rands::StdRand, @@ -1784,6 +2077,7 @@ mod tests { fuzzer::Fuzzer, inputs::BytesInput, mutators::BitFlipMutator, + observers::TimeObserver, schedulers::RandScheduler, stages::StdMutationalStage, state::StdState, @@ -1796,6 +2090,10 @@ mod tests { fn test_mgr_state_restore() { let rand = StdRand::with_seed(0); + let time = TimeObserver::new("time"); + #[cfg(feature = "adaptive_serialization")] + let time_ref = time.handle(); + let mut corpus = InMemoryCorpus::::new(); let testcase = Testcase::new(vec![0; 4].into()); corpus.add(testcase).unwrap(); @@ -1822,7 +2120,11 @@ mod tests { llmp_client.mark_safe_to_unmap(); } + #[cfg(not(feature = "adaptive_serialization"))] let mut llmp_mgr = LlmpEventManager::new(llmp_client, "fuzzer".into()).unwrap(); + #[cfg(feature = "adaptive_serialization")] + let mut llmp_mgr = + LlmpEventManager::new(llmp_client, "fuzzer".into(), time_ref.clone()).unwrap(); let scheduler = RandScheduler::new(); @@ -1834,7 +2136,7 @@ mod tests { let mut harness = |_buf: &BytesInput| ExitKind::Ok; let mut executor = InProcessExecutor::new( &mut harness, - tuple_list!(), + tuple_list!(time), &mut fuzzer, &mut state, &mut llmp_mgr, @@ -1864,22 +2166,29 @@ mod tests { assert!(sc_cpy.has_content()); let (mut state_clone, mgr_description) = staterestorer.restore().unwrap().unwrap(); + #[cfg(not(feature = "adaptive_serialization"))] let mut llmp_clone = LlmpEventManager::existing_client_from_description( shmem_provider, &mgr_description, "fuzzer".into(), ) .unwrap(); + #[cfg(feature = "adaptive_serialization")] + let mut llmp_clone = LlmpEventManager::existing_client_from_description( + shmem_provider, + &mgr_description, + "fuzzer".into(), + time_ref, + ) + .unwrap(); - if false { - fuzzer - .fuzz_one( - &mut stages, - &mut executor, - &mut state_clone, - &mut llmp_clone, - ) - .unwrap(); - } + fuzzer + .fuzz_one( + &mut stages, + &mut executor, + &mut state_clone, + &mut llmp_clone, + ) + .unwrap(); } } diff --git a/libafl/src/events/mod.rs b/libafl/src/events/mod.rs index 536bb3177f..de3ef0d6af 100644 --- a/libafl/src/events/mod.rs +++ b/libafl/src/events/mod.rs @@ -1,6 +1,8 @@ //! An [`EventManager`] manages all events that go to other instances of the fuzzer. //! The messages are commonly information about new Testcases as well as stats and other [`Event`]s. +pub mod hooks; + pub mod simple; pub use simple::*; #[cfg(all(unix, feature = "std"))] @@ -12,12 +14,12 @@ pub use centralized::*; pub mod launcher; #[allow(clippy::ignored_unit_patterns)] pub mod llmp; +pub use llmp::*; + #[cfg(feature = "tcp_manager")] #[allow(clippy::ignored_unit_patterns)] pub mod tcp; -#[cfg(feature = "scalability_introspection")] -use alloc::string::ToString; -use alloc::{boxed::Box, string::String, vec::Vec}; +use alloc::{borrow::Cow, boxed::Box, string::String, vec::Vec}; use core::{ fmt, hash::{BuildHasher, Hasher}, @@ -30,8 +32,9 @@ use ahash::RandomState; pub use launcher::*; #[cfg(all(unix, feature = "std"))] use libafl_bolts::os::unix_signals::{siginfo_t, ucontext_t, Handler, Signal}; +#[cfg(feature = "adaptive_serialization")] +use libafl_bolts::tuples::{Handle, MatchNameRef}; use libafl_bolts::{current_time, ClientId}; -pub use llmp::*; use serde::{Deserialize, Serialize}; #[cfg(feature = "std")] use uuid::Uuid; @@ -43,8 +46,8 @@ use crate::{ inputs::Input, monitors::UserStats, observers::ObserversTuple, - state::{HasExecutions, HasLastReportTime, HasMetadata, State}, - Error, + state::{HasExecutions, HasLastReportTime, State}, + Error, HasMetadata, }; #[cfg(feature = "scalability_introspection")] use crate::{ @@ -52,31 +55,23 @@ use crate::{ state::HasScalabilityMonitor, }; +/// The special exit code when the target exited throught ctrl-c +#[cfg(unix)] +pub const CTRL_C_EXIT: i32 = 100; +/// The special exit code when the target exited throught ctrl-c +#[cfg(windows)] +pub const CTRL_C_EXIT: i32 = -1073741510; + /// Check if ctrl-c is sent with this struct #[cfg(all(unix, feature = "std"))] -pub static mut EVENTMGR_SIGHANDLER_STATE: ShutdownSignalData = ShutdownSignalData { - shutting_down: false, - exit_from_main: false, -}; +pub static mut EVENTMGR_SIGHANDLER_STATE: ShutdownSignalData = ShutdownSignalData {}; -/// A signal handler for releasing `StateRestore` `ShMem` -/// This struct holds a pointer to `StateRestore` and clean up the `ShMem` segment used by it. +/// A signal handler for catching ctrl-c. +/// The purpose of this signal handler is solely for calling `exit()` with a specific exit code 100 +/// In this way, the restarting manager can tell that we really want to exit #[cfg(all(unix, feature = "std"))] #[derive(Debug, Clone)] -pub struct ShutdownSignalData { - shutting_down: bool, - exit_from_main: bool, -} - -#[cfg(all(unix, feature = "std"))] -impl ShutdownSignalData { - /// Set the flag to true, indicating that this process has allocated shmem - pub fn set_exit_from_main(&mut self) { - unsafe { - core::ptr::write_volatile(core::ptr::addr_of_mut!(self.exit_from_main), true); - } - } -} +pub struct ShutdownSignalData {} /// Shutdown handler. `SigTerm`, `SigInterrupt`, `SigQuit` call this /// We can't handle SIGKILL in the signal handler, this means that you shouldn't kill your fuzzer with `kill -9` because then the shmem segments are never freed @@ -88,27 +83,15 @@ impl Handler for ShutdownSignalData { _info: &mut siginfo_t, _context: Option<&mut ucontext_t>, ) { - /* - println!( - "in handler! {} {}", - self.exit_from_main, - std::process::id() - ); - */ - // if this process has not allocated any shmem. then simply exit() - if !self.exit_from_main { - unsafe { - #[cfg(unix)] - libc::_exit(0); - - #[cfg(windows)] - windows::Win32::System::Threading::ExitProcess(1); - } - } - - // else wait till the next is_shutting_down() is called. then the process will exit throught main(). + // println!("in handler! {}", std::process::id()); unsafe { - core::ptr::write_volatile(core::ptr::addr_of_mut!(self.shutting_down), true); + // println!("Exiting from the handler...."); + + #[cfg(unix)] + libc::_exit(CTRL_C_EXIT); + + #[cfg(windows)] + windows::Win32::System::Threading::ExitProcess(100); } } @@ -118,7 +101,7 @@ impl Handler for ShutdownSignalData { } /// A per-fuzzer unique `ID`, usually starting with `0` and increasing -/// by `1` in multiprocessed [`EventManager`]s, such as [`self::llmp::LlmpEventManager`]. +/// by `1` in multiprocessed [`EventManager`]s, such as [`LlmpEventManager`]. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] #[repr(transparent)] pub struct EventManagerId( @@ -128,6 +111,8 @@ pub struct EventManagerId( #[cfg(feature = "introspection")] use crate::monitors::ClientPerfMonitor; +#[cfg(feature = "adaptive_serialization")] +use crate::observers::TimeObserver; use crate::{inputs::UsesInput, stages::HasCurrentStage, state::UsesState}; /// The log event severity @@ -296,7 +281,7 @@ where /// The time of generation of the event time: Duration, /// The executions of this client - executions: usize, + executions: u64, /// The original sender if, if forwarded forward_id: Option, }, @@ -305,14 +290,14 @@ where /// The time of generation of the [`Event`] time: Duration, /// The executions of this client - executions: usize, + executions: u64, /// [`PhantomData`] phantom: PhantomData, }, /// New user stats event to monitor. UpdateUserStats { /// Custom user monitor name - name: String, + name: Cow<'static, str>, /// Custom user monitor value value: UserStats, /// [`PhantomData`] @@ -324,7 +309,7 @@ where /// The time of generation of the event time: Duration, /// The executions of this client - executions: usize, + executions: u64, /// Current performance statistics introspection_monitor: Box, @@ -335,6 +320,10 @@ where Objective { /// Objective corpus size objective_size: usize, + /// The total number of executions when this objective is found + executions: u64, + /// The time when this event was created + time: Duration, }, /// Write a new log Log { @@ -379,12 +368,12 @@ where time: _, executions: _, phantom: _, - } - | Event::UpdateUserStats { + } => "Client Heartbeat", + Event::UpdateUserStats { name: _, value: _, phantom: _, - } => "Stats", + } => "UserStats", #[cfg(feature = "introspection")] Event::UpdatePerfMonitor { time: _, @@ -410,7 +399,7 @@ where pub trait EventFirer: UsesState { /// Send off an [`Event`] to the broker /// - /// For multi-processed managers, such as [`llmp::LlmpEventManager`], + /// For multi-processed managers, such as [`LlmpEventManager`], /// this serializes the [`Event`] and commits it to the [`llmp`] page. /// In this case, if you `fire` faster than the broker can consume /// (for example for each [`Input`], on multiple cores) @@ -461,7 +450,7 @@ where { /// Given the last time, if `monitor_timeout` seconds passed, send off an info/monitor/heartbeat message to the broker. /// Returns the new `last` time (so the old one, unless `monitor_timeout` time has passed and monitor have been sent) - /// Will return an [`crate::Error`], if the stats could not be sent. + /// Will return an [`Error`], if the stats could not be sent. fn maybe_report_progress( &mut self, state: &mut Self::State, @@ -482,7 +471,7 @@ where } /// Send off an info/monitor/heartbeat message to the broker. - /// Will return an [`crate::Error`], if the stats could not be sent. + /// Will return an [`Error`], if the stats could not be sent. fn report_progress(&mut self, state: &mut Self::State) -> Result<(), Error> { let executions = *state.executions(); let cur = current_time(); @@ -527,7 +516,7 @@ where self.fire( state, Event::UpdateUserStats { - name: "total imported".to_string(), + name: Cow::from("total imported"), value: UserStats::new( UserStatsValue::Number( (imported_with_observer + imported_without_observer) as u64, @@ -598,8 +587,7 @@ where } /// The handler function for custom buffers exchanged via [`EventManager`] -type CustomBufHandlerFn = - dyn FnMut(&mut S, &String, &[u8]) -> Result; +type CustomBufHandlerFn = dyn FnMut(&mut S, &str, &[u8]) -> Result; /// Supports custom buf handlers to handle `CustomBuf` events. pub trait HasCustomBufHandlers: UsesState { @@ -677,7 +665,7 @@ where fn add_custom_buf_handler( &mut self, _handler: Box< - dyn FnMut(&mut Self::State, &String, &[u8]) -> Result, + dyn FnMut(&mut Self::State, &str, &[u8]) -> Result, >, ) { } @@ -808,7 +796,7 @@ where fn add_custom_buf_handler( &mut self, handler: Box< - dyn FnMut(&mut Self::State, &String, &[u8]) -> Result, + dyn FnMut(&mut Self::State, &str, &[u8]) -> Result, >, ) { self.inner.add_custom_buf_handler(handler); @@ -846,6 +834,79 @@ where } } +/// Collected stats to decide if observers must be serialized or not +#[cfg(not(feature = "adaptive_serialization"))] +pub trait AdaptiveSerializer {} + +/// Collected stats to decide if observers must be serialized or not +#[cfg(feature = "adaptive_serialization")] +pub trait AdaptiveSerializer { + /// Expose the collected observers serialization time + fn serialization_time(&self) -> Duration; + /// Expose the collected observers deserialization time + fn deserialization_time(&self) -> Duration; + /// How many times observers were serialized + fn serializations_cnt(&self) -> usize; + /// How many times shoukd have been serialized an observer + fn should_serialize_cnt(&self) -> usize; + + /// Expose the collected observers serialization time (mut) + fn serialization_time_mut(&mut self) -> &mut Duration; + /// Expose the collected observers deserialization time (mut) + fn deserialization_time_mut(&mut self) -> &mut Duration; + /// How many times observers were serialized (mut) + fn serializations_cnt_mut(&mut self) -> &mut usize; + /// How many times shoukd have been serialized an observer (mut) + fn should_serialize_cnt_mut(&mut self) -> &mut usize; + + /// A [`Handle`] to the time observer to determine the `time_factor` + fn time_ref(&self) -> &Handle; + + /// Serialize the observer using the `time_factor` and `percentage_threshold`. + /// These parameters are unique to each of the different types of `EventManager` + fn serialize_observers_adaptive( + &mut self, + observers: &OT, + time_factor: u32, + percentage_threshold: usize, + ) -> Result>, Error> + where + OT: ObserversTuple + Serialize, + S: UsesInput, + { + let exec_time = observers + .get(self.time_ref()) + .map(|o| o.last_runtime().unwrap_or(Duration::ZERO)) + .unwrap(); + + let mut must_ser = + (self.serialization_time() + self.deserialization_time()) * time_factor < exec_time; + if must_ser { + *self.should_serialize_cnt_mut() += 1; + } + + if self.serializations_cnt() > 32 { + must_ser = (self.should_serialize_cnt() * 100 / self.serializations_cnt()) + > percentage_threshold; + } + + if self.serialization_time() == Duration::ZERO + || must_ser + || self.serializations_cnt().trailing_zeros() >= 8 + { + let start = current_time(); + let ser = postcard::to_allocvec(observers)?; + *self.serialization_time_mut() = current_time() - start; + + *self.serializations_cnt_mut() += 1; + Ok(Some(ser)) + } else { + *self.serializations_cnt_mut() += 1; + Ok(None) + } + } +} + #[cfg(test)] mod tests { @@ -905,100 +966,3 @@ mod tests { }; } } - -/// `EventManager` Python bindings -#[cfg(feature = "python")] -#[allow(missing_docs)] -pub mod pybind { - use pyo3::prelude::*; - - use crate::{ - events::{ - simple::pybind::PythonSimpleEventManager, Event, EventFirer, EventManager, - EventManagerId, EventProcessor, EventRestarter, HasEventManagerId, ProgressReporter, - }, - executors::pybind::PythonExecutor, - fuzzer::pybind::PythonStdFuzzer, - inputs::BytesInput, - state::{pybind::PythonStdState, UsesState}, - Error, - }; - - #[derive(Debug, Clone)] - pub enum PythonEventManagerWrapper { - Simple(Py), - } - - /// EventManager Trait binding - #[pyclass(unsendable, name = "EventManager")] - #[derive(Debug, Clone)] - pub struct PythonEventManager { - pub wrapper: PythonEventManagerWrapper, - } - - macro_rules! unwrap_me { - ($wrapper:expr, $name:ident, $body:block) => { - libafl_bolts::unwrap_me_body!($wrapper, $name, $body, PythonEventManagerWrapper, { - Simple - }) - }; - } - - macro_rules! unwrap_me_mut { - ($wrapper:expr, $name:ident, $body:block) => { - libafl_bolts::unwrap_me_mut_body!($wrapper, $name, $body, PythonEventManagerWrapper, { - Simple - }) - }; - } - - #[pymethods] - impl PythonEventManager { - #[staticmethod] - #[must_use] - pub fn new_simple(mgr: Py) -> Self { - Self { - wrapper: PythonEventManagerWrapper::Simple(mgr), - } - } - } - - impl UsesState for PythonEventManager { - type State = PythonStdState; - } - - impl EventFirer for PythonEventManager { - fn fire(&mut self, state: &mut Self::State, event: Event) -> Result<(), Error> { - unwrap_me_mut!(self.wrapper, e, { e.fire(state, event) }) - } - } - - impl EventRestarter for PythonEventManager {} - - impl EventProcessor for PythonEventManager { - fn process( - &mut self, - fuzzer: &mut PythonStdFuzzer, - state: &mut PythonStdState, - executor: &mut PythonExecutor, - ) -> Result { - unwrap_me_mut!(self.wrapper, e, { e.process(fuzzer, state, executor) }) - } - } - - impl ProgressReporter for PythonEventManager {} - - impl HasEventManagerId for PythonEventManager { - fn mgr_id(&self) -> EventManagerId { - unwrap_me!(self.wrapper, e, { e.mgr_id() }) - } - } - - impl EventManager for PythonEventManager {} - - /// Register the classes to the python module - pub fn register(_py: Python, m: &PyModule) -> PyResult<()> { - m.add_class::()?; - Ok(()) - } -} diff --git a/libafl/src/events/simple.rs b/libafl/src/events/simple.rs index eca401d7d4..c77878ea4f 100644 --- a/libafl/src/events/simple.rs +++ b/libafl/src/events/simple.rs @@ -1,10 +1,6 @@ //! A very simple event manager, that just supports log outputs, but no multiprocessing -use alloc::{ - boxed::Box, - string::{String, ToString}, - vec::Vec, -}; +use alloc::{boxed::Box, vec::Vec}; #[cfg(all(unix, not(miri), feature = "std"))] use core::ptr::addr_of_mut; use core::{fmt::Debug, marker::PhantomData}; @@ -27,7 +23,7 @@ use libafl_bolts::{shmem::ShMemProvider, staterestore::StateRestorer}; use serde::{de::DeserializeOwned, Serialize}; use super::{CustomBufEventResult, CustomBufHandlerFn, HasCustomBufHandlers, ProgressReporter}; -#[cfg(all(unix, feature = "std"))] +#[cfg(all(unix, feature = "std", not(miri)))] use crate::events::EVENTMGR_SIGHANDLER_STATE; use crate::{ events::{ @@ -36,8 +32,8 @@ use crate::{ }, inputs::UsesInput, monitors::Monitor, - state::{HasExecutions, HasLastReportTime, HasMetadata, State, UsesState}, - Error, + state::{HasExecutions, HasLastReportTime, State, UsesState}, + Error, HasMetadata, }; #[cfg(feature = "std")] use crate::{ @@ -58,7 +54,7 @@ where { /// The monitor monitor: MT, - /// The events that happened since the last handle_in_broker + /// The events that happened since the last `handle_in_broker` events: Vec>, /// The custom buf handler custom_buf_handlers: Vec>>, @@ -146,7 +142,7 @@ where fn add_custom_buf_handler( &mut self, handler: Box< - dyn FnMut(&mut Self::State, &String, &[u8]) -> Result, + dyn FnMut(&mut Self::State, &str, &[u8]) -> Result, >, ) { self.custom_buf_handlers.push(handler); @@ -220,8 +216,8 @@ where .update_corpus_size(*corpus_size as u64); monitor .client_stats_mut_for(ClientId(0)) - .update_executions(*executions as u64, *time); - monitor.display(event.name().to_string(), ClientId(0)); + .update_executions(*executions, *time); + monitor.display(event.name(), ClientId(0)); Ok(BrokerEventResult::Handled) } Event::UpdateExecStats { @@ -233,9 +229,9 @@ where monitor.client_stats_insert(ClientId(0)); let client = monitor.client_stats_mut_for(ClientId(0)); - client.update_executions(*executions as u64, *time); + client.update_executions(*executions, *time); - monitor.display(event.name().to_string(), ClientId(0)); + monitor.display(event.name(), ClientId(0)); Ok(BrokerEventResult::Handled) } Event::UpdateUserStats { @@ -248,7 +244,7 @@ where .client_stats_mut_for(ClientId(0)) .update_user_stats(name.clone(), value.clone()); monitor.aggregate(name); - monitor.display(event.name().to_string(), ClientId(0)); + monitor.display(event.name(), ClientId(0)); Ok(BrokerEventResult::Handled) } #[cfg(feature = "introspection")] @@ -261,17 +257,24 @@ where // TODO: The monitor buffer should be added on client add. monitor.client_stats_insert(ClientId(0)); let client = monitor.client_stats_mut_for(ClientId(0)); - client.update_executions(*executions as u64, *time); + client.update_executions(*executions, *time); client.update_introspection_monitor((**introspection_monitor).clone()); - monitor.display(event.name().to_string(), ClientId(0)); + monitor.display(event.name(), ClientId(0)); Ok(BrokerEventResult::Handled) } - Event::Objective { objective_size } => { + Event::Objective { + objective_size, + executions, + time, + } => { monitor.client_stats_insert(ClientId(0)); monitor .client_stats_mut_for(ClientId(0)) .update_objective_size(*objective_size as u64); - monitor.display(event.name().to_string(), ClientId(0)); + monitor + .client_stats_mut_for(ClientId(0)) + .update_executions(*executions, *time); + monitor.display(event.name(), ClientId(0)); Ok(BrokerEventResult::Handled) } Event::Log { @@ -407,7 +410,7 @@ where { fn add_custom_buf_handler( &mut self, - handler: Box Result>, + handler: Box Result>, ) { self.simple_event_mgr.add_custom_buf_handler(handler); } @@ -450,19 +453,6 @@ where } } - /// Internal function, returns true when shuttdown is requested by a `SIGINT` signal - #[inline] - #[allow(clippy::unused_self)] - fn is_shutting_down() -> bool { - #[cfg(unix)] - unsafe { - core::ptr::read_volatile(core::ptr::addr_of!(EVENTMGR_SIGHANDLER_STATE.shutting_down)) - } - - #[cfg(windows)] - false - } - /// Launch the simple restarting manager. /// This [`EventManager`] is simple and single threaded, /// but can still used shared maps to recover from crashes and timeouts. @@ -485,15 +475,6 @@ where //let staterestorer = { LlmpSender::new(shmem_provider.clone(), 0, false)? }; staterestorer.write_to_env(_ENV_FUZZER_SENDER)?; - // We setup signal handlers to clean up shmem segments used by state restorer - #[cfg(all(unix, not(miri)))] - if let Err(_e) = - unsafe { setup_signal_handler(addr_of_mut!(EVENTMGR_SIGHANDLER_STATE)) } - { - // We can live without a proper ctrl+c signal handler. Print and ignore. - log::error!("Failed to setup signal handlers: {_e}"); - } - let mut ctr: u64 = 0; // Client->parent loop loop { @@ -506,9 +487,7 @@ where match unsafe { fork() }? { ForkResult::Parent(handle) => { unsafe { - // The parent will later exit through is_shutting down below - // if the process exits gracefully, it cleans up the shmem. - EVENTMGR_SIGHANDLER_STATE.set_exit_from_main(); + libc::signal(libc::SIGINT, libc::SIG_IGN); } shmem_provider.post_fork(false)?; handle.status() @@ -520,23 +499,28 @@ where } }; - // Same, as fork version, mark this main thread as the shmem allocator - // then it will not call exit or exitprocess in the sigint handler - // so that it exits after cleaning up the shmem segments - #[cfg(all(unix, not(feature = "fork")))] + // If this guy wants to fork, then ignore sigit + #[cfg(any(windows, not(feature = "fork")))] unsafe { - EVENTMGR_SIGHANDLER_STATE.set_exit_from_main(); + #[cfg(windows)] + libafl_bolts::os::windows_exceptions::signal( + libafl_bolts::os::windows_exceptions::SIGINT, + libafl_bolts::os::windows_exceptions::sig_ign(), + ); + + #[cfg(unix)] + libc::signal(libc::SIGINT, libc::SIG_IGN); } // On Windows (or in any case without forks), we spawn ourself again #[cfg(any(windows, not(feature = "fork")))] let child_status = startable_self()?.status()?; - #[cfg(all(unix, not(feature = "fork")))] + #[cfg(any(windows, not(feature = "fork")))] let child_status = child_status.code().unwrap_or_default(); compiler_fence(Ordering::SeqCst); - if staterestorer.wants_to_exit() || Self::is_shutting_down() { + if child_status == crate::events::CTRL_C_EXIT || staterestorer.wants_to_exit() { return Err(Error::shutting_down()); } @@ -556,6 +540,16 @@ where ctr = ctr.wrapping_add(1); } } else { + // At this point we are the fuzzer *NOT* the restarter. + // We setup signal handlers to clean up shmem segments used by state restorer + #[cfg(all(unix, not(miri)))] + if let Err(_e) = + unsafe { setup_signal_handler(addr_of_mut!(EVENTMGR_SIGHANDLER_STATE)) } + { + // We can live without a proper ctrl+c signal handler. Print and ignore. + log::error!("Failed to setup signal handlers: {_e}"); + } + // We are the newly started fuzzing instance (i.e. on Windows), first, connect to our own restore map. // We get here *only on Windows*, if we were started by a restarting fuzzer. // A staterestorer and a receiver for single communication @@ -598,45 +592,3 @@ where Ok((state, mgr)) } } - -/// `SimpleEventManager` Python bindings -#[cfg(feature = "python")] -#[allow(missing_docs)] -#[allow(clippy::unnecessary_fallible_conversions)] -pub mod pybind { - use pyo3::prelude::*; - - use crate::{ - events::{pybind::PythonEventManager, SimpleEventManager}, - monitors::pybind::PythonMonitor, - state::pybind::PythonStdState, - }; - - #[pyclass(unsendable, name = "SimpleEventManager")] - #[derive(Debug)] - /// Python class for SimpleEventManager - pub struct PythonSimpleEventManager { - /// Rust wrapped SimpleEventManager object - pub inner: SimpleEventManager, - } - - #[pymethods] - impl PythonSimpleEventManager { - #[new] - fn new(py_monitor: PythonMonitor) -> Self { - Self { - inner: SimpleEventManager::new(py_monitor), - } - } - - fn as_manager(slf: Py) -> PythonEventManager { - PythonEventManager::new_simple(slf) - } - } - - /// Register the classes to the python module - pub fn register(_py: Python, m: &PyModule) -> PyResult<()> { - m.add_class::()?; - Ok(()) - } -} diff --git a/libafl/src/events/tcp.rs b/libafl/src/events/tcp.rs index 6b8d336ebd..7c7e723128 100644 --- a/libafl/src/events/tcp.rs +++ b/libafl/src/events/tcp.rs @@ -1,10 +1,6 @@ //! TCP-backed event manager for scalable multi-processed fuzzing -use alloc::{ - boxed::Box, - string::{String, ToString}, - vec::Vec, -}; +use alloc::{boxed::Box, vec::Vec}; #[cfg(all(unix, feature = "std", not(miri)))] use core::ptr::addr_of_mut; use core::{ @@ -27,7 +23,7 @@ use libafl_bolts::os::startable_self; use libafl_bolts::os::unix_signals::setup_signal_handler; #[cfg(all(feature = "std", feature = "fork", unix))] use libafl_bolts::os::{fork, ForkResult}; -use libafl_bolts::{shmem::ShMemProvider, ClientId}; +use libafl_bolts::{shmem::ShMemProvider, tuples::tuple_list, ClientId}; #[cfg(feature = "std")] use libafl_bolts::{shmem::StdShMemProvider, staterestore::StateRestorer}; use serde::{de::DeserializeOwned, Deserialize}; @@ -40,25 +36,26 @@ use tokio::{ use typed_builder::TypedBuilder; use super::{CustomBufEventResult, CustomBufHandlerFn}; -#[cfg(all(unix, feature = "std"))] +#[cfg(all(unix, feature = "std", not(miri)))] use crate::events::EVENTMGR_SIGHANDLER_STATE; use crate::{ events::{ - BrokerEventResult, Event, EventConfig, EventFirer, EventManager, EventManagerId, - EventProcessor, EventRestarter, HasCustomBufHandlers, HasEventManagerId, ProgressReporter, + hooks::EventManagerHooksTuple, BrokerEventResult, Event, EventConfig, EventFirer, + EventManager, EventManagerId, EventProcessor, EventRestarter, HasCustomBufHandlers, + HasEventManagerId, ProgressReporter, }, executors::{Executor, HasObservers}, fuzzer::{EvaluatorObservers, ExecutionProcessor}, inputs::{Input, UsesInput}, monitors::Monitor, - state::{HasExecutions, HasLastReportTime, HasMetadata, State, UsesState}, - Error, + state::{HasExecutions, HasLastReportTime, State, UsesState}, + Error, HasMetadata, }; /// Tries to create (synchronously) a [`TcpListener`] that is `nonblocking` (for later use in tokio). /// Will error if the port is already in use (or other errors occur) fn create_nonblocking_listener(addr: A) -> Result { - let listener = std::net::TcpListener::bind(addr)?; + let listener = TcpListener::bind(addr)?; listener.set_nonblocking(true)?; Ok(listener) } @@ -336,8 +333,8 @@ where monitor.client_stats_insert(id); let client = monitor.client_stats_mut_for(id); client.update_corpus_size(*corpus_size as u64); - client.update_executions(*executions as u64, *time); - monitor.display(event.name().to_string(), id); + client.update_executions(*executions, *time); + monitor.display(event.name(), id); Ok(BrokerEventResult::Forward) } Event::UpdateExecStats { @@ -348,8 +345,8 @@ where // TODO: The monitor buffer should be added on client add. monitor.client_stats_insert(client_id); let client = monitor.client_stats_mut_for(client_id); - client.update_executions(*executions as u64, *time); - monitor.display(event.name().to_string(), client_id); + client.update_executions(*executions, *time); + monitor.display(event.name(), client_id); Ok(BrokerEventResult::Handled) } Event::UpdateUserStats { @@ -361,7 +358,7 @@ where let client = monitor.client_stats_mut_for(client_id); client.update_user_stats(name.clone(), value.clone()); monitor.aggregate(name); - monitor.display(event.name().to_string(), client_id); + monitor.display(event.name(), client_id); Ok(BrokerEventResult::Handled) } #[cfg(feature = "introspection")] @@ -378,22 +375,27 @@ where let client = monitor.client_stats_mut_for(client_id); // Update the normal monitor for this client - client.update_executions(*executions as u64, *time); + client.update_executions(*executions, *time); // Update the performance monitor for this client client.update_introspection_monitor((**introspection_monitor).clone()); // Display the monitor via `.display` only on core #1 - monitor.display(event.name().to_string(), client_id); + monitor.display(event.name(), client_id); // Correctly handled the event Ok(BrokerEventResult::Handled) } - Event::Objective { objective_size } => { + Event::Objective { + objective_size, + executions, + time, + } => { monitor.client_stats_insert(client_id); let client = monitor.client_stats_mut_for(client_id); client.update_objective_size(*objective_size as u64); - monitor.display(event.name().to_string(), client_id); + client.update_executions(*executions, *time); + monitor.display(event.name(), client_id); Ok(BrokerEventResult::Handled) } Event::Log { @@ -413,10 +415,12 @@ where } /// An [`EventManager`] that forwards all events to other attached via tcp. -pub struct TcpEventManager +pub struct TcpEventManager where + EMH: EventManagerHooksTuple, S: State, { + hooks: EMH, /// The TCP stream for inter process communication tcp: TcpStream, /// Our `CientId` @@ -432,8 +436,9 @@ where phantom: PhantomData, } -impl core::fmt::Debug for TcpEventManager +impl core::fmt::Debug for TcpEventManager where + EMH: EventManagerHooksTuple, S: State, { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { @@ -449,8 +454,9 @@ where } } -impl Drop for TcpEventManager +impl Drop for TcpEventManager where + EMH: EventManagerHooksTuple, S: State, { /// TCP clients will have to wait until their pages are mapped by somebody. @@ -459,15 +465,74 @@ where } } -impl TcpEventManager +impl TcpEventManager<(), S> where - S: State + HasExecutions, + S: State + HasExecutions + HasMetadata, { /// Create a manager from a raw TCP client specifying the client id pub fn existing( addr: &A, client_id: ClientId, configuration: EventConfig, + ) -> Result { + Self::existing_with_hooks(addr, client_id, configuration, tuple_list!()) + } + + /// Create a manager from a raw TCP client + pub fn new(addr: &A, configuration: EventConfig) -> Result { + Self::existing_with_hooks(addr, UNDEFINED_CLIENT_ID, configuration, tuple_list!()) + } + + /// Create an TCP event manager on a port specifying the client id + /// + /// If the port is not yet bound, it will act as a broker; otherwise, it + /// will act as a client. + pub fn existing_on_port( + port: u16, + client_id: ClientId, + configuration: EventConfig, + ) -> Result { + Self::existing_with_hooks( + &("127.0.0.1", port), + client_id, + configuration, + tuple_list!(), + ) + } + + /// Create an TCP event manager on a port with hooks + /// + /// If the port is not yet bound, it will act as a broker; otherwise, it + /// will act as a client. + pub fn on_port(port: u16, configuration: EventConfig) -> Result { + Self::with_hooks(&("127.0.0.1", port), configuration, tuple_list!()) + } + + /// Create an TCP event manager on a port specifying the client id from env + /// + /// If the port is not yet bound, it will act as a broker; otherwise, it + /// will act as a client. + pub fn existing_from_env( + addr: &A, + env_name: &str, + configuration: EventConfig, + ) -> Result { + let this_id = ClientId(str::parse::(&env::var(env_name)?)?); + Self::existing_with_hooks(addr, this_id, configuration, tuple_list!()) + } +} + +impl TcpEventManager +where + EMH: EventManagerHooksTuple, + S: State + HasExecutions + HasMetadata, +{ + /// Create a manager from a raw TCP client specifying the client id with hooks + pub fn existing_with_hooks( + addr: &A, + client_id: ClientId, + configuration: EventConfig, + hooks: EMH, ) -> Result { let mut tcp = TcpStream::connect(addr)?; @@ -482,6 +547,7 @@ where println!("Our client id: {client_id:?}"); Ok(Self { + hooks, tcp, client_id, #[cfg(feature = "tcp_compression")] @@ -492,42 +558,52 @@ where }) } - /// Create a manager from a raw TCP client - pub fn new(addr: &A, configuration: EventConfig) -> Result { - Self::existing(addr, UNDEFINED_CLIENT_ID, configuration) + /// Create a manager from a raw TCP client with hooks + pub fn with_hooks( + addr: &A, + configuration: EventConfig, + hooks: EMH, + ) -> Result { + Self::existing_with_hooks(addr, UNDEFINED_CLIENT_ID, configuration, hooks) } - /// Create an TCP event manager on a port specifying the client id + /// Create an TCP event manager on a port specifying the client id with hooks /// /// If the port is not yet bound, it will act as a broker; otherwise, it /// will act as a client. - pub fn existing_on_port( + pub fn existing_on_port_with_hooks( port: u16, client_id: ClientId, configuration: EventConfig, + hooks: EMH, ) -> Result { - Self::existing(&("127.0.0.1", port), client_id, configuration) + Self::existing_with_hooks(&("127.0.0.1", port), client_id, configuration, hooks) } - /// Create an TCP event manager on a port + /// Create an TCP event manager on a port with hooks /// /// If the port is not yet bound, it will act as a broker; otherwise, it /// will act as a client. - pub fn on_port(port: u16, configuration: EventConfig) -> Result { - Self::new(&("127.0.0.1", port), configuration) + pub fn on_port_with_hooks( + port: u16, + configuration: EventConfig, + hooks: EMH, + ) -> Result { + Self::with_hooks(&("127.0.0.1", port), configuration, hooks) } - /// Create an TCP event manager on a port specifying the client id from env + /// Create an TCP event manager on a port specifying the client id from env with hooks /// /// If the port is not yet bound, it will act as a broker; otherwise, it /// will act as a client. - pub fn existing_from_env( + pub fn existing_from_env_with_hooks( addr: &A, env_name: &str, configuration: EventConfig, + hooks: EMH, ) -> Result { let this_id = ClientId(str::parse::(&env::var(env_name)?)?); - Self::existing(addr, this_id, configuration) + Self::existing_with_hooks(addr, this_id, configuration, hooks) } /// Write the client id for a client [`EventManager`] to env vars @@ -550,6 +626,9 @@ where for<'a> E::Observers: Deserialize<'a>, Z: ExecutionProcessor + EvaluatorObservers, { + if !self.hooks.pre_exec_all(state, client_id, &event)? { + return Ok(()); + } match event { Event::NewTestcase { input, @@ -572,7 +651,7 @@ where { state.scalability_monitor_mut().testcase_with_observers += 1; } - fuzzer.process_execution(state, self, input, &observers, &exit_kind, false)? + fuzzer.execute_and_process(state, self, input, &observers, &exit_kind, false)? } else { #[cfg(feature = "scalability_introspection")] { @@ -585,7 +664,6 @@ where if let Some(item) = _res.1 { log::info!("Added received Testcase as item #{item}"); } - Ok(()) } Event::CustomBuf { tag, buf } => { for handler in &mut self.custom_buf_handlers { @@ -593,18 +671,22 @@ where break; } } - Ok(()) } - _ => Err(Error::unknown(format!( - "Received illegal message that message should not have arrived: {:?}.", - event.name() - ))), + _ => { + return Err(Error::unknown(format!( + "Received illegal message that message should not have arrived: {:?}.", + event.name() + ))) + } } + self.hooks.post_exec_all(state, client_id)?; + Ok(()) } } -impl TcpEventManager +impl TcpEventManager where + EMH: EventManagerHooksTuple, S: State, { /// Send information that this client is exiting. @@ -617,15 +699,17 @@ where } } -impl UsesState for TcpEventManager +impl UsesState for TcpEventManager where + EMH: EventManagerHooksTuple, S: State, { type State = S; } -impl EventFirer for TcpEventManager +impl EventFirer for TcpEventManager where + EMH: EventManagerHooksTuple, S: State, { #[cfg(feature = "tcp_compression")] @@ -671,8 +755,9 @@ where } } -impl EventRestarter for TcpEventManager +impl EventRestarter for TcpEventManager where + EMH: EventManagerHooksTuple, S: State, { /// The TCP client needs to wait until a broker has mapped all pages before shutting down. @@ -683,11 +768,12 @@ where } } -impl EventProcessor for TcpEventManager +impl EventProcessor for TcpEventManager where - S: State + HasExecutions, E: HasObservers + Executor, for<'a> E::Observers: Deserialize<'a>, + EMH: EventManagerHooksTuple, + S: State + HasExecutions + HasMetadata, Z: EvaluatorObservers + ExecutionProcessor, { fn process( @@ -744,34 +830,39 @@ where } } -impl EventManager for TcpEventManager +impl EventManager for TcpEventManager where E: HasObservers + Executor, for<'a> E::Observers: Deserialize<'a>, + EMH: EventManagerHooksTuple, S: State + HasExecutions + HasMetadata + HasLastReportTime, Z: EvaluatorObservers + ExecutionProcessor, { } -impl HasCustomBufHandlers for TcpEventManager +impl HasCustomBufHandlers for TcpEventManager where + EMH: EventManagerHooksTuple, S: State, { fn add_custom_buf_handler( &mut self, - handler: Box Result>, + handler: Box Result>, ) { self.custom_buf_handlers.push(handler); } } -impl ProgressReporter for TcpEventManager where - S: State + HasExecutions + HasMetadata + HasLastReportTime +impl ProgressReporter for TcpEventManager +where + EMH: EventManagerHooksTuple, + S: State + HasExecutions + HasMetadata + HasLastReportTime, { } -impl HasEventManagerId for TcpEventManager +impl HasEventManagerId for TcpEventManager where + EMH: EventManagerHooksTuple, S: State, { /// Gets the id assigned to this staterestorer. @@ -783,14 +874,15 @@ where /// A manager that can restart on the fly, storing states in-between (in `on_restart`) #[cfg(feature = "std")] #[derive(Debug)] -pub struct TcpRestartingEventManager +pub struct TcpRestartingEventManager where + EMH: EventManagerHooksTuple, S: State, SP: ShMemProvider + 'static, //CE: CustomEvent, { /// The embedded TCP event manager - tcp_mgr: TcpEventManager, + tcp_mgr: TcpEventManager, /// The staterestorer to serialize the state for the next runner staterestorer: StateRestorer, /// Decide if the state restorer must save the serialized state @@ -798,8 +890,9 @@ where } #[cfg(feature = "std")] -impl UsesState for TcpRestartingEventManager +impl UsesState for TcpRestartingEventManager where + EMH: EventManagerHooksTuple, S: State, SP: ShMemProvider + 'static, { @@ -807,16 +900,18 @@ where } #[cfg(feature = "std")] -impl ProgressReporter for TcpRestartingEventManager +impl ProgressReporter for TcpRestartingEventManager where + EMH: EventManagerHooksTuple, S: State + HasExecutions + HasMetadata + HasLastReportTime, SP: ShMemProvider, { } #[cfg(feature = "std")] -impl EventFirer for TcpRestartingEventManager +impl EventFirer for TcpRestartingEventManager where + EMH: EventManagerHooksTuple, SP: ShMemProvider, S: State, //CE: CustomEvent, @@ -836,8 +931,9 @@ where } #[cfg(feature = "std")] -impl EventRestarter for TcpRestartingEventManager +impl EventRestarter for TcpRestartingEventManager where + EMH: EventManagerHooksTuple, S: State + HasExecutions, SP: ShMemProvider, //CE: CustomEvent, @@ -874,11 +970,12 @@ where } #[cfg(feature = "std")] -impl EventProcessor for TcpRestartingEventManager +impl EventProcessor for TcpRestartingEventManager where - E: HasObservers + Executor, Z>, + E: HasObservers + Executor, Z>, for<'a> E::Observers: Deserialize<'a>, - S: State + HasExecutions, + EMH: EventManagerHooksTuple, + S: State + HasExecutions + HasMetadata, SP: ShMemProvider + 'static, Z: EvaluatorObservers + ExecutionProcessor, //CE: CustomEvent, { @@ -888,10 +985,11 @@ where } #[cfg(feature = "std")] -impl EventManager for TcpRestartingEventManager +impl EventManager for TcpRestartingEventManager where - E: HasObservers + Executor, Z>, + E: HasObservers + Executor, Z>, for<'a> E::Observers: Deserialize<'a>, + EMH: EventManagerHooksTuple, S: State + HasExecutions + HasMetadata + HasLastReportTime, SP: ShMemProvider + 'static, Z: EvaluatorObservers + ExecutionProcessor, //CE: CustomEvent, @@ -899,8 +997,9 @@ where } #[cfg(feature = "std")] -impl HasEventManagerId for TcpRestartingEventManager +impl HasEventManagerId for TcpRestartingEventManager where + EMH: EventManagerHooksTuple, S: State, SP: ShMemProvider + 'static, { @@ -916,14 +1015,15 @@ const _ENV_FUZZER_RECEIVER: &str = "_AFL_ENV_FUZZER_RECEIVER"; const _ENV_FUZZER_BROKER_CLIENT_INITIAL: &str = "_AFL_ENV_FUZZER_BROKER_CLIENT"; #[cfg(feature = "std")] -impl TcpRestartingEventManager +impl TcpRestartingEventManager where + EMH: EventManagerHooksTuple, S: State, SP: ShMemProvider + 'static, //CE: CustomEvent, { /// Create a new runner, the executed child doing the actual fuzzing. - pub fn new(tcp_mgr: TcpEventManager, staterestorer: StateRestorer) -> Self { + pub fn new(tcp_mgr: TcpEventManager, staterestorer: StateRestorer) -> Self { Self { tcp_mgr, staterestorer, @@ -933,7 +1033,7 @@ where /// Create a new runner specifying if it must save the serialized state on restart. pub fn with_save_state( - tcp_mgr: TcpEventManager, + tcp_mgr: TcpEventManager, staterestorer: StateRestorer, save_state: bool, ) -> Self { @@ -979,16 +1079,23 @@ pub fn setup_restarting_mgr_tcp( monitor: MT, broker_port: u16, configuration: EventConfig, -) -> Result<(Option, TcpRestartingEventManager), Error> +) -> Result< + ( + Option, + TcpRestartingEventManager<(), S, StdShMemProvider>, + ), + Error, +> where MT: Monitor + Clone, - S: State + HasExecutions, + S: State + HasExecutions + HasMetadata, { TcpRestartingMgr::builder() .shmem_provider(StdShMemProvider::new()?) .monitor(Some(monitor)) .broker_port(broker_port) .configuration(configuration) + .hooks(tuple_list!()) .build() .launch() } @@ -999,7 +1106,7 @@ where #[cfg(feature = "std")] #[allow(clippy::default_trait_access, clippy::ignored_unit_patterns)] #[derive(TypedBuilder, Debug)] -pub struct TcpRestartingMgr +pub struct TcpRestartingMgr where S: UsesInput + DeserializeOwned, SP: ShMemProvider + 'static, @@ -1034,36 +1141,25 @@ where /// Tell the manager to serialize or not the state on restart #[builder(default = true)] serialize_state: bool, + /// The hooks for `handle_in_client` + hooks: EMH, #[builder(setter(skip), default = PhantomData)] phantom_data: PhantomData, } #[cfg(feature = "std")] #[allow(clippy::type_complexity, clippy::too_many_lines)] -impl TcpRestartingMgr +impl TcpRestartingMgr where + EMH: EventManagerHooksTuple + Copy + Clone, SP: ShMemProvider, - S: State + HasExecutions, + S: State + HasExecutions + HasMetadata, MT: Monitor + Clone, { - /// Internal function, returns true when shuttdown is requested by a `SIGINT` signal - #[inline] - #[allow(clippy::unused_self)] - fn is_shutting_down() -> bool { - #[cfg(unix)] - unsafe { - core::ptr::read_volatile(core::ptr::addr_of!(EVENTMGR_SIGHANDLER_STATE.shutting_down)) - } - - #[cfg(windows)] - false - } - /// Launch the restarting manager - pub fn launch(&mut self) -> Result<(Option, TcpRestartingEventManager), Error> { + pub fn launch(&mut self) -> Result<(Option, TcpRestartingEventManager), Error> { // We start ourself as child process to actually fuzz - let (staterestorer, _new_shmem_provider, core_id) = if std::env::var(_ENV_FUZZER_SENDER) - .is_err() + let (staterestorer, _new_shmem_provider, core_id) = if env::var(_ENV_FUZZER_SENDER).is_err() { let broker_things = |mut broker: TcpEventBroker, _remote_broker_addr| { if let Some(exit_cleanly_after) = self.exit_cleanly_after { @@ -1093,11 +1189,12 @@ where return Err(Error::shutting_down()); } - Err(Error::File(_, _)) => { + Err(Error::OsError(..)) => { // port was likely already bound - let mgr = TcpEventManager::::new( + let mgr = TcpEventManager::::with_hooks( &("127.0.0.1", self.broker_port), self.configuration, + self.hooks, )?; (mgr, None) } @@ -1117,7 +1214,11 @@ where } TcpManagerKind::Client { cpu_core } => { // We are a client - let mgr = TcpEventManager::::on_port(self.broker_port, self.configuration)?; + let mgr = TcpEventManager::::on_port_with_hooks( + self.broker_port, + self.configuration, + self.hooks, + )?; (mgr, cpu_core) } @@ -1143,15 +1244,6 @@ where // Store the information to a map. staterestorer.write_to_env(_ENV_FUZZER_SENDER)?; - // We setup signal handlers to clean up shmem segments used by state restorer - #[cfg(all(unix, not(miri)))] - if let Err(_e) = - unsafe { setup_signal_handler(addr_of_mut!(EVENTMGR_SIGHANDLER_STATE)) } - { - // We can live without a proper ctrl+c signal handler. Print and ignore. - log::error!("Failed to setup signal handlers: {_e}"); - } - let mut ctr: u64 = 0; // Client->parent loop loop { @@ -1165,7 +1257,7 @@ where match unsafe { fork() }? { ForkResult::Parent(handle) => { unsafe { - EVENTMGR_SIGHANDLER_STATE.set_exit_from_main(); + libc::signal(libc::SIGINT, libc::SIG_IGN); } self.shmem_provider.post_fork(false)?; handle.status() @@ -1177,19 +1269,31 @@ where } }; - #[cfg(all(unix, not(feature = "fork")))] + // If this guy wants to fork, then ignore sigit + #[cfg(any(windows, not(feature = "fork")))] unsafe { - EVENTMGR_SIGHANDLER_STATE.set_exit_from_main(); + #[cfg(windows)] + libafl_bolts::os::windows_exceptions::signal( + libafl_bolts::os::windows_exceptions::SIGINT, + libafl_bolts::os::windows_exceptions::sig_ign(), + ); + + #[cfg(unix)] + libc::signal(libc::SIGINT, libc::SIG_IGN); } // On Windows (or in any case without fork), we spawn ourself again #[cfg(any(windows, not(feature = "fork")))] let child_status = startable_self()?.status()?; - #[cfg(all(unix, not(feature = "fork")))] + #[cfg(any(windows, not(feature = "fork")))] let child_status = child_status.code().unwrap_or_default(); compiler_fence(Ordering::SeqCst); + if child_status == crate::events::CTRL_C_EXIT || staterestorer.wants_to_exit() { + return Err(Error::shutting_down()); + } + #[allow(clippy::manual_assert)] if !staterestorer.has_content() && self.serialize_state { #[cfg(unix)] @@ -1203,13 +1307,19 @@ where panic!("Fuzzer-respawner: Storing state in crashed fuzzer instance did not work, no point to spawn the next client! This can happen if the child calls `exit()`, in that case make sure it uses `abort()`, if it got killed unrecoverable (OOM), or if there is a bug in the fuzzer itself. (Child exited with: {child_status})"); } - if staterestorer.wants_to_exit() || Self::is_shutting_down() { - return Err(Error::shutting_down()); - } - ctr = ctr.wrapping_add(1); } } else { + // At this point we are the fuzzer *NOT* the restarter. + // We setup signal handlers to clean up shmem segments used by state restorer + #[cfg(all(unix, not(miri)))] + if let Err(_e) = + unsafe { setup_signal_handler(addr_of_mut!(EVENTMGR_SIGHANDLER_STATE)) } + { + // We can live without a proper ctrl+c signal handler. Print and ignore. + log::error!("Failed to setup signal handlers: {_e}"); + } + // We are the newly started fuzzing instance (i.e. on Windows), first, connect to our own restore map. // We get here *only on Windows*, if we were started by a restarting fuzzer. // A staterestorer and a receiver for single communication @@ -1230,10 +1340,11 @@ where ( state_opt, TcpRestartingEventManager::with_save_state( - TcpEventManager::existing_on_port( + TcpEventManager::existing_on_port_with_hooks( self.broker_port, this_id, self.configuration, + self.hooks, )?, staterestorer, self.serialize_state, @@ -1242,10 +1353,11 @@ where } else { log::info!("First run. Let's set it all up"); // Mgr to send and receive msgs from/to all other fuzzer instances - let mgr = TcpEventManager::::existing_from_env( + let mgr = TcpEventManager::::existing_from_env_with_hooks( &("127.0.0.1", self.broker_port), _ENV_FUZZER_BROKER_CLIENT_INITIAL, self.configuration, + self.hooks, )?; ( diff --git a/libafl/src/executors/combined.rs b/libafl/src/executors/combined.rs index eeff368eed..e48df9ec5c 100644 --- a/libafl/src/executors/combined.rs +++ b/libafl/src/executors/combined.rs @@ -3,6 +3,8 @@ use core::fmt::Debug; +use libafl_bolts::tuples::RefIndexable; + use crate::{ executors::{Executor, ExitKind, HasObservers}, observers::UsesObservers, @@ -80,12 +82,12 @@ where A: HasObservers, { #[inline] - fn observers(&self) -> &Self::Observers { + fn observers(&self) -> RefIndexable<&Self::Observers, Self::Observers> { self.primary.observers() } #[inline] - fn observers_mut(&mut self) -> &mut Self::Observers { + fn observers_mut(&mut self) -> RefIndexable<&mut Self::Observers, Self::Observers> { self.primary.observers_mut() } } diff --git a/libafl/src/executors/command.rs b/libafl/src/executors/command.rs index a37344ccfc..124d8a936f 100644 --- a/libafl/src/executors/command.rs +++ b/libafl/src/executors/command.rs @@ -18,7 +18,7 @@ use std::{ use libafl_bolts::{ fs::{get_unique_std_input_file, InputFile}, - tuples::MatchName, + tuples::{MatchName, RefIndexable}, AsSlice, }; @@ -87,11 +87,11 @@ pub struct StdCommandConfigurator { command: Command, } -impl CommandConfigurator for StdCommandConfigurator { - fn spawn_child(&mut self, input: &I) -> Result - where - I: Input + HasTargetBytes, - { +impl CommandConfigurator for StdCommandConfigurator +where + I: HasTargetBytes, +{ + fn spawn_child(&mut self, input: &I) -> Result { match &mut self.input_location { InputLocation::Arg { argnum } => { let args = self.command.get_args(); @@ -217,6 +217,7 @@ impl CommandExecutor where OT: MatchName + ObserversTuple, S: UsesInput, + S::Input: HasTargetBytes, { /// Creates a new `CommandExecutor`. /// Instead of parsing the Command for `@@`, it will @@ -314,8 +315,7 @@ impl Executor for CommandExecutor where EM: UsesState, S: State + HasExecutions, - S::Input: HasTargetBytes, - T: CommandConfigurator, + T: CommandConfigurator, OT: Debug + MatchName + ObserversTuple, Z: UsesState, { @@ -397,12 +397,12 @@ where T: Debug, OT: ObserversTuple, { - fn observers(&self) -> &OT { - &self.observers + fn observers(&self) -> RefIndexable<&Self::Observers, Self::Observers> { + RefIndexable::from(&self.observers) } - fn observers_mut(&mut self) -> &mut OT { - &mut self.observers + fn observers_mut(&mut self) -> RefIndexable<&mut Self::Observers, Self::Observers> { + RefIndexable::from(&mut self.observers) } } @@ -567,6 +567,7 @@ impl CommandExecutorBuilder { where OT: MatchName + ObserversTuple, S: UsesInput, + S::Input: Input + HasTargetBytes, { let Some(program) = &self.program else { return Err(Error::illegal_argument( @@ -612,7 +613,12 @@ impl CommandExecutorBuilder { timeout: self.timeout, command, }; - Ok(configurator.into_executor::(observers)) + Ok( + >::into_executor::( + configurator, + observers, + ), + ) } } @@ -621,15 +627,15 @@ impl CommandExecutorBuilder { #[cfg_attr(all(feature = "std", unix), doc = " ```")] #[cfg_attr(not(all(feature = "std", unix)), doc = " ```ignore")] /// use std::{io::Write, process::{Stdio, Command, Child}, time::Duration}; -/// use libafl::{Error, inputs::{HasTargetBytes, Input, UsesInput}, executors::{Executor, command::CommandConfigurator}, state::{UsesState, HasExecutions}}; +/// use libafl::{Error, inputs::{BytesInput, HasTargetBytes, Input, UsesInput}, executors::{Executor, command::CommandConfigurator}, state::{UsesState, HasExecutions}}; /// use libafl_bolts::AsSlice; /// #[derive(Debug)] /// struct MyExecutor; /// -/// impl CommandConfigurator for MyExecutor { -/// fn spawn_child( +/// impl CommandConfigurator for MyExecutor { +/// fn spawn_child( /// &mut self, -/// input: &I, +/// input: &BytesInput, /// ) -> Result { /// let mut command = Command::new("../if"); /// command @@ -652,19 +658,16 @@ impl CommandExecutorBuilder { /// where /// EM: UsesState, /// Z: UsesState, -/// EM::State: UsesInput + HasExecutions, -/// EM::Input: HasTargetBytes +/// EM::State: UsesInput + HasExecutions, /// { /// MyExecutor.into_executor(()) /// } /// ``` #[cfg(all(feature = "std", any(unix, doc)))] -pub trait CommandConfigurator: Sized { +pub trait CommandConfigurator: Sized { /// Spawns a new process with the given configuration. - fn spawn_child(&mut self, input: &I) -> Result - where - I: Input + HasTargetBytes; + fn spawn_child(&mut self, input: &I) -> Result; /// Provides timeout duration for execution of the child process. fn exec_timeout(&self) -> Duration; @@ -693,7 +696,7 @@ mod tests { fuzzer::test::NopFuzzer, inputs::BytesInput, monitors::SimpleMonitor, - state::test::NopState, + state::NopState, }; #[test] diff --git a/libafl/src/executors/differential.rs b/libafl/src/executors/differential.rs index 745c4ed1b7..c4e2620e97 100644 --- a/libafl/src/executors/differential.rs +++ b/libafl/src/executors/differential.rs @@ -2,9 +2,12 @@ //! It wraps two executors that will be run after each other with the same input. //! In comparison to the [`crate::executors::CombinedExecutor`] it also runs the secondary executor in `run_target`. //! -use core::{cell::UnsafeCell, fmt::Debug}; +use core::{cell::UnsafeCell, fmt::Debug, ptr}; -use libafl_bolts::{ownedref::OwnedMutPtr, tuples::MatchName}; +use libafl_bolts::{ + ownedref::OwnedMutPtr, + tuples::{MatchName, RefIndexable}, +}; use serde::{Deserialize, Serialize}; use crate::{ @@ -17,13 +20,13 @@ use crate::{ /// A [`DiffExecutor`] wraps a primary executor, forwarding its methods, and a secondary one #[derive(Debug)] -pub struct DiffExecutor { +pub struct DiffExecutor { primary: A, secondary: B, observers: UnsafeCell>, } -impl DiffExecutor { +impl DiffExecutor { /// Create a new `DiffExecutor`, wrapping the given `executor`s. pub fn new(primary: A, secondary: B, observers: DOT) -> Self where @@ -37,8 +40,8 @@ impl DiffExecutor { primary, secondary, observers: UnsafeCell::new(ProxyObserversTuple { - primary: OwnedMutPtr::Ptr(core::ptr::null_mut()), - secondary: OwnedMutPtr::Ptr(core::ptr::null_mut()), + primary: OwnedMutPtr::Ptr(ptr::null_mut()), + secondary: OwnedMutPtr::Ptr(ptr::null_mut()), differential: observers, }), } @@ -55,7 +58,7 @@ impl DiffExecutor { } } -impl Executor for DiffExecutor +impl Executor for DiffExecutor where A: Executor + HasObservers, B: Executor + HasObservers, @@ -183,6 +186,7 @@ where B: MatchName, DOT: MatchName, { + #[allow(deprecated)] fn match_name(&self, name: &str) -> Option<&T> { if let Some(t) = self.primary.as_ref().match_name::(name) { Some(t) @@ -192,6 +196,8 @@ where self.differential.match_name::(name) } } + + #[allow(deprecated)] fn match_name_mut(&mut self, name: &str) -> Option<&mut T> { if let Some(t) = self.primary.as_mut().match_name_mut::(name) { Some(t) @@ -205,12 +211,12 @@ where impl ProxyObserversTuple { fn set(&mut self, primary: &A, secondary: &B) { - self.primary = OwnedMutPtr::Ptr(core::ptr::from_ref::(primary) as *mut A); - self.secondary = OwnedMutPtr::Ptr(core::ptr::from_ref::(secondary) as *mut B); + self.primary = OwnedMutPtr::Ptr(ptr::from_ref(primary) as *mut A); + self.secondary = OwnedMutPtr::Ptr(ptr::from_ref(secondary) as *mut B); } } -impl UsesObservers for DiffExecutor +impl UsesObservers for DiffExecutor where A: HasObservers, B: HasObservers, @@ -221,7 +227,7 @@ where type Observers = ProxyObserversTuple; } -impl UsesState for DiffExecutor +impl UsesState for DiffExecutor where A: UsesState, B: UsesState, @@ -229,7 +235,7 @@ where type State = A::State; } -impl HasObservers for DiffExecutor +impl HasObservers for DiffExecutor where A: HasObservers, B: HasObservers, @@ -238,26 +244,25 @@ where DOT: DifferentialObserversTuple, { #[inline] - fn observers(&self) -> &ProxyObserversTuple { + fn observers(&self) -> RefIndexable<&Self::Observers, Self::Observers> { unsafe { self.observers .get() .as_mut() .unwrap() - .set(self.primary.observers(), self.secondary.observers()); - self.observers.get().as_ref().unwrap() + .set(&*self.primary.observers(), &*self.secondary.observers()); + RefIndexable::from(self.observers.get().as_ref().unwrap()) } } #[inline] - fn observers_mut(&mut self) -> &mut ProxyObserversTuple { + fn observers_mut(&mut self) -> RefIndexable<&mut Self::Observers, Self::Observers> { unsafe { - self.observers - .get() - .as_mut() - .unwrap() - .set(self.primary.observers(), self.secondary.observers()); - self.observers.get().as_mut().unwrap() + self.observers.get().as_mut().unwrap().set( + &*self.primary.observers_mut(), + &*self.secondary.observers_mut(), + ); + RefIndexable::from(self.observers.get().as_mut().unwrap()) } } } diff --git a/libafl/src/executors/forkserver.rs b/libafl/src/executors/forkserver.rs index 5e042a204b..39f9d0f0d6 100644 --- a/libafl/src/executors/forkserver.rs +++ b/libafl/src/executors/forkserver.rs @@ -7,6 +7,7 @@ use core::{ time::Duration, }; use std::{ + env, ffi::{OsStr, OsString}, io::{self, prelude::*, ErrorKind}, os::{ @@ -21,8 +22,8 @@ use libafl_bolts::{ fs::{get_unique_std_input_file, InputFile}, os::{dup2, pipes::Pipe}, shmem::{ShMem, ShMemProvider, UnixShMemProvider}, - tuples::Prepend, - AsMutSlice, AsSlice, Truncate, + tuples::{Handle, Handler, MatchNameRef, Prepend, RefIndexable}, + AsSlice, AsSliceMut, Truncate, }; use nix::{ sys::{ @@ -290,6 +291,14 @@ impl Forkserver { debug_output: bool, kill_signal: Signal, ) -> Result { + if env::var("AFL_MAP_SIZE").is_err() { + log::warn!("AFL_MAP_SIZE not set. If it is unset, the forkserver may fail to start up"); + } + + if env::var("__AFL_SHM_ID").is_err() { + log::warn!("__AFL_SHM_ID not set. It is necessary to set this env, otherwise the forkserver cannot communicate with the fuzzer"); + } + let mut st_pipe = Pipe::new().unwrap(); let mut ctl_pipe = Pipe::new().unwrap(); @@ -436,10 +445,10 @@ impl Forkserver { pub fn read_st_timed(&mut self, timeout: &TimeSpec) -> Result, Error> { let mut buf: [u8; 4] = [0_u8; 4]; let Some(st_read) = self.st_pipe.read_end() else { - return Err(Error::file(io::Error::new( - ErrorKind::BrokenPipe, - "Read pipe end was already closed", - ))); + return Err(Error::os_error( + io::Error::new(ErrorKind::BrokenPipe, "Read pipe end was already closed"), + "read_st_timed failed", + )); }; // # Safety @@ -488,7 +497,10 @@ where map: Option, phantom: PhantomData, map_size: Option, + #[cfg(feature = "regex")] + asan_obs: Handle, timeout: TimeSpec, + crash_exitcode: Option, } impl Debug for ForkserverExecutor @@ -574,6 +586,9 @@ pub struct ForkserverExecutorBuilder<'a, SP> { real_map_size: i32, kill_signal: Option, timeout: Option, + #[cfg(feature = "regex")] + asan_obs: Option>, + crash_exitcode: Option, } impl<'a, SP> ForkserverExecutorBuilder<'a, SP> { @@ -622,18 +637,24 @@ impl<'a, SP> ForkserverExecutorBuilder<'a, SP> { phantom: PhantomData, map_size: self.map_size, timeout, + asan_obs: self + .asan_obs + .clone() + .unwrap_or(AsanBacktraceObserver::default().handle()), + crash_exitcode: self.crash_exitcode, }) } /// Builds `ForkserverExecutor` downsizing the coverage map to fit exaclty the AFL++ map size. #[allow(clippy::pedantic)] - pub fn build_dynamic_map( + pub fn build_dynamic_map( &mut self, - mut map_observer: MO, + mut map_observer: A, other_observers: OT, - ) -> Result, Error> + ) -> Result, Error> where - MO: Observer + MapObserver + Truncate, // TODO maybe enforce Entry = u8 for the cov map + MO: MapObserver + Truncate, // TODO maybe enforce Entry = u8 for the cov map + A: Observer + AsRef + AsMut, OT: ObserversTuple + Prepend, S: UsesInput, S::Input: Input + HasTargetBytes, @@ -651,10 +672,10 @@ impl<'a, SP> ForkserverExecutorBuilder<'a, SP> { ); if let Some(dynamic_map_size) = self.map_size { - map_observer.truncate(dynamic_map_size); + map_observer.as_mut().truncate(dynamic_map_size); } - let observers: (MO, OT) = other_observers.prepend(map_observer); + let observers = (map_observer, other_observers); if self.uses_shmem_testcase && map.is_none() { return Err(Error::illegal_state( @@ -678,6 +699,11 @@ impl<'a, SP> ForkserverExecutorBuilder<'a, SP> { phantom: PhantomData, map_size: self.map_size, timeout, + asan_obs: self + .asan_obs + .clone() + .unwrap_or(AsanBacktraceObserver::default().handle()), + crash_exitcode: self.crash_exitcode, }) } @@ -704,7 +730,7 @@ impl<'a, SP> ForkserverExecutorBuilder<'a, SP> { shmem.write_to_env("__AFL_SHM_FUZZ_ID")?; let size_in_bytes = (self.max_input_size + SHMEM_FUZZ_HDR_SIZE).to_ne_bytes(); - shmem.as_mut_slice()[..4].clone_from_slice(&size_in_bytes[..4]); + shmem.as_slice_mut()[..4].clone_from_slice(&size_in_bytes[..4]); Some(shmem) } }; @@ -989,6 +1015,13 @@ impl<'a, SP> ForkserverExecutorBuilder<'a, SP> { self } + /// Treats an execution as a crash if the provided exitcode is returned + #[must_use] + pub fn crash_exitcode(mut self, exitcode: i8) -> Self { + self.crash_exitcode = Some(exitcode); + self + } + /// Call this if the harness uses deferred forkserver mode; default is false #[must_use] pub fn is_deferred_frksrv(mut self, is_deferred_frksrv: bool) -> Self { @@ -1037,6 +1070,8 @@ impl<'a> ForkserverExecutorBuilder<'a, UnixShMemProvider> { max_input_size: MAX_INPUT_SIZE_DEFAULT, kill_signal: None, timeout: None, + asan_obs: None, + crash_exitcode: None, } } @@ -1062,6 +1097,8 @@ impl<'a> ForkserverExecutorBuilder<'a, UnixShMemProvider> { max_input_size: MAX_INPUT_SIZE_DEFAULT, kill_signal: None, timeout: None, + asan_obs: None, + crash_exitcode: None, } } } @@ -1112,9 +1149,9 @@ where } let size_in_bytes = size.to_ne_bytes(); // The first four bytes tells the size of the shmem. - map.as_mut_slice()[..SHMEM_FUZZ_HDR_SIZE] + map.as_slice_mut()[..SHMEM_FUZZ_HDR_SIZE] .copy_from_slice(&size_in_bytes[..SHMEM_FUZZ_HDR_SIZE]); - map.as_mut_slice()[SHMEM_FUZZ_HDR_SIZE..(SHMEM_FUZZ_HDR_SIZE + size)] + map.as_slice_mut()[SHMEM_FUZZ_HDR_SIZE..(SHMEM_FUZZ_HDR_SIZE + size)] .copy_from_slice(&target_bytes.as_slice()[..size]); } else { self.input_file.write_buf(input.target_bytes().as_slice())?; @@ -1147,13 +1184,15 @@ where if let Some(status) = self.forkserver.read_st_timed(&self.timeout)? { self.forkserver.set_status(status); - if libc::WIFSIGNALED(self.forkserver().status()) { + let exitcode_is_crash = if let Some(crash_exitcode) = self.crash_exitcode { + (libc::WEXITSTATUS(self.forkserver().status()) as i8) == crash_exitcode + } else { + false + }; + if libc::WIFSIGNALED(self.forkserver().status()) || exitcode_is_crash { exit_kind = ExitKind::Crash; #[cfg(feature = "regex")] - if let Some(asan_observer) = self - .observers_mut() - .match_name_mut::("AsanBacktraceObserver") - { + if let Some(asan_observer) = self.observers.get_mut(&self.asan_obs) { asan_observer.parse_asan_output_from_asan_log_file(pid)?; } } @@ -1201,13 +1240,13 @@ where SP: ShMemProvider, { #[inline] - fn observers(&self) -> &OT { - &self.observers + fn observers(&self) -> RefIndexable<&Self::Observers, Self::Observers> { + RefIndexable::from(&self.observers) } #[inline] - fn observers_mut(&mut self) -> &mut OT { - &mut self.observers + fn observers_mut(&mut self) -> RefIndexable<&mut Self::Observers, Self::Observers> { + RefIndexable::from(&mut self.observers) } } @@ -1218,7 +1257,7 @@ mod tests { use libafl_bolts::{ shmem::{ShMem, ShMemProvider, UnixShMemProvider}, tuples::tuple_list, - AsMutSlice, + AsSliceMut, }; use serial_test::serial; @@ -1240,7 +1279,7 @@ mod tests { let mut shmem = shmem_provider.new_shmem(MAP_SIZE).unwrap(); shmem.write_to_env("__AFL_SHM_ID").unwrap(); - let shmem_buf = shmem.as_mut_slice(); + let shmem_buf = shmem.as_slice_mut(); let edges_observer = HitcountsMapObserver::new(ConstMapObserver::<_, MAP_SIZE>::new( "shared_mem", diff --git a/libafl/src/executors/hooks/inprocess.rs b/libafl/src/executors/hooks/inprocess.rs index 30a92c233f..8733c37a36 100644 --- a/libafl/src/executors/hooks/inprocess.rs +++ b/libafl/src/executors/hooks/inprocess.rs @@ -5,6 +5,7 @@ use core::ptr::addr_of_mut; use core::sync::atomic::{compiler_fence, Ordering}; use core::{ ffi::c_void, + marker::PhantomData, ptr::{self, null_mut}, time::Duration, }; @@ -30,21 +31,26 @@ use crate::{ events::{EventFirer, EventRestarter}, executors::{hooks::ExecutorHook, inprocess::HasInProcessHooks, Executor, HasObservers}, feedbacks::Feedback, + inputs::UsesInput, state::{HasCorpus, HasExecutions, HasSolutions}, Error, HasObjective, }; /// The inmem executor's handlers. #[allow(missing_debug_implementations)] -pub struct InProcessHooks { +pub struct InProcessHooks +where + S: UsesInput, +{ /// On crash C function pointer #[cfg(feature = "std")] pub crash_handler: *const c_void, /// On timeout C function pointer #[cfg(feature = "std")] pub timeout_handler: *const c_void, - /// TImer struct + /// `TImer` struct #[cfg(feature = "std")] pub timer: TimerStruct, + phantom: PhantomData, } /// Any hooks that is about timeout @@ -80,7 +86,10 @@ pub trait HasTimeout { fn handle_timeout(&mut self, data: &mut InProcessExecutorHandlerData) -> bool; } -impl HasTimeout for InProcessHooks { +impl HasTimeout for InProcessHooks +where + S: UsesInput, +{ #[cfg(feature = "std")] fn timer(&self) -> &TimerStruct { &self.timer @@ -184,13 +193,15 @@ impl HasTimeout for InProcessHooks { } } -impl ExecutorHook for InProcessHooks { - fn init(&mut self, _state: &mut S) {} - +impl ExecutorHook for InProcessHooks +where + S: UsesInput, +{ + fn init(&mut self, _state: &mut S) {} /// Call before running a target. #[allow(clippy::unused_self)] #[allow(unused_variables)] - fn pre_exec(&mut self, fuzzer: &mut Z, state: &mut S, mgr: &mut EM, input: &I) { + fn pre_exec(&mut self, state: &mut S, input: &S::Input) { #[cfg(feature = "std")] unsafe { let data = addr_of_mut!(GLOBAL_STATE); @@ -198,32 +209,29 @@ impl ExecutorHook for InProcessHooks { (*data).timeout_handler = self.timeout_handler; } - #[cfg(feature = "std")] + #[cfg(all(feature = "std", not(all(miri, target_vendor = "apple"))))] self.timer_mut().set_timer(); } /// Call after running a target. #[allow(clippy::unused_self)] - fn post_exec( - &mut self, - _fuzzer: &mut Z, - _state: &mut S, - _mgr: &mut EM, - _input: &I, - ) { + fn post_exec(&mut self, _state: &mut S, _input: &S::Input) { // timeout stuff - #[cfg(feature = "std")] + #[cfg(all(feature = "std", not(all(miri, target_vendor = "apple"))))] self.timer_mut().unset_timer(); } } -impl InProcessHooks { +impl InProcessHooks +where + S: UsesInput, +{ /// Create new [`InProcessHooks`]. #[cfg(unix)] #[allow(unused_variables)] pub fn new(exec_tmout: Duration) -> Result where - E: Executor + HasObservers + HasInProcessHooks, + E: Executor + HasObservers + HasInProcessHooks, EM: EventFirer + EventRestarter, OF: Feedback, E::State: HasExecutions + HasSolutions + HasCorpus, @@ -246,6 +254,7 @@ impl InProcessHooks { as *const _, #[cfg(feature = "std")] timer: TimerStruct::new(exec_tmout), + phantom: PhantomData, }) } } @@ -255,7 +264,7 @@ impl InProcessHooks { #[allow(unused)] pub fn new(exec_tmout: Duration) -> Result where - E: Executor + HasObservers + HasInProcessHooks, + E: Executor + HasObservers + HasInProcessHooks, EM: EventFirer + EventRestarter, OF: Feedback, E::State: State + HasExecutions + HasSolutions + HasCorpus, @@ -292,11 +301,14 @@ impl InProcessHooks { crash_handler, timeout_handler, timer, + phantom: PhantomData, }); } #[cfg(not(feature = "std"))] { - ret = Ok(Self {}); + ret = Ok(Self { + phantom: PhantomData, + }); } ret @@ -307,14 +319,16 @@ impl InProcessHooks { #[allow(unused_variables)] pub fn new(exec_tmout: Duration) -> Result where - E: Executor + HasObservers + HasInProcessHooks, + E: Executor + HasObservers + HasInProcessHooks, EM: EventFirer + EventRestarter, OF: Feedback, E::State: HasExecutions + HasSolutions + HasCorpus, Z: HasObjective, { #[cfg_attr(miri, allow(unused_variables))] - let ret = Self {}; + let ret = Self { + phantom: PhantomData, + }; Ok(ret) } @@ -329,6 +343,7 @@ impl InProcessHooks { timeout_handler: ptr::null(), #[cfg(feature = "std")] timer: TimerStruct::new(Duration::from_millis(5000)), + phantom: PhantomData, } } } diff --git a/libafl/src/executors/hooks/inprocess_fork.rs b/libafl/src/executors/hooks/inprocess_fork.rs index 48e64e3db1..819bbe30df 100644 --- a/libafl/src/executors/hooks/inprocess_fork.rs +++ b/libafl/src/executors/hooks/inprocess_fork.rs @@ -2,6 +2,7 @@ use alloc::vec::Vec; use core::{ ffi::c_void, + marker::PhantomData, ptr::{addr_of_mut, null}, sync::atomic::{compiler_fence, Ordering}, }; @@ -19,30 +20,29 @@ use crate::{ inprocess_fork::{child_signal_handlers, ForkHandlerFuncPtr}, HasObservers, }, + inputs::UsesInput, Error, }; /// The inmem fork executor's hooks. #[derive(Debug)] -pub struct InChildProcessHooks { +pub struct InChildProcessHooks { /// On crash C function pointer pub crash_handler: *const c_void, /// On timeout C function pointer pub timeout_handler: *const c_void, + phantom: PhantomData, } -impl ExecutorHook for InChildProcessHooks { +impl ExecutorHook for InChildProcessHooks +where + S: UsesInput, +{ /// Init this hook - fn init(&mut self, _state: &mut S) {} + fn init(&mut self, _state: &mut S) {} /// Call before running a target. - fn pre_exec( - &mut self, - _fuzzer: &mut Z, - _state: &mut S, - _mgr: &mut EM, - _input: &I, - ) { + fn pre_exec(&mut self, _state: &mut S, _input: &S::Input) { unsafe { let data = addr_of_mut!(FORK_EXECUTOR_GLOBAL_DATA); (*data).crash_handler = self.crash_handler; @@ -51,17 +51,10 @@ impl ExecutorHook for InChildProcessHooks { } } - fn post_exec( - &mut self, - _fuzzer: &mut Z, - _state: &mut S, - _mgr: &mut EM, - _input: &I, - ) { - } + fn post_exec(&mut self, _state: &mut S, _input: &S::Input) {} } -impl InChildProcessHooks { +impl InChildProcessHooks { /// Create new [`InChildProcessHooks`]. pub fn new() -> Result where @@ -77,6 +70,7 @@ impl InChildProcessHooks { Ok(Self { crash_handler: child_signal_handlers::child_crash_handler:: as *const c_void, timeout_handler: child_signal_handlers::child_timeout_handler:: as *const c_void, + phantom: PhantomData, }) } } @@ -87,6 +81,7 @@ impl InChildProcessHooks { Self { crash_handler: null(), timeout_handler: null(), + phantom: PhantomData, } } } @@ -101,9 +96,9 @@ pub(crate) struct InProcessForkExecutorGlobalData { pub state_ptr: *const c_void, /// Stores a pointer to the current input pub current_input_ptr: *const c_void, - /// Stores a pointer to the crash_handler function + /// Stores a pointer to the `crash_handler` function pub crash_handler: *const c_void, - /// Stores a pointer to the timeout_handler function + /// Stores a pointer to the `timeout_handler` function pub timeout_handler: *const c_void, } diff --git a/libafl/src/executors/hooks/mod.rs b/libafl/src/executors/hooks/mod.rs index add21cde11..35453192fa 100644 --- a/libafl/src/executors/hooks/mod.rs +++ b/libafl/src/executors/hooks/mod.rs @@ -1,7 +1,7 @@ //! Hooks for the executors. //! These will be executed right before and after the executor's harness run. -use crate::executors::HasObservers; +use crate::{executors::HasObservers, inputs::UsesInput}; /// windows crash/timeout handler and asan death callback #[cfg(windows)] @@ -23,80 +23,58 @@ pub mod inprocess; pub mod timer; /// The hook that runs before and after the executor runs the target -pub trait ExecutorHook { +pub trait ExecutorHook +where + S: UsesInput, +{ /// Init this hook - fn init(&mut self, state: &mut S); + fn init(&mut self, state: &mut S); /// The hook that runs before runs the target - fn pre_exec(&mut self, fuzzer: &mut Z, state: &mut S, mgr: &mut EM, input: &I); + fn pre_exec(&mut self, state: &mut S, input: &S::Input); /// The hook that runs before runs the target - fn post_exec(&mut self, fuzzer: &mut Z, state: &mut S, mgr: &mut EM, input: &I); + fn post_exec(&mut self, state: &mut S, input: &S::Input); } /// The hook that runs before and after the executor runs the target -pub trait ExecutorHooksTuple { +pub trait ExecutorHooksTuple +where + S: UsesInput, +{ /// Init these hooks - fn init_all(&mut self, state: &mut S); + fn init_all(&mut self, state: &mut S); /// The hooks that runs before runs the target - fn pre_exec_all(&mut self, fuzzer: &mut Z, state: &mut S, mgr: &mut EM, input: &I); + fn pre_exec_all(&mut self, state: &mut S, input: &S::Input); /// The hooks that runs after runs the target - fn post_exec_all( - &mut self, - fuzzer: &mut Z, - state: &mut S, - mgr: &mut EM, - input: &I, - ); + fn post_exec_all(&mut self, state: &mut S, input: &S::Input); } -impl ExecutorHooksTuple for () { - fn init_all(&mut self, _state: &mut S) {} - fn pre_exec_all( - &mut self, - _fuzzer: &mut Z, - _state: &mut S, - _mgr: &mut EM, - _input: &I, - ) { - } - fn post_exec_all( - &mut self, - _fuzzer: &mut Z, - _state: &mut S, - _mgr: &mut EM, - _input: &I, - ) { - } +impl ExecutorHooksTuple for () +where + S: UsesInput, +{ + fn init_all(&mut self, _state: &mut S) {} + fn pre_exec_all(&mut self, _state: &mut S, _input: &S::Input) {} + fn post_exec_all(&mut self, _state: &mut S, _input: &S::Input) {} } -impl ExecutorHooksTuple for (Head, Tail) +impl ExecutorHooksTuple for (Head, Tail) where - Head: ExecutorHook, - Tail: ExecutorHooksTuple, + S: UsesInput, + Head: ExecutorHook, + Tail: ExecutorHooksTuple, { - fn init_all(&mut self, state: &mut S) { - self.0.init::(state); - self.1.init_all::(state); + fn init_all(&mut self, state: &mut S) { + self.0.init::(state); + self.1.init_all::(state); } - fn pre_exec_all( - &mut self, - fuzzer: &mut Z, - state: &mut S, - mgr: &mut EM, - input: &I, - ) { - self.0.pre_exec(fuzzer, state, mgr, input); - self.1.pre_exec_all(fuzzer, state, mgr, input); + fn pre_exec_all(&mut self, state: &mut S, input: &S::Input) { + self.0.pre_exec(state, input); + self.1.pre_exec_all(state, input); } - fn post_exec_all( - &mut self, - fuzzer: &mut Z, - state: &mut S, - mgr: &mut EM, - input: &I, - ) { - self.0.post_exec(fuzzer, state, mgr, input); - self.1.post_exec_all(fuzzer, state, mgr, input); + fn post_exec_all(&mut self, state: &mut S, input: &S::Input) { + self.0.post_exec(state, input); + self.1.post_exec_all(state, input); } } diff --git a/libafl/src/executors/hooks/timer.rs b/libafl/src/executors/hooks/timer.rs index 2f93f2192e..f81202f191 100644 --- a/libafl/src/executors/hooks/timer.rs +++ b/libafl/src/executors/hooks/timer.rs @@ -1,10 +1,7 @@ //! The struct `TimerStruct` will absorb all the difference in timeout implementation in various system. -use core::time::Duration; #[cfg(any(windows, target_os = "linux"))] -use core::{ - ffi::c_void, - ptr::{addr_of_mut, write_volatile}, -}; +use core::ptr::addr_of_mut; +use core::time::Duration; #[cfg(target_os = "linux")] use core::{ mem::zeroed, @@ -15,7 +12,11 @@ use core::{ pub(crate) const ITIMER_REAL: core::ffi::c_int = 0; #[cfg(windows)] -use core::sync::atomic::{compiler_fence, Ordering}; +use core::{ + ffi::c_void, + ptr::write_volatile, + sync::atomic::{compiler_fence, Ordering}, +}; #[cfg(target_os = "linux")] use libafl_bolts::current_time; @@ -29,7 +30,7 @@ use windows::Win32::{ }, }; -#[cfg(any(windows, target_os = "linux"))] +#[cfg(windows)] use crate::executors::hooks::inprocess::GLOBAL_STATE; #[repr(C)] @@ -296,12 +297,6 @@ impl TimerStruct { pub fn set_timer(&mut self) { unsafe { if self.batch_mode { - let data = addr_of_mut!(GLOBAL_STATE); - write_volatile( - addr_of_mut!((*data).executor_ptr), - core::ptr::from_mut(self) as *mut c_void, - ); - if self.executions == 0 { libc::timer_settime(self.timerid, 0, addr_of_mut!(self.itimerspec), null_mut()); self.tmout_start_time = current_time(); @@ -329,10 +324,11 @@ impl TimerStruct { pub fn unset_timer(&mut self) { if self.batch_mode { unsafe { - let elapsed = current_time() - self.tmout_start_time; + let elapsed = current_time().saturating_sub(self.tmout_start_time); + let elapsed_since_signal = current_time().saturating_sub(self.tmout_start_time); // elapsed may be > than tmout in case of received but ingored signal if elapsed > self.exec_tmout - || self.exec_tmout - elapsed < self.avg_exec_time * self.avg_mul_k + || self.exec_tmout.saturating_sub(elapsed) < self.avg_exec_time * self.avg_mul_k { let disarmed: libc::itimerspec = zeroed(); libc::timer_settime(self.timerid, 0, addr_of!(disarmed), null_mut()); @@ -342,8 +338,7 @@ impl TimerStruct { self.executions = 0; } // readjust K - if self.last_signal_time > self.exec_tmout * self.avg_mul_k - && self.avg_mul_k > 1 + if elapsed_since_signal > self.exec_tmout * self.avg_mul_k && self.avg_mul_k > 1 { self.avg_mul_k -= 1; } diff --git a/libafl/src/executors/hooks/unix.rs b/libafl/src/executors/hooks/unix.rs index a47fde335e..2f127ccd80 100644 --- a/libafl/src/executors/hooks/unix.rs +++ b/libafl/src/executors/hooks/unix.rs @@ -122,7 +122,7 @@ pub mod unix_signal_handler { _context: Option<&mut ucontext_t>, data: &mut InProcessExecutorHandlerData, ) where - E: HasObservers + HasInProcessHooks, + E: HasObservers + HasInProcessHooks, EM: EventFirer + EventRestarter, OF: Feedback, E::State: HasExecutions + HasSolutions + HasCorpus, @@ -185,8 +185,8 @@ pub mod unix_signal_handler { { #[cfg(all(target_os = "android", target_arch = "aarch64"))] let _context = _context.map(|p| { - &mut *(((p as *mut _ as *mut libc::c_void as usize) + 128) as *mut libc::c_void - as *mut ucontext_t) + &mut *(((core::ptr::from_mut(p) as *mut libc::c_void as usize) + 128) + as *mut libc::c_void as *mut ucontext_t) }); log::error!("Crashed with {signal}"); @@ -204,17 +204,21 @@ pub mod unix_signal_handler { let mut bsod = Vec::new(); { let mut writer = std::io::BufWriter::new(&mut bsod); - writeln!(writer, "input: {:?}", input.generate_name(0)).unwrap(); - libafl_bolts::minibsod::generate_minibsod( + let _ = writeln!(writer, "input: {:?}", input.generate_name(0)); + let bsod = libafl_bolts::minibsod::generate_minibsod( &mut writer, signal, _info, _context.as_deref(), - ) - .unwrap(); - writer.flush().unwrap(); + ); + if bsod.is_err() { + log::error!("generate_minibsod failed"); + } + let _ = writer.flush(); + } + if let Ok(r) = std::str::from_utf8(&bsod) { + log::error!("{}", r); } - log::error!("{}", std::str::from_utf8(&bsod).unwrap()); } run_observers_and_save_state::( @@ -241,16 +245,20 @@ pub mod unix_signal_handler { let mut bsod = Vec::new(); { let mut writer = std::io::BufWriter::new(&mut bsod); - libafl_bolts::minibsod::generate_minibsod( + let bsod = libafl_bolts::minibsod::generate_minibsod( &mut writer, signal, _info, _context.as_deref(), - ) - .unwrap(); - writer.flush().unwrap(); + ); + if bsod.is_err() { + log::error!("generate_minibsod failed"); + } + let _ = writer.flush(); + } + if let Ok(r) = std::str::from_utf8(&bsod) { + log::error!("{}", r); } - log::error!("{}", std::str::from_utf8(&bsod).unwrap()); } } @@ -258,7 +266,7 @@ pub mod unix_signal_handler { log::error!("Type QUIT to restart the child"); let mut line = String::new(); while line.trim() != "QUIT" { - std::io::stdin().read_line(&mut line).unwrap(); + let _ = std::io::stdin().read_line(&mut line); } } diff --git a/libafl/src/executors/hooks/windows.rs b/libafl/src/executors/hooks/windows.rs index 34854e2532..ec5be577de 100644 --- a/libafl/src/executors/hooks/windows.rs +++ b/libafl/src/executors/hooks/windows.rs @@ -64,7 +64,7 @@ pub mod windows_asan_handler { log::error!("Type QUIT to restart the child"); let mut line = String::new(); while line.trim() != "QUIT" { - std::io::stdin().read_line(&mut line).unwrap(); + let _ = std::io::stdin().read_line(&mut line); } } @@ -237,7 +237,7 @@ pub mod windows_exception_handler { global_state: *mut c_void, _p1: *mut u8, ) where - E: HasObservers + HasInProcessHooks, + E: HasObservers + HasInProcessHooks, EM: EventFirer + EventRestarter, OF: Feedback, E::State: State + HasExecutions + HasSolutions + HasCorpus, @@ -333,15 +333,14 @@ pub mod windows_exception_handler { let mut is_crash = true; #[cfg(feature = "std")] if let Some(exception_pointers) = exception_pointers.as_mut() { - let code = ExceptionCode::try_from( + let code: ExceptionCode = ExceptionCode::from( exception_pointers .ExceptionRecord .as_mut() .unwrap() .ExceptionCode .0, - ) - .unwrap(); + ); let exception_list = data.exceptions(); if exception_list.contains(&code) { @@ -374,7 +373,7 @@ pub mod windows_exception_handler { log::error!("Type QUIT to restart the child"); let mut line = String::new(); while line.trim() != "QUIT" { - std::io::stdin().read_line(&mut line).unwrap(); + let _ = std::io::stdin().read_line(&mut line); } } diff --git a/libafl/src/executors/inprocess/inner.rs b/libafl/src/executors/inprocess/inner.rs new file mode 100644 index 0000000000..492e474dae --- /dev/null +++ b/libafl/src/executors/inprocess/inner.rs @@ -0,0 +1,302 @@ +use core::{ + ffi::c_void, + fmt::{self, Debug, Formatter}, + marker::PhantomData, + ptr::{self, addr_of_mut, null, write_volatile}, + sync::atomic::{compiler_fence, Ordering}, + time::Duration, +}; + +use libafl_bolts::tuples::{tuple_list, Merge, RefIndexable}; +#[cfg(windows)] +use windows::Win32::System::Threading::SetThreadStackGuarantee; + +#[cfg(all(feature = "std", target_os = "linux"))] +use crate::executors::hooks::inprocess::HasTimeout; +#[cfg(all(windows, feature = "std"))] +use crate::executors::hooks::inprocess::HasTimeout; +use crate::{ + events::{EventFirer, EventRestarter}, + executors::{ + hooks::{ + inprocess::{InProcessHooks, GLOBAL_STATE}, + ExecutorHooksTuple, + }, + inprocess::HasInProcessHooks, + Executor, HasObservers, + }, + feedbacks::Feedback, + fuzzer::HasObjective, + inputs::UsesInput, + observers::{ObserversTuple, UsesObservers}, + state::{HasCorpus, HasExecutions, HasSolutions, State, UsesState}, + Error, +}; + +/// The internal state of `GenericInProcessExecutor`. +pub struct GenericInProcessExecutorInner +where + HT: ExecutorHooksTuple, + OT: ObserversTuple, + S: State, +{ + /// The observers, observing each run + pub(super) observers: OT, + // Crash and timeout hah + pub(super) hooks: (InProcessHooks, HT), + phantom: PhantomData, +} + +impl Debug for GenericInProcessExecutorInner +where + HT: ExecutorHooksTuple, + OT: ObserversTuple + Debug, + S: State, +{ + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct("GenericInProcessExecutorState") + .field("observers", &self.observers) + .finish_non_exhaustive() + } +} + +impl UsesState for GenericInProcessExecutorInner +where + HT: ExecutorHooksTuple, + OT: ObserversTuple, + S: State, +{ + type State = S; +} + +impl UsesObservers for GenericInProcessExecutorInner +where + HT: ExecutorHooksTuple, + OT: ObserversTuple, + S: State, +{ + type Observers = OT; +} + +impl HasObservers for GenericInProcessExecutorInner +where + HT: ExecutorHooksTuple, + OT: ObserversTuple, + S: State, +{ + #[inline] + fn observers(&self) -> RefIndexable<&Self::Observers, Self::Observers> { + RefIndexable::from(&self.observers) + } + + #[inline] + fn observers_mut(&mut self) -> RefIndexable<&mut Self::Observers, Self::Observers> { + RefIndexable::from(&mut self.observers) + } +} + +impl GenericInProcessExecutorInner +where + HT: ExecutorHooksTuple, + OT: ObserversTuple, + S: State, +{ + /// This function marks the boundary between the fuzzer and the target + /// + /// # Safety + /// This function sets a bunch of raw pointers in global variables, reused in other parts of + /// the code. + #[inline] + pub unsafe fn enter_target( + &mut self, + fuzzer: &mut Z, + state: &mut ::State, + mgr: &mut EM, + input: &::Input, + executor_ptr: *const c_void, + ) { + unsafe { + let data = addr_of_mut!(GLOBAL_STATE); + write_volatile( + addr_of_mut!((*data).current_input_ptr), + ptr::from_ref(input) as *const c_void, + ); + write_volatile(addr_of_mut!((*data).executor_ptr), executor_ptr); + // Direct raw pointers access /aliasing is pretty undefined behavior. + // Since the state and event may have moved in memory, refresh them right before the signal may happen + write_volatile( + addr_of_mut!((*data).state_ptr), + ptr::from_mut(state) as *mut c_void, + ); + write_volatile( + addr_of_mut!((*data).event_mgr_ptr), + ptr::from_mut(mgr) as *mut c_void, + ); + write_volatile( + addr_of_mut!((*data).fuzzer_ptr), + ptr::from_mut(fuzzer) as *mut c_void, + ); + compiler_fence(Ordering::SeqCst); + } + } + + /// This function marks the boundary between the fuzzer and the target + #[inline] + pub fn leave_target( + &mut self, + _fuzzer: &mut Z, + _state: &mut ::State, + _mgr: &mut EM, + _input: &::Input, + ) { + unsafe { + let data = addr_of_mut!(GLOBAL_STATE); + + write_volatile(addr_of_mut!((*data).current_input_ptr), null()); + compiler_fence(Ordering::SeqCst); + } + } +} + +impl GenericInProcessExecutorInner +where + HT: ExecutorHooksTuple, + OT: ObserversTuple, + S: HasExecutions + HasSolutions + HasCorpus + State, +{ + /// Create a new in mem executor with the default timeout (5 sec) + pub fn generic( + user_hooks: HT, + observers: OT, + fuzzer: &mut Z, + state: &mut S, + event_mgr: &mut EM, + ) -> Result + where + E: Executor + HasObservers + HasInProcessHooks, + EM: EventFirer + EventRestarter, + OF: Feedback, + S: State, + Z: HasObjective, + { + Self::with_timeout_generic::( + user_hooks, + observers, + fuzzer, + state, + event_mgr, + Duration::from_millis(5000), + ) + } + + /// Create a new in mem executor with the default timeout and use batch mode(5 sec) + #[cfg(all(feature = "std", target_os = "linux"))] + pub fn batched_timeout_generic( + user_hooks: HT, + observers: OT, + fuzzer: &mut Z, + state: &mut S, + event_mgr: &mut EM, + exec_tmout: Duration, + ) -> Result + where + E: Executor + HasObservers + HasInProcessHooks, + EM: EventFirer + EventRestarter, + OF: Feedback, + S: State, + Z: HasObjective, + { + let mut me = Self::with_timeout_generic::( + user_hooks, observers, fuzzer, state, event_mgr, exec_tmout, + )?; + me.hooks_mut().0.timer_mut().batch_mode = true; + Ok(me) + } + + /// Create a new in mem executor. + /// Caution: crash and restart in one of them will lead to odd behavior if multiple are used, + /// depending on different corpus or state. + /// * `user_hooks` - the hooks run before and after the harness's execution + /// * `harness_fn` - the harness, executing the function + /// * `observers` - the observers observing the target during execution + /// This may return an error on unix, if signal handler setup fails + pub fn with_timeout_generic( + user_hooks: HT, + observers: OT, + _fuzzer: &mut Z, + state: &mut S, + _event_mgr: &mut EM, + timeout: Duration, + ) -> Result + where + E: Executor + HasObservers + HasInProcessHooks, + EM: EventFirer + EventRestarter, + OF: Feedback, + S: State, + Z: HasObjective, + { + let default = InProcessHooks::new::(timeout)?; + let mut hooks = tuple_list!(default).merge(user_hooks); + hooks.init_all::(state); + + #[cfg(windows)] + // Some initialization necessary for windows. + unsafe { + /* + See https://github.com/AFLplusplus/LibAFL/pull/403 + This one reserves certain amount of memory for the stack. + If stack overflow happens during fuzzing on windows, the program is transferred to our exception handler for windows. + However, if we run out of the stack memory again in this exception handler, we'll crash with STATUS_ACCESS_VIOLATION. + We need this API call because with the llmp_compression + feature enabled, the exception handler uses a lot of stack memory (in the compression lib code) on release build. + As far as I have observed, the compression uses around 0x10000 bytes, but for safety let's just reserve 0x20000 bytes for our exception handlers. + This number 0x20000 could vary depending on the compilers optimization for future compression library changes. + */ + let mut stack_reserved = 0x20000; + SetThreadStackGuarantee(&mut stack_reserved)?; + } + + #[cfg(all(feature = "std", windows))] + { + // set timeout for the handler + *hooks.0.millis_sec_mut() = timeout.as_millis() as i64; + } + + Ok(Self { + observers, + hooks, + phantom: PhantomData, + }) + } + + /// The inprocess handlers + #[inline] + pub fn hooks(&self) -> &(InProcessHooks, HT) { + &self.hooks + } + + /// The inprocess handlers (mutable) + #[inline] + pub fn hooks_mut(&mut self) -> &mut (InProcessHooks, HT) { + &mut self.hooks + } +} + +impl HasInProcessHooks for GenericInProcessExecutorInner +where + HT: ExecutorHooksTuple, + OT: ObserversTuple, + S: State + HasExecutions + HasSolutions + HasCorpus, +{ + /// the timeout handler + #[inline] + fn inprocess_hooks(&self) -> &InProcessHooks { + &self.hooks.0 + } + + /// the timeout handler + #[inline] + fn inprocess_hooks_mut(&mut self) -> &mut InProcessHooks { + &mut self.hooks.0 + } +} diff --git a/libafl/src/executors/inprocess.rs b/libafl/src/executors/inprocess/mod.rs similarity index 54% rename from libafl/src/executors/inprocess.rs rename to libafl/src/executors/inprocess/mod.rs index f405118507..97df8116b0 100644 --- a/libafl/src/executors/inprocess.rs +++ b/libafl/src/executors/inprocess/mod.rs @@ -5,45 +5,48 @@ #![allow(clippy::needless_pass_by_value)] use alloc::boxed::Box; +#[cfg(any(unix, feature = "std"))] +use core::ptr::addr_of_mut; use core::{ borrow::BorrowMut, ffi::c_void, fmt::{self, Debug, Formatter}, marker::PhantomData, - ptr::{addr_of_mut, null, write_volatile}, - sync::atomic::{compiler_fence, Ordering}, + ptr, time::Duration, }; -use libafl_bolts::tuples::{tuple_list, Merge}; -#[cfg(windows)] -use windows::Win32::System::Threading::SetThreadStackGuarantee; +use libafl_bolts::tuples::{tuple_list, RefIndexable}; -#[cfg(all(feature = "std", target_os = "linux"))] -use crate::executors::hooks::inprocess::HasTimeout; -#[cfg(all(windows, feature = "std"))] -use crate::executors::hooks::inprocess::HasTimeout; +#[cfg(any(unix, feature = "std"))] +use crate::executors::hooks::inprocess::GLOBAL_STATE; use crate::{ corpus::{Corpus, Testcase}, events::{Event, EventFirer, EventRestarter}, executors::{ - hooks::{ - inprocess::{InProcessHooks, GLOBAL_STATE}, - ExecutorHooksTuple, - }, + hooks::{inprocess::InProcessHooks, ExecutorHooksTuple}, + inprocess::inner::GenericInProcessExecutorInner, Executor, ExitKind, HasObservers, }, feedbacks::Feedback, fuzzer::HasObjective, inputs::UsesInput, observers::{ObserversTuple, UsesObservers}, - state::{HasCorpus, HasExecutions, HasMetadata, HasSolutions, State, UsesState}, - Error, + state::{HasCorpus, HasCurrentTestcase, HasExecutions, HasSolutions, State, UsesState}, + Error, HasMetadata, }; -/// The process executor simply calls a target function, as mutable reference to a closure +/// The inner structure of `InProcessExecutor`. +pub mod inner; +/// A version of `InProcessExecutor` with a state accessible from the harness. +pub mod stateful; + +/// The process executor simply calls a target function, as mutable reference to a closure. pub type InProcessExecutor<'a, H, OT, S> = GenericInProcessExecutor; +/// The inprocess executor that allows hooks +pub type HookableInProcessExecutor<'a, H, HT, OT, S> = + GenericInProcessExecutor; /// The process executor simply calls a target function, as boxed `FnMut` trait object pub type OwnedInProcessExecutor = GenericInProcessExecutor< dyn FnMut(&::Input) -> ExitKind, @@ -59,40 +62,36 @@ pub struct GenericInProcessExecutor where H: FnMut(&S::Input) -> ExitKind + ?Sized, HB: BorrowMut, - HT: ExecutorHooksTuple, + HT: ExecutorHooksTuple, OT: ObserversTuple, S: State, { - /// The harness function, being executed for each fuzzing loop execution harness_fn: HB, - /// The observers, observing each run - observers: OT, - // Crash and timeout hah - hooks: (InProcessHooks, HT), - phantom: PhantomData<(S, *const H)>, + inner: GenericInProcessExecutorInner, + phantom: PhantomData<(*const H, HB)>, } impl Debug for GenericInProcessExecutor where H: FnMut(&S::Input) -> ExitKind + ?Sized, HB: BorrowMut, - HT: ExecutorHooksTuple, + HT: ExecutorHooksTuple, OT: ObserversTuple + Debug, S: State, { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { f.debug_struct("GenericInProcessExecutor") + .field("inner", &self.inner) .field("harness_fn", &"") - .field("observers", &self.observers) .finish_non_exhaustive() } } impl UsesState for GenericInProcessExecutor where - H: ?Sized + FnMut(&S::Input) -> ExitKind, + H: FnMut(&S::Input) -> ExitKind + ?Sized, HB: BorrowMut, - HT: ExecutorHooksTuple, + HT: ExecutorHooksTuple, OT: ObserversTuple, S: State, { @@ -101,9 +100,9 @@ where impl UsesObservers for GenericInProcessExecutor where - H: ?Sized + FnMut(&S::Input) -> ExitKind, + H: FnMut(&S::Input) -> ExitKind + ?Sized, HB: BorrowMut, - HT: ExecutorHooksTuple, + HT: ExecutorHooksTuple, OT: ObserversTuple, S: State, { @@ -115,7 +114,7 @@ where EM: UsesState, H: FnMut(&S::Input) -> ExitKind + ?Sized, HB: BorrowMut, - HT: ExecutorHooksTuple, + HT: ExecutorHooksTuple, OT: ObserversTuple, S: State + HasExecutions, Z: UsesState, @@ -128,13 +127,17 @@ where input: &Self::Input, ) -> Result { *state.executions_mut() += 1; - self.enter_target(fuzzer, state, mgr, input); - self.hooks.pre_exec_all(fuzzer, state, mgr, input); + unsafe { + let executor_ptr = ptr::from_ref(self) as *const c_void; + self.inner + .enter_target(fuzzer, state, mgr, input, executor_ptr); + } + self.inner.hooks.pre_exec_all(state, input); - let ret = (self.harness_fn.borrow_mut())(input); + let ret = self.harness_fn.borrow_mut()(input); - self.hooks.post_exec_all(fuzzer, state, mgr, input); - self.leave_target(fuzzer, state, mgr, input); + self.inner.hooks.post_exec_all(state, input); + self.inner.leave_target(fuzzer, state, mgr, input); Ok(ret) } } @@ -143,86 +146,24 @@ impl HasObservers for GenericInProcessExecutor ExitKind + ?Sized, HB: BorrowMut, - HT: ExecutorHooksTuple, - OT: ObserversTuple, - S: State, -{ - #[inline] - fn observers(&self) -> &OT { - &self.observers - } - - #[inline] - fn observers_mut(&mut self) -> &mut OT { - &mut self.observers - } -} -impl GenericInProcessExecutor -where - H: FnMut(&S::Input) -> ExitKind + ?Sized, - HB: BorrowMut, - HT: ExecutorHooksTuple, + HT: ExecutorHooksTuple, OT: ObserversTuple, S: State, { - /// This function marks the boundary between the fuzzer and the target #[inline] - pub fn enter_target( - &mut self, - fuzzer: &mut Z, - state: &mut ::State, - mgr: &mut EM, - input: &::Input, - ) { - unsafe { - let data = addr_of_mut!(GLOBAL_STATE); - write_volatile( - addr_of_mut!((*data).current_input_ptr), - core::ptr::from_ref(input) as *const c_void, - ); - write_volatile( - addr_of_mut!((*data).executor_ptr), - core::ptr::from_ref(self) as *const c_void, - ); - // Direct raw pointers access /aliasing is pretty undefined behavior. - // Since the state and event may have moved in memory, refresh them right before the signal may happen - write_volatile( - addr_of_mut!((*data).state_ptr), - core::ptr::from_mut(state) as *mut c_void, - ); - write_volatile( - addr_of_mut!((*data).event_mgr_ptr), - core::ptr::from_mut(mgr) as *mut c_void, - ); - write_volatile( - addr_of_mut!((*data).fuzzer_ptr), - core::ptr::from_mut(fuzzer) as *mut c_void, - ); - compiler_fence(Ordering::SeqCst); - } + fn observers(&self) -> RefIndexable<&Self::Observers, Self::Observers> { + self.inner.observers() } - /// This function marks the boundary between the fuzzer and the target #[inline] - pub fn leave_target( - &mut self, - _fuzzer: &mut Z, - _state: &mut ::State, - _mgr: &mut EM, - _input: &::Input, - ) { - unsafe { - let data = addr_of_mut!(GLOBAL_STATE); - - write_volatile(addr_of_mut!((*data).current_input_ptr), null()); - compiler_fence(Ordering::SeqCst); - } + fn observers_mut(&mut self) -> RefIndexable<&mut Self::Observers, Self::Observers> { + self.inner.observers_mut() } } impl<'a, H, OT, S> InProcessExecutor<'a, H, OT, S> where - H: FnMut(&::Input) -> ExitKind + ?Sized, + H: FnMut(&S::Input) -> ExitKind + ?Sized, OT: ObserversTuple, S: HasExecutions + HasSolutions + HasCorpus + State, { @@ -235,7 +176,7 @@ where event_mgr: &mut EM, ) -> Result where - Self: Executor, + Self: Executor + HasObservers, EM: EventFirer + EventRestarter, OF: Feedback, S: State, @@ -252,10 +193,9 @@ where ) } - /// Create a new in mem executor with the default timeout and use batch mode (5 sec) - /// Do not use batched mode timeouts with cmplog cores. It is not supported + /// Create a new in mem executor with the default timeout and use batch mode(5 sec) #[cfg(all(feature = "std", target_os = "linux"))] - pub fn batched_timeouts( + pub fn batched_timeout( harness_fn: &'a mut H, observers: OT, fuzzer: &mut Z, @@ -270,17 +210,20 @@ where S: State, Z: HasObjective, { - let mut me = Self::with_timeout_generic( + let inner = GenericInProcessExecutorInner::batched_timeout_generic::( tuple_list!(), - harness_fn, observers, fuzzer, state, event_mgr, exec_tmout, )?; - me.hooks_mut().0.timer_mut().batch_mode = true; - Ok(me) + + Ok(Self { + harness_fn, + inner, + phantom: PhantomData, + }) } /// Create a new in mem executor. @@ -293,49 +236,30 @@ where pub fn with_timeout( harness_fn: &'a mut H, observers: OT, - _fuzzer: &mut Z, + fuzzer: &mut Z, state: &mut S, - _event_mgr: &mut EM, + event_mgr: &mut EM, timeout: Duration, ) -> Result where - Self: Executor, + Self: Executor + HasObservers, EM: EventFirer + EventRestarter, OF: Feedback, S: State, Z: HasObjective, { - let default = InProcessHooks::new::(timeout)?; - let mut hooks = tuple_list!(default).merge(tuple_list!()); - hooks.init_all::(state); - - #[cfg(windows)] - // Some initialization necessary for windows. - unsafe { - /* - See https://github.com/AFLplusplus/LibAFL/pull/403 - This one reserves certain amount of memory for the stack. - If stack overflow happens during fuzzing on windows, the program is transferred to our exception handler for windows. - However, if we run out of the stack memory again in this exception handler, we'll crash with STATUS_ACCESS_VIOLATION. - We need this API call because with the llmp_compression - feature enabled, the exception handler uses a lot of stack memory (in the compression lib code) on release build. - As far as I have observed, the compression uses around 0x10000 bytes, but for safety let's just reserve 0x20000 bytes for our exception handlers. - This number 0x20000 could vary depending on the compilers optimization for future compression library changes. - */ - let mut stack_reserved = 0x20000; - SetThreadStackGuarantee(&mut stack_reserved)?; - } - - #[cfg(all(feature = "std", windows))] - { - // set timeout for the handler - *hooks.0.millis_sec_mut() = timeout.as_millis() as i64; - } + let inner = GenericInProcessExecutorInner::with_timeout_generic::( + tuple_list!(), + observers, + fuzzer, + state, + event_mgr, + timeout, + )?; Ok(Self { harness_fn, - observers, - hooks, + inner, phantom: PhantomData, }) } @@ -343,11 +267,11 @@ where impl GenericInProcessExecutor where - H: FnMut(&::Input) -> ExitKind + ?Sized, + H: FnMut(&S::Input) -> ExitKind + ?Sized, HB: BorrowMut, - HT: ExecutorHooksTuple, + HT: ExecutorHooksTuple, OT: ObserversTuple, - S: HasExecutions + HasSolutions + HasCorpus + State, + S: State + HasExecutions + HasSolutions + HasCorpus, { /// Create a new in mem executor with the default timeout (5 sec) pub fn generic( @@ -359,7 +283,7 @@ where event_mgr: &mut EM, ) -> Result where - Self: Executor, + Self: Executor + HasObservers, EM: EventFirer + EventRestarter, OF: Feedback, S: State, @@ -388,17 +312,21 @@ where exec_tmout: Duration, ) -> Result where - Self: Executor, + Self: Executor + HasObservers, EM: EventFirer + EventRestarter, OF: Feedback, S: State, Z: HasObjective, { - let mut me = Self::with_timeout_generic( - user_hooks, harness_fn, observers, fuzzer, state, event_mgr, exec_tmout, + let inner = GenericInProcessExecutorInner::batched_timeout_generic::( + user_hooks, observers, fuzzer, state, event_mgr, exec_tmout, )?; - me.hooks_mut().0.timer_mut().batch_mode = true; - Ok(me) + + Ok(Self { + harness_fn, + inner, + phantom: PhantomData, + }) } /// Create a new in mem executor. @@ -412,49 +340,25 @@ where user_hooks: HT, harness_fn: HB, observers: OT, - _fuzzer: &mut Z, + fuzzer: &mut Z, state: &mut S, - _event_mgr: &mut EM, + event_mgr: &mut EM, timeout: Duration, ) -> Result where - Self: Executor, + Self: Executor + HasObservers, EM: EventFirer + EventRestarter, OF: Feedback, S: State, Z: HasObjective, { - let default = InProcessHooks::new::(timeout)?; - let mut hooks = tuple_list!(default).merge(user_hooks); - hooks.init_all::(state); - - #[cfg(windows)] - // Some initialization necessary for windows. - unsafe { - /* - See https://github.com/AFLplusplus/LibAFL/pull/403 - This one reserves certain amount of memory for the stack. - If stack overflow happens during fuzzing on windows, the program is transferred to our exception handler for windows. - However, if we run out of the stack memory again in this exception handler, we'll crash with STATUS_ACCESS_VIOLATION. - We need this API call because with the llmp_compression - feature enabled, the exception handler uses a lot of stack memory (in the compression lib code) on release build. - As far as I have observed, the compression uses around 0x10000 bytes, but for safety let's just reserve 0x20000 bytes for our exception handlers. - This number 0x20000 could vary depending on the compilers optimization for future compression library changes. - */ - let mut stack_reserved = 0x20000; - SetThreadStackGuarantee(&mut stack_reserved)?; - } - - #[cfg(all(feature = "std", windows))] - { - // set timeout for the handler - *hooks.0.millis_sec_mut() = timeout.as_millis() as i64; - } + let inner = GenericInProcessExecutorInner::with_timeout_generic::( + user_hooks, observers, fuzzer, state, event_mgr, timeout, + )?; Ok(Self { harness_fn, - observers, - hooks, + inner, phantom: PhantomData, }) } @@ -473,44 +377,47 @@ where /// The inprocess handlers #[inline] - pub fn hooks(&self) -> &(InProcessHooks, HT) { - &self.hooks + pub fn hooks(&self) -> &(InProcessHooks, HT) { + self.inner.hooks() } /// The inprocess handlers (mutable) #[inline] - pub fn hooks_mut(&mut self) -> &mut (InProcessHooks, HT) { - &mut self.hooks + pub fn hooks_mut(&mut self) -> &mut (InProcessHooks, HT) { + self.inner.hooks_mut() } } /// The struct has [`InProcessHooks`]. -pub trait HasInProcessHooks { +pub trait HasInProcessHooks +where + S: UsesInput, +{ /// Get the in-process handlers. - fn inprocess_hooks(&self) -> &InProcessHooks; + fn inprocess_hooks(&self) -> &InProcessHooks; /// Get the mut in-process handlers. - fn inprocess_hooks_mut(&mut self) -> &mut InProcessHooks; + fn inprocess_hooks_mut(&mut self) -> &mut InProcessHooks; } -impl HasInProcessHooks for GenericInProcessExecutor +impl HasInProcessHooks for GenericInProcessExecutor where H: FnMut(&::Input) -> ExitKind + ?Sized, HB: BorrowMut, - HT: ExecutorHooksTuple, + HT: ExecutorHooksTuple, OT: ObserversTuple, S: State + HasExecutions + HasSolutions + HasCorpus, { /// the timeout handler #[inline] - fn inprocess_hooks(&self) -> &InProcessHooks { - &self.hooks.0 + fn inprocess_hooks(&self) -> &InProcessHooks { + self.inner.inprocess_hooks() } /// the timeout handler #[inline] - fn inprocess_hooks_mut(&mut self) -> &mut InProcessHooks { - &mut self.hooks.0 + fn inprocess_hooks_mut(&mut self) -> &mut InProcessHooks { + self.inner.inprocess_hooks_mut() } } @@ -531,7 +438,7 @@ pub fn run_observers_and_save_state( E::State: HasExecutions + HasSolutions + HasCorpus, Z: HasObjective, { - let observers = executor.observers_mut(); + let mut observers = executor.observers_mut(); observers .post_exec_all(state, input, &exitkind) @@ -539,16 +446,22 @@ pub fn run_observers_and_save_state( let interesting = fuzzer .objective_mut() - .is_interesting(state, event_mgr, input, observers, &exitkind) + .is_interesting(state, event_mgr, input, &*observers, &exitkind) .expect("In run_observers_and_save_state objective failure."); if interesting { - let mut new_testcase = Testcase::with_executions(input.clone(), *state.executions()); + let executions = *state.executions(); + let mut new_testcase = Testcase::with_executions(input.clone(), executions); new_testcase.add_metadata(exitkind); new_testcase.set_parent_id_optional(*state.corpus().current()); + + if let Ok(mut tc) = state.current_testcase_mut() { + tc.found_objective(); + } + fuzzer .objective_mut() - .append_metadata(state, observers, &mut new_testcase) + .append_metadata(state, event_mgr, &*observers, &mut new_testcase) .expect("Failed adding metadata"); state .solutions_mut() @@ -559,6 +472,8 @@ pub fn run_observers_and_save_state( state, Event::Objective { objective_size: state.solutions().count(), + executions, + time: libafl_bolts::current_time(), }, ) .expect("Could not save state in run_observers_and_save_state"); @@ -655,74 +570,3 @@ mod tests { .unwrap(); } } - -#[cfg(feature = "python")] -#[allow(missing_docs)] -#[allow(clippy::unnecessary_fallible_conversions)] -/// `InProcess` Python bindings -pub mod pybind { - use alloc::boxed::Box; - - use libafl_bolts::tuples::tuple_list; - use pyo3::{prelude::*, types::PyBytes}; - - use crate::{ - events::pybind::PythonEventManager, - executors::{inprocess::OwnedInProcessExecutor, pybind::PythonExecutor, ExitKind}, - fuzzer::pybind::PythonStdFuzzerWrapper, - inputs::{BytesInput, HasBytesVec}, - observers::pybind::PythonObserversTuple, - state::pybind::{PythonStdState, PythonStdStateWrapper}, - }; - - #[pyclass(unsendable, name = "InProcessExecutor")] - #[derive(Debug)] - /// Python class for OwnedInProcessExecutor (i.e. InProcessExecutor with owned harness) - pub struct PythonOwnedInProcessExecutor { - /// Rust wrapped OwnedInProcessExecutor object - pub inner: OwnedInProcessExecutor, - } - - #[pymethods] - impl PythonOwnedInProcessExecutor { - #[new] - fn new( - harness: PyObject, - py_observers: PythonObserversTuple, - py_fuzzer: &mut PythonStdFuzzerWrapper, - py_state: &mut PythonStdStateWrapper, - py_event_manager: &mut PythonEventManager, - ) -> Self { - Self { - inner: OwnedInProcessExecutor::generic( - tuple_list!(), - Box::new(move |input: &BytesInput| { - Python::with_gil(|py| -> PyResult<()> { - let args = (PyBytes::new(py, input.bytes()),); - harness.call1(py, args)?; - Ok(()) - }) - .unwrap(); - ExitKind::Ok - }), - py_observers, - py_fuzzer.unwrap_mut(), - py_state.unwrap_mut(), - py_event_manager, - ) - .expect("Failed to create the Executor"), - } - } - - #[must_use] - pub fn as_executor(slf: Py) -> PythonExecutor { - PythonExecutor::new_inprocess(slf) - } - } - - /// Register the classes to the python module - pub fn register(_py: Python, m: &PyModule) -> PyResult<()> { - m.add_class::()?; - Ok(()) - } -} diff --git a/libafl/src/executors/inprocess/stateful.rs b/libafl/src/executors/inprocess/stateful.rs new file mode 100644 index 0000000000..b84a7e0793 --- /dev/null +++ b/libafl/src/executors/inprocess/stateful.rs @@ -0,0 +1,432 @@ +use alloc::boxed::Box; +use core::{ + borrow::BorrowMut, + ffi::c_void, + fmt::{self, Debug, Formatter}, + marker::PhantomData, + ptr, + time::Duration, +}; + +use libafl_bolts::tuples::{tuple_list, RefIndexable}; + +use crate::{ + events::{EventFirer, EventRestarter}, + executors::{ + hooks::{inprocess::InProcessHooks, ExecutorHooksTuple}, + inprocess::{GenericInProcessExecutorInner, HasInProcessHooks}, + Executor, ExitKind, HasObservers, + }, + feedbacks::Feedback, + fuzzer::HasObjective, + inputs::UsesInput, + observers::{ObserversTuple, UsesObservers}, + state::{HasCorpus, HasExecutions, HasSolutions, State, UsesState}, + Error, +}; + +/// The process executor simply calls a target function, as mutable reference to a closure +/// The internal state of the executor is made available to the harness. +pub type StatefulInProcessExecutor<'a, H, OT, S, ES> = + StatefulGenericInProcessExecutor; + +/// The process executor simply calls a target function, as boxed `FnMut` trait object +/// The internal state of the executor is made available to the harness. +pub type OwnedInProcessExecutor = StatefulGenericInProcessExecutor< + dyn FnMut(&::Input, &mut ES) -> ExitKind, + Box::Input, &mut ES) -> ExitKind>, + (), + OT, + S, + ES, +>; + +/// The inmem executor simply calls a target function, then returns afterwards. +/// The harness can access the internal state of the executor. +#[allow(dead_code)] +pub struct StatefulGenericInProcessExecutor +where + H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized, + HB: BorrowMut, + HT: ExecutorHooksTuple, + OT: ObserversTuple, + S: State, +{ + /// The harness function, being executed for each fuzzing loop execution + harness_fn: HB, + /// The state used as argument of the harness + pub exposed_executor_state: ES, + /// Inner state of the executor + pub inner: GenericInProcessExecutorInner, + phantom: PhantomData<(ES, *const H)>, +} + +impl Debug for StatefulGenericInProcessExecutor +where + H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized, + HB: BorrowMut, + HT: ExecutorHooksTuple, + OT: ObserversTuple + Debug, + S: State, +{ + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct("StatefulGenericInProcessExecutor") + .field("harness_fn", &"") + .field("inner", &self.inner) + .finish_non_exhaustive() + } +} + +impl UsesState for StatefulGenericInProcessExecutor +where + H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized, + HB: BorrowMut, + HT: ExecutorHooksTuple, + OT: ObserversTuple, + S: State, +{ + type State = S; +} + +impl UsesObservers for StatefulGenericInProcessExecutor +where + H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized, + HB: BorrowMut, + HT: ExecutorHooksTuple, + OT: ObserversTuple, + S: State, +{ + type Observers = OT; +} + +impl Executor + for StatefulGenericInProcessExecutor +where + EM: UsesState, + H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized, + HB: BorrowMut, + HT: ExecutorHooksTuple, + OT: ObserversTuple, + S: State + HasExecutions, + Z: UsesState, +{ + fn run_target( + &mut self, + fuzzer: &mut Z, + state: &mut Self::State, + mgr: &mut EM, + input: &Self::Input, + ) -> Result { + *state.executions_mut() += 1; + unsafe { + let executor_ptr = ptr::from_ref(self) as *const c_void; + self.inner + .enter_target(fuzzer, state, mgr, input, executor_ptr); + } + self.inner.hooks.pre_exec_all(state, input); + + let ret = (self.harness_fn.borrow_mut())(input, &mut self.exposed_executor_state); + + self.inner.hooks.post_exec_all(state, input); + self.inner.leave_target(fuzzer, state, mgr, input); + Ok(ret) + } +} + +impl HasObservers for StatefulGenericInProcessExecutor +where + H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized, + HB: BorrowMut, + HT: ExecutorHooksTuple, + OT: ObserversTuple, + S: State, +{ + #[inline] + fn observers(&self) -> RefIndexable<&Self::Observers, Self::Observers> { + self.inner.observers() + } + + #[inline] + fn observers_mut(&mut self) -> RefIndexable<&mut Self::Observers, Self::Observers> { + self.inner.observers_mut() + } +} + +impl<'a, H, OT, S, ES> StatefulInProcessExecutor<'a, H, OT, S, ES> +where + H: FnMut(&::Input, &mut ES) -> ExitKind + ?Sized, + OT: ObserversTuple, + S: HasExecutions + HasSolutions + HasCorpus + State, +{ + /// Create a new in mem executor with the default timeout (5 sec) + pub fn new( + harness_fn: &'a mut H, + exposed_executor_state: ES, + observers: OT, + fuzzer: &mut Z, + state: &mut S, + event_mgr: &mut EM, + ) -> Result + where + Self: Executor, + EM: EventFirer + EventRestarter, + OF: Feedback, + S: State, + Z: HasObjective, + { + Self::with_timeout_generic( + tuple_list!(), + harness_fn, + exposed_executor_state, + observers, + fuzzer, + state, + event_mgr, + Duration::from_millis(5000), + ) + } + + /// Create a new in mem executor with the default timeout and use batch mode(5 sec) + #[cfg(all(feature = "std", target_os = "linux"))] + pub fn batched_timeout( + harness_fn: &'a mut H, + exposed_executor_state: ES, + observers: OT, + fuzzer: &mut Z, + state: &mut S, + event_mgr: &mut EM, + exec_tmout: Duration, + ) -> Result + where + Self: Executor, + EM: EventFirer + EventRestarter, + OF: Feedback, + S: State, + Z: HasObjective, + { + let inner = GenericInProcessExecutorInner::batched_timeout_generic::( + tuple_list!(), + observers, + fuzzer, + state, + event_mgr, + exec_tmout, + )?; + + Ok(Self { + harness_fn, + exposed_executor_state, + inner, + phantom: PhantomData, + }) + } + + /// Create a new in mem executor. + /// Caution: crash and restart in one of them will lead to odd behavior if multiple are used, + /// depending on different corpus or state. + /// * `user_hooks` - the hooks run before and after the harness's execution + /// * `harness_fn` - the harness, executing the function + /// * `observers` - the observers observing the target during execution + /// This may return an error on unix, if signal handler setup fails + pub fn with_timeout( + harness_fn: &'a mut H, + exposed_executor_state: ES, + observers: OT, + fuzzer: &mut Z, + state: &mut S, + event_mgr: &mut EM, + timeout: Duration, + ) -> Result + where + Self: Executor, + EM: EventFirer + EventRestarter, + OF: Feedback, + S: State, + Z: HasObjective, + { + let inner = GenericInProcessExecutorInner::with_timeout_generic::( + tuple_list!(), + observers, + fuzzer, + state, + event_mgr, + timeout, + )?; + + Ok(Self { + harness_fn, + exposed_executor_state, + inner, + phantom: PhantomData, + }) + } +} + +impl StatefulGenericInProcessExecutor +where + H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized, + HB: BorrowMut, + HT: ExecutorHooksTuple, + OT: ObserversTuple, + S: State, +{ + /// The executor state given to the harness + pub fn exposed_executor_state(&self) -> &ES { + &self.exposed_executor_state + } + + /// The mutable executor state given to the harness + pub fn exposed_executor_state_mut(&mut self) -> &mut ES { + &mut self.exposed_executor_state + } +} + +impl StatefulGenericInProcessExecutor +where + H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized, + HB: BorrowMut, + HT: ExecutorHooksTuple, + OT: ObserversTuple, + S: State + HasExecutions + HasSolutions + HasCorpus, +{ + /// Create a new in mem executor with the default timeout (5 sec) + pub fn generic( + user_hooks: HT, + harness_fn: HB, + exposed_executor_state: ES, + observers: OT, + fuzzer: &mut Z, + state: &mut S, + event_mgr: &mut EM, + ) -> Result + where + EM: EventFirer + EventRestarter, + OF: Feedback, + S: State, + Z: HasObjective, + { + Self::with_timeout_generic( + user_hooks, + harness_fn, + exposed_executor_state, + observers, + fuzzer, + state, + event_mgr, + Duration::from_millis(5000), + ) + } + + /// Create a new in mem executor with the default timeout and use batch mode(5 sec) + #[cfg(all(feature = "std", target_os = "linux"))] + #[allow(clippy::too_many_arguments)] + pub fn batched_timeout_generic( + user_hooks: HT, + harness_fn: HB, + exposed_executor_state: ES, + observers: OT, + fuzzer: &mut Z, + state: &mut S, + event_mgr: &mut EM, + exec_tmout: Duration, + ) -> Result + where + EM: EventFirer + EventRestarter, + OF: Feedback, + S: State, + Z: HasObjective, + { + let inner = GenericInProcessExecutorInner::batched_timeout_generic::( + user_hooks, observers, fuzzer, state, event_mgr, exec_tmout, + )?; + + Ok(Self { + harness_fn, + exposed_executor_state, + inner, + phantom: PhantomData, + }) + } + + /// Create a new in mem executor. + /// Caution: crash and restart in one of them will lead to odd behavior if multiple are used, + /// depending on different corpus or state. + /// * `user_hooks` - the hooks run before and after the harness's execution + /// * `harness_fn` - the harness, executing the function + /// * `observers` - the observers observing the target during execution + /// This may return an error on unix, if signal handler setup fails + #[allow(clippy::too_many_arguments)] + pub fn with_timeout_generic( + user_hooks: HT, + harness_fn: HB, + exposed_executor_state: ES, + observers: OT, + fuzzer: &mut Z, + state: &mut S, + event_mgr: &mut EM, + timeout: Duration, + ) -> Result + where + EM: EventFirer + EventRestarter, + OF: Feedback, + S: State, + Z: HasObjective, + { + let inner = GenericInProcessExecutorInner::with_timeout_generic::( + user_hooks, observers, fuzzer, state, event_mgr, timeout, + )?; + + Ok(Self { + harness_fn, + exposed_executor_state, + inner, + phantom: PhantomData, + }) + } + + /// Retrieve the harness function. + #[inline] + pub fn harness(&self) -> &H { + self.harness_fn.borrow() + } + + /// Retrieve the harness function for a mutable reference. + #[inline] + pub fn harness_mut(&mut self) -> &mut H { + self.harness_fn.borrow_mut() + } + + /// The inprocess handlers + #[inline] + pub fn hooks(&self) -> &(InProcessHooks, HT) { + self.inner.hooks() + } + + /// The inprocess handlers (mutable) + #[inline] + pub fn hooks_mut(&mut self) -> &mut (InProcessHooks, HT) { + self.inner.hooks_mut() + } +} + +impl HasInProcessHooks + for StatefulGenericInProcessExecutor +where + H: FnMut(&::Input, &mut ES) -> ExitKind + ?Sized, + HB: BorrowMut, + HT: ExecutorHooksTuple, + OT: ObserversTuple, + S: State + HasExecutions + HasSolutions + HasCorpus, +{ + /// the timeout handler + #[inline] + fn inprocess_hooks(&self) -> &InProcessHooks { + self.inner.inprocess_hooks() + } + + /// the timeout handler + #[inline] + fn inprocess_hooks_mut(&mut self) -> &mut InProcessHooks { + self.inner.inprocess_hooks_mut() + } +} diff --git a/libafl/src/executors/inprocess_fork.rs b/libafl/src/executors/inprocess_fork.rs deleted file mode 100644 index 51e03ee862..0000000000 --- a/libafl/src/executors/inprocess_fork.rs +++ /dev/null @@ -1,618 +0,0 @@ -//! The `GenericInProcessForkExecutor` to do forking before executing the harness in-processly -use core::{ - ffi::c_void, - fmt::{self, Debug, Formatter}, - marker::PhantomData, - ptr::{addr_of_mut, null_mut, write_volatile}, - sync::atomic::{compiler_fence, Ordering}, - time::Duration, -}; - -use libafl_bolts::{ - os::unix_signals::{ucontext_t, Signal}, - shmem::ShMemProvider, - tuples::{tuple_list, Merge}, -}; -use libc::siginfo_t; -use nix::{ - sys::wait::{waitpid, WaitStatus}, - unistd::{fork, ForkResult}, -}; - -use super::hooks::ExecutorHooksTuple; -use crate::{ - events::{EventFirer, EventRestarter}, - executors::{ - hooks::inprocess_fork::{ - InChildProcessHooks, InProcessForkExecutorGlobalData, FORK_EXECUTOR_GLOBAL_DATA, - }, - Executor, ExitKind, HasObservers, - }, - feedbacks::Feedback, - fuzzer::HasObjective, - inputs::UsesInput, - observers::{ObserversTuple, UsesObservers}, - state::{HasExecutions, HasSolutions, State, UsesState}, - Error, -}; -/// The signature of the crash handler function -pub(crate) type ForkHandlerFuncPtr = unsafe fn( - Signal, - &mut siginfo_t, - Option<&mut ucontext_t>, - data: *mut InProcessForkExecutorGlobalData, -); - -#[cfg(all(unix, not(target_os = "linux")))] -use crate::executors::hooks::timer::{setitimer, Itimerval, Timeval, ITIMER_REAL}; - -/// The `InProcessForkExecutor` with no user hooks -pub type InProcessForkExecutor<'a, H, OT, S, SP> = - GenericInProcessForkExecutor<'a, H, (), OT, S, SP>; - -impl<'a, H, OT, S, SP> InProcessForkExecutor<'a, H, OT, S, SP> -where - H: FnMut(&S::Input) -> ExitKind + ?Sized, - S: State, - OT: ObserversTuple, - SP: ShMemProvider, -{ - #[allow(clippy::too_many_arguments)] - /// The constructor for `InProcessForkExecutor` - pub fn new( - harness_fn: &'a mut H, - observers: OT, - fuzzer: &mut Z, - state: &mut S, - event_mgr: &mut EM, - timeout: Duration, - shmem_provider: SP, - ) -> Result - where - EM: EventFirer + EventRestarter, - OF: Feedback, - S: HasSolutions, - Z: HasObjective, - { - Self::with_hooks( - tuple_list!(), - harness_fn, - observers, - fuzzer, - state, - event_mgr, - timeout, - shmem_provider, - ) - } -} - -/// [`GenericInProcessForkExecutor`] is an executor that forks the current process before each execution. -pub struct GenericInProcessForkExecutor<'a, H, HT, OT, S, SP> -where - H: FnMut(&S::Input) -> ExitKind + ?Sized, - OT: ObserversTuple, - S: UsesInput, - SP: ShMemProvider, - HT: ExecutorHooksTuple, -{ - hooks: (InChildProcessHooks, HT), - harness_fn: &'a mut H, - shmem_provider: SP, - observers: OT, - #[cfg(target_os = "linux")] - itimerspec: libc::itimerspec, - #[cfg(all(unix, not(target_os = "linux")))] - itimerval: Itimerval, - phantom: PhantomData, -} - -impl<'a, H, HT, OT, S, SP> Debug for GenericInProcessForkExecutor<'a, H, HT, OT, S, SP> -where - H: FnMut(&S::Input) -> ExitKind + ?Sized, - OT: ObserversTuple + Debug, - S: UsesInput, - SP: ShMemProvider, - HT: ExecutorHooksTuple, -{ - #[cfg(target_os = "linux")] - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - f.debug_struct("GenericInProcessForkExecutor") - .field("observers", &self.observers) - .field("shmem_provider", &self.shmem_provider) - .field("itimerspec", &self.itimerspec) - .finish() - } - - #[cfg(not(target_os = "linux"))] - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - #[cfg(not(target_os = "linux"))] - return f - .debug_struct("GenericInProcessForkExecutor") - .field("observers", &self.observers) - .field("shmem_provider", &self.shmem_provider) - .field("itimerval", &self.itimerval) - .finish(); - } -} - -impl<'a, H, HT, OT, S, SP> UsesState for GenericInProcessForkExecutor<'a, H, HT, OT, S, SP> -where - H: ?Sized + FnMut(&S::Input) -> ExitKind, - OT: ObserversTuple, - S: State, - SP: ShMemProvider, - HT: ExecutorHooksTuple, -{ - type State = S; -} - -impl<'a, EM, H, HT, OT, S, SP, Z> Executor - for GenericInProcessForkExecutor<'a, H, HT, OT, S, SP> -where - EM: UsesState, - H: FnMut(&S::Input) -> ExitKind + ?Sized, - OT: ObserversTuple, - S: State + HasExecutions, - SP: ShMemProvider, - HT: ExecutorHooksTuple, - Z: UsesState, -{ - #[allow(unreachable_code)] - #[inline] - fn run_target( - &mut self, - fuzzer: &mut Z, - state: &mut Self::State, - mgr: &mut EM, - input: &Self::Input, - ) -> Result { - *state.executions_mut() += 1; - - unsafe { - self.shmem_provider.pre_fork()?; - match fork() { - Ok(ForkResult::Child) => { - // Child - self.shmem_provider.post_fork(true)?; - - self.enter_target(fuzzer, state, mgr, input); - self.hooks.pre_exec_all(fuzzer, state, mgr, input); - - self.observers - .pre_exec_child_all(state, input) - .expect("Failed to run post_exec on observers"); - - #[cfg(target_os = "linux")] - { - let mut timerid: libc::timer_t = null_mut(); - // creates a new per-process interval timer - // we can't do this from the parent, timerid is unique to each process. - libc::timer_create( - libc::CLOCK_MONOTONIC, - null_mut(), - addr_of_mut!(timerid), - ); - - // log::info!("Set timer! {:#?} {timerid:#?}", self.itimerspec); - let _: i32 = libc::timer_settime( - timerid, - 0, - addr_of_mut!(self.itimerspec), - null_mut(), - ); - } - #[cfg(not(target_os = "linux"))] - { - setitimer(ITIMER_REAL, &mut self.itimerval, null_mut()); - } - // log::trace!("{v:#?} {}", nix::errno::errno()); - (self.harness_fn)(input); - - self.observers - .post_exec_child_all(state, input, &ExitKind::Ok) - .expect("Failed to run post_exec on observers"); - - self.hooks.post_exec_all(fuzzer, state, mgr, input); - self.leave_target(fuzzer, state, mgr, input); - - libc::_exit(0); - - Ok(ExitKind::Ok) - } - Ok(ForkResult::Parent { child }) => { - // Parent - // log::trace!("from parent {} child is {}", std::process::id(), child); - self.shmem_provider.post_fork(false)?; - - let res = waitpid(child, None)?; - log::trace!("{res:#?}"); - match res { - WaitStatus::Signaled(_, signal, _) => match signal { - nix::sys::signal::Signal::SIGALRM - | nix::sys::signal::Signal::SIGUSR2 => Ok(ExitKind::Timeout), - _ => Ok(ExitKind::Crash), - }, - WaitStatus::Exited(_, code) => { - if code > 128 && code < 160 { - // Signal exit codes - let signal = code - 128; - if signal == Signal::SigAlarm as libc::c_int - || signal == Signal::SigUser2 as libc::c_int - { - Ok(ExitKind::Timeout) - } else { - Ok(ExitKind::Crash) - } - } else { - Ok(ExitKind::Ok) - } - } - _ => Ok(ExitKind::Ok), - } - } - Err(e) => Err(Error::from(e)), - } - } - } -} - -impl<'a, H, HT, OT, S, SP> GenericInProcessForkExecutor<'a, H, HT, OT, S, SP> -where - H: FnMut(&S::Input) -> ExitKind + ?Sized, - HT: ExecutorHooksTuple, - S: State, - OT: ObserversTuple, - SP: ShMemProvider, -{ - #[inline] - /// This function marks the boundary between the fuzzer and the target. - pub fn enter_target( - &mut self, - _fuzzer: &mut Z, - state: &mut ::State, - _event_mgr: &mut EM, - input: &::Input, - ) { - unsafe { - let data = addr_of_mut!(FORK_EXECUTOR_GLOBAL_DATA); - write_volatile( - addr_of_mut!((*data).executor_ptr), - core::ptr::from_ref(self) as *const c_void, - ); - write_volatile( - addr_of_mut!((*data).current_input_ptr), - core::ptr::from_ref(input) as *const c_void, - ); - write_volatile( - addr_of_mut!((*data).state_ptr), - core::ptr::from_mut(state) as *mut c_void, - ); - compiler_fence(Ordering::SeqCst); - } - } - - #[inline] - /// This function marks the boundary between the fuzzer and the target. - pub fn leave_target( - &mut self, - _fuzzer: &mut Z, - _state: &mut ::State, - _event_mgr: &mut EM, - _input: &::Input, - ) { - // do nothing - } - - /// Creates a new [`GenericInProcessForkExecutor`] with custom hooks - #[cfg(target_os = "linux")] - #[allow(clippy::too_many_arguments)] - pub fn with_hooks( - userhooks: HT, - harness_fn: &'a mut H, - observers: OT, - _fuzzer: &mut Z, - state: &mut S, - _event_mgr: &mut EM, - timeout: Duration, - shmem_provider: SP, - ) -> Result - where - EM: EventFirer + EventRestarter, - OF: Feedback, - S: HasSolutions, - Z: HasObjective, - { - let default_hooks = InChildProcessHooks::new::()?; - let mut hooks = tuple_list!(default_hooks).merge(userhooks); - hooks.init_all::(state); - - let milli_sec = timeout.as_millis(); - let it_value = libc::timespec { - tv_sec: (milli_sec / 1000) as _, - tv_nsec: ((milli_sec % 1000) * 1000 * 1000) as _, - }; - let it_interval = libc::timespec { - tv_sec: 0, - tv_nsec: 0, - }; - let itimerspec = libc::itimerspec { - it_interval, - it_value, - }; - - Ok(Self { - harness_fn, - shmem_provider, - observers, - hooks, - itimerspec, - phantom: PhantomData, - }) - } - - /// Creates a new [`GenericInProcessForkExecutor`], non linux - #[cfg(not(target_os = "linux"))] - #[allow(clippy::too_many_arguments)] - pub fn with_hooks( - userhooks: HT, - harness_fn: &'a mut H, - observers: OT, - _fuzzer: &mut Z, - state: &mut S, - _event_mgr: &mut EM, - timeout: Duration, - shmem_provider: SP, - ) -> Result - where - EM: EventFirer + EventRestarter, - OF: Feedback, - S: HasSolutions, - Z: HasObjective, - { - let default_hooks = InChildProcessHooks::new::()?; - let mut hooks = tuple_list!(default_hooks).merge(userhooks); - hooks.init_all::(state); - - let milli_sec = timeout.as_millis(); - let it_value = Timeval { - tv_sec: (milli_sec / 1000) as i64, - tv_usec: (milli_sec % 1000) as i64, - }; - let it_interval = Timeval { - tv_sec: 0, - tv_usec: 0, - }; - let itimerval = Itimerval { - it_interval, - it_value, - }; - - Ok(Self { - harness_fn, - shmem_provider, - observers, - hooks, - itimerval, - phantom: PhantomData, - }) - } - - /// Retrieve the harness function. - #[inline] - pub fn harness(&self) -> &H { - self.harness_fn - } - - /// Retrieve the harness function for a mutable reference. - #[inline] - pub fn harness_mut(&mut self) -> &mut H { - self.harness_fn - } -} - -impl<'a, H, HT, OT, S, SP> UsesObservers for GenericInProcessForkExecutor<'a, H, HT, OT, S, SP> -where - H: ?Sized + FnMut(&S::Input) -> ExitKind, - HT: ExecutorHooksTuple, - OT: ObserversTuple, - S: State, - SP: ShMemProvider, -{ - type Observers = OT; -} - -impl<'a, H, HT, OT, S, SP> HasObservers for GenericInProcessForkExecutor<'a, H, HT, OT, S, SP> -where - H: FnMut(&S::Input) -> ExitKind + ?Sized, - HT: ExecutorHooksTuple, - S: State, - OT: ObserversTuple, - SP: ShMemProvider, -{ - #[inline] - fn observers(&self) -> &OT { - &self.observers - } - - #[inline] - fn observers_mut(&mut self) -> &mut OT { - &mut self.observers - } -} -/// signal hooks and `panic_hooks` for the child process - -pub mod child_signal_handlers { - use alloc::boxed::Box; - use core::ptr::addr_of_mut; - use std::panic; - - use libafl_bolts::os::unix_signals::{ucontext_t, Signal}; - use libc::siginfo_t; - - use crate::{ - executors::{ - hooks::inprocess_fork::{InProcessForkExecutorGlobalData, FORK_EXECUTOR_GLOBAL_DATA}, - ExitKind, HasObservers, - }, - inputs::UsesInput, - observers::ObserversTuple, - }; - - /// invokes the `post_exec_child` hook on all observer in case the child process panics - pub fn setup_child_panic_hook() - where - E: HasObservers, - { - let old_hook = panic::take_hook(); - panic::set_hook(Box::new(move |panic_info| unsafe { - old_hook(panic_info); - let data = addr_of_mut!(FORK_EXECUTOR_GLOBAL_DATA); - if !data.is_null() && (*data).is_valid() { - let executor = (*data).executor_mut::(); - let observers = executor.observers_mut(); - let state = (*data).state_mut::(); - // Invalidate data to not execute again the observer hooks in the crash handler - let input = (*data).take_current_input::<::Input>(); - observers - .post_exec_child_all(state, input, &ExitKind::Crash) - .expect("Failed to run post_exec on observers"); - - // std::process::abort(); - libc::_exit(128 + 6); // ABORT exit code - } - })); - } - - /// invokes the `post_exec` hook on all observer in case the child process crashes - /// - /// # Safety - /// The function should only be called from a child crash handler. - /// It will dereference the `data` pointer and assume it's valid. - #[cfg(unix)] - #[allow(clippy::needless_pass_by_value)] - pub(crate) unsafe fn child_crash_handler( - _signal: Signal, - _info: &mut siginfo_t, - _context: Option<&mut ucontext_t>, - data: &mut InProcessForkExecutorGlobalData, - ) where - E: HasObservers, - { - if data.is_valid() { - let executor = data.executor_mut::(); - let observers = executor.observers_mut(); - let state = data.state_mut::(); - let input = data.take_current_input::<::Input>(); - observers - .post_exec_child_all(state, input, &ExitKind::Crash) - .expect("Failed to run post_exec on observers"); - } - - libc::_exit(128 + (_signal as i32)); - } - - #[cfg(unix)] - #[allow(clippy::needless_pass_by_value)] - pub(crate) unsafe fn child_timeout_handler( - _signal: Signal, - _info: &mut siginfo_t, - _context: Option<&mut ucontext_t>, - data: &mut InProcessForkExecutorGlobalData, - ) where - E: HasObservers, - { - if data.is_valid() { - let executor = data.executor_mut::(); - let observers = executor.observers_mut(); - let state = data.state_mut::(); - let input = data.take_current_input::<::Input>(); - observers - .post_exec_child_all(state, input, &ExitKind::Timeout) - .expect("Failed to run post_exec on observers"); - } - libc::_exit(128 + (_signal as i32)); - } -} - -#[cfg(test)] -mod tests { - use libafl_bolts::tuples::tuple_list; - - use crate::{executors::ExitKind, inputs::NopInput}; - - #[test] - #[cfg_attr(miri, ignore)] - #[cfg(all(feature = "std", feature = "fork", unix))] - fn test_inprocessfork_exec() { - use core::marker::PhantomData; - - use libafl_bolts::shmem::{ShMemProvider, StdShMemProvider}; - #[cfg(target_os = "linux")] - use libc::{itimerspec, timespec}; - - #[cfg(not(target_os = "linux"))] - use crate::executors::hooks::timer::{Itimerval, Timeval}; - use crate::{ - events::SimpleEventManager, - executors::{ - hooks::inprocess_fork::InChildProcessHooks, - inprocess_fork::GenericInProcessForkExecutor, Executor, - }, - fuzzer::test::NopFuzzer, - state::test::NopState, - }; - - let provider = StdShMemProvider::new().unwrap(); - - #[cfg(target_os = "linux")] - let timespec = timespec { - tv_sec: 5, - tv_nsec: 0, - }; - #[cfg(target_os = "linux")] - let itimerspec = itimerspec { - it_interval: timespec, - it_value: timespec, - }; - - #[cfg(not(target_os = "linux"))] - let timespec = Timeval { - tv_sec: 5, - tv_usec: 0, - }; - #[cfg(not(target_os = "linux"))] - let itimerspec = Itimerval { - it_interval: timespec, - it_value: timespec, - }; - - let mut harness = |_buf: &NopInput| ExitKind::Ok; - let default = InChildProcessHooks::nop(); - #[cfg(target_os = "linux")] - let mut in_process_fork_executor = GenericInProcessForkExecutor::<_, (), (), _, _> { - hooks: tuple_list!(default), - harness_fn: &mut harness, - shmem_provider: provider, - observers: tuple_list!(), - itimerspec, - phantom: PhantomData, - }; - #[cfg(not(target_os = "linux"))] - let mut in_process_fork_executor = GenericInProcessForkExecutor::<_, (), (), _, _> { - harness_fn: &mut harness, - shmem_provider: provider, - observers: tuple_list!(), - hooks: tuple_list!(default), - itimerval: itimerspec, - phantom: PhantomData, - }; - let input = NopInput {}; - let mut fuzzer = NopFuzzer::new(); - let mut state = NopState::new(); - let mut mgr = SimpleEventManager::printing(); - in_process_fork_executor - .run_target(&mut fuzzer, &mut state, &mut mgr, &input) - .unwrap(); - } -} diff --git a/libafl/src/executors/inprocess_fork/inner.rs b/libafl/src/executors/inprocess_fork/inner.rs new file mode 100644 index 0000000000..103bba4401 --- /dev/null +++ b/libafl/src/executors/inprocess_fork/inner.rs @@ -0,0 +1,350 @@ +use core::{ + ffi::c_void, + fmt::{self, Debug, Formatter}, + marker::PhantomData, + ptr::{self, addr_of_mut, null_mut, write_volatile}, + sync::atomic::{compiler_fence, Ordering}, + time::Duration, +}; + +use libafl_bolts::{ + os::unix_signals::Signal, + shmem::ShMemProvider, + tuples::{tuple_list, Merge, RefIndexable}, +}; +use nix::{ + sys::wait::{waitpid, WaitStatus}, + unistd::Pid, +}; + +#[cfg(all(unix, not(target_os = "linux")))] +use crate::executors::hooks::timer::{setitimer, Itimerval, Timeval, ITIMER_REAL}; +use crate::{ + events::{EventFirer, EventRestarter}, + executors::{ + hooks::{ + inprocess_fork::{InChildProcessHooks, FORK_EXECUTOR_GLOBAL_DATA}, + ExecutorHooksTuple, + }, + ExitKind, HasObservers, + }, + inputs::UsesInput, + observers::{ObserversTuple, UsesObservers}, + state::{State, UsesState}, + Error, +}; + +/// Inner state of GenericInProcessExecutor-like structures. +pub struct GenericInProcessForkExecutorInner +where + OT: ObserversTuple, + S: UsesInput, + SP: ShMemProvider, + HT: ExecutorHooksTuple, + EM: UsesState, + Z: UsesState, +{ + pub(super) hooks: (InChildProcessHooks, HT), + pub(super) shmem_provider: SP, + pub(super) observers: OT, + #[cfg(target_os = "linux")] + pub(super) itimerspec: libc::itimerspec, + #[cfg(all(unix, not(target_os = "linux")))] + pub(super) itimerval: Itimerval, + pub(super) phantom: PhantomData<(S, EM, Z)>, +} + +impl Debug for GenericInProcessForkExecutorInner +where + OT: ObserversTuple + Debug, + S: UsesInput, + SP: ShMemProvider, + HT: ExecutorHooksTuple + Debug, + EM: UsesState, + Z: UsesState, +{ + #[cfg(target_os = "linux")] + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct("GenericInProcessForkExecutorInner") + .field("observers", &self.observers) + .field("shmem_provider", &self.shmem_provider) + .field("itimerspec", &self.itimerspec) + .finish_non_exhaustive() + } + + #[cfg(not(target_os = "linux"))] + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + #[cfg(not(target_os = "linux"))] + return f + .debug_struct("GenericInProcessForkExecutorInner") + .field("observers", &self.observers) + .field("shmem_provider", &self.shmem_provider) + .field("itimerval", &self.itimerval) + .finish_non_exhaustive(); + } +} + +impl UsesState for GenericInProcessForkExecutorInner +where + OT: ObserversTuple, + S: State, + SP: ShMemProvider, + HT: ExecutorHooksTuple, + EM: UsesState, + Z: UsesState, +{ + type State = S; +} + +impl GenericInProcessForkExecutorInner +where + OT: ObserversTuple + Debug, + S: State + UsesInput, + SP: ShMemProvider, + HT: ExecutorHooksTuple, + EM: EventFirer + EventRestarter, + Z: UsesState, +{ + pub(super) unsafe fn pre_run_target_child( + &mut self, + fuzzer: &mut Z, + state: &mut as UsesState>::State, + mgr: &mut EM, + input: & as UsesInput>::Input, + ) -> Result<(), Error> { + self.shmem_provider.post_fork(true)?; + + self.enter_target(fuzzer, state, mgr, input); + self.hooks.pre_exec_all(state, input); + + self.observers + .pre_exec_child_all(state, input) + .expect("Failed to run post_exec on observers"); + + #[cfg(target_os = "linux")] + { + let mut timerid: libc::timer_t = null_mut(); + // creates a new per-process interval timer + // we can't do this from the parent, timerid is unique to each process. + libc::timer_create(libc::CLOCK_MONOTONIC, null_mut(), addr_of_mut!(timerid)); + + // log::info!("Set timer! {:#?} {timerid:#?}", self.itimerspec); + let _: i32 = libc::timer_settime(timerid, 0, addr_of_mut!(self.itimerspec), null_mut()); + } + #[cfg(not(target_os = "linux"))] + { + setitimer(ITIMER_REAL, &mut self.itimerval, null_mut()); + } + // log::trace!("{v:#?} {}", nix::errno::errno()); + + Ok(()) + } + + pub(super) unsafe fn post_run_target_child( + &mut self, + fuzzer: &mut Z, + state: &mut as UsesState>::State, + mgr: &mut EM, + input: & as UsesInput>::Input, + ) { + self.observers + .post_exec_child_all(state, input, &ExitKind::Ok) + .expect("Failed to run post_exec on observers"); + + self.hooks.post_exec_all(state, input); + self.leave_target(fuzzer, state, mgr, input); + + libc::_exit(0); + } + + pub(super) fn parent(&mut self, child: Pid) -> Result { + // log::trace!("from parent {} child is {}", std::process::id(), child); + self.shmem_provider.post_fork(false)?; + + let res = waitpid(child, None)?; + log::trace!("{res:#?}"); + match res { + WaitStatus::Signaled(_, signal, _) => match signal { + nix::sys::signal::Signal::SIGALRM | nix::sys::signal::Signal::SIGUSR2 => { + Ok(ExitKind::Timeout) + } + _ => Ok(ExitKind::Crash), + }, + WaitStatus::Exited(_, code) => { + if code > 128 && code < 160 { + // Signal exit codes + let signal = code - 128; + if signal == Signal::SigAlarm as libc::c_int + || signal == Signal::SigUser2 as libc::c_int + { + Ok(ExitKind::Timeout) + } else { + Ok(ExitKind::Crash) + } + } else { + Ok(ExitKind::Ok) + } + } + _ => Ok(ExitKind::Ok), + } + } +} + +impl GenericInProcessForkExecutorInner +where + HT: ExecutorHooksTuple, + S: State, + OT: ObserversTuple, + SP: ShMemProvider, + EM: EventFirer + EventRestarter, + Z: UsesState, +{ + #[inline] + /// This function marks the boundary between the fuzzer and the target. + pub fn enter_target( + &mut self, + _fuzzer: &mut Z, + state: &mut ::State, + _event_mgr: &mut EM, + input: &::Input, + ) { + unsafe { + let data = addr_of_mut!(FORK_EXECUTOR_GLOBAL_DATA); + write_volatile( + addr_of_mut!((*data).executor_ptr), + ptr::from_ref(self) as *const c_void, + ); + write_volatile( + addr_of_mut!((*data).current_input_ptr), + ptr::from_ref(input) as *const c_void, + ); + write_volatile( + addr_of_mut!((*data).state_ptr), + ptr::from_mut(state) as *mut c_void, + ); + compiler_fence(Ordering::SeqCst); + } + } + + #[inline] + /// This function marks the boundary between the fuzzer and the target. + pub fn leave_target( + &mut self, + _fuzzer: &mut Z, + _state: &mut ::State, + _event_mgr: &mut EM, + _input: &::Input, + ) { + // do nothing + } + + /// Creates a new [`GenericInProcessForkExecutorInner`] with custom hooks + #[cfg(target_os = "linux")] + #[allow(clippy::too_many_arguments)] + pub fn with_hooks( + userhooks: HT, + observers: OT, + _fuzzer: &mut Z, + state: &mut S, + _event_mgr: &mut EM, + timeout: Duration, + shmem_provider: SP, + ) -> Result { + let default_hooks = InChildProcessHooks::new::()?; + let mut hooks = tuple_list!(default_hooks).merge(userhooks); + hooks.init_all::(state); + + let milli_sec = timeout.as_millis(); + let it_value = libc::timespec { + tv_sec: (milli_sec / 1000) as _, + tv_nsec: ((milli_sec % 1000) * 1000 * 1000) as _, + }; + let it_interval = libc::timespec { + tv_sec: 0, + tv_nsec: 0, + }; + let itimerspec = libc::itimerspec { + it_interval, + it_value, + }; + + Ok(Self { + shmem_provider, + observers, + hooks, + itimerspec, + phantom: PhantomData, + }) + } + + /// Creates a new [`GenericInProcessForkExecutorInner`], non linux + #[cfg(not(target_os = "linux"))] + #[allow(clippy::too_many_arguments)] + pub fn with_hooks( + userhooks: HT, + observers: OT, + _fuzzer: &mut Z, + state: &mut S, + _event_mgr: &mut EM, + timeout: Duration, + shmem_provider: SP, + ) -> Result { + let default_hooks = InChildProcessHooks::new::()?; + let mut hooks = tuple_list!(default_hooks).merge(userhooks); + hooks.init_all::(state); + + let milli_sec = timeout.as_millis(); + let it_value = Timeval { + tv_sec: (milli_sec / 1000) as i64, + tv_usec: (milli_sec % 1000) as i64, + }; + let it_interval = Timeval { + tv_sec: 0, + tv_usec: 0, + }; + let itimerval = Itimerval { + it_interval, + it_value, + }; + + Ok(Self { + shmem_provider, + observers, + hooks, + itimerval, + phantom: PhantomData, + }) + } +} + +impl UsesObservers for GenericInProcessForkExecutorInner +where + HT: ExecutorHooksTuple, + OT: ObserversTuple, + S: State, + SP: ShMemProvider, + EM: UsesState, + Z: UsesState, +{ + type Observers = OT; +} + +impl HasObservers for GenericInProcessForkExecutorInner +where + HT: ExecutorHooksTuple, + S: State, + OT: ObserversTuple, + SP: ShMemProvider, + EM: UsesState, + Z: UsesState, +{ + #[inline] + fn observers(&self) -> RefIndexable<&Self::Observers, Self::Observers> { + RefIndexable::from(&self.observers) + } + + #[inline] + fn observers_mut(&mut self) -> RefIndexable<&mut Self::Observers, Self::Observers> { + RefIndexable::from(&mut self.observers) + } +} diff --git a/libafl/src/executors/inprocess_fork/mod.rs b/libafl/src/executors/inprocess_fork/mod.rs new file mode 100644 index 0000000000..29959a4b48 --- /dev/null +++ b/libafl/src/executors/inprocess_fork/mod.rs @@ -0,0 +1,450 @@ +//! The `GenericInProcessForkExecutor` to do forking before executing the harness in-processly +use core::{ + fmt::{self, Debug, Formatter}, + time::Duration, +}; + +use libafl_bolts::{ + os::unix_signals::{ucontext_t, Signal}, + shmem::ShMemProvider, + tuples::{tuple_list, RefIndexable}, +}; +use libc::siginfo_t; +use nix::unistd::{fork, ForkResult}; + +use super::hooks::ExecutorHooksTuple; +use crate::{ + events::{EventFirer, EventRestarter}, + executors::{ + hooks::inprocess_fork::InProcessForkExecutorGlobalData, + inprocess_fork::inner::GenericInProcessForkExecutorInner, Executor, ExitKind, HasObservers, + }, + feedbacks::Feedback, + fuzzer::HasObjective, + inputs::UsesInput, + observers::{ObserversTuple, UsesObservers}, + state::{HasExecutions, HasSolutions, State, UsesState}, + Error, +}; + +/// The signature of the crash handler function +pub(crate) type ForkHandlerFuncPtr = unsafe fn( + Signal, + &mut siginfo_t, + Option<&mut ucontext_t>, + data: *mut InProcessForkExecutorGlobalData, +); + +/// The inner structure of `InProcessForkExecutor`. +pub mod inner; +/// A version of `InProcessForkExecutor` with a state accessible from the harness. +pub mod stateful; + +/// The `InProcessForkExecutor` with no user hooks +pub type InProcessForkExecutor<'a, H, OT, S, SP, EM, Z> = + GenericInProcessForkExecutor<'a, H, (), OT, S, SP, EM, Z>; + +impl<'a, H, OT, S, SP, EM, Z, OF> InProcessForkExecutor<'a, H, OT, S, SP, EM, Z> +where + H: FnMut(&S::Input) -> ExitKind + ?Sized, + S: State, + OT: ObserversTuple, + SP: ShMemProvider, + EM: EventFirer + EventRestarter, + OF: Feedback, + S: HasSolutions, + Z: HasObjective, +{ + #[allow(clippy::too_many_arguments)] + /// The constructor for `InProcessForkExecutor` + pub fn new( + harness_fn: &'a mut H, + observers: OT, + fuzzer: &mut Z, + state: &mut S, + event_mgr: &mut EM, + timeout: Duration, + shmem_provider: SP, + ) -> Result { + Self::with_hooks( + tuple_list!(), + harness_fn, + observers, + fuzzer, + state, + event_mgr, + timeout, + shmem_provider, + ) + } +} + +/// [`GenericInProcessForkExecutor`] is an executor that forks the current process before each execution. +pub struct GenericInProcessForkExecutor<'a, H, HT, OT, S, SP, EM, Z> +where + H: FnMut(&S::Input) -> ExitKind + ?Sized, + OT: ObserversTuple, + S: UsesInput, + SP: ShMemProvider, + HT: ExecutorHooksTuple, + EM: UsesState, + Z: UsesState, +{ + harness_fn: &'a mut H, + inner: GenericInProcessForkExecutorInner, +} + +impl<'a, H, HT, OT, S, SP, EM, Z> Debug + for GenericInProcessForkExecutor<'a, H, HT, OT, S, SP, EM, Z> +where + H: FnMut(&S::Input) -> ExitKind + ?Sized, + OT: ObserversTuple + Debug, + S: UsesInput, + SP: ShMemProvider, + HT: ExecutorHooksTuple + Debug, + EM: UsesState, + Z: UsesState, +{ + #[cfg(target_os = "linux")] + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct("GenericInProcessForkExecutor") + .field("GenericInProcessForkExecutorInner", &self.inner) + .finish() + } + + #[cfg(not(target_os = "linux"))] + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + #[cfg(not(target_os = "linux"))] + return f + .debug_struct("GenericInProcessForkExecutor") + .field("GenericInProcessForkExecutorInner", &self.inner) + .finish(); + } +} + +impl<'a, H, HT, OT, S, SP, EM, Z> UsesState + for GenericInProcessForkExecutor<'a, H, HT, OT, S, SP, EM, Z> +where + H: FnMut(&S::Input) -> ExitKind + ?Sized, + OT: ObserversTuple, + S: State, + SP: ShMemProvider, + HT: ExecutorHooksTuple, + EM: UsesState, + Z: UsesState, +{ + type State = S; +} + +impl<'a, EM, H, HT, OT, S, SP, Z> Executor + for GenericInProcessForkExecutor<'a, H, HT, OT, S, SP, EM, Z> +where + H: FnMut(&S::Input) -> ExitKind + ?Sized, + OT: ObserversTuple + Debug, + S: State + HasExecutions, + SP: ShMemProvider, + HT: ExecutorHooksTuple, + EM: EventFirer + EventRestarter, + Z: UsesState, +{ + #[allow(unreachable_code)] + #[inline] + fn run_target( + &mut self, + fuzzer: &mut Z, + state: &mut Self::State, + mgr: &mut EM, + input: &Self::Input, + ) -> Result { + *state.executions_mut() += 1; + + unsafe { + self.inner.shmem_provider.pre_fork()?; + match fork() { + Ok(ForkResult::Child) => { + // Child + self.inner.pre_run_target_child(fuzzer, state, mgr, input)?; + (self.harness_fn)(input); + self.inner.post_run_target_child(fuzzer, state, mgr, input); + Ok(ExitKind::Ok) + } + Ok(ForkResult::Parent { child }) => { + // Parent + self.inner.parent(child) + } + Err(e) => Err(Error::from(e)), + } + } + } +} + +impl<'a, H, HT, OT, S, SP, EM, Z, OF> GenericInProcessForkExecutor<'a, H, HT, OT, S, SP, EM, Z> +where + H: FnMut(&S::Input) -> ExitKind + ?Sized, + HT: ExecutorHooksTuple, + OT: ObserversTuple, + SP: ShMemProvider, + EM: EventFirer + EventRestarter, + OF: Feedback, + S: State + HasSolutions, + Z: HasObjective, +{ + /// Creates a new [`GenericInProcessForkExecutor`] with custom hooks + #[allow(clippy::too_many_arguments)] + pub fn with_hooks( + userhooks: HT, + harness_fn: &'a mut H, + observers: OT, + fuzzer: &mut Z, + state: &mut S, + event_mgr: &mut EM, + timeout: Duration, + shmem_provider: SP, + ) -> Result +where { + Ok(Self { + harness_fn, + inner: GenericInProcessForkExecutorInner::with_hooks( + userhooks, + observers, + fuzzer, + state, + event_mgr, + timeout, + shmem_provider, + )?, + }) + } + + /// Retrieve the harness function. + #[inline] + pub fn harness(&self) -> &H { + self.harness_fn + } + + /// Retrieve the harness function for a mutable reference. + #[inline] + pub fn harness_mut(&mut self) -> &mut H { + self.harness_fn + } +} + +impl<'a, H, HT, OT, S, SP, EM, Z> UsesObservers + for GenericInProcessForkExecutor<'a, H, HT, OT, S, SP, EM, Z> +where + H: FnMut(&S::Input) -> ExitKind + ?Sized, + HT: ExecutorHooksTuple, + OT: ObserversTuple, + S: State, + SP: ShMemProvider, + EM: UsesState, + Z: UsesState, +{ + type Observers = OT; +} + +impl<'a, H, HT, OT, S, SP, EM, Z> HasObservers + for GenericInProcessForkExecutor<'a, H, HT, OT, S, SP, EM, Z> +where + H: FnMut(&S::Input) -> ExitKind + ?Sized, + HT: ExecutorHooksTuple, + S: State, + OT: ObserversTuple, + SP: ShMemProvider, + EM: UsesState, + Z: UsesState, +{ + #[inline] + fn observers(&self) -> RefIndexable<&Self::Observers, Self::Observers> { + self.inner.observers() + } + + #[inline] + fn observers_mut(&mut self) -> RefIndexable<&mut Self::Observers, Self::Observers> { + self.inner.observers_mut() + } +} + +/// signal hooks and `panic_hooks` for the child process + +pub mod child_signal_handlers { + use alloc::boxed::Box; + use core::ptr::addr_of_mut; + use std::panic; + + use libafl_bolts::os::unix_signals::{ucontext_t, Signal}; + use libc::siginfo_t; + + use crate::{ + executors::{ + hooks::inprocess_fork::{InProcessForkExecutorGlobalData, FORK_EXECUTOR_GLOBAL_DATA}, + ExitKind, HasObservers, + }, + inputs::UsesInput, + observers::ObserversTuple, + }; + + /// invokes the `post_exec_child` hook on all observer in case the child process panics + pub fn setup_child_panic_hook() + where + E: HasObservers, + { + let old_hook = panic::take_hook(); + panic::set_hook(Box::new(move |panic_info| unsafe { + old_hook(panic_info); + let data = addr_of_mut!(FORK_EXECUTOR_GLOBAL_DATA); + if !data.is_null() && (*data).is_valid() { + let executor = (*data).executor_mut::(); + let mut observers = executor.observers_mut(); + let state = (*data).state_mut::(); + // Invalidate data to not execute again the observer hooks in the crash handler + let input = (*data).take_current_input::<::Input>(); + observers + .post_exec_child_all(state, input, &ExitKind::Crash) + .expect("Failed to run post_exec on observers"); + + // std::process::abort(); + libc::_exit(128 + 6); // ABORT exit code + } + })); + } + + /// invokes the `post_exec` hook on all observer in case the child process crashes + /// + /// # Safety + /// The function should only be called from a child crash handler. + /// It will dereference the `data` pointer and assume it's valid. + #[cfg(unix)] + #[allow(clippy::needless_pass_by_value)] + pub(crate) unsafe fn child_crash_handler( + _signal: Signal, + _info: &mut siginfo_t, + _context: Option<&mut ucontext_t>, + data: &mut InProcessForkExecutorGlobalData, + ) where + E: HasObservers, + { + if data.is_valid() { + let executor = data.executor_mut::(); + let mut observers = executor.observers_mut(); + let state = data.state_mut::(); + let input = data.take_current_input::<::Input>(); + observers + .post_exec_child_all(state, input, &ExitKind::Crash) + .expect("Failed to run post_exec on observers"); + } + + libc::_exit(128 + (_signal as i32)); + } + + #[cfg(unix)] + #[allow(clippy::needless_pass_by_value)] + pub(crate) unsafe fn child_timeout_handler( + _signal: Signal, + _info: &mut siginfo_t, + _context: Option<&mut ucontext_t>, + data: &mut InProcessForkExecutorGlobalData, + ) where + E: HasObservers, + { + if data.is_valid() { + let executor = data.executor_mut::(); + let mut observers = executor.observers_mut(); + let state = data.state_mut::(); + let input = data.take_current_input::<::Input>(); + observers + .post_exec_child_all(state, input, &ExitKind::Timeout) + .expect("Failed to run post_exec on observers"); + } + libc::_exit(128 + (_signal as i32)); + } +} + +#[cfg(test)] +mod tests { + use libafl_bolts::tuples::tuple_list; + + use crate::{ + executors::{inprocess_fork::GenericInProcessForkExecutorInner, Executor, ExitKind}, + inputs::NopInput, + }; + + #[test] + #[cfg_attr(miri, ignore)] + #[cfg(all(feature = "std", feature = "fork", unix))] + fn test_inprocessfork_exec() { + use core::marker::PhantomData; + + use libafl_bolts::shmem::{ShMemProvider, StdShMemProvider}; + #[cfg(target_os = "linux")] + use libc::{itimerspec, timespec}; + + #[cfg(not(target_os = "linux"))] + use crate::executors::hooks::timer::{Itimerval, Timeval}; + use crate::{ + events::SimpleEventManager, + executors::{ + hooks::inprocess_fork::InChildProcessHooks, + inprocess_fork::GenericInProcessForkExecutor, + }, + fuzzer::test::NopFuzzer, + state::NopState, + }; + + let provider = StdShMemProvider::new().unwrap(); + + #[cfg(target_os = "linux")] + let timespec = timespec { + tv_sec: 5, + tv_nsec: 0, + }; + #[cfg(target_os = "linux")] + let itimerspec = itimerspec { + it_interval: timespec, + it_value: timespec, + }; + + #[cfg(not(target_os = "linux"))] + let timespec = Timeval { + tv_sec: 5, + tv_usec: 0, + }; + #[cfg(not(target_os = "linux"))] + let itimerspec = Itimerval { + it_interval: timespec, + it_value: timespec, + }; + + let mut harness = |_buf: &NopInput| ExitKind::Ok; + let default = InChildProcessHooks::nop(); + #[cfg(target_os = "linux")] + let mut in_process_fork_executor = GenericInProcessForkExecutor { + harness_fn: &mut harness, + inner: GenericInProcessForkExecutorInner { + hooks: tuple_list!(default), + shmem_provider: provider, + observers: tuple_list!(), + itimerspec, + phantom: PhantomData, + }, + }; + #[cfg(not(target_os = "linux"))] + let mut in_process_fork_executor = GenericInProcessForkExecutor { + harness_fn: &mut harness, + inner: GenericInProcessForkExecutorInner { + hooks: tuple_list!(default), + shmem_provider: provider, + observers: tuple_list!(), + itimerval: itimerspec, + phantom: PhantomData, + }, + }; + let input = NopInput {}; + let mut fuzzer = NopFuzzer::new(); + let mut state = NopState::new(); + let mut mgr = SimpleEventManager::printing(); + in_process_fork_executor + .run_target(&mut fuzzer, &mut state, &mut mgr, &input) + .unwrap(); + } +} diff --git a/libafl/src/executors/inprocess_fork/stateful.rs b/libafl/src/executors/inprocess_fork/stateful.rs new file mode 100644 index 0000000000..d2ae815f99 --- /dev/null +++ b/libafl/src/executors/inprocess_fork/stateful.rs @@ -0,0 +1,259 @@ +//! The `StatefulGenericInProcessForkExecutor` to do forking before executing the harness in-processly. Harness can access internal state. +use core::{ + fmt::{self, Debug, Formatter}, + marker::PhantomData, + time::Duration, +}; + +use libafl_bolts::{ + shmem::ShMemProvider, + tuples::{tuple_list, RefIndexable}, +}; +use nix::unistd::{fork, ForkResult}; + +use super::super::hooks::ExecutorHooksTuple; +use crate::{ + events::{EventFirer, EventRestarter}, + executors::{ + inprocess_fork::GenericInProcessForkExecutorInner, Executor, ExitKind, HasObservers, + }, + feedbacks::Feedback, + fuzzer::HasObjective, + inputs::UsesInput, + observers::{ObserversTuple, UsesObservers}, + state::{HasExecutions, HasSolutions, State, UsesState}, + Error, +}; + +/// The `StatefulInProcessForkExecutor` with no user hooks +pub type StatefulInProcessForkExecutor<'a, H, OT, S, SP, ES, EM, Z> = + StatefulGenericInProcessForkExecutor<'a, H, (), OT, S, SP, ES, EM, Z>; + +impl<'a, H, OT, S, SP, ES, EM, Z, OF> StatefulInProcessForkExecutor<'a, H, OT, S, SP, ES, EM, Z> +where + H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized, + OT: ObserversTuple, + SP: ShMemProvider, + EM: EventFirer + EventRestarter, + OF: Feedback, + S: State + HasSolutions, + Z: HasObjective, +{ + #[allow(clippy::too_many_arguments)] + /// The constructor for `InProcessForkExecutor` + pub fn new( + harness_fn: &'a mut H, + exposed_executor_state: ES, + observers: OT, + fuzzer: &mut Z, + state: &mut S, + event_mgr: &mut EM, + timeout: Duration, + shmem_provider: SP, + ) -> Result { + Self::with_hooks( + tuple_list!(), + harness_fn, + exposed_executor_state, + observers, + fuzzer, + state, + event_mgr, + timeout, + shmem_provider, + ) + } +} + +/// [`StatefulGenericInProcessForkExecutor`] is an executor that forks the current process before each execution. Harness can access some internal state. +pub struct StatefulGenericInProcessForkExecutor<'a, H, HT, OT, S, SP, ES, EM, Z> +where + H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized, + OT: ObserversTuple, + S: UsesInput, + SP: ShMemProvider, + HT: ExecutorHooksTuple, + EM: UsesState, + Z: UsesState, +{ + harness_fn: &'a mut H, + exposed_executor_state: ES, + inner: GenericInProcessForkExecutorInner, + phantom: PhantomData, +} + +impl<'a, H, HT, OT, S, SP, ES, EM, Z> Debug + for StatefulGenericInProcessForkExecutor<'a, H, HT, OT, S, SP, ES, EM, Z> +where + H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized, + OT: ObserversTuple + Debug, + S: UsesInput, + SP: ShMemProvider, + HT: ExecutorHooksTuple + Debug, + EM: UsesState, + Z: UsesState, +{ + #[cfg(target_os = "linux")] + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct("GenericInProcessForkExecutor") + .field("GenericInProcessForkExecutionInner", &self.inner) + .finish() + } + + #[cfg(not(target_os = "linux"))] + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + #[cfg(not(target_os = "linux"))] + return f + .debug_struct("GenericInProcessForkExecutor") + .field("GenericInProcessForkExecutionInner", &self.inner) + .finish(); + } +} + +impl<'a, H, HT, OT, S, SP, ES, EM, Z> UsesState + for StatefulGenericInProcessForkExecutor<'a, H, HT, OT, S, SP, ES, EM, Z> +where + H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized, + OT: ObserversTuple, + S: State, + SP: ShMemProvider, + HT: ExecutorHooksTuple, + EM: UsesState, + Z: UsesState, +{ + type State = S; +} + +impl<'a, EM, H, HT, OT, S, SP, Z, ES, OF> Executor + for StatefulGenericInProcessForkExecutor<'a, H, HT, OT, S, SP, ES, EM, Z> +where + H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized, + OT: ObserversTuple + Debug, + S: State + HasExecutions, + SP: ShMemProvider, + HT: ExecutorHooksTuple, + EM: EventFirer + EventRestarter, + Z: HasObjective, + OF: Feedback, +{ + #[allow(unreachable_code)] + #[inline] + fn run_target( + &mut self, + fuzzer: &mut Z, + state: &mut Self::State, + mgr: &mut EM, + input: &Self::Input, + ) -> Result { + *state.executions_mut() += 1; + + unsafe { + self.inner.shmem_provider.pre_fork()?; + match fork() { + Ok(ForkResult::Child) => { + // Child + self.inner.pre_run_target_child(fuzzer, state, mgr, input)?; + (self.harness_fn)(input, &mut self.exposed_executor_state); + self.inner.post_run_target_child(fuzzer, state, mgr, input); + Ok(ExitKind::Ok) + } + Ok(ForkResult::Parent { child }) => { + // Parent + self.inner.parent(child) + } + Err(e) => Err(Error::from(e)), + } + } + } +} + +impl<'a, H, HT, OT, S, SP, ES, EM, Z, OF> + StatefulGenericInProcessForkExecutor<'a, H, HT, OT, S, SP, ES, EM, Z> +where + H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized, + HT: ExecutorHooksTuple, + OT: ObserversTuple, + SP: ShMemProvider, + Z: UsesState, + EM: EventFirer + EventRestarter, + OF: Feedback, + S: State + HasSolutions, + Z: HasObjective, +{ + /// Creates a new [`StatefulGenericInProcessForkExecutor`] with custom hooks + #[allow(clippy::too_many_arguments)] + pub fn with_hooks( + userhooks: HT, + harness_fn: &'a mut H, + exposed_executor_state: ES, + observers: OT, + fuzzer: &mut Z, + state: &mut S, + event_mgr: &mut EM, + timeout: Duration, + shmem_provider: SP, + ) -> Result { + Ok(Self { + harness_fn, + exposed_executor_state, + inner: GenericInProcessForkExecutorInner::with_hooks( + userhooks, + observers, + fuzzer, + state, + event_mgr, + timeout, + shmem_provider, + )?, + phantom: PhantomData, + }) + } + + /// Retrieve the harness function. + #[inline] + pub fn harness(&self) -> &H { + self.harness_fn + } + + /// Retrieve the harness function for a mutable reference. + #[inline] + pub fn harness_mut(&mut self) -> &mut H { + self.harness_fn + } +} + +impl<'a, H, HT, OT, S, SP, ES, EM, Z> UsesObservers + for StatefulGenericInProcessForkExecutor<'a, H, HT, OT, S, SP, ES, EM, Z> +where + H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized, + HT: ExecutorHooksTuple, + OT: ObserversTuple, + S: State, + SP: ShMemProvider, + EM: UsesState, + Z: UsesState, +{ + type Observers = OT; +} + +impl<'a, H, HT, OT, S, SP, ES, EM, Z> HasObservers + for StatefulGenericInProcessForkExecutor<'a, H, HT, OT, S, SP, ES, EM, Z> +where + H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized, + HT: ExecutorHooksTuple, + S: State, + OT: ObserversTuple, + SP: ShMemProvider, + EM: UsesState, + Z: UsesState, +{ + #[inline] + fn observers(&self) -> RefIndexable<&Self::Observers, Self::Observers> { + self.inner.observers() + } + + #[inline] + fn observers_mut(&mut self) -> RefIndexable<&mut Self::Observers, Self::Observers> { + self.inner.observers_mut() + } +} diff --git a/libafl/src/executors/mod.rs b/libafl/src/executors/mod.rs index 9fed40d22f..5e9d462b36 100644 --- a/libafl/src/executors/mod.rs +++ b/libafl/src/executors/mod.rs @@ -15,6 +15,7 @@ pub use inprocess::InProcessExecutor; pub use inprocess_fork::InProcessForkExecutor; #[cfg(unix)] use libafl_bolts::os::unix_signals::Signal; +use libafl_bolts::tuples::RefIndexable; use serde::{Deserialize, Serialize}; pub use shadow::ShadowExecutor; pub use with_observers::WithObservers; @@ -110,10 +111,10 @@ libafl_bolts::impl_serdeany!(DiffExitKind); /// Holds a tuple of Observers pub trait HasObservers: UsesObservers { /// Get the linked observers - fn observers(&self) -> &Self::Observers; + fn observers(&self) -> RefIndexable<&Self::Observers, Self::Observers>; /// Get the linked observers (mutable) - fn observers_mut(&mut self) -> &mut Self::Observers; + fn observers_mut(&mut self) -> RefIndexable<&mut Self::Observers, Self::Observers>; } /// An executor takes the given inputs, and runs the harness/target. @@ -174,7 +175,7 @@ pub mod test { executors::{Executor, ExitKind}, fuzzer::test::NopFuzzer, inputs::{BytesInput, HasTargetBytes}, - state::{test::NopState, HasExecutions, State, UsesState}, + state::{HasExecutions, NopState, State, UsesState}, }; /// A simple executor that does nothing. @@ -257,278 +258,3 @@ pub mod test { .unwrap(); } } - -#[cfg(feature = "python")] -#[allow(missing_docs)] -/// `Executor` Python bindings -pub mod pybind { - use pyo3::prelude::*; - use serde::{Deserialize, Serialize}; - - use crate::{ - events::pybind::PythonEventManager, - executors::{ - inprocess::pybind::PythonOwnedInProcessExecutor, Executor, ExitKind, HasObservers, - }, - fuzzer::pybind::{PythonStdFuzzer, PythonStdFuzzerWrapper}, - inputs::HasBytesVec, - observers::{pybind::PythonObserversTuple, UsesObservers}, - state::{ - pybind::{PythonStdState, PythonStdStateWrapper}, - UsesState, - }, - Error, - }; - - #[pyclass(unsendable, name = "ExitKind")] - #[allow(clippy::unsafe_derive_deserialize)] - #[derive(Clone, Debug, Serialize, Deserialize)] - pub struct PythonExitKind { - pub inner: ExitKind, - } - - impl From for PythonExitKind { - fn from(inner: ExitKind) -> Self { - Self { inner } - } - } - - #[pymethods] - impl PythonExitKind { - fn __eq__(&self, other: &PythonExitKind) -> bool { - self.inner == other.inner - } - - #[must_use] - fn is_ok(&self) -> bool { - self.inner == ExitKind::Ok - } - - #[must_use] - fn is_crash(&self) -> bool { - self.inner == ExitKind::Crash - } - - #[must_use] - fn is_oom(&self) -> bool { - self.inner == ExitKind::Oom - } - - #[must_use] - fn is_timeout(&self) -> bool { - self.inner == ExitKind::Timeout - } - - #[staticmethod] - #[must_use] - fn ok() -> Self { - Self { - inner: ExitKind::Ok, - } - } - - #[staticmethod] - #[must_use] - fn crash() -> Self { - Self { - inner: ExitKind::Crash, - } - } - - #[staticmethod] - #[must_use] - fn oom() -> Self { - Self { - inner: ExitKind::Oom, - } - } - - #[staticmethod] - #[must_use] - fn timeout() -> Self { - Self { - inner: ExitKind::Timeout, - } - } - } - - #[derive(Clone, Debug)] - pub struct PyObjectExecutor { - inner: PyObject, - tuple: PythonObserversTuple, - } - - impl PyObjectExecutor { - #[must_use] - pub fn new(obj: PyObject) -> Self { - let tuple = Python::with_gil(|py| -> PyResult { - obj.call_method1(py, "observers", ())?.extract(py) - }) - .unwrap(); - PyObjectExecutor { inner: obj, tuple } - } - } - - impl UsesState for PyObjectExecutor { - type State = PythonStdState; - } - - impl UsesObservers for PyObjectExecutor { - type Observers = PythonObserversTuple; - } - - impl HasObservers for PyObjectExecutor { - #[inline] - fn observers(&self) -> &PythonObserversTuple { - &self.tuple - } - - #[inline] - fn observers_mut(&mut self) -> &mut PythonObserversTuple { - &mut self.tuple - } - } - - impl Executor for PyObjectExecutor { - #[inline] - fn run_target( - &mut self, - fuzzer: &mut PythonStdFuzzer, - state: &mut Self::State, - mgr: &mut PythonEventManager, - input: &Self::Input, - ) -> Result { - let ek = Python::with_gil(|py| -> PyResult<_> { - let ek: PythonExitKind = self - .inner - .call_method1( - py, - "run_target", - ( - PythonStdFuzzerWrapper::wrap(fuzzer), - PythonStdStateWrapper::wrap(state), - mgr.clone(), - input.bytes(), - ), - )? - .extract(py)?; - Ok(ek) - })?; - Ok(ek.inner) - } - } - - #[derive(Clone, Debug)] - enum PythonExecutorWrapper { - InProcess(Py), - Python(PyObjectExecutor), - } - - #[pyclass(unsendable, name = "Executor")] - #[derive(Clone, Debug)] - /// Executor + HasObservers Trait binding - pub struct PythonExecutor { - wrapper: PythonExecutorWrapper, - } - - macro_rules! unwrap_me { - ($wrapper:expr, $name:ident, $body:block) => { - libafl_bolts::unwrap_me_body!($wrapper, $name, $body, PythonExecutorWrapper, - { InProcess }, - { - Python(py_wrapper) => { - let $name = py_wrapper; - $body - } - } - ) - }; - } - - macro_rules! unwrap_me_mut { - ($wrapper:expr, $name:ident, $body:block) => { - libafl_bolts::unwrap_me_mut_body!($wrapper, $name, $body, PythonExecutorWrapper, - { InProcess }, - { - Python(py_wrapper) => { - let $name = py_wrapper; - $body - } - } - ) - }; - } - - #[pymethods] - impl PythonExecutor { - #[staticmethod] - #[must_use] - pub fn new_inprocess(owned_inprocess_executor: Py) -> Self { - Self { - wrapper: PythonExecutorWrapper::InProcess(owned_inprocess_executor), - } - } - - #[staticmethod] - #[must_use] - pub fn new_py(obj: PyObject) -> Self { - Self { - wrapper: PythonExecutorWrapper::Python(PyObjectExecutor::new(obj)), - } - } - - #[must_use] - pub fn unwrap_py(&self) -> Option { - match &self.wrapper { - PythonExecutorWrapper::Python(pyo) => Some(pyo.inner.clone()), - PythonExecutorWrapper::InProcess(_) => None, - } - } - } - - impl UsesState for PythonExecutor { - type State = PythonStdState; - } - - impl UsesObservers for PythonExecutor { - type Observers = PythonObserversTuple; - } - - impl HasObservers for PythonExecutor { - #[inline] - fn observers(&self) -> &PythonObserversTuple { - let ptr = unwrap_me!(self.wrapper, e, { - core::ptr::from_ref::(e.observers()) - }); - unsafe { ptr.as_ref().unwrap() } - } - - #[inline] - fn observers_mut(&mut self) -> &mut PythonObserversTuple { - let ptr = unwrap_me_mut!(self.wrapper, e, { - core::ptr::from_mut::(e.observers_mut()) - }); - unsafe { ptr.as_mut().unwrap() } - } - } - - impl Executor for PythonExecutor { - #[inline] - fn run_target( - &mut self, - fuzzer: &mut PythonStdFuzzer, - state: &mut Self::State, - mgr: &mut PythonEventManager, - input: &Self::Input, - ) -> Result { - unwrap_me_mut!(self.wrapper, e, { e.run_target(fuzzer, state, mgr, input) }) - } - } - - /// Register the classes to the python module - pub fn register(_py: Python, m: &PyModule) -> PyResult<()> { - m.add_class::()?; - m.add_class::()?; - Ok(()) - } -} diff --git a/libafl/src/executors/shadow.rs b/libafl/src/executors/shadow.rs index f2b30c0fec..242fca8413 100644 --- a/libafl/src/executors/shadow.rs +++ b/libafl/src/executors/shadow.rs @@ -2,6 +2,8 @@ use core::fmt::{self, Debug, Formatter}; +use libafl_bolts::tuples::RefIndexable; + use crate::{ executors::{Executor, ExitKind, HasObservers}, observers::{ObserversTuple, UsesObservers}, @@ -45,14 +47,14 @@ where /// The shadow observers are not considered by the feedbacks and the manager, mutable #[inline] - pub fn shadow_observers(&self) -> &SOT { - &self.shadow_observers + pub fn shadow_observers(&self) -> RefIndexable<&SOT, SOT> { + RefIndexable::from(&self.shadow_observers) } /// The shadow observers are not considered by the feedbacks and the manager, mutable #[inline] - pub fn shadow_observers_mut(&mut self) -> &mut SOT { - &mut self.shadow_observers + pub fn shadow_observers_mut(&mut self) -> RefIndexable<&mut SOT, SOT> { + RefIndexable::from(&mut self.shadow_observers) } } @@ -94,12 +96,12 @@ where SOT: ObserversTuple, { #[inline] - fn observers(&self) -> &Self::Observers { + fn observers(&self) -> RefIndexable<&Self::Observers, Self::Observers> { self.executor.observers() } #[inline] - fn observers_mut(&mut self) -> &mut Self::Observers { + fn observers_mut(&mut self) -> RefIndexable<&mut Self::Observers, Self::Observers> { self.executor.observers_mut() } } diff --git a/libafl/src/executors/with_observers.rs b/libafl/src/executors/with_observers.rs index 78594ad776..6924b9e401 100644 --- a/libafl/src/executors/with_observers.rs +++ b/libafl/src/executors/with_observers.rs @@ -2,6 +2,8 @@ use core::fmt::Debug; +use libafl_bolts::tuples::RefIndexable; + use crate::{ executors::{Executor, ExitKind, HasObservers}, observers::{ObserversTuple, UsesObservers}, @@ -53,12 +55,12 @@ where E: UsesState, OT: ObserversTuple, { - fn observers(&self) -> &OT { - &self.observers + fn observers(&self) -> RefIndexable<&Self::Observers, Self::Observers> { + RefIndexable::from(&self.observers) } - fn observers_mut(&mut self) -> &mut OT { - &mut self.observers + fn observers_mut(&mut self) -> RefIndexable<&mut Self::Observers, Self::Observers> { + RefIndexable::from(&mut self.observers) } } diff --git a/libafl/src/feedbacks/concolic.rs b/libafl/src/feedbacks/concolic.rs index 42f16b433a..7d6d6d9310 100644 --- a/libafl/src/feedbacks/concolic.rs +++ b/libafl/src/feedbacks/concolic.rs @@ -3,10 +3,13 @@ //! This feedback should be used in combination with another feedback as this feedback always considers testcases //! to be not interesting. //! Requires a [`ConcolicObserver`] to observe the concolic trace. -use alloc::{borrow::ToOwned, string::String}; +use alloc::borrow::Cow; use core::{fmt::Debug, marker::PhantomData}; -use libafl_bolts::Named; +use libafl_bolts::{ + tuples::{Handle, Handler, MatchNameRef}, + Named, +}; use crate::{ corpus::Testcase, @@ -15,8 +18,8 @@ use crate::{ feedbacks::Feedback, inputs::UsesInput, observers::{concolic::ConcolicObserver, ObserversTuple}, - state::{HasMetadata, State}, - Error, + state::State, + Error, HasMetadata, }; /// The concolic feedback. It is used to attach concolic tracing metadata to the testcase. @@ -24,30 +27,30 @@ use crate::{ /// to be not interesting. /// Requires a [`ConcolicObserver`] to observe the concolic trace. #[derive(Debug)] -pub struct ConcolicFeedback { - name: String, +pub struct ConcolicFeedback<'map, S> { + obs_ref: Handle>, phantom: PhantomData, } -impl ConcolicFeedback { +impl<'map, S> ConcolicFeedback<'map, S> { /// Creates a concolic feedback from an observer #[allow(unused)] #[must_use] - pub fn from_observer(observer: &ConcolicObserver) -> Self { + pub fn from_observer(observer: &ConcolicObserver<'map>) -> Self { Self { - name: observer.name().to_owned(), + obs_ref: observer.handle(), phantom: PhantomData, } } } -impl Named for ConcolicFeedback { - fn name(&self) -> &str { - &self.name +impl Named for ConcolicFeedback<'_, S> { + fn name(&self) -> &Cow<'static, str> { + self.obs_ref.name() } } -impl Feedback for ConcolicFeedback +impl Feedback for ConcolicFeedback<'_, S> where S: State, { @@ -67,17 +70,19 @@ where Ok(false) } - fn append_metadata( + fn append_metadata( &mut self, _state: &mut S, + _manager: &mut EM, observers: &OT, testcase: &mut Testcase, ) -> Result<(), Error> where OT: ObserversTuple, + EM: EventFirer, { if let Some(metadata) = observers - .match_name::(&self.name) + .get(&self.obs_ref) .map(ConcolicObserver::create_metadata_from_current_map) { testcase.metadata_map_mut().insert(metadata); diff --git a/libafl/src/feedbacks/differential.rs b/libafl/src/feedbacks/differential.rs index b944df2438..5b69c54d9c 100644 --- a/libafl/src/feedbacks/differential.rs +++ b/libafl/src/feedbacks/differential.rs @@ -1,23 +1,26 @@ //! Diff Feedback, comparing the content of two observers of the same type. //! -use alloc::string::{String, ToString}; +use alloc::borrow::Cow; use core::{ fmt::{self, Debug, Formatter}, marker::PhantomData, }; -use libafl_bolts::{tuples::MatchName, Named}; +use libafl_bolts::{ + tuples::{Handle, Handler, MatchName, MatchNameRef}, + Named, +}; use serde::{Deserialize, Serialize}; use crate::{ events::EventFirer, executors::ExitKind, - feedbacks::Feedback, + feedbacks::{Feedback, FeedbackFactory}, inputs::Input, observers::{Observer, ObserversTuple}, - state::{HasMetadata, State}, - Error, + state::State, + Error, HasMetadata, }; /// The result of a differential test between two observers. @@ -53,14 +56,14 @@ where F: FnMut(&O1, &O2) -> DiffResult, { /// This feedback's name - name: String, + name: Cow<'static, str>, /// The first observer to compare against - o1_name: String, + o1_ref: Handle, /// The second observer to compare against - o2_name: String, + o2_ref: Handle, /// The function used to compare the two observers compare_fn: F, - phantomm: PhantomData<(O1, O2, I, S)>, + phantomm: PhantomData<(I, S)>, } impl DiffFeedback @@ -70,18 +73,19 @@ where O2: Named, { /// Create a new [`DiffFeedback`] using two observers and a test function. - pub fn new(name: &str, o1: &O1, o2: &O2, compare_fn: F) -> Result { - let o1_name = o1.name().to_string(); - let o2_name = o2.name().to_string(); - if o1_name == o2_name { + pub fn new(name: &'static str, o1: &O1, o2: &O2, compare_fn: F) -> Result { + let o1_ref = o1.handle(); + let o2_ref = o2.handle(); + if o1_ref.name() == o2_ref.name() { Err(Error::illegal_argument(format!( - "DiffFeedback: observer names must be different (both were {o1_name})" + "DiffFeedback: observer names must be different (both were {})", + o1_ref.name() ))) } else { Ok(Self { - o1_name, - o2_name, - name: name.to_string(), + o1_ref, + o2_ref, + name: Cow::from(name), compare_fn, phantomm: PhantomData, }) @@ -89,13 +93,33 @@ where } } +impl FeedbackFactory, S, T> + for DiffFeedback +where + F: FnMut(&O1, &O2) -> DiffResult + Clone, + I: Input, + O1: Observer + Named, + O2: Observer + Named, + S: HasMetadata + State, +{ + fn create_feedback(&self, _ctx: &T) -> DiffFeedback { + Self { + name: self.name.clone(), + o1_ref: self.o1_ref.clone(), + o2_ref: self.o2_ref.clone(), + compare_fn: self.compare_fn.clone(), + phantomm: self.phantomm, + } + } +} + impl Named for DiffFeedback where F: FnMut(&O1, &O2) -> DiffResult, O1: Named, O2: Named, { - fn name(&self) -> &str { + fn name(&self) -> &Cow<'static, str> { &self.name } } @@ -107,11 +131,11 @@ where O2: Named, { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - write!( - f, - "DiffFeedback {{ name: {}, o1: {}, o2: {} }}", - self.name, self.o1_name, self.o2_name - ) + f.debug_struct("DiffFeedback") + .field("name", self.name()) + .field("o1", &self.o1_ref) + .field("o2", &self.o2_ref) + .finish_non_exhaustive() } } @@ -140,11 +164,11 @@ where Error::illegal_argument(format!("DiffFeedback: observer {name} not found")) } let o1: &O1 = observers - .match_name(&self.o1_name) - .ok_or_else(|| err(&self.o1_name))?; + .get(&self.o1_ref) + .ok_or_else(|| err(self.o1_ref.name()))?; let o2: &O2 = observers - .match_name(&self.o2_name) - .ok_or_else(|| err(&self.o2_name))?; + .get(&self.o2_ref) + .ok_or_else(|| err(self.o2_ref.name()))?; Ok((self.compare_fn)(o1, o2) == DiffResult::Diff) } @@ -152,7 +176,7 @@ where #[cfg(test)] mod tests { - use alloc::string::{String, ToString}; + use alloc::borrow::Cow; use core::marker::PhantomData; use libafl_bolts::{tuples::tuple_list, Named}; @@ -163,18 +187,18 @@ mod tests { feedbacks::{differential::DiffResult, DiffFeedback, Feedback}, inputs::{BytesInput, UsesInput}, observers::Observer, - state::{test::NopState, State, UsesState}, + state::{NopState, State, UsesState}, }; #[derive(Debug)] struct NopObserver { - name: String, + name: Cow<'static, str>, value: bool, } impl NopObserver { - fn new(name: &str, value: bool) -> Self { + fn new(name: &'static str, value: bool) -> Self { Self { - name: name.to_string(), + name: Cow::from(name), value, } } @@ -186,7 +210,7 @@ mod tests { } } impl Named for NopObserver { - fn name(&self) -> &str { + fn name(&self) -> &Cow<'static, str> { &self.name } } diff --git a/libafl/src/feedbacks/list.rs b/libafl/src/feedbacks/list.rs new file mode 100644 index 0000000000..c433fc372c --- /dev/null +++ b/libafl/src/feedbacks/list.rs @@ -0,0 +1,170 @@ +use alloc::borrow::Cow; +use core::{fmt::Debug, hash::Hash}; + +use hashbrown::HashSet; +use libafl_bolts::{ + tuples::{Handle, Handler, MatchNameRef}, + Error, HasRefCnt, Named, +}; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; + +use crate::{ + events::EventFirer, + executors::ExitKind, + feedbacks::Feedback, + observers::{ListObserver, ObserversTuple}, + state::State, + HasNamedMetadata, +}; + +/// The metadata to remember past observed value +#[derive(Default, Serialize, Deserialize, Clone, Debug)] +#[serde(bound = "T: DeserializeOwned")] +#[cfg_attr( + any(not(feature = "serdeany_autoreg"), miri), + allow(clippy::unsafe_derive_deserialize) +)] +pub struct ListFeedbackMetadata +where + T: Default + Copy + 'static + Serialize + Eq + Hash, +{ + /// Contains the information of past observed set of values. + pub set: HashSet, + /// A refcount used to know when we can remove this metadata + pub tcref: isize, +} + +impl ListFeedbackMetadata +where + T: Default + Copy + 'static + Serialize + Eq + Hash, +{ + /// The constructor + #[must_use] + pub fn new() -> Self { + Self { + set: HashSet::::new(), + tcref: 0, + } + } + + /// Reset the inner hashset + pub fn reset(&mut self) -> Result<(), Error> { + self.set.clear(); + Ok(()) + } +} + +impl HasRefCnt for ListFeedbackMetadata +where + T: Default + Copy + 'static + Serialize + Eq + Hash, +{ + fn refcnt(&self) -> isize { + self.tcref + } + + fn refcnt_mut(&mut self) -> &mut isize { + &mut self.tcref + } +} + +/// Consider interesting a testcase if the list in `ListObserver` is not empty. +#[derive(Clone, Debug)] +pub struct ListFeedback +where + T: Hash + Eq, +{ + obs_ref: Handle>, + novelty: HashSet, +} + +libafl_bolts::impl_serdeany!( + ListFeedbackMetadata, + ,,,,,,,,,, +); + +impl Feedback for ListFeedback +where + S: State + HasNamedMetadata, + T: Debug + Serialize + Hash + Eq + DeserializeOwned + Default + Copy + 'static, +{ + fn init_state(&mut self, state: &mut S) -> Result<(), Error> { + // eprintln!("self.name {:#?}", &self.name); + state.add_named_metadata(self.name(), ListFeedbackMetadata::::default()); + Ok(()) + } + #[allow(clippy::wrong_self_convention)] + fn is_interesting( + &mut self, + state: &mut S, + _manager: &mut EM, + _input: &S::Input, + observers: &OT, + _exit_kind: &ExitKind, + ) -> Result + where + EM: EventFirer, + OT: ObserversTuple, + { + // TODO Replace with match_name_type when stable + let observer = observers.get(&self.obs_ref).unwrap(); + // TODO register the list content in a testcase metadata + self.novelty.clear(); + // can't fail + let history_set = state + .named_metadata_map_mut() + .get_mut::>(self.name()) + .unwrap(); + for v in observer.list() { + if !history_set.set.contains(v) { + self.novelty.insert(*v); + } + } + Ok(!self.novelty.is_empty()) + } + + fn append_metadata( + &mut self, + state: &mut S, + _manager: &mut EM, + _observers: &OT, + _testcase: &mut crate::corpus::Testcase<::Input>, + ) -> Result<(), Error> + where + OT: ObserversTuple, + EM: EventFirer, + { + let history_set = state + .named_metadata_map_mut() + .get_mut::>(self.name()) + .unwrap(); + + for v in &self.novelty { + history_set.set.insert(*v); + } + Ok(()) + } +} + +impl Named for ListFeedback +where + T: Debug + Serialize + Hash + Eq + DeserializeOwned, +{ + #[inline] + fn name(&self) -> &Cow<'static, str> { + self.obs_ref.name() + } +} + +impl ListFeedback +where + T: Debug + Serialize + Hash + Eq + DeserializeOwned, +{ + /// Creates a new [`ListFeedback`], deciding if the given [`ListObserver`] value of a run is interesting. + #[must_use] + pub fn new(observer: &ListObserver) -> Self { + Self { + obs_ref: observer.handle(), + novelty: HashSet::::new(), + } + } +} diff --git a/libafl/src/feedbacks/map.rs b/libafl/src/feedbacks/map.rs index 7365180cff..14294406fe 100644 --- a/libafl/src/feedbacks/map.rs +++ b/libafl/src/feedbacks/map.rs @@ -1,18 +1,20 @@ //! Map feedback, maximizing or minimizing maps, for example the afl-style map observer. -use alloc::{ - string::{String, ToString}, - vec::Vec, -}; +use alloc::{borrow::Cow, vec::Vec}; #[rustversion::nightly] use core::simd::prelude::SimdOrd; use core::{ fmt::Debug, marker::PhantomData, - ops::{BitAnd, BitOr}, + ops::{BitAnd, BitOr, Deref, DerefMut}, }; -use libafl_bolts::{AsIter, AsMutSlice, AsSlice, HasRefCnt, Named}; +#[rustversion::nightly] +use libafl_bolts::AsSlice; +use libafl_bolts::{ + tuples::{Handle, Handler, MatchNameRef}, + AsIter, HasRefCnt, Named, +}; use num_traits::PrimInt; use serde::{de::DeserializeOwned, Deserialize, Serialize}; @@ -20,34 +22,31 @@ use crate::{ corpus::Testcase, events::{Event, EventFirer}, executors::ExitKind, - feedbacks::{Feedback, HasObserverName}, + feedbacks::{Feedback, HasObserverReference}, inputs::UsesInput, monitors::{AggregatorOps, UserStats, UserStatsValue}, - observers::{MapObserver, Observer, ObserversTuple, UsesObserver}, - state::{HasMetadata, HasNamedMetadata, State}, - Error, + observers::{CanTrack, MapObserver, Observer, ObserversTuple}, + state::State, + Error, HasMetadata, HasNamedMetadata, }; -/// The prefix of the metadata names -pub const MAPFEEDBACK_PREFIX: &str = "mapfeedback_metadata_"; - -/// A [`MapFeedback`] that implements the AFL algorithm using an [`OrReducer`] combining the bits for the history map and the bit from ``HitcountsMapObserver``. -pub type AflMapFeedback = MapFeedback; +/// A [`MapFeedback`] that implements the AFL algorithm using an [`OrReducer`] combining the bits for the history map and the bit from (`HitcountsMapObserver`)[`crate::observers::HitcountsMapObserver`]. +pub type AflMapFeedback = MapFeedback; /// A [`MapFeedback`] that strives to maximize the map contents. -pub type MaxMapFeedback = MapFeedback; +pub type MaxMapFeedback = MapFeedback; /// A [`MapFeedback`] that strives to minimize the map contents. -pub type MinMapFeedback = MapFeedback; +pub type MinMapFeedback = MapFeedback; /// A [`MapFeedback`] that always returns `true` for `is_interesting`. Useful for tracing all executions. -pub type AlwaysInterestingMapFeedback = MapFeedback; +pub type AlwaysInterestingMapFeedback = MapFeedback; /// A [`MapFeedback`] that strives to maximize the map contents, /// but only, if a value is larger than `pow2` of the previous. -pub type MaxMapPow2Feedback = MapFeedback; +pub type MaxMapPow2Feedback = MapFeedback; /// A [`MapFeedback`] that strives to maximize the map contents, /// but only, if a value is larger than `pow2` of the previous. -pub type MaxMapOneOrFilledFeedback = MapFeedback; +pub type MaxMapOneOrFilledFeedback = MapFeedback; /// A `Reducer` function is used to aggregate values for the novelty search pub trait Reducer: 'static @@ -177,6 +176,7 @@ fn saturating_next_power_of_two(n: T) -> T { /// Consider as novelty if the reduced value is different from the old value. #[derive(Clone, Debug)] pub struct DifferentIsNovel {} + impl IsNovel for DifferentIsNovel where T: PartialEq + Default + Copy + 'static, @@ -190,6 +190,7 @@ where /// Only consider as novel the values which are at least the next pow2 class of the old value #[derive(Clone, Debug)] pub struct NextPow2IsNovel {} + impl IsNovel for NextPow2IsNovel where T: PrimInt + Default + Copy + 'static, @@ -210,6 +211,7 @@ where /// Only consider `T::one()` or `T::max_value()`, if they are bigger than the old value, as novel #[derive(Clone, Debug)] pub struct OneOrFilledIsNovel {} + impl IsNovel for OneOrFilledIsNovel where T: PrimInt + Default + Copy + 'static, @@ -235,18 +237,18 @@ pub struct MapIndexesMetadata { libafl_bolts::impl_serdeany!(MapIndexesMetadata); -impl AsSlice for MapIndexesMetadata { - type Entry = usize; +impl Deref for MapIndexesMetadata { + type Target = [usize]; /// Convert to a slice - fn as_slice(&self) -> &[usize] { - self.list.as_slice() + fn deref(&self) -> &[usize] { + &self.list } } -impl AsMutSlice for MapIndexesMetadata { - type Entry = usize; + +impl DerefMut for MapIndexesMetadata { /// Convert to a slice - fn as_mut_slice(&mut self) -> &mut [usize] { - self.list.as_mut_slice() + fn deref_mut(&mut self) -> &mut [usize] { + &mut self.list } } @@ -281,22 +283,23 @@ pub struct MapNoveltiesMetadata { libafl_bolts::impl_serdeany!(MapNoveltiesMetadata); -impl AsSlice for MapNoveltiesMetadata { - type Entry = usize; +impl Deref for MapNoveltiesMetadata { + type Target = [usize]; /// Convert to a slice #[must_use] - fn as_slice(&self) -> &[usize] { - self.list.as_slice() + fn deref(&self) -> &[usize] { + &self.list } } -impl AsMutSlice for MapNoveltiesMetadata { - type Entry = usize; + +impl DerefMut for MapNoveltiesMetadata { /// Convert to a slice #[must_use] - fn as_mut_slice(&mut self) -> &mut [usize] { - self.list.as_mut_slice() + fn deref_mut(&mut self) -> &mut [usize] { + &mut self.list } } + impl MapNoveltiesMetadata { /// Creates a new [`struct@MapNoveltiesMetadata`] #[must_use] @@ -318,6 +321,8 @@ where { /// Contains information about untouched entries pub history_map: Vec, + /// Tells us how many non-initial entries there are in `history_map` + pub num_covered_map_indexes: usize, } libafl_bolts::impl_serdeany!( @@ -327,21 +332,29 @@ libafl_bolts::impl_serdeany!( impl MapFeedbackMetadata where - T: Default + Copy + 'static + Serialize + DeserializeOwned, + T: Default + Copy + 'static + Serialize + DeserializeOwned + PartialEq, { /// Create new `MapFeedbackMetadata` #[must_use] pub fn new(map_size: usize) -> Self { Self { history_map: vec![T::default(); map_size], + num_covered_map_indexes: 0, } } /// Create new `MapFeedbackMetadata` using a name and a map. /// The map can be shared. + /// `initial_elem_value` is used to calculate `Self.num_covered_map_indexes` #[must_use] - pub fn with_history_map(history_map: Vec) -> Self { - Self { history_map } + pub fn with_history_map(history_map: Vec, initial_elem_value: T) -> Self { + let num_covered_map_indexes = history_map + .iter() + .fold(0, |acc, x| acc + usize::from(*x != initial_elem_value)); + Self { + history_map, + num_covered_map_indexes, + } } /// Reset the map @@ -350,6 +363,7 @@ where for i in 0..cnt { self.history_map[i] = T::default(); } + self.num_covered_map_indexes = 0; Ok(()) } @@ -359,49 +373,41 @@ where for i in 0..cnt { self.history_map[i] = value; } + // assume that resetting the map should indicate no coverage, + // regardless of value + self.num_covered_map_indexes = 0; Ok(()) } } /// The most common AFL-like feedback type #[derive(Clone, Debug)] -pub struct MapFeedback { - /// For tracking, always keep indexes and/or novelties, even if the map isn't considered `interesting`. - always_track: bool, - /// Indexes used in the last observation - indexes: bool, +pub struct MapFeedback { /// New indexes observed in the last observation novelties: Option>, /// Name identifier of this instance - name: String, + name: Cow<'static, str>, /// Name identifier of the observer - observer_name: String, + map_ref: Handle, /// Name of the feedback as shown in the `UserStats` - stats_name: String, + stats_name: Cow<'static, str>, /// Phantom Data of Reducer - phantom: PhantomData<(N, O, R, S, T)>, + phantom: PhantomData<(C, N, O, R, T)>, } -impl UsesObserver for MapFeedback -where - S: UsesInput, - O: Observer, -{ - type Observer = O; -} - -impl Feedback for MapFeedback +impl Feedback for MapFeedback where N: IsNovel, O: MapObserver + for<'it> AsIter<'it, Item = T>, R: Reducer, S: State + HasNamedMetadata, T: Default + Copy + Serialize + for<'de> Deserialize<'de> + PartialEq + Debug + 'static, + C: CanTrack + AsRef + Observer, { fn init_state(&mut self, state: &mut S) -> Result<(), Error> { // Initialize `MapFeedbackMetadata` with an empty vector and add it to the state. // The `MapFeedbackMetadata` would be resized on-demand in `is_interesting` - state.add_named_metadata(MapFeedbackMetadata::::default(), &self.name); + state.add_named_metadata(&self.name, MapFeedbackMetadata::::default()); Ok(()) } @@ -418,7 +424,7 @@ where EM: EventFirer, OT: ObserversTuple, { - self.is_interesting_default(state, manager, input, observers, exit_kind) + Ok(self.is_interesting_default(state, manager, input, observers, exit_kind)) } #[rustversion::not(nightly)] @@ -434,23 +440,25 @@ where EM: EventFirer, OT: ObserversTuple, { - self.is_interesting_default(state, manager, input, observers, exit_kind) + Ok(self.is_interesting_default(state, manager, input, observers, exit_kind)) } - fn append_metadata( + fn append_metadata( &mut self, state: &mut S, + manager: &mut EM, observers: &OT, testcase: &mut Testcase, ) -> Result<(), Error> where OT: ObserversTuple, + EM: EventFirer, { if let Some(novelties) = self.novelties.as_mut().map(core::mem::take) { let meta = MapNoveltiesMetadata::new(novelties); testcase.add_metadata(meta); } - let observer = observers.match_name::(&self.observer_name).unwrap(); + let observer = observers.get(&self.map_ref).unwrap().as_ref(); let initial = observer.initial(); let map_state = state .named_metadata_map_mut() @@ -461,16 +469,19 @@ where map_state.history_map.resize(len, observer.initial()); } - let history_map = map_state.history_map.as_mut_slice(); - if self.indexes { + let history_map = &mut map_state.history_map; + if C::INDICES { let mut indices = Vec::new(); for (i, value) in observer .as_iter() - .copied() + .map(|x| *x) .enumerate() .filter(|(_, value)| *value != initial) { + if history_map[i] == initial { + map_state.num_covered_map_indexes += 1; + } history_map[i] = R::reduce(history_map[i], value); indices.push(i); } @@ -479,31 +490,65 @@ where } else { for (i, value) in observer .as_iter() - .copied() + .map(|x| *x) .enumerate() .filter(|(_, value)| *value != initial) { + if history_map[i] == initial { + map_state.num_covered_map_indexes += 1; + } history_map[i] = R::reduce(history_map[i], value); } } + + debug_assert!( + history_map + .iter() + .fold(0, |acc, x| acc + usize::from(*x != initial)) + == map_state.num_covered_map_indexes, + "history_map had {} filled, but map_state.num_covered_map_indexes was {}", + history_map + .iter() + .fold(0, |acc, x| acc + usize::from(*x != initial)), + map_state.num_covered_map_indexes, + ); + + // at this point you are executing this code, the testcase is always interesting + let covered = map_state.num_covered_map_indexes; + let len = history_map.len(); + // opt: if not tracking optimisations, we technically don't show the *current* history + // map but the *last* history map; this is better than walking over and allocating + // unnecessarily + manager.fire( + state, + Event::UpdateUserStats { + name: self.stats_name.clone(), + value: UserStats::new( + UserStatsValue::Ratio(covered as u64, len as u64), + AggregatorOps::Avg, + ), + phantom: PhantomData, + }, + )?; + Ok(()) } } /// Specialize for the common coverage map size, maximization of u8s #[rustversion::nightly] -impl Feedback for MapFeedback +impl Feedback for MapFeedback where - O: MapObserver + AsSlice, - for<'it> O: AsIter<'it, Item = u8>, + O: MapObserver + for<'a> AsSlice<'a, Entry = u8> + for<'a> AsIter<'a, Item = u8>, S: State + HasNamedMetadata, + C: CanTrack + AsRef + Observer, { #[allow(clippy::wrong_self_convention)] #[allow(clippy::needless_range_loop)] fn is_interesting( &mut self, state: &mut S, - manager: &mut EM, + _manager: &mut EM, _input: &S::Input, observers: &OT, _exit_kind: &ExitKind, @@ -517,7 +562,7 @@ where let mut interesting = false; // TODO Replace with match_name_type when stable - let observer = observers.match_name::(&self.observer_name).unwrap(); + let observer = observers.get(&self.map_ref).unwrap().as_ref(); let map_state = state .named_metadata_map_mut() @@ -604,151 +649,71 @@ where } } - let initial = observer.initial(); - if interesting { - let len = history_map.len(); - let filled = history_map.iter().filter(|&&i| i != initial).count(); - // opt: if not tracking optimisations, we technically don't show the *current* history - // map but the *last* history map; this is better than walking over and allocating - // unnecessarily - manager.fire( - state, - Event::UpdateUserStats { - name: self.stats_name.to_string(), - value: UserStats::new( - UserStatsValue::Ratio( - self.novelties - .as_ref() - .map_or(filled, |novelties| filled + novelties.len()) - as u64, - len as u64, - ), - AggregatorOps::Avg, - ), - phantom: PhantomData, - }, - )?; - } - Ok(interesting) } } -impl Named for MapFeedback { +impl Named for MapFeedback { #[inline] - fn name(&self) -> &str { - self.name.as_str() + fn name(&self) -> &Cow<'static, str> { + &self.name } } -impl HasObserverName for MapFeedback +impl HasObserverReference for MapFeedback where - T: PartialEq + Default + Copy + 'static + Serialize + DeserializeOwned + Debug, - R: Reducer, - N: IsNovel, - O: MapObserver, - for<'it> O: AsIter<'it, Item = T>, - S: HasNamedMetadata, + O: Named, + C: AsRef, { + type Observer = C; + #[inline] - fn observer_name(&self) -> &str { - self.observer_name.as_str() + fn observer_ref(&self) -> &Handle { + &self.map_ref } } -fn create_stats_name(name: &str) -> String { - name.to_lowercase() +#[allow(clippy::ptr_arg)] +fn create_stats_name(name: &Cow<'static, str>) -> Cow<'static, str> { + if name.chars().all(char::is_lowercase) { + name.clone() + } else { + name.to_lowercase().into() + } } -impl MapFeedback +impl MapFeedback where T: PartialEq + Default + Copy + 'static + Serialize + DeserializeOwned + Debug, R: Reducer, O: MapObserver, for<'it> O: AsIter<'it, Item = T>, N: IsNovel, - S: UsesInput + HasNamedMetadata, + C: CanTrack + AsRef + Named, { /// Create new `MapFeedback` #[must_use] - pub fn new(map_observer: &O) -> Self { + pub fn new(map_observer: &C) -> Self { Self { - indexes: false, - novelties: None, - name: MAPFEEDBACK_PREFIX.to_string() + map_observer.name(), - observer_name: map_observer.name().to_string(), + novelties: if C::NOVELTIES { Some(vec![]) } else { None }, + name: map_observer.name().clone(), + map_ref: map_observer.handle(), stats_name: create_stats_name(map_observer.name()), - always_track: false, phantom: PhantomData, } } - /// Create new `MapFeedback` specifying if it must track indexes of used entries and/or novelties - #[must_use] - pub fn tracking(map_observer: &O, track_indexes: bool, track_novelties: bool) -> Self { - Self { - indexes: track_indexes, - novelties: if track_novelties { Some(vec![]) } else { None }, - name: MAPFEEDBACK_PREFIX.to_string() + map_observer.name(), - observer_name: map_observer.name().to_string(), - stats_name: create_stats_name(map_observer.name()), - always_track: false, - phantom: PhantomData, - } - } - - /// Create new `MapFeedback` - #[must_use] - pub fn with_names(name: &'static str, observer_name: &'static str) -> Self { - Self { - indexes: false, - novelties: None, - name: name.to_string(), - observer_name: observer_name.to_string(), - stats_name: create_stats_name(name), - phantom: PhantomData, - always_track: false, - } - } - - /// For tracking, enable `always_track` mode, that also adds `novelties` or `indexes`, - /// even if the map is not novel for this feedback. - /// This is useful in combination with `load_initial_inputs_forced`, or other feedbacks. - pub fn set_always_track(&mut self, always_track: bool) { - self.always_track = always_track; - } - /// Creating a new `MapFeedback` with a specific name. This is usefully whenever the same /// feedback is needed twice, but with a different history. Using `new()` always results in the /// same name and therefore also the same history. #[must_use] - pub fn with_name(name: &'static str, map_observer: &O) -> Self { + pub fn with_name(name: &'static str, map_observer: &C) -> Self { + let name = Cow::from(name); Self { - indexes: false, - novelties: None, - name: name.to_string(), - observer_name: map_observer.name().to_string(), - stats_name: create_stats_name(name), - always_track: false, - phantom: PhantomData, - } - } - - /// Create new `MapFeedback` specifying if it must track indexes of used entries and/or novelties - #[must_use] - pub fn with_names_tracking( - name: &'static str, - observer_name: &'static str, - track_indexes: bool, - track_novelties: bool, - ) -> Self { - Self { - indexes: track_indexes, - novelties: if track_novelties { Some(vec![]) } else { None }, - observer_name: observer_name.to_string(), - stats_name: create_stats_name(name), - name: name.to_string(), - always_track: false, + novelties: if C::NOVELTIES { Some(vec![]) } else { None }, + map_ref: map_observer.handle(), + stats_name: create_stats_name(&name), + name, phantom: PhantomData, } } @@ -756,21 +721,22 @@ where #[allow(clippy::wrong_self_convention)] #[allow(clippy::needless_range_loop)] #[allow(clippy::trivially_copy_pass_by_ref)] - fn is_interesting_default( + fn is_interesting_default( &mut self, state: &mut S, - manager: &mut EM, + _manager: &mut EM, _input: &S::Input, observers: &OT, _exit_kind: &ExitKind, - ) -> Result + ) -> bool where EM: EventFirer, OT: ObserversTuple, + S: UsesInput + HasNamedMetadata, { let mut interesting = false; // TODO Replace with match_name_type when stable - let observer = observers.match_name::(&self.observer_name).unwrap(); + let observer = observers.get(&self.map_ref).unwrap().as_ref(); let map_state = state .named_metadata_map_mut() @@ -789,7 +755,7 @@ where novelties.clear(); for (i, item) in observer .as_iter() - .copied() + .map(|x| *x) .enumerate() .filter(|(_, item)| *item != initial) { @@ -803,7 +769,7 @@ where } else { for (i, item) in observer .as_iter() - .copied() + .map(|x| *x) .enumerate() .filter(|(_, item)| *item != initial) { @@ -816,139 +782,7 @@ where } } - if interesting || self.always_track { - let len = history_map.len(); - let filled = history_map.iter().filter(|&&i| i != initial).count(); - // opt: if not tracking optimisations, we technically don't show the *current* history - // map but the *last* history map; this is better than walking over and allocating - // unnecessarily - manager.fire( - state, - Event::UpdateUserStats { - name: self.stats_name.to_string(), - value: UserStats::new( - UserStatsValue::Ratio( - self.novelties - .as_ref() - .map_or(filled, |novelties| filled + novelties.len()) - as u64, - len as u64, - ), - AggregatorOps::Avg, - ), - phantom: PhantomData, - }, - )?; - } - - Ok(interesting) - } -} - -/// A [`ReachabilityFeedback`] reports if a target has been reached. -#[derive(Clone, Debug)] -pub struct ReachabilityFeedback { - name: String, - target_idx: Vec, - phantom: PhantomData<(O, S)>, -} - -impl ReachabilityFeedback -where - O: MapObserver, - for<'it> O: AsIter<'it, Item = usize>, -{ - /// Creates a new [`ReachabilityFeedback`] for a [`MapObserver`]. - #[must_use] - pub fn new(map_observer: &O) -> Self { - Self { - name: map_observer.name().to_string(), - target_idx: vec![], - phantom: PhantomData, - } - } - - /// Creates a new [`ReachabilityFeedback`] for a [`MapObserver`] with the given `name`. - #[must_use] - pub fn with_name(name: &'static str) -> Self { - Self { - name: name.to_string(), - target_idx: vec![], - phantom: PhantomData, - } - } -} - -impl Feedback for ReachabilityFeedback -where - S: State, - O: MapObserver, - for<'it> O: AsIter<'it, Item = usize>, -{ - #[allow(clippy::wrong_self_convention)] - fn is_interesting( - &mut self, - _state: &mut S, - _manager: &mut EM, - _input: &S::Input, - observers: &OT, - _exit_kind: &ExitKind, - ) -> Result - where - EM: EventFirer, - OT: ObserversTuple, - { - // TODO Replace with match_name_type when stable - let observer = observers.match_name::(&self.name).unwrap(); - let mut hit_target: bool = false; - //check if we've hit any targets. - for (i, &elem) in observer.as_iter().enumerate() { - if elem > 0 { - self.target_idx.push(i); - hit_target = true; - } - } - if hit_target { - Ok(true) - } else { - Ok(false) - } - } - - fn append_metadata( - &mut self, - _state: &mut S, - _observers: &OT, - testcase: &mut Testcase, - ) -> Result<(), Error> - where - OT: ObserversTuple, - { - if !self.target_idx.is_empty() { - let meta = MapIndexesMetadata::new(core::mem::take(self.target_idx.as_mut())); - testcase.add_metadata(meta); - }; - Ok(()) - } - - fn discard_metadata( - &mut self, - _state: &mut S, - _input: &::Input, - ) -> Result<(), Error> { - self.target_idx.clear(); - Ok(()) - } -} - -impl Named for ReachabilityFeedback -where - O: MapObserver, - for<'it> O: AsIter<'it, Item = usize>, -{ - #[inline] - fn name(&self) -> &str { - self.name.as_str() + interesting } } @@ -976,127 +810,3 @@ mod tests { assert!(!NextPow2IsNovel::is_novel(255_u8, 255)); } } - -/// `MapFeedback` Python bindings -#[cfg(feature = "python")] -#[allow(missing_docs)] -pub mod pybind { - use concat_idents::concat_idents; - use pyo3::prelude::*; - - use super::{Debug, HasObserverName, MaxMapFeedback}; - use crate::{feedbacks::pybind::PythonFeedback, state::pybind::PythonStdState}; - - macro_rules! define_python_map_feedback { - ($struct_name:ident, $py_name:tt, $datatype:ty, $map_observer_type_name: ident, $my_std_state_type_name: ident) => { - use crate::observers::map::pybind::$map_observer_type_name; - - #[pyclass(unsendable, name = $py_name)] - #[derive(Debug, Clone)] - /// Python class for MaxMapFeedback - pub struct $struct_name { - /// Rust wrapped MaxMapFeedback object - pub inner: MaxMapFeedback< - $map_observer_type_name, /* PythonMapObserverI8 */ - $my_std_state_type_name, - $datatype, - >, - } - - #[pymethods] - impl $struct_name { - #[new] - fn new(observer: &$map_observer_type_name) -> Self { - Self { - inner: MaxMapFeedback::new(observer), - } - } - - #[must_use] - pub fn as_feedback(slf: Py) -> PythonFeedback { - concat_idents!(func = new_max_map_,$datatype { - PythonFeedback::func(slf) - }) - } - } - - impl HasObserverName for $struct_name { - fn observer_name(&self) -> &str { - self.inner.observer_name() - } - } - }; - } - - define_python_map_feedback!( - PythonMaxMapFeedbackI8, - "MaxMapFeedbackI8", - i8, - PythonMapObserverI8, - PythonStdState - ); - define_python_map_feedback!( - PythonMaxMapFeedbackI16, - "MaxMapFeedbackI16", - i16, - PythonMapObserverI16, - PythonStdState - ); - define_python_map_feedback!( - PythonMaxMapFeedbackI32, - "MaxMapFeedbackI32", - i32, - PythonMapObserverI32, - PythonStdState - ); - define_python_map_feedback!( - PythonMaxMapFeedbackI64, - "MaxMapFeedbackI64", - i64, - PythonMapObserverI64, - PythonStdState - ); - - define_python_map_feedback!( - PythonMaxMapFeedbackU8, - "MaxMapFeedbackU8", - u8, - PythonMapObserverU8, - PythonStdState - ); - define_python_map_feedback!( - PythonMaxMapFeedbackU16, - "MaxMapFeedbackU16", - u16, - PythonMapObserverU16, - PythonStdState - ); - define_python_map_feedback!( - PythonMaxMapFeedbackU32, - "MaxMapFeedbackU32", - u32, - PythonMapObserverU32, - PythonStdState - ); - define_python_map_feedback!( - PythonMaxMapFeedbackU64, - "MaxMapFeedbackU64", - u64, - PythonMapObserverU64, - PythonStdState - ); - - /// Register the classes to the python module - pub fn register(_py: Python, m: &PyModule) -> PyResult<()> { - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - Ok(()) - } -} diff --git a/libafl/src/feedbacks/mod.rs b/libafl/src/feedbacks/mod.rs index af57449d6d..83b8598982 100644 --- a/libafl/src/feedbacks/mod.rs +++ b/libafl/src/feedbacks/mod.rs @@ -4,45 +4,54 @@ // TODO: make S of Feedback an associated type when specialisation + AT is stable -pub mod map; -pub use map::*; - -pub mod differential; -pub use differential::DiffFeedback; -#[cfg(feature = "std")] -pub mod concolic; -#[cfg(feature = "std")] -pub use concolic::ConcolicFeedback; - -#[cfg(feature = "std")] -pub mod new_hash_feedback; -#[cfg(feature = "std")] -pub use new_hash_feedback::NewHashFeedback; -#[cfg(feature = "std")] -pub use new_hash_feedback::NewHashFeedbackMetadata; - -#[cfg(feature = "nautilus")] -pub mod nautilus; -use alloc::string::{String, ToString}; +use alloc::borrow::Cow; use core::{ fmt::{self, Debug, Formatter}, marker::PhantomData, }; -use libafl_bolts::Named; +#[cfg(feature = "std")] +pub use concolic::ConcolicFeedback; +pub use differential::DiffFeedback; +use libafl_bolts::{ + tuples::{Handle, Handler, MatchNameRef}, + Named, +}; +pub use list::*; +pub use map::*; #[cfg(feature = "nautilus")] pub use nautilus::*; +#[cfg(feature = "std")] +pub use new_hash_feedback::NewHashFeedback; +#[cfg(feature = "std")] +pub use new_hash_feedback::NewHashFeedbackMetadata; use serde::{Deserialize, Serialize}; use crate::{ corpus::Testcase, events::EventFirer, executors::ExitKind, - observers::{ListObserver, ObserversTuple, TimeObserver}, + observers::{ObserversTuple, TimeObserver}, state::State, Error, }; +pub mod map; + +#[cfg(feature = "std")] +pub mod concolic; +pub mod differential; +#[cfg(feature = "nautilus")] +pub mod nautilus; +#[cfg(feature = "std")] +pub mod new_hash_feedback; +#[cfg(feature = "std")] +pub mod stdio; +pub mod transferred; + +/// The module for list feedback +pub mod list; + /// Feedbacks evaluate the observers. /// Basically, they reduce the information provided by an observer to a value, /// indicating the "interestingness" of the last run. @@ -107,14 +116,16 @@ where /// Append to the testcase the generated metadata in case of a new corpus item #[inline] #[allow(unused_variables)] - fn append_metadata( + fn append_metadata( &mut self, state: &mut S, + manager: &mut EM, observers: &OT, testcase: &mut Testcase, ) -> Result<(), Error> where OT: ObserversTuple, + EM: EventFirer, { Ok(()) } @@ -127,9 +138,12 @@ where } /// Has an associated observer name (mostly used to retrieve the observer with `MatchName` from an `ObserverTuple`) -pub trait HasObserverName { +pub trait HasObserverReference { + /// The observer for which we hold a reference + type Observer: ?Sized; + /// The name associated with the observer - fn observer_name(&self) -> &str; + fn observer_ref(&self) -> &Handle; } /// A combined feedback consisting of multiple [`Feedback`]s @@ -145,7 +159,7 @@ where pub first: A, /// Second [`Feedback`] pub second: B, - name: String, + name: Cow<'static, str>, phantom: PhantomData<(S, FL)>, } @@ -156,8 +170,8 @@ where FL: FeedbackLogic, S: State, { - fn name(&self) -> &str { - self.name.as_ref() + fn name(&self) -> &Cow<'static, str> { + &self.name } } @@ -170,7 +184,12 @@ where { /// Create a new combined feedback pub fn new(first: A, second: B) -> Self { - let name = format!("{} ({},{})", FL::name(), first.name(), second.name()); + let name = Cow::from(format!( + "{} ({},{})", + FL::name(), + first.name(), + second.name() + )); Self { first, second, @@ -243,17 +262,21 @@ where } #[inline] - fn append_metadata( + fn append_metadata( &mut self, state: &mut S, + manager: &mut EM, observers: &OT, testcase: &mut Testcase, ) -> Result<(), Error> where OT: ObserversTuple, + EM: EventFirer, { - self.first.append_metadata(state, observers, testcase)?; - self.second.append_metadata(state, observers, testcase) + self.first + .append_metadata(state, manager, observers, testcase)?; + self.second + .append_metadata(state, manager, observers, testcase) } #[inline] @@ -263,6 +286,22 @@ where } } +impl FeedbackFactory, S, T> + for CombinedFeedback +where + A: Feedback + FeedbackFactory, + B: Feedback + FeedbackFactory, + FL: FeedbackLogic, + S: State, +{ + fn create_feedback(&self, ctx: &T) -> CombinedFeedback { + CombinedFeedback::new( + self.first.create_feedback(ctx), + self.second.create_feedback(ctx), + ) + } +} + /// Logical combination of two feedbacks pub trait FeedbackLogic: 'static where @@ -612,7 +651,7 @@ where /// The feedback to invert pub first: A, /// The name - name: String, + name: Cow<'static, str>, phantom: PhantomData, } @@ -657,16 +696,19 @@ where } #[inline] - fn append_metadata( + fn append_metadata( &mut self, state: &mut S, + manager: &mut EM, observers: &OT, testcase: &mut Testcase, ) -> Result<(), Error> where OT: ObserversTuple, + EM: EventFirer, { - self.first.append_metadata(state, observers, testcase) + self.first + .append_metadata(state, manager, observers, testcase) } #[inline] @@ -681,7 +723,7 @@ where S: State, { #[inline] - fn name(&self) -> &str { + fn name(&self) -> &Cow<'static, str> { &self.name } } @@ -693,7 +735,7 @@ where { /// Creates a new [`NotFeedback`]. pub fn new(first: A) -> Self { - let name = format!("Not({})", first.name()); + let name = Cow::from(format!("Not({})", first.name())); Self { first, name, @@ -807,8 +849,9 @@ where impl Named for CrashFeedback { #[inline] - fn name(&self) -> &str { - "CrashFeedback" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("CrashFeedback"); + &NAME } } @@ -826,8 +869,11 @@ impl Default for CrashFeedback { } } -/// A feedback factory for crash feedbacks -pub type CrashFeedbackFactory = DefaultFeedbackFactory; +impl FeedbackFactory for CrashFeedback { + fn create_feedback(&self, _ctx: &T) -> CrashFeedback { + CrashFeedback::new() + } +} /// A [`TimeoutFeedback`] reduces the timeout value of a run. #[derive(Serialize, Deserialize, Clone, Debug)] @@ -860,8 +906,9 @@ where impl Named for TimeoutFeedback { #[inline] - fn name(&self) -> &str { - "TimeoutFeedback" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("TimeoutFeedback"); + &NAME } } @@ -887,7 +934,7 @@ pub type TimeoutFeedbackFactory = DefaultFeedbackFactory; /// It decides, if the given [`TimeObserver`] value of a run is interesting. #[derive(Serialize, Deserialize, Clone, Debug)] pub struct TimeFeedback { - name: String, + obs_ref: Handle, } impl Feedback for TimeFeedback @@ -913,16 +960,18 @@ where /// Append to the testcase the generated metadata in case of a new corpus item #[inline] - fn append_metadata( + fn append_metadata( &mut self, _state: &mut S, + _manager: &mut EM, observers: &OT, testcase: &mut Testcase, ) -> Result<(), Error> where OT: ObserversTuple, + EM: EventFirer, { - let observer = observers.match_name::(self.name()).unwrap(); + let observer = observers.get(&self.obs_ref).unwrap(); *testcase.exec_time_mut() = *observer.last_runtime(); Ok(()) } @@ -936,98 +985,17 @@ where impl Named for TimeFeedback { #[inline] - fn name(&self) -> &str { - self.name.as_str() + fn name(&self) -> &Cow<'static, str> { + self.obs_ref.name() } } impl TimeFeedback { - /// Creates a new [`TimeFeedback`], deciding if the value of a [`TimeObserver`] with the given `name` of a run is interesting. - #[must_use] - pub fn new(name: &'static str) -> Self { - Self { - name: name.to_string(), - } - } - /// Creates a new [`TimeFeedback`], deciding if the given [`TimeObserver`] value of a run is interesting. #[must_use] - pub fn with_observer(observer: &TimeObserver) -> Self { + pub fn new(observer: &TimeObserver) -> Self { Self { - name: observer.name().to_string(), - } - } -} - -/// Consider interesting a testcase if the list in `ListObserver` is not empty. -#[derive(Serialize, Deserialize, Clone, Debug)] -pub struct ListFeedback -where - T: Debug + Serialize + serde::de::DeserializeOwned, -{ - name: String, - last_addr: usize, - phantom: PhantomData, -} - -impl Feedback for ListFeedback -where - S: State, - T: Debug + Serialize + serde::de::DeserializeOwned, -{ - #[allow(clippy::wrong_self_convention)] - fn is_interesting( - &mut self, - _state: &mut S, - _manager: &mut EM, - _input: &S::Input, - observers: &OT, - _exit_kind: &ExitKind, - ) -> Result - where - EM: EventFirer, - OT: ObserversTuple, - { - // TODO Replace with match_name_type when stable - let observer = observers - .match_name::>(self.name()) - .unwrap(); - // TODO register the list content in a testcase metadata - Ok(!observer.list().is_empty()) - } -} - -impl Named for ListFeedback -where - T: Debug + Serialize + serde::de::DeserializeOwned, -{ - #[inline] - fn name(&self) -> &str { - self.name.as_str() - } -} - -impl ListFeedback -where - T: Debug + Serialize + serde::de::DeserializeOwned, -{ - /// Creates a new [`ListFeedback`], deciding if the value of a [`ListObserver`] with the given `name` of a run is interesting. - #[must_use] - pub fn new(name: &'static str) -> Self { - Self { - name: name.to_string(), - last_addr: 0, - phantom: PhantomData, - } - } - - /// Creates a new [`TimeFeedback`], deciding if the given [`ListObserver`] value of a run is interesting. - #[must_use] - pub fn with_observer(observer: &ListObserver) -> Self { - Self { - name: observer.name().to_string(), - last_addr: 0, - phantom: PhantomData, + obs_ref: observer.handle(), } } } @@ -1069,8 +1037,9 @@ where impl Named for ConstFeedback { #[inline] - fn name(&self) -> &str { - "ConstFeedback" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("ConstFeedback"); + &NAME } } @@ -1091,600 +1060,3 @@ impl From for ConstFeedback { } } } - -/// `Feedback` Python bindings -#[cfg(feature = "python")] -#[allow(clippy::unnecessary_fallible_conversions)] -#[allow(missing_docs)] -pub mod pybind { - use std::cell::UnsafeCell; - - use libafl_bolts::Named; - use pyo3::prelude::*; - - use super::{ - ConstFeedback, CrashFeedback, Debug, EagerAndFeedback, EagerOrFeedback, FastAndFeedback, - FastOrFeedback, Feedback, NotFeedback, String, ToString, - }; - use crate::{ - corpus::{testcase::pybind::PythonTestcaseWrapper, Testcase}, - events::{pybind::PythonEventManager, EventFirer}, - executors::{pybind::PythonExitKind, ExitKind}, - feedbacks::map::pybind::{ - PythonMaxMapFeedbackI16, PythonMaxMapFeedbackI32, PythonMaxMapFeedbackI64, - PythonMaxMapFeedbackI8, PythonMaxMapFeedbackU16, PythonMaxMapFeedbackU32, - PythonMaxMapFeedbackU64, PythonMaxMapFeedbackU8, - }, - inputs::{BytesInput, HasBytesVec}, - observers::{pybind::PythonObserversTuple, ObserversTuple}, - state::pybind::{PythonStdState, PythonStdStateWrapper}, - Error, - }; - - #[derive(Debug)] - pub struct PyObjectFeedback { - inner: PyObject, - name: UnsafeCell, - } - - impl Clone for PyObjectFeedback { - fn clone(&self) -> PyObjectFeedback { - PyObjectFeedback { - inner: self.inner.clone(), - name: UnsafeCell::new(String::new()), - } - } - } - - impl PyObjectFeedback { - #[must_use] - pub fn new(obj: PyObject) -> Self { - PyObjectFeedback { - inner: obj, - name: UnsafeCell::new(String::new()), - } - } - } - - // crate::impl_serde_pyobjectwrapper!(PyObjectObserver, inner); - - impl Named for PyObjectFeedback { - fn name(&self) -> &str { - let s = Python::with_gil(|py| -> PyResult { - let s: String = self.inner.call_method0(py, "name")?.extract(py)?; - Ok(s) - }) - .unwrap(); - unsafe { - *self.name.get() = s; - &*self.name.get() - } - } - } - - impl Feedback for PyObjectFeedback { - fn init_state(&mut self, state: &mut PythonStdState) -> Result<(), Error> { - Python::with_gil(|py| -> PyResult<()> { - self.inner - .call_method1(py, "init_state", (PythonStdStateWrapper::wrap(state),))?; - Ok(()) - })?; - Ok(()) - } - - fn is_interesting( - &mut self, - state: &mut PythonStdState, - manager: &mut EM, - input: &BytesInput, - observers: &OT, - exit_kind: &ExitKind, - ) -> Result - where - EM: EventFirer, - OT: ObserversTuple, - { - // # Safety - // We use this observer in Python ony when the ObserverTuple is PythonObserversTuple - let dont_look_at_this: &PythonObserversTuple = - unsafe { &*(core::ptr::from_ref::(observers) as *const PythonObserversTuple) }; - let dont_look_at_this2: &PythonEventManager = - unsafe { &*(core::ptr::from_mut::(manager) as *const PythonEventManager) }; - Ok(Python::with_gil(|py| -> PyResult { - let r: bool = self - .inner - .call_method1( - py, - "is_interesting", - ( - PythonStdStateWrapper::wrap(state), - dont_look_at_this2.clone(), - input.bytes(), - dont_look_at_this.clone(), - PythonExitKind::from(*exit_kind), - ), - )? - .extract(py)?; - Ok(r) - })?) - } - - fn append_metadata( - &mut self, - state: &mut PythonStdState, - observers: &OT, - testcase: &mut Testcase, - ) -> Result<(), Error> - where - OT: ObserversTuple, - { - // # Safety - // We use this observer in Python ony when the ObserverTuple is PythonObserversTuple - let dont_look_at_this: &PythonObserversTuple = - unsafe { &*(core::ptr::from_ref::(observers) as *const PythonObserversTuple) }; - Python::with_gil(|py| -> PyResult<()> { - self.inner.call_method1( - py, - "append_metadata", - ( - PythonStdStateWrapper::wrap(state), - dont_look_at_this.clone(), - PythonTestcaseWrapper::wrap(testcase), - ), - )?; - Ok(()) - })?; - Ok(()) - } - - fn discard_metadata( - &mut self, - state: &mut PythonStdState, - input: &BytesInput, - ) -> Result<(), Error> { - Python::with_gil(|py| -> PyResult<()> { - self.inner.call_method1( - py, - "discard_metadata", - (PythonStdStateWrapper::wrap(state), input.bytes()), - )?; - Ok(()) - })?; - Ok(()) - } - } - - #[derive(Clone, Debug)] - #[pyclass(unsendable, name = "CrashFeedback")] - pub struct PythonCrashFeedback { - pub inner: CrashFeedback, - } - - #[pymethods] - impl PythonCrashFeedback { - #[new] - fn new() -> Self { - Self { - inner: CrashFeedback::new(), - } - } - - #[must_use] - pub fn as_feedback(slf: Py) -> PythonFeedback { - PythonFeedback::new_crash(slf) - } - } - - #[derive(Clone, Debug)] - #[pyclass(unsendable, name = "ConstFeedback")] - pub struct PythonConstFeedback { - pub inner: ConstFeedback, - } - - #[pymethods] - impl PythonConstFeedback { - #[new] - fn new(v: bool) -> Self { - Self { - inner: ConstFeedback::new(v), - } - } - - #[must_use] - pub fn as_feedback(slf: Py) -> PythonFeedback { - PythonFeedback::new_const(slf) - } - } - - #[derive(Debug)] - #[pyclass(unsendable, name = "NotFeedback")] - pub struct PythonNotFeedback { - pub inner: NotFeedback, - } - - #[pymethods] - impl PythonNotFeedback { - #[new] - fn new(feedback: PythonFeedback) -> Self { - Self { - inner: NotFeedback::new(feedback), - } - } - - #[must_use] - pub fn as_feedback(slf: Py) -> PythonFeedback { - PythonFeedback::new_not(slf) - } - } - - macro_rules! define_combined { - ($feed:ident, $pyname:ident, $pystring:tt, $method:ident) => { - #[derive(Debug)] - #[pyclass(unsendable, name = $pystring)] - pub struct $pyname { - pub inner: $feed, - } - - #[pymethods] - impl $pyname { - #[new] - fn new(a: PythonFeedback, b: PythonFeedback) -> Self { - Self { - inner: $feed::new(a, b), - } - } - - #[must_use] - pub fn as_feedback(slf: Py) -> PythonFeedback { - PythonFeedback::$method(slf) - } - } - }; - } - - define_combined!( - EagerAndFeedback, - PythonEagerAndFeedback, - "EagerAndFeedback", - new_and - ); - define_combined!( - FastAndFeedback, - PythonFastAndFeedback, - "FastAndFeedback", - new_fast_and - ); - define_combined!( - EagerOrFeedback, - PythonEagerOrFeedback, - "EagerOrFeedback", - new_or - ); - define_combined!( - FastOrFeedback, - PythonFastOrFeedback, - "FastOrFeedback", - new_fast_or - ); - - #[derive(Clone, Debug)] - pub enum PythonFeedbackWrapper { - MaxMapI8(Py), - MaxMapI16(Py), - MaxMapI32(Py), - MaxMapI64(Py), - MaxMapU8(Py), - MaxMapU16(Py), - MaxMapU32(Py), - MaxMapU64(Py), - Crash(Py), - Const(Py), - Not(Py), - And(Py), - FastAnd(Py), - Or(Py), - FastOr(Py), - Python(PyObjectFeedback), - } - - #[pyclass(unsendable, name = "Feedback")] - #[derive(Debug)] - /// Observer Trait binding - pub struct PythonFeedback { - pub wrapper: PythonFeedbackWrapper, - name: UnsafeCell, - } - - macro_rules! unwrap_me { - ($wrapper:expr, $name:ident, $body:block) => { - libafl_bolts::unwrap_me_body!($wrapper, $name, $body, PythonFeedbackWrapper, - { - MaxMapI8, - MaxMapI16, - MaxMapI32, - MaxMapI64, - MaxMapU8, - MaxMapU16, - MaxMapU32, - MaxMapU64, - Crash, - Const, - Not, - And, - FastAnd, - Or, - FastOr - }, - { - Python(py_wrapper) => { - let $name = py_wrapper; - $body - } - } - ) - }; - } - - macro_rules! unwrap_me_mut { - ($wrapper:expr, $name:ident, $body:block) => { - libafl_bolts::unwrap_me_mut_body!($wrapper, $name, $body, PythonFeedbackWrapper, - { - MaxMapI8, - MaxMapI16, - MaxMapI32, - MaxMapI64, - MaxMapU8, - MaxMapU16, - MaxMapU32, - MaxMapU64, - Crash, - Const, - Not, - And, - FastAnd, - Or, - FastOr - }, - { - Python(py_wrapper) => { - let $name = py_wrapper; - $body - } - } - ) - }; - } - - impl Clone for PythonFeedback { - fn clone(&self) -> PythonFeedback { - PythonFeedback { - wrapper: self.wrapper.clone(), - name: UnsafeCell::new(String::new()), - } - } - } - - #[pymethods] - impl PythonFeedback { - #[staticmethod] - #[must_use] - pub fn new_max_map_i8(map_feedback: Py) -> Self { - Self { - wrapper: PythonFeedbackWrapper::MaxMapI8(map_feedback), - name: UnsafeCell::new(String::new()), - } - } - - #[staticmethod] - #[must_use] - pub fn new_max_map_i16(map_feedback: Py) -> Self { - Self { - wrapper: PythonFeedbackWrapper::MaxMapI16(map_feedback), - name: UnsafeCell::new(String::new()), - } - } - - #[staticmethod] - #[must_use] - pub fn new_max_map_i32(map_feedback: Py) -> Self { - Self { - wrapper: PythonFeedbackWrapper::MaxMapI32(map_feedback), - name: UnsafeCell::new(String::new()), - } - } - - #[staticmethod] - #[must_use] - pub fn new_max_map_i64(map_feedback: Py) -> Self { - Self { - wrapper: PythonFeedbackWrapper::MaxMapI64(map_feedback), - name: UnsafeCell::new(String::new()), - } - } - - #[staticmethod] - #[must_use] - pub fn new_max_map_u8(map_feedback: Py) -> Self { - Self { - wrapper: PythonFeedbackWrapper::MaxMapU8(map_feedback), - name: UnsafeCell::new(String::new()), - } - } - - #[staticmethod] - #[must_use] - pub fn new_max_map_u16(map_feedback: Py) -> Self { - Self { - wrapper: PythonFeedbackWrapper::MaxMapU16(map_feedback), - name: UnsafeCell::new(String::new()), - } - } - - #[staticmethod] - #[must_use] - pub fn new_max_map_u32(map_feedback: Py) -> Self { - Self { - wrapper: PythonFeedbackWrapper::MaxMapU32(map_feedback), - name: UnsafeCell::new(String::new()), - } - } - - #[staticmethod] - #[must_use] - pub fn new_max_map_u64(map_feedback: Py) -> Self { - Self { - wrapper: PythonFeedbackWrapper::MaxMapU64(map_feedback), - name: UnsafeCell::new(String::new()), - } - } - - #[staticmethod] - #[must_use] - pub fn new_crash(feedback: Py) -> Self { - Self { - wrapper: PythonFeedbackWrapper::Crash(feedback), - name: UnsafeCell::new(String::new()), - } - } - - #[staticmethod] - #[must_use] - pub fn new_const(feedback: Py) -> Self { - Self { - wrapper: PythonFeedbackWrapper::Const(feedback), - name: UnsafeCell::new(String::new()), - } - } - - #[staticmethod] - #[must_use] - pub fn new_not(feedback: Py) -> Self { - Self { - wrapper: PythonFeedbackWrapper::Not(feedback), - name: UnsafeCell::new(String::new()), - } - } - - #[staticmethod] - #[must_use] - pub fn new_and(feedback: Py) -> Self { - Self { - wrapper: PythonFeedbackWrapper::And(feedback), - name: UnsafeCell::new(String::new()), - } - } - - #[staticmethod] - #[must_use] - pub fn new_fast_and(feedback: Py) -> Self { - Self { - wrapper: PythonFeedbackWrapper::FastAnd(feedback), - name: UnsafeCell::new(String::new()), - } - } - - #[staticmethod] - #[must_use] - pub fn new_or(feedback: Py) -> Self { - Self { - wrapper: PythonFeedbackWrapper::Or(feedback), - name: UnsafeCell::new(String::new()), - } - } - - #[staticmethod] - #[must_use] - pub fn new_fast_or(feedback: Py) -> Self { - Self { - wrapper: PythonFeedbackWrapper::FastOr(feedback), - name: UnsafeCell::new(String::new()), - } - } - - #[staticmethod] - #[must_use] - pub fn new_py(obj: PyObject) -> Self { - Self { - wrapper: PythonFeedbackWrapper::Python(PyObjectFeedback::new(obj)), - name: UnsafeCell::new(String::new()), - } - } - - pub fn unwrap_py(&self) -> Option { - match &self.wrapper { - PythonFeedbackWrapper::Python(pyo) => Some(pyo.inner.clone()), - _ => None, - } - } - } - - impl Named for PythonFeedback { - fn name(&self) -> &str { - let s = unwrap_me!(self.wrapper, f, { f.name().to_string() }); - unsafe { - *self.name.get() = s; - &*self.name.get() - } - } - } - - impl Feedback for PythonFeedback { - fn init_state(&mut self, state: &mut PythonStdState) -> Result<(), Error> { - unwrap_me_mut!(self.wrapper, f, { - Feedback::::init_state(f, state) - }) - } - - fn is_interesting( - &mut self, - state: &mut PythonStdState, - manager: &mut EM, - input: &BytesInput, - observers: &OT, - exit_kind: &ExitKind, - ) -> Result - where - EM: EventFirer, - OT: ObserversTuple, - { - unwrap_me_mut!(self.wrapper, f, { - f.is_interesting(state, manager, input, observers, exit_kind) - }) - } - - fn append_metadata( - &mut self, - state: &mut PythonStdState, - observers: &OT, - testcase: &mut Testcase, - ) -> Result<(), Error> - where - OT: ObserversTuple, - { - unwrap_me_mut!(self.wrapper, f, { - f.append_metadata(state, observers, testcase) - }) - } - - fn discard_metadata( - &mut self, - state: &mut PythonStdState, - input: &BytesInput, - ) -> Result<(), Error> { - unwrap_me_mut!(self.wrapper, f, { f.discard_metadata(state, input) }) - } - } - - /// Register the classes to the python module - pub fn register(_py: Python, m: &PyModule) -> PyResult<()> { - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - Ok(()) - } -} diff --git a/libafl/src/feedbacks/nautilus.rs b/libafl/src/feedbacks/nautilus.rs index bfde84436f..a2407577ce 100644 --- a/libafl/src/feedbacks/nautilus.rs +++ b/libafl/src/feedbacks/nautilus.rs @@ -1,12 +1,11 @@ //! Nautilus grammar mutator, see -use alloc::string::String; +use alloc::{borrow::Cow, string::String}; use core::{fmt::Debug, marker::PhantomData}; use std::fs::create_dir_all; use grammartec::{chunkstore::ChunkStore, context::Context}; use libafl_bolts::Named; use serde::{Deserialize, Serialize}; -use serde_json; use crate::{ corpus::{Corpus, Testcase}, @@ -16,8 +15,8 @@ use crate::{ generators::NautilusContext, inputs::NautilusInput, observers::ObserversTuple, - state::{HasCorpus, HasMetadata, State}, - Error, + state::{HasCorpus, State}, + Error, HasMetadata, }; /// Metadata for Nautilus grammar mutator chunks @@ -75,8 +74,9 @@ impl<'a, S> NautilusFeedback<'a, S> { } impl<'a, S> Named for NautilusFeedback<'a, S> { - fn name(&self) -> &str { - "NautilusFeedback" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("NautilusFeedback"); + &NAME } } @@ -100,9 +100,10 @@ where Ok(false) } - fn append_metadata( + fn append_metadata( &mut self, state: &mut S, + _manager: &mut EM, _observers: &OT, testcase: &mut Testcase, ) -> Result<(), Error> diff --git a/libafl/src/feedbacks/new_hash_feedback.rs b/libafl/src/feedbacks/new_hash_feedback.rs index 13b7f31742..4b90b18422 100644 --- a/libafl/src/feedbacks/new_hash_feedback.rs +++ b/libafl/src/feedbacks/new_hash_feedback.rs @@ -1,20 +1,23 @@ //! The ``NewHashFeedback`` uses the backtrace hash and a hashset to only keep novel cases -use alloc::string::{String, ToString}; +use alloc::{borrow::Cow, string::ToString}; use std::{fmt::Debug, marker::PhantomData}; use hashbrown::HashSet; -use libafl_bolts::Named; +use libafl_bolts::{ + tuples::{Handle, Handler, MatchNameRef}, + Named, +}; use serde::{Deserialize, Serialize}; use crate::{ events::EventFirer, executors::ExitKind, - feedbacks::{Feedback, HasObserverName}, + feedbacks::{Feedback, HasObserverReference}, inputs::UsesInput, observers::{ObserverWithHashField, ObserversTuple}, - state::{HasNamedMetadata, State}, - Error, + state::State, + Error, HasNamedMetadata, }; /// The prefix of the metadata names @@ -78,11 +81,11 @@ impl HashSetState for NewHashFeedbackMetadata { /// A [`NewHashFeedback`] maintains a hashset of already seen stacktraces and considers interesting unseen ones #[derive(Serialize, Deserialize, Clone, Debug)] pub struct NewHashFeedback { - name: String, - observer_name: String, + name: Cow<'static, str>, + o_ref: Handle, /// Initial capacity of hash set capacity: usize, - o_type: PhantomData<(O, S)>, + phantom: PhantomData, } impl Feedback for NewHashFeedback @@ -92,8 +95,8 @@ where { fn init_state(&mut self, state: &mut S) -> Result<(), Error> { state.add_named_metadata( - NewHashFeedbackMetadata::with_capacity(self.capacity), &self.name, + NewHashFeedbackMetadata::with_capacity(self.capacity), ); Ok(()) } @@ -112,7 +115,7 @@ where OT: ObserversTuple, { let observer = observers - .match_name::(&self.observer_name) + .get(&self.o_ref) .expect("A NewHashFeedback needs a BacktraceObserver"); let backtrace_state = state @@ -137,15 +140,17 @@ where impl Named for NewHashFeedback { #[inline] - fn name(&self) -> &str { + fn name(&self) -> &Cow<'static, str> { &self.name } } -impl HasObserverName for NewHashFeedback { +impl HasObserverReference for NewHashFeedback { + type Observer = O; + #[inline] - fn observer_name(&self) -> &str { - &self.observer_name + fn observer_ref(&self) -> &Handle { + &self.o_ref } } @@ -159,18 +164,6 @@ impl NewHashFeedback where O: ObserverWithHashField + Named, { - /// Returns a new [`NewHashFeedback`]. - /// Setting an observer name that doesn't exist would eventually trigger a panic. - #[must_use] - pub fn with_names(name: &str, observer_name: &str) -> Self { - Self { - name: name.to_string(), - observer_name: observer_name.to_string(), - capacity: DEFAULT_CAPACITY, - o_type: PhantomData, - } - } - /// Returns a new [`NewHashFeedback`]. #[must_use] pub fn new(observer: &O) -> Self { @@ -182,10 +175,10 @@ where #[must_use] pub fn with_capacity(observer: &O, capacity: usize) -> Self { Self { - name: NEWHASHFEEDBACK_PREFIX.to_string() + observer.name(), - observer_name: observer.name().to_string(), + name: Cow::from(NEWHASHFEEDBACK_PREFIX.to_string() + observer.name()), + o_ref: observer.handle(), capacity, - o_type: PhantomData, + phantom: PhantomData, } } } diff --git a/libafl/src/feedbacks/stdio.rs b/libafl/src/feedbacks/stdio.rs new file mode 100644 index 0000000000..bda7619421 --- /dev/null +++ b/libafl/src/feedbacks/stdio.rs @@ -0,0 +1,200 @@ +//! Feedback and metatadata for stderr and stdout. + +use alloc::{borrow::Cow, string::String}; + +use libafl_bolts::{ + impl_serdeany, + tuples::{Handle, Handler, MatchNameRef}, + Named, +}; +use serde::{Deserialize, Serialize}; + +use crate::{ + corpus::Testcase, + events::EventFirer, + executors::ExitKind, + feedbacks::Feedback, + observers::{ObserversTuple, StdErrObserver, StdOutObserver}, + state::State, + Error, HasMetadata, +}; + +/// Metadata for [`StdOutToMetadataFeedback`]. +#[derive(Debug, Serialize, Deserialize)] +pub struct StdOutMetadata { + #[allow(missing_docs)] + pub stdout: String, +} + +impl_serdeany!(StdOutMetadata); + +/// Nop feedback that annotates stdout in the new testcase. The testcase +/// is never interesting (use with an OR). +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct StdOutToMetadataFeedback { + o_ref: Handle, +} + +impl Feedback for StdOutToMetadataFeedback +where + S: State, +{ + #[allow(clippy::wrong_self_convention)] + #[inline] + fn is_interesting( + &mut self, + _state: &mut S, + _manager: &mut EM, + _input: &S::Input, + _observers: &OT, + _exit_kind: &ExitKind, + ) -> Result + where + EM: EventFirer, + OT: ObserversTuple, + { + Ok(false) + } + + /// Append to the testcase the generated metadata in case of a new corpus item. + #[inline] + fn append_metadata( + &mut self, + _state: &mut S, + _manager: &mut EM, + observers: &OT, + testcase: &mut Testcase, + ) -> Result<(), Error> + where + OT: ObserversTuple, + EM: EventFirer, + { + let observer = observers + .get(&self.o_ref) + .ok_or(Error::illegal_state("StdOutObserver is missing"))?; + let buffer = observer + .stdout + .as_ref() + .ok_or(Error::illegal_state("StdOutObserver has no stdout"))?; + let stdout = String::from_utf8_lossy(buffer).into_owned(); + + testcase + .metadata_map_mut() + .insert(StdOutMetadata { stdout }); + + Ok(()) + } + + /// Discard the stored metadata in case that the testcase is not added to the corpus. + #[inline] + fn discard_metadata(&mut self, _state: &mut S, _input: &S::Input) -> Result<(), Error> { + Ok(()) + } +} + +impl Named for StdOutToMetadataFeedback { + #[inline] + fn name(&self) -> &Cow<'static, str> { + self.o_ref.name() + } +} + +impl StdOutToMetadataFeedback { + /// Creates a new [`StdOutToMetadataFeedback`]. + #[must_use] + pub fn new(observer: &StdOutObserver) -> Self { + Self { + o_ref: observer.handle(), + } + } +} + +/// Metadata for [`StdErrToMetadataFeedback`]. +#[derive(Debug, Serialize, Deserialize)] +pub struct StdErrMetadata { + #[allow(missing_docs)] + pub stderr: String, +} + +impl_serdeany!(StdErrMetadata); + +/// Nop feedback that annotates stderr in the new testcase. The testcase +/// is never interesting (use with an OR). +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct StdErrToMetadataFeedback { + o_ref: Handle, +} + +impl Feedback for StdErrToMetadataFeedback +where + S: State, +{ + #[allow(clippy::wrong_self_convention)] + #[inline] + fn is_interesting( + &mut self, + _state: &mut S, + _manager: &mut EM, + _input: &S::Input, + _observers: &OT, + _exit_kind: &ExitKind, + ) -> Result + where + EM: EventFirer, + OT: ObserversTuple, + { + Ok(false) + } + + /// Append to the testcase the generated metadata in case of a new corpus item. + #[inline] + fn append_metadata( + &mut self, + _state: &mut S, + _manager: &mut EM, + observers: &OT, + testcase: &mut Testcase, + ) -> Result<(), Error> + where + OT: ObserversTuple, + EM: EventFirer, + { + let observer = observers + .get(&self.o_ref) + .ok_or(Error::illegal_state("StdErrObserver is missing"))?; + let buffer = observer + .stderr + .as_ref() + .ok_or(Error::illegal_state("StdErrObserver has no stderr"))?; + let stderr = String::from_utf8_lossy(buffer).into_owned(); + + testcase + .metadata_map_mut() + .insert(StdErrMetadata { stderr }); + + Ok(()) + } + + /// Discard the stored metadata in case that the testcase is not added to the corpus. + #[inline] + fn discard_metadata(&mut self, _state: &mut S, _input: &S::Input) -> Result<(), Error> { + Ok(()) + } +} + +impl Named for StdErrToMetadataFeedback { + #[inline] + fn name(&self) -> &Cow<'static, str> { + self.o_ref.name() + } +} + +impl StdErrToMetadataFeedback { + /// Creates a new [`StdErrToMetadataFeedback`]. + #[must_use] + pub fn new(observer: &StdErrObserver) -> Self { + Self { + o_ref: observer.handle(), + } + } +} diff --git a/libafl/src/feedbacks/transferred.rs b/libafl/src/feedbacks/transferred.rs new file mode 100644 index 0000000000..d6d2c36b5a --- /dev/null +++ b/libafl/src/feedbacks/transferred.rs @@ -0,0 +1,71 @@ +//! Feedbacks and associated metadata for detecting whether a given testcase was transferred from +//! another node. + +use alloc::borrow::Cow; + +use libafl_bolts::{impl_serdeany, Error, Named}; +use serde::{Deserialize, Serialize}; + +use crate::{ + events::EventFirer, executors::ExitKind, feedbacks::Feedback, observers::ObserversTuple, + state::State, HasMetadata, +}; + +/// Constant name of the [`TransferringMetadata`]. +pub const TRANSFERRED_FEEDBACK_NAME: Cow<'static, str> = + Cow::Borrowed("transferred_feedback_internal"); + +/// Metadata which denotes whether we are currently transferring an input. Implementors of +/// multi-node communication systems (like [`crate::events::LlmpEventManager`]) should wrap any +/// [`crate::EvaluatorObservers::evaluate_input_with_observers`] or +/// [`crate::ExecutionProcessor::process_execution`] calls with setting this metadata to true/false +/// before and after. +#[derive(Copy, Clone, Debug, Deserialize, Serialize)] +pub struct TransferringMetadata { + transferring: bool, +} + +impl_serdeany!(TransferringMetadata); + +impl TransferringMetadata { + /// Indicate to the metadata that we are currently transferring data. + pub fn set_transferring(&mut self, transferring: bool) { + self.transferring = transferring; + } +} + +/// Simple feedback which may be used to test whether the testcase was transferred from another node +/// in a multi-node fuzzing arrangement. +#[derive(Copy, Clone, Debug)] +pub struct TransferredFeedback; + +impl Named for TransferredFeedback { + fn name(&self) -> &Cow<'static, str> { + &TRANSFERRED_FEEDBACK_NAME + } +} + +impl Feedback for TransferredFeedback +where + S: HasMetadata + State, +{ + fn init_state(&mut self, state: &mut S) -> Result<(), Error> { + state.add_metadata(TransferringMetadata { transferring: true }); + Ok(()) + } + + fn is_interesting( + &mut self, + state: &mut S, + _manager: &mut EM, + _input: &S::Input, + _observers: &OT, + _exit_kind: &ExitKind, + ) -> Result + where + EM: EventFirer, + OT: ObserversTuple, + { + Ok(state.metadata::()?.transferring) + } +} diff --git a/libafl/src/fuzzer/mod.rs b/libafl/src/fuzzer/mod.rs index 3613e598a5..f55016573e 100644 --- a/libafl/src/fuzzer/mod.rs +++ b/libafl/src/fuzzer/mod.rs @@ -18,10 +18,10 @@ use crate::{ stages::{HasCurrentStage, StagesTuple}, start_timer, state::{ - HasCorpus, HasExecutions, HasImported, HasLastReportTime, HasMetadata, HasSolutions, + HasCorpus, HasCurrentTestcase, HasExecutions, HasImported, HasLastReportTime, HasSolutions, UsesState, }, - Error, + Error, HasMetadata, }; #[cfg(feature = "introspection")] use crate::{monitors::PerfFeature, state::HasClientPerfMonitor}; @@ -71,7 +71,34 @@ pub trait HasObjective: UsesState { /// Evaluates if an input is interesting using the feedback pub trait ExecutionProcessor: UsesState { /// Evaluate if a set of observation channels has an interesting state + fn execute_no_process( + &mut self, + state: &mut Self::State, + manager: &mut EM, + input: &::Input, + observers: &OT, + exit_kind: &ExitKind, + ) -> Result + where + EM: EventFirer; + + /// Process `ExecuteInputResult`. Add to corpus, solution or ignore + #[allow(clippy::too_many_arguments)] fn process_execution( + &mut self, + state: &mut Self::State, + manager: &mut EM, + input: ::Input, + exec_res: &ExecuteInputResult, + observers: &OT, + exit_kind: &ExitKind, + send_events: bool, + ) -> Result, Error> + where + EM: EventFirer; + + /// Evaluate if a set of observation channels has an interesting state + fn execute_and_process( &mut self, state: &mut Self::State, manager: &mut EM, @@ -143,6 +170,17 @@ where manager: &mut EM, input: ::Input, ) -> Result; + + /// Adds the input to the corpus as disabled a input. + /// Used during initial corpus loading. + /// Disabled testcases are only used for splicing + /// Returns the `index` of the new testcase in the corpus. + /// Usually, you want to use [`Evaluator::evaluate_input`], unless you know what you are doing. + fn add_disabled_input( + &mut self, + state: &mut Self::State, + input: ::Input, + ) -> Result; } /// The main fuzzer trait. @@ -180,6 +218,7 @@ where ) -> Result<(), Error> { let monitor_timeout = STATS_TIMEOUT_DEFAULT; loop { + // log::info!("Starting another fuzz_loop"); manager.maybe_report_progress(state, monitor_timeout)?; self.fuzz_one(stages, executor, state, manager)?; } @@ -212,6 +251,7 @@ where let monitor_timeout = STATS_TIMEOUT_DEFAULT; for _ in 0..iters { + // log::info!("Starting another fuzz_loop"); manager.maybe_report_progress(state, monitor_timeout)?; ret = Some(self.fuzz_one(stages, executor, state, manager)?); } @@ -323,18 +363,22 @@ where F: Feedback, OF: Feedback, OT: ObserversTuple + Serialize + DeserializeOwned, - CS::State: HasCorpus + HasSolutions + HasExecutions + HasCorpus + HasImported, + CS::State: HasCorpus + + HasSolutions + + HasExecutions + + HasCorpus + + HasImported + + HasCurrentTestcase<::Input> + + HasCurrentCorpusIdx, { - /// Evaluate if a set of observation channels has an interesting state - fn process_execution( + fn execute_no_process( &mut self, state: &mut Self::State, manager: &mut EM, - input: ::Input, + input: &::Input, observers: &OT, exit_kind: &ExitKind, - send_events: bool, - ) -> Result<(ExecuteInputResult, Option), Error> + ) -> Result where EM: EventFirer, { @@ -343,36 +387,77 @@ where #[cfg(not(feature = "introspection"))] let is_solution = self .objective_mut() - .is_interesting(state, manager, &input, observers, exit_kind)?; + .is_interesting(state, manager, input, observers, exit_kind)?; #[cfg(feature = "introspection")] let is_solution = self .objective_mut() - .is_interesting_introspection(state, manager, &input, observers, exit_kind)?; + .is_interesting_introspection(state, manager, input, observers, exit_kind)?; if is_solution { res = ExecuteInputResult::Solution; } else { #[cfg(not(feature = "introspection"))] - let is_corpus = self + let corpus_worthy = self .feedback_mut() - .is_interesting(state, manager, &input, observers, exit_kind)?; + .is_interesting(state, manager, input, observers, exit_kind)?; #[cfg(feature = "introspection")] - let is_corpus = self + let corpus_worthy = self .feedback_mut() - .is_interesting_introspection(state, manager, &input, observers, exit_kind)?; + .is_interesting_introspection(state, manager, input, observers, exit_kind)?; - if is_corpus { + if corpus_worthy { res = ExecuteInputResult::Corpus; } } + Ok(res) + } + + fn execute_and_process( + &mut self, + state: &mut Self::State, + manager: &mut EM, + input: ::Input, + observers: &OT, + exit_kind: &ExitKind, + send_events: bool, + ) -> Result<(ExecuteInputResult, Option), Error> + where + EM: EventFirer, + { + let exec_res = self.execute_no_process(state, manager, &input, observers, exit_kind)?; + let corpus_idx = self.process_execution( + state, + manager, + input, + &exec_res, + observers, + exit_kind, + send_events, + )?; + Ok((exec_res, corpus_idx)) + } - match res { + /// Evaluate if a set of observation channels has an interesting state + fn process_execution( + &mut self, + state: &mut Self::State, + manager: &mut EM, + input: ::Input, + exec_res: &ExecuteInputResult, + observers: &OT, + exit_kind: &ExitKind, + send_events: bool, + ) -> Result, Error> + where + EM: EventFirer, + { + match exec_res { ExecuteInputResult::None => { self.feedback_mut().discard_metadata(state, &input)?; self.objective_mut().discard_metadata(state, &input)?; - Ok((res, None)) + Ok(None) } ExecuteInputResult::Corpus => { // Not a solution @@ -381,7 +466,7 @@ where // Add the input to the main corpus let mut testcase = Testcase::with_executions(input.clone(), *state.executions()); self.feedback_mut() - .append_metadata(state, observers, &mut testcase)?; + .append_metadata(state, manager, observers, &mut testcase)?; let idx = state.corpus_mut().add(testcase)?; self.scheduler_mut().on_add(state, idx)?; @@ -409,17 +494,21 @@ where // This testcase is from the other fuzzers. *state.imported_mut() += 1; } - Ok((res, Some(idx))) + Ok(Some(idx)) } ExecuteInputResult::Solution => { // Not interesting self.feedback_mut().discard_metadata(state, &input)?; + let executions = *state.executions(); // The input is a solution, add it to the respective corpus - let mut testcase = Testcase::with_executions(input, *state.executions()); + let mut testcase = Testcase::with_executions(input, executions); testcase.set_parent_id_optional(*state.corpus().current()); + if let Ok(mut tc) = state.current_testcase_mut() { + tc.found_objective(); + } self.objective_mut() - .append_metadata(state, observers, &mut testcase)?; + .append_metadata(state, manager, observers, &mut testcase)?; state.solutions_mut().add(testcase)?; if send_events { @@ -427,11 +516,13 @@ where state, Event::Objective { objective_size: state.solutions().count(), + executions, + time: current_time(), }, )?; } - Ok((res, None)) + Ok(None) } } } @@ -462,9 +553,9 @@ where let exit_kind = self.execute_input(state, executor, manager, &input)?; let observers = executor.observers(); - self.scheduler.on_evaluation(state, &input, observers)?; + self.scheduler.on_evaluation(state, &input, &*observers)?; - self.process_execution(state, manager, input, observers, &exit_kind, send_events) + self.execute_and_process(state, manager, input, &*observers, &exit_kind, send_events) } } @@ -490,7 +581,17 @@ where ) -> Result<(ExecuteInputResult, Option), Error> { self.evaluate_input_with_observers(state, executor, manager, input, send_events) } - + fn add_disabled_input( + &mut self, + state: &mut Self::State, + input: ::Input, + ) -> Result { + let mut testcase = Testcase::with_executions(input.clone(), *state.executions()); + testcase.set_disabled(true); + // Add the disabled input to the main corpus + let idx = state.corpus_mut().add_disabled(testcase)?; + Ok(idx) + } /// Adds an input, even if it's not considered `interesting` by any of the executors fn add_input( &mut self, @@ -506,24 +607,31 @@ where // Maybe a solution #[cfg(not(feature = "introspection"))] - let is_solution = self - .objective_mut() - .is_interesting(state, manager, &input, observers, &exit_kind)?; + let is_solution = + self.objective_mut() + .is_interesting(state, manager, &input, &*observers, &exit_kind)?; #[cfg(feature = "introspection")] - let is_solution = self - .objective_mut() - .is_interesting_introspection(state, manager, &input, observers, &exit_kind)?; + let is_solution = self.objective_mut().is_interesting_introspection( + state, + manager, + &input, + &*observers, + &exit_kind, + )?; if is_solution { self.objective_mut() - .append_metadata(state, observers, &mut testcase)?; + .append_metadata(state, manager, &*observers, &mut testcase)?; let idx = state.solutions_mut().add(testcase)?; + let executions = *state.executions(); manager.fire( state, Event::Objective { objective_size: state.solutions().count(), + executions, + time: current_time(), }, )?; return Ok(idx); @@ -535,25 +643,29 @@ where // several is_interesting implementations collect some data about the run, later used in // append_metadata; we *must* invoke is_interesting here to collect it #[cfg(not(feature = "introspection"))] - let _is_corpus = self - .feedback_mut() - .is_interesting(state, manager, &input, observers, &exit_kind)?; + let _corpus_worthy = + self.feedback_mut() + .is_interesting(state, manager, &input, &*observers, &exit_kind)?; #[cfg(feature = "introspection")] - let _is_corpus = self - .feedback_mut() - .is_interesting_introspection(state, manager, &input, observers, &exit_kind)?; + let _corpus_worthy = self.feedback_mut().is_interesting_introspection( + state, + manager, + &input, + &*observers, + &exit_kind, + )?; // Add the input to the main corpus self.feedback_mut() - .append_metadata(state, observers, &mut testcase)?; + .append_metadata(state, manager, &*observers, &mut testcase)?; let idx = state.corpus_mut().add(testcase)?; self.scheduler_mut().on_add(state, idx)?; let observers_buf = if manager.configuration() == EventConfig::AlwaysUnique { None } else { - manager.serialize_observers::(observers)? + manager.serialize_observers::(&*observers)? }; manager.fire( state, @@ -754,8 +866,8 @@ pub mod test { corpus::CorpusId, events::ProgressReporter, stages::{HasCurrentStage, StagesTuple}, - state::{HasExecutions, HasLastReportTime, HasMetadata, State, UsesState}, - Fuzzer, + state::{HasExecutions, HasLastReportTime, State, UsesState}, + Fuzzer, HasMetadata, }; #[derive(Clone, Debug)] @@ -803,110 +915,3 @@ pub mod test { } } } - -#[cfg(feature = "python")] -#[allow(missing_docs)] -/// `Fuzzer` Python bindings -pub mod pybind { - use alloc::{boxed::Box, vec::Vec}; - - use libafl_bolts::ownedref::OwnedMutPtr; - use pyo3::prelude::*; - - use crate::{ - events::pybind::PythonEventManager, - executors::pybind::PythonExecutor, - feedbacks::pybind::PythonFeedback, - fuzzer::{Evaluator, Fuzzer, StdFuzzer}, - inputs::BytesInput, - observers::pybind::PythonObserversTuple, - schedulers::QueueScheduler, - stages::pybind::PythonStagesTuple, - state::pybind::{PythonStdState, PythonStdStateWrapper}, - }; - - /// `StdFuzzer` with fixed generics - pub type PythonStdFuzzer = StdFuzzer< - QueueScheduler, - PythonFeedback, - PythonFeedback, - PythonObserversTuple, - >; - - /// Python class for StdFuzzer - #[pyclass(unsendable, name = "StdFuzzer")] - #[derive(Debug)] - pub struct PythonStdFuzzerWrapper { - /// Rust wrapped StdFuzzer object - pub inner: OwnedMutPtr, - } - - impl PythonStdFuzzerWrapper { - pub fn wrap(r: &mut PythonStdFuzzer) -> Self { - Self { - inner: OwnedMutPtr::Ptr(r), - } - } - - #[must_use] - pub fn unwrap(&self) -> &PythonStdFuzzer { - self.inner.as_ref() - } - - pub fn unwrap_mut(&mut self) -> &mut PythonStdFuzzer { - self.inner.as_mut() - } - } - - #[pymethods] - impl PythonStdFuzzerWrapper { - #[new] - fn new(py_feedback: PythonFeedback, py_objective: PythonFeedback) -> Self { - Self { - inner: OwnedMutPtr::Owned(Box::new(StdFuzzer::new( - QueueScheduler::new(), - py_feedback, - py_objective, - ))), - } - } - - fn add_input( - &mut self, - py_state: &mut PythonStdStateWrapper, - py_executor: &mut PythonExecutor, - py_mgr: &mut PythonEventManager, - input: Vec, - ) -> usize { - self.inner - .as_mut() - .add_input( - py_state.unwrap_mut(), - py_executor, - py_mgr, - BytesInput::new(input), - ) - .expect("Failed to add input") - .0 - } - - fn fuzz_loop( - &mut self, - py_executor: &mut PythonExecutor, - py_state: &mut PythonStdStateWrapper, - py_mgr: &mut PythonEventManager, - stages_tuple: &mut PythonStagesTuple, - ) { - self.inner - .as_mut() - .fuzz_loop(stages_tuple, py_executor, py_state.unwrap_mut(), py_mgr) - .expect("Failed to generate the initial corpus"); - } - } - - /// Register the classes to the python module - pub fn register(_py: Python, m: &PyModule) -> PyResult<()> { - m.add_class::()?; - Ok(()) - } -} diff --git a/libafl/src/generators/gramatron.rs b/libafl/src/generators/gramatron.rs index edf5b87d5e..9bec0e4dd6 100644 --- a/libafl/src/generators/gramatron.rs +++ b/libafl/src/generators/gramatron.rs @@ -76,13 +76,13 @@ where .last() .map_or(self.automaton.init_state, |last| { let triggers = &self.automaton.pda[last.state]; - let idx = state.rand_mut().below(triggers.len() as u64) as usize; + let idx = state.rand_mut().below(triggers.len()); triggers[idx].dest }); while current_state != final_state { let triggers = &self.automaton.pda[current_state]; - let idx = state.rand_mut().below(triggers.len() as u64) as usize; + let idx = state.rand_mut().below(triggers.len()); let trigger = &triggers[idx]; input .terminals_mut() diff --git a/libafl/src/generators/mod.rs b/libafl/src/generators/mod.rs index 82cb0d6d4a..54c89adccd 100644 --- a/libafl/src/generators/mod.rs +++ b/libafl/src/generators/mod.rs @@ -101,7 +101,7 @@ where S: HasRand, { fn generate(&mut self, state: &mut S) -> Result { - let mut size = state.rand_mut().below(self.max_size as u64); + let mut size = state.rand_mut().below(self.max_size); if size == 0 { size = 1; } @@ -141,7 +141,7 @@ where S: HasRand, { fn generate(&mut self, state: &mut S) -> Result { - let mut size = state.rand_mut().below(self.max_size as u64); + let mut size = state.rand_mut().below(self.max_size); if size == 0 { size = 1; } @@ -166,179 +166,3 @@ where } } } - -/// `Generator` Python bindings -#[allow(missing_docs)] -#[cfg(feature = "python")] -#[allow(clippy::unnecessary_fallible_conversions)] -pub mod pybind { - use alloc::vec::Vec; - - use pyo3::prelude::*; - - use crate::{ - generators::{Generator, RandBytesGenerator, RandPrintablesGenerator}, - inputs::{BytesInput, HasBytesVec}, - state::pybind::{PythonStdState, PythonStdStateWrapper}, - Error, - }; - - #[derive(Clone, Debug)] - pub struct PyObjectGenerator { - inner: PyObject, - } - - impl PyObjectGenerator { - #[must_use] - pub fn new(obj: PyObject) -> Self { - PyObjectGenerator { inner: obj } - } - } - - impl Generator for PyObjectGenerator { - fn generate(&mut self, state: &mut PythonStdState) -> Result { - let bytes = Python::with_gil(|py| -> PyResult> { - self.inner - .call_method1(py, "generate", (PythonStdStateWrapper::wrap(state),))? - .extract(py) - }) - .unwrap(); - Ok(BytesInput::new(bytes)) - } - } - - #[pyclass(unsendable, name = "RandBytesGenerator")] - #[derive(Debug, Clone)] - /// Python class for RandBytesGenerator - pub struct PythonRandBytesGenerator { - /// Rust wrapped RandBytesGenerator object - pub inner: RandBytesGenerator, - } - - #[pymethods] - impl PythonRandBytesGenerator { - #[new] - fn new(max_size: usize) -> Self { - Self { - inner: RandBytesGenerator::new(max_size), - } - } - - fn generate(&mut self, state: &mut PythonStdStateWrapper) -> Vec { - self.inner - .generate(state.unwrap_mut()) - .expect("PythonRandBytesGenerator::generate failed") - .bytes() - .to_vec() - } - - fn as_generator(slf: Py) -> PythonGenerator { - PythonGenerator::new_rand_bytes(slf) - } - } - - #[pyclass(unsendable, name = "RandPrintablesGenerator")] - #[derive(Debug, Clone)] - /// Python class for RandPrintablesGenerator - pub struct PythonRandPrintablesGenerator { - /// Rust wrapped RandPrintablesGenerator object - pub inner: RandPrintablesGenerator, - } - - #[pymethods] - impl PythonRandPrintablesGenerator { - #[new] - fn new(max_size: usize) -> Self { - Self { - inner: RandPrintablesGenerator::new(max_size), - } - } - - fn generate(&mut self, state: &mut PythonStdStateWrapper) -> Vec { - self.inner - .generate(state.unwrap_mut()) - .expect("PythonRandPrintablesGenerator::generate failed") - .bytes() - .to_vec() - } - - fn as_generator(slf: Py) -> PythonGenerator { - PythonGenerator::new_rand_printables(slf) - } - } - - #[derive(Debug, Clone)] - enum PythonGeneratorWrapper { - RandBytes(Py), - RandPrintables(Py), - Python(PyObjectGenerator), - } - - /// Rand Trait binding - #[pyclass(unsendable, name = "Generator")] - #[derive(Debug, Clone)] - pub struct PythonGenerator { - wrapper: PythonGeneratorWrapper, - } - - macro_rules! unwrap_me_mut { - ($wrapper:expr, $name:ident, $body:block) => { - libafl_bolts::unwrap_me_mut_body!($wrapper, $name, $body, PythonGeneratorWrapper, - { RandBytes, RandPrintables }, - { - Python(py_wrapper) => { - let $name = py_wrapper; - $body - } - } - ) - }; - } - - #[pymethods] - impl PythonGenerator { - #[staticmethod] - fn new_rand_bytes(py_gen: Py) -> Self { - Self { - wrapper: PythonGeneratorWrapper::RandBytes(py_gen), - } - } - - #[staticmethod] - fn new_rand_printables(py_gen: Py) -> Self { - Self { - wrapper: PythonGeneratorWrapper::RandPrintables(py_gen), - } - } - - #[staticmethod] - #[must_use] - pub fn new_py(obj: PyObject) -> Self { - Self { - wrapper: PythonGeneratorWrapper::Python(PyObjectGenerator::new(obj)), - } - } - - #[must_use] - pub fn unwrap_py(&self) -> Option { - match &self.wrapper { - PythonGeneratorWrapper::Python(pyo) => Some(pyo.inner.clone()), - _ => None, - } - } - } - - impl Generator for PythonGenerator { - fn generate(&mut self, state: &mut PythonStdState) -> Result { - unwrap_me_mut!(self.wrapper, g, { g.generate(state) }) - } - } - - /// Register the classes to the python module - pub fn register(_py: Python, m: &PyModule) -> PyResult<()> { - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - Ok(()) - } -} diff --git a/libafl/src/generators/nautilus.rs b/libafl/src/generators/nautilus.rs index fb842000d5..0514d0493a 100644 --- a/libafl/src/generators/nautilus.rs +++ b/libafl/src/generators/nautilus.rs @@ -39,6 +39,47 @@ impl NautilusContext { Self { ctx } } + /// Returns a new [`NautilusContext`] with support for non UTF-8 rules. + /// + /// - This has the same behaviour as [`NautilusContext::new`] but returns `None` if the `rules` are empty. + /// - The starting rule is `rules[0]`. + /// + /// # Examples + /// + /// ``` + /// use libafl::generators::nautilus::NautilusContext; + /// + /// // Create a simple grammar for a series of null-terminated data. + /// let null = vec![0]; + /// let mut rules = vec![ + /// // rules[0] is considered the starting rule + /// ("SERIES", "{DATA}{NULL}".as_bytes()), + /// ("SERIES", "{SERIES}{SERIES}".as_bytes()), + /// ("DATA", "".as_bytes()), + /// ("DATA", "{BYTE}{DATA}".as_bytes()), + /// ("NULL", &null), + /// ]; + /// + /// let bytes: Vec = (1..=u8::MAX).collect(); + /// for i in 0..bytes.len() { + /// rules.push(("BYTE", &bytes[i..=i])); + /// } + /// + /// let context = NautilusContext::with_rules(100, &rules).unwrap(); + /// ``` + #[must_use] + pub fn with_rules(tree_depth: usize, rules: &[(&str, &[u8])]) -> Option { + let mut ctx = Context::new(); + for (symbol, rule) in rules { + ctx.add_rule(symbol, rule); + } + + let root = format!("{{{}}}", rules.first()?.0); + ctx.add_rule("START", root.as_bytes()); + ctx.initialize(tree_depth); + Some(Self { ctx }) + } + /// Create a new [`NautilusContext`] from a file #[must_use] pub fn from_file>(tree_depth: usize, grammar_file: P) -> Self { diff --git a/libafl/src/inputs/bytes.rs b/libafl/src/inputs/bytes.rs index a70bd40d14..bcb846bd09 100644 --- a/libafl/src/inputs/bytes.rs +++ b/libafl/src/inputs/bytes.rs @@ -4,7 +4,6 @@ use alloc::{borrow::ToOwned, rc::Rc, string::String, vec::Vec}; use core::{ cell::RefCell, - convert::From, hash::{BuildHasher, Hasher}, }; #[cfg(feature = "std")] diff --git a/libafl/src/inputs/encoded.rs b/libafl/src/inputs/encoded.rs index 7690527291..f5b9c9ba59 100644 --- a/libafl/src/inputs/encoded.rs +++ b/libafl/src/inputs/encoded.rs @@ -9,7 +9,6 @@ use alloc::{borrow::ToOwned, rc::Rc, string::String, vec::Vec}; use core::str::from_utf8; use core::{ cell::RefCell, - convert::From, hash::{BuildHasher, Hasher}, }; diff --git a/libafl/src/inputs/generalized.rs b/libafl/src/inputs/generalized.rs index 6eb4739f05..48c7e09b45 100644 --- a/libafl/src/inputs/generalized.rs +++ b/libafl/src/inputs/generalized.rs @@ -6,11 +6,11 @@ use libafl_bolts::impl_serdeany; use serde::{Deserialize, Serialize}; use crate::{ - corpus::{CorpusId, Testcase}, + corpus::Testcase, inputs::BytesInput, stages::mutational::{MutatedTransform, MutatedTransformPost}, - state::{HasCorpus, HasMetadata}, - Error, + state::HasCorpus, + Error, HasMetadata, }; /// An item of the generalized input @@ -111,17 +111,13 @@ where { type Post = Self; - fn try_transform_from( - base: &mut Testcase, - _state: &S, - corpus_idx: CorpusId, - ) -> Result { + fn try_transform_from(base: &mut Testcase, _state: &S) -> Result { let meta = base .metadata_map() .get::() .ok_or_else(|| { Error::key_not_found(format!( - "Couldn't find the GeneralizedInputMetadata for corpus entry {corpus_idx}", + "Couldn't find the GeneralizedInputMetadata for corpus entry {base:?}", )) }) .cloned()?; diff --git a/libafl/src/inputs/gramatron.rs b/libafl/src/inputs/gramatron.rs index f11cf2287f..1b1577b7ef 100644 --- a/libafl/src/inputs/gramatron.rs +++ b/libafl/src/inputs/gramatron.rs @@ -2,7 +2,6 @@ use alloc::{rc::Rc, string::String, vec::Vec}; use core::{ cell::RefCell, - convert::From, hash::{BuildHasher, Hasher}, }; diff --git a/libafl/src/inputs/nautilus.rs b/libafl/src/inputs/nautilus.rs index a1a3ef46cc..836efcfaf1 100644 --- a/libafl/src/inputs/nautilus.rs +++ b/libafl/src/inputs/nautilus.rs @@ -5,7 +5,7 @@ //use core::hash::Hasher; use alloc::{rc::Rc, string::String, vec::Vec}; -use core::{cell::RefCell, convert::From}; +use core::cell::RefCell; use std::hash::{Hash, Hasher}; use grammartec::{ diff --git a/libafl/src/lib.rs b/libafl/src/lib.rs index c3660ad9bb..9ad2d543fa 100644 --- a/libafl/src/lib.rs +++ b/libafl/src/lib.rs @@ -27,7 +27,9 @@ Welcome to `LibAFL` clippy::missing_docs_in_private_items, clippy::module_name_repetitions, clippy::ptr_cast_constness, - clippy::unsafe_derive_deserialize + clippy::unsafe_derive_deserialize, + clippy::similar_names, + clippy::too_many_lines )] #![cfg_attr(not(test), warn( missing_debug_implementations, @@ -96,6 +98,8 @@ pub mod bolts {} #[doc(hidden)] pub use libafl_derive::*; +pub mod common; +pub use common::*; pub mod corpus; pub mod events; pub mod executors; @@ -134,8 +138,12 @@ pub unsafe extern "C" fn external_current_millis() -> u64 { #[cfg(test)] mod tests { + #[cfg(miri)] + use libafl_bolts::serdeany::RegistryBuilder; use libafl_bolts::{rands::StdRand, tuples::tuple_list}; + #[cfg(miri)] + use crate::stages::ExecutionCountRestartHelperMetadata; use crate::{ corpus::{Corpus, InMemoryCorpus, Testcase}, events::NopEventManager, @@ -154,6 +162,13 @@ mod tests { #[test] #[allow(clippy::similar_names)] fn test_fuzzer() { + // # Safety + // No concurrency per testcase + #[cfg(miri)] + unsafe { + RegistryBuilder::register::(); + } + let rand = StdRand::with_seed(0); let mut corpus = InMemoryCorpus::::new(); @@ -220,58 +235,3 @@ mod tests { assert_eq!(state.corpus().count(), corpus_deserialized.count()); } } - -#[cfg(feature = "python")] -#[allow(missing_docs)] -pub mod pybind { - use pyo3::prelude::*; - - use super::{ - corpus, events, executors, feedbacks, fuzzer, generators, monitors, mutators, observers, - stages, state, - }; - - #[derive(Debug, Clone)] - pub struct PythonMetadata { - pub map: PyObject, - } - - libafl_bolts::impl_serde_pyobjectwrapper!(PythonMetadata, map); - libafl_bolts::impl_serdeany!(PythonMetadata); - - impl PythonMetadata { - #[must_use] - pub fn new(map: PyObject) -> Self { - Self { map } - } - } - - #[pymodule] - #[pyo3(name = "libafl")] - /// Register the classes to the python module - pub fn python_module(py: Python, m: &PyModule) -> PyResult<()> { - libafl_bolts::rands::pybind::register(py, m)?; - observers::map::pybind::register(py, m)?; - observers::pybind::register(py, m)?; - feedbacks::map::pybind::register(py, m)?; - feedbacks::pybind::register(py, m)?; - state::pybind::register(py, m)?; - monitors::pybind::register(py, m)?; - events::pybind::register(py, m)?; - events::simple::pybind::register(py, m)?; - fuzzer::pybind::register(py, m)?; - executors::pybind::register(py, m)?; - executors::inprocess::pybind::register(py, m)?; - generators::pybind::register(py, m)?; - mutators::pybind::register(py, m)?; - mutators::scheduled::pybind::register(py, m)?; - corpus::pybind::register(py, m)?; - corpus::testcase::pybind::register(py, m)?; - corpus::ondisk::pybind::register(py, m)?; - corpus::inmemory::pybind::register(py, m)?; - corpus::cached::pybind::register(py, m)?; - stages::pybind::register(py, m)?; - stages::mutational::pybind::register(py, m)?; - Ok(()) - } -} diff --git a/libafl/src/monitors/disk.rs b/libafl/src/monitors/disk.rs index d87e735a6c..6df195bc63 100644 --- a/libafl/src/monitors/disk.rs +++ b/libafl/src/monitors/disk.rs @@ -22,6 +22,7 @@ where base: M, filename: PathBuf, last_update: Duration, + update_interval: Duration, } impl Monitor for OnDiskTOMLMonitor @@ -52,10 +53,10 @@ where self.base.aggregate(name); } - fn display(&mut self, event_msg: String, sender_id: ClientId) { + fn display(&mut self, event_msg: &str, sender_id: ClientId) { let cur_time = current_time(); - if (cur_time - self.last_update).as_secs() >= 60 { + if cur_time - self.last_update >= self.update_interval { self.last_update = cur_time; let mut file = File::create(&self.filename).expect("Failed to open the TOML file"); @@ -72,7 +73,7 @@ executions = {} exec_sec = {} ", format_duration_hms(&(cur_time - self.start_time())), - self.client_stats().len(), + self.client_stats_count(), self.corpus_size(), self.objective_size(), self.total_execs(), @@ -80,7 +81,7 @@ exec_sec = {} ) .expect("Failed to write to the TOML file"); - for (i, client) in self.client_stats_mut().iter_mut().skip(1).enumerate() { + for (i, client) in self.client_stats_mut().iter_mut().enumerate() { let exec_sec = client.execs_per_sec(cur_time); write!( @@ -92,11 +93,7 @@ objectives = {} executions = {} exec_sec = {} ", - i + 1, - client.corpus_size, - client.objective_size, - client.executions, - exec_sec + i, client.corpus_size, client.objective_size, client.executions, exec_sec ) .expect("Failed to write to the TOML file"); @@ -125,13 +122,23 @@ where /// Create new [`OnDiskTOMLMonitor`] #[must_use] pub fn new

(filename: P, base: M) -> Self + where + P: Into, + { + Self::with_update_interval(filename, base, Duration::from_secs(60)) + } + + /// Create new [`OnDiskTOMLMonitor`] with custom update interval + #[must_use] + pub fn with_update_interval

(filename: P, base: M, update_interval: Duration) -> Self where P: Into, { Self { base, filename: filename.into(), - last_update: current_time(), + last_update: current_time() - update_interval, + update_interval, } } } @@ -201,7 +208,7 @@ where self.base.set_start_time(time); } - fn display(&mut self, event_msg: String, sender_id: ClientId) { + fn display(&mut self, event_msg: &str, sender_id: ClientId) { if (self.log_record)(&mut self.base) { let file = OpenOptions::new() .append(true) @@ -211,12 +218,11 @@ where let line = json!({ "run_time": current_time() - self.base.start_time(), - "clients": self.base.client_stats().len(), + "clients": self.client_stats_count(), "corpus": self.base.corpus_size(), "objectives": self.base.objective_size(), "executions": self.base.total_execs(), "exec_sec": self.base.execs_per_sec(), - "clients": &self.client_stats()[1..] }); writeln!(&file, "{line}").expect("Unable to write JSON to file"); } diff --git a/libafl/src/monitors/mod.rs b/libafl/src/monitors/mod.rs index e86ca7f8b8..5b3a3081d3 100644 --- a/libafl/src/monitors/mod.rs +++ b/libafl/src/monitors/mod.rs @@ -16,7 +16,7 @@ use alloc::string::ToString; pub use prometheus::PrometheusMonitor; #[cfg(feature = "std")] pub mod disk; -use alloc::{fmt::Debug, string::String, vec::Vec}; +use alloc::{borrow::Cow, fmt::Debug, string::String, vec::Vec}; use core::{fmt, fmt::Write, time::Duration}; #[cfg(feature = "std")] @@ -62,22 +62,20 @@ impl Aggregator { /// takes the key and the ref to clients stats then aggregate them all. fn aggregate(&mut self, name: &str, client_stats: &[ClientStats]) { - let mut gather = vec![]; + let mut gather = client_stats + .iter() + .filter_map(|client| client.user_monitor.get(name)); - for client in client_stats { - if let Some(x) = client.user_monitor.get(name) { - gather.push(x); - } - } + let gather_count = gather.clone().count(); - let (mut init, op) = match gather.first() { + let (mut init, op) = match gather.next() { Some(x) => (x.value().clone(), x.aggregator_op().clone()), _ => { return; } }; - for item in gather.iter().skip(1) { + for item in gather { match op { AggregatorOps::None => { // Nothing @@ -112,7 +110,7 @@ impl Aggregator { if let AggregatorOps::Avg = op { // if avg then divide last. - init = match init.stats_div(gather.len()) { + init = match init.stats_div(gather_count) { Some(x) => x, _ => { return; @@ -160,7 +158,7 @@ pub enum UserStatsValue { /// A Float value Float(f64), /// A `String` - String(String), + String(Cow<'static, str>), /// A ratio of two values Ratio(u64, u64), /// Percent @@ -340,6 +338,8 @@ fn prettify_float(value: f64) -> String { /// A simple struct to keep track of client monitor #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct ClientStats { + /// If this client is enabled. This is set to `true` the first time we see this client. + pub enabled: bool, // monitor (maybe we need a separated struct?) /// The corpus size for this client pub corpus_size: u64, @@ -364,7 +364,7 @@ pub struct ClientStats { /// the start time of the client pub start_time: Duration, /// User-defined monitor - pub user_monitor: HashMap, + pub user_monitor: HashMap, UserStats>, /// Client performance statistics #[cfg(feature = "introspection")] pub introspection_monitor: ClientPerfMonitor, @@ -465,7 +465,11 @@ impl ClientStats { } /// Update the user-defined stat with name and value - pub fn update_user_stats(&mut self, name: String, value: UserStats) -> Option { + pub fn update_user_stats( + &mut self, + name: Cow<'static, str>, + value: UserStats, + ) -> Option { self.user_monitor.insert(name, value) } @@ -497,7 +501,7 @@ pub trait Monitor { fn set_start_time(&mut self, time: Duration); /// Show the monitor to the user - fn display(&mut self, event_msg: String, sender_id: ClientId); + fn display(&mut self, event_msg: &str, sender_id: ClientId); /// Amount of elements in the corpus (combined for all children) fn corpus_size(&self) -> u64 { @@ -506,6 +510,14 @@ pub trait Monitor { .fold(0_u64, |acc, x| acc + x.corpus_size) } + /// Count the number of enabled client stats + fn client_stats_count(&self) -> usize { + self.client_stats() + .iter() + .filter(|client| client.enabled) + .count() + } + /// Amount of elements in the objectives (combined for all children) fn objective_size(&self) -> u64 { self.client_stats() @@ -538,14 +550,23 @@ pub trait Monitor { /// The client monitor for a specific id, creating new if it doesn't exist fn client_stats_insert(&mut self, client_id: ClientId) { - let client_stat_count = self.client_stats().len(); - for _ in client_stat_count..(client_id.0 + 1) as usize { + let total_client_stat_count = self.client_stats().len(); + for _ in total_client_stat_count..=(client_id.0) as usize { self.client_stats_mut().push(ClientStats { - last_window_time: current_time(), - start_time: current_time(), + enabled: false, + last_window_time: Duration::from_secs(0), + start_time: Duration::from_secs(0), ..ClientStats::default() }); } + let new_stat = self.client_stats_mut_for(client_id); + if !new_stat.enabled { + let timestamp = current_time(); + // I have never seen this man in my life + new_stat.start_time = timestamp; + new_stat.last_window_time = timestamp; + new_stat.enabled = true; + } } /// Get mutable reference to client stats @@ -591,7 +612,8 @@ impl Monitor for NopMonitor { self.start_time = time; } - fn display(&mut self, _event_msg: String, _sender_id: ClientId) {} + #[inline] + fn display(&mut self, _event_msg: &str, _sender_id: ClientId) {} } impl NopMonitor { @@ -660,7 +682,7 @@ impl Monitor for SimplePrintingMonitor { self.start_time = time; } - fn display(&mut self, event_msg: String, sender_id: ClientId) { + fn display(&mut self, event_msg: &str, sender_id: ClientId) { let mut userstats = self.client_stats()[sender_id.0 as usize] .user_monitor .iter() @@ -672,7 +694,7 @@ impl Monitor for SimplePrintingMonitor { event_msg, sender_id.0, format_duration_hms(&(current_time() - self.start_time)), - self.client_stats().len(), + self.client_stats_count(), self.corpus_size(), self.objective_size(), self.total_execs(), @@ -698,7 +720,7 @@ impl Monitor for SimplePrintingMonitor { #[derive(Clone)] pub struct SimpleMonitor where - F: FnMut(String), + F: FnMut(&str), { print_fn: F, start_time: Duration, @@ -708,7 +730,7 @@ where impl Debug for SimpleMonitor where - F: FnMut(String), + F: FnMut(&str), { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("SimpleMonitor") @@ -720,7 +742,7 @@ where impl Monitor for SimpleMonitor where - F: FnMut(String), + F: FnMut(&str), { /// the client monitor, mutable fn client_stats_mut(&mut self) -> &mut Vec { @@ -742,13 +764,13 @@ where self.start_time = time; } - fn display(&mut self, event_msg: String, sender_id: ClientId) { + fn display(&mut self, event_msg: &str, sender_id: ClientId) { let mut fmt = format!( "[{} #{}] run time: {}, clients: {}, corpus: {}, objectives: {}, executions: {}, exec/sec: {}", event_msg, sender_id.0, format_duration_hms(&(current_time() - self.start_time)), - self.client_stats().len(), + self.client_stats_count(), self.corpus_size(), self.objective_size(), self.total_execs(), @@ -763,7 +785,7 @@ where } } - (self.print_fn)(fmt); + (self.print_fn)(&fmt); // Only print perf monitor if the feature is enabled #[cfg(feature = "introspection")] @@ -773,17 +795,17 @@ where "Client {:03}:\n{}", sender_id.0, self.client_stats[sender_id.0 as usize].introspection_monitor ); - (self.print_fn)(fmt); + (self.print_fn)(&fmt); // Separate the spacing just a bit - (self.print_fn)(String::new()); + (self.print_fn)(""); } } } impl SimpleMonitor where - F: FnMut(String), + F: FnMut(&str), { /// Creates the monitor, using the `current_time` as `start_time`. pub fn new(print_fn: F) -> Self { @@ -806,11 +828,11 @@ where } /// Creates the monitor that also prints the user monitor - pub fn with_user_monitor(print_fn: F, print_user_monitor: bool) -> Self { + pub fn with_user_monitor(print_fn: F) -> Self { Self { print_fn, start_time: current_time(), - print_user_monitor, + print_user_monitor: true, client_stats: vec![], } } @@ -1210,9 +1232,9 @@ impl ClientPerfMonitor { } #[cfg(feature = "introspection")] -impl core::fmt::Display for ClientPerfMonitor { +impl fmt::Display for ClientPerfMonitor { #[allow(clippy::cast_precision_loss)] - fn fmt(&self, f: &mut core::fmt::Formatter) -> Result<(), core::fmt::Error> { + fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { // Calculate the elapsed time from the monitor let elapsed: f64 = self.elapsed_cycles() as f64; @@ -1288,174 +1310,3 @@ impl Default for ClientPerfMonitor { Self::new() } } -/// `Monitor` Python bindings -#[cfg(feature = "python")] -#[allow(clippy::unnecessary_fallible_conversions)] -#[allow(missing_docs)] -pub mod pybind { - use alloc::{boxed::Box, string::String, vec::Vec}; - use core::time::Duration; - - use libafl_bolts::ClientId; - use pyo3::{prelude::*, types::PyUnicode}; - - use super::ClientStats; - use crate::monitors::{Monitor, SimpleMonitor}; - - // TODO create a PyObjectFnMut to pass, track stabilization of https://github.com/rust-lang/rust/issues/29625 - - #[pyclass(unsendable, name = "SimpleMonitor")] - /// Python class for SimpleMonitor - pub struct PythonSimpleMonitor { - /// Rust wrapped SimpleMonitor object - pub inner: SimpleMonitor>, - print_fn: PyObject, - } - - impl std::fmt::Debug for PythonSimpleMonitor { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("PythonSimpleMonitor") - .field("print_fn", &self.print_fn) - .finish_non_exhaustive() - } - } - - impl Clone for PythonSimpleMonitor { - fn clone(&self) -> PythonSimpleMonitor { - let py_print_fn = self.print_fn.clone(); - let closure = move |s: String| { - Python::with_gil(|py| -> PyResult<()> { - py_print_fn.call1(py, (PyUnicode::new(py, &s),))?; - Ok(()) - }) - .unwrap(); - }; - - PythonSimpleMonitor { - inner: SimpleMonitor { - print_fn: Box::new(closure), - start_time: self.inner.start_time, - print_user_monitor: false, - client_stats: self.inner.client_stats.clone(), - }, - print_fn: self.print_fn.clone(), - } - } - } - - #[pymethods] - impl PythonSimpleMonitor { - #[new] - fn new(py_print_fn: PyObject) -> Self { - let py_print_fn1 = py_print_fn.clone(); - let closure = move |s: String| { - Python::with_gil(|py| -> PyResult<()> { - py_print_fn1.call1(py, (PyUnicode::new(py, &s),))?; - Ok(()) - }) - .unwrap(); - }; - Self { - inner: SimpleMonitor::new(Box::new(closure)), - print_fn: py_print_fn, - } - } - - #[must_use] - pub fn as_monitor(slf: Py) -> PythonMonitor { - PythonMonitor::new_simple(slf) - } - } - - #[derive(Clone, Debug)] - enum PythonMonitorWrapper { - Simple(Py), - } - - #[pyclass(unsendable, name = "Monitor")] - #[derive(Clone, Debug)] - /// EventManager Trait binding - pub struct PythonMonitor { - wrapper: PythonMonitorWrapper, - } - - macro_rules! unwrap_me { - ($wrapper:expr, $name:ident, $body:block) => { - libafl_bolts::unwrap_me_body!($wrapper, $name, $body, PythonMonitorWrapper, { Simple }) - }; - } - - macro_rules! unwrap_me_mut { - ($wrapper:expr, $name:ident, $body:block) => { - libafl_bolts::unwrap_me_mut_body!($wrapper, $name, $body, PythonMonitorWrapper, { - Simple - }) - }; - } - - #[pymethods] - impl PythonMonitor { - #[staticmethod] - #[must_use] - pub fn new_simple(simple_monitor: Py) -> Self { - Self { - wrapper: PythonMonitorWrapper::Simple(simple_monitor), - } - } - } - - impl Monitor for PythonMonitor { - fn client_stats_mut(&mut self) -> &mut Vec { - let ptr = unwrap_me_mut!(self.wrapper, m, { - core::ptr::from_mut::>(m.client_stats_mut()) - }); - unsafe { ptr.as_mut().unwrap() } - } - - fn client_stats(&self) -> &[ClientStats] { - let ptr = unwrap_me!(self.wrapper, m, { - core::ptr::from_ref::<[ClientStats]>(m.client_stats()) - }); - unsafe { ptr.as_ref().unwrap() } - } - - /// Time this fuzzing run stated - fn start_time(&self) -> Duration { - unwrap_me!(self.wrapper, m, { m.start_time() }) - } - - /// set start time - fn set_start_time(&mut self, time: Duration) { - unwrap_me_mut!(self.wrapper, m, { m.set_start_time(time) }); - } - - fn display(&mut self, event_msg: String, sender_id: ClientId) { - unwrap_me_mut!(self.wrapper, m, { m.display(event_msg, sender_id) }); - } - } - /// Register the classes to the python module - pub fn register(_py: Python, m: &PyModule) -> PyResult<()> { - m.add_class::()?; - m.add_class::()?; - Ok(()) - } -} - -#[cfg(test)] -mod test { - use crate::monitors::prettify_float; - #[test] - fn test_prettify_float() { - assert_eq!(prettify_float(123423123.0), "123.4M"); - assert_eq!(prettify_float(12342312.3), "12.34M"); - assert_eq!(prettify_float(1234231.23), "1.234M"); - assert_eq!(prettify_float(123423.123), "123.4k"); - assert_eq!(prettify_float(12342.3123), "12.34k"); - assert_eq!(prettify_float(1234.23123), "1.234k"); - assert_eq!(prettify_float(123.423123), "123.4"); - assert_eq!(prettify_float(12.3423123), "12.34"); - assert_eq!(prettify_float(1.23423123), "1.234"); - assert_eq!(prettify_float(0.123423123), "0.123"); - assert_eq!(prettify_float(0.0123423123), "0.012"); - } -} diff --git a/libafl/src/monitors/multi.rs b/libafl/src/monitors/multi.rs index 226b68431a..cf338781f3 100644 --- a/libafl/src/monitors/multi.rs +++ b/libafl/src/monitors/multi.rs @@ -1,7 +1,5 @@ //! Monitor to display both cumulative and per-client monitor -#[cfg(feature = "introspection")] -use alloc::string::ToString; use alloc::{string::String, vec::Vec}; use core::{ fmt::{Debug, Formatter, Write}, @@ -17,7 +15,7 @@ use crate::monitors::{ClientStats, Monitor}; #[derive(Clone)] pub struct MultiMonitor where - F: FnMut(String), + F: FnMut(&str), { print_fn: F, start_time: Duration, @@ -27,7 +25,7 @@ where impl Debug for MultiMonitor where - F: FnMut(String), + F: FnMut(&str), { fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { f.debug_struct("MultiMonitor") @@ -39,7 +37,7 @@ where impl Monitor for MultiMonitor where - F: FnMut(String), + F: FnMut(&str), { /// the client monitor, mutable fn client_stats_mut(&mut self) -> &mut Vec { @@ -65,7 +63,7 @@ where self.aggregator.aggregate(name, &self.client_stats); } - fn display(&mut self, event_msg: String, sender_id: ClientId) { + fn display(&mut self, event_msg: &str, sender_id: ClientId) { let sender = format!("#{}", sender_id.0); let pad = if event_msg.len() + sender.len() < 13 { " ".repeat(13 - event_msg.len() - sender.len()) @@ -77,7 +75,7 @@ where "[{}] (GLOBAL) run time: {}, clients: {}, corpus: {}, objectives: {}, executions: {}, exec/sec: {}", head, format_duration_hms(&(current_time() - self.start_time)), - self.client_stats().len(), + self.client_stats_count(), self.corpus_size(), self.objective_size(), self.total_execs(), @@ -87,7 +85,7 @@ where write!(global_fmt, ", {key}: {val}").unwrap(); } - (self.print_fn)(global_fmt); + (self.print_fn)(&global_fmt); self.client_stats_insert(sender_id); let client = self.client_stats_mut_for(sender_id); @@ -102,26 +100,26 @@ where for (key, val) in &client.user_monitor { write!(fmt, ", {key}: {val}").unwrap(); } - (self.print_fn)(fmt); + (self.print_fn)(&fmt); // Only print perf monitor if the feature is enabled #[cfg(feature = "introspection")] { // Print the client performance monitor. Skip the Client 0 which is the broker - for (i, client) in self.client_stats.iter().skip(1).enumerate() { + for (i, client) in self.client_stats.iter().filter(|x| x.enabled).enumerate() { let fmt = format!("Client {:03}:\n{}", i + 1, client.introspection_monitor); - (self.print_fn)(fmt); + (self.print_fn)(&fmt); } // Separate the spacing just a bit - (self.print_fn)("\n".to_string()); + (self.print_fn)("\n"); } } } impl MultiMonitor where - F: FnMut(String), + F: FnMut(&str), { /// Creates the monitor, using the `current_time` as `start_time`. pub fn new(print_fn: F) -> Self { diff --git a/libafl/src/monitors/prometheus.rs b/libafl/src/monitors/prometheus.rs index ab6e3b904c..2ef8227b9b 100644 --- a/libafl/src/monitors/prometheus.rs +++ b/libafl/src/monitors/prometheus.rs @@ -22,7 +22,7 @@ // When using docker, you may need to point prometheus.yml to the docker0 interface or host.docker.internal // ==================== -use alloc::{fmt::Debug, string::String, vec::Vec}; +use alloc::{borrow::Cow, fmt::Debug, string::String, vec::Vec}; use core::{fmt, time::Duration}; use std::{ sync::{atomic::AtomicU64, Arc}, @@ -47,7 +47,7 @@ use crate::monitors::{ClientStats, Monitor, UserStatsValue}; #[derive(Clone)] pub struct PrometheusMonitor where - F: FnMut(String), + F: FnMut(&str), { print_fn: F, start_time: Duration, @@ -63,7 +63,7 @@ where impl Debug for PrometheusMonitor where - F: FnMut(String), + F: FnMut(&str), { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("PrometheusMonitor") @@ -75,7 +75,7 @@ where impl Monitor for PrometheusMonitor where - F: FnMut(String), + F: FnMut(&str), { /// the client monitor, mutable fn client_stats_mut(&mut self) -> &mut Vec { @@ -98,7 +98,7 @@ where } #[allow(clippy::cast_sign_loss)] - fn display(&mut self, event_msg: String, sender_id: ClientId) { + fn display(&mut self, event_msg: &str, sender_id: ClientId) { // Update the prometheus metrics // Label each metric with the sender / client_id // The gauges must take signed i64's, with max value of 2^63-1 so it is @@ -111,42 +111,42 @@ where self.corpus_count .get_or_create(&Labels { client: sender_id.0, - stat: String::new(), + stat: Cow::from(""), }) .set(corpus_size.try_into().unwrap()); let objective_size = self.objective_size(); self.objective_count .get_or_create(&Labels { client: sender_id.0, - stat: String::new(), + stat: Cow::from(""), }) .set(objective_size.try_into().unwrap()); let total_execs = self.total_execs(); self.executions .get_or_create(&Labels { client: sender_id.0, - stat: String::new(), + stat: Cow::from(""), }) .set(total_execs.try_into().unwrap()); let execs_per_sec = self.execs_per_sec(); self.exec_rate .get_or_create(&Labels { client: sender_id.0, - stat: String::new(), + stat: Cow::from(""), }) .set(execs_per_sec); let run_time = (current_time() - self.start_time).as_secs(); self.runtime .get_or_create(&Labels { client: sender_id.0, - stat: String::new(), + stat: Cow::from(""), }) .set(run_time.try_into().unwrap()); // run time in seconds, which can be converted to a time format by Grafana or similar - let total_clients = self.client_stats().len().try_into().unwrap(); // convert usize to u64 (unlikely that # of clients will be > 2^64 -1...) + let total_clients = self.client_stats_count().try_into().unwrap(); // convert usize to u64 (unlikely that # of clients will be > 2^64 -1...) self.clients_count .get_or_create(&Labels { client: sender_id.0, - stat: String::new(), + stat: Cow::from(""), }) .set(total_clients); @@ -156,13 +156,13 @@ where event_msg, sender_id.0, format_duration_hms(&(current_time() - self.start_time)), - self.client_stats().len(), + self.client_stats_count(), self.corpus_size(), self.objective_size(), self.total_execs(), self.execs_per_sec_pretty() ); - (self.print_fn)(fmt); + (self.print_fn)(&fmt); self.client_stats_insert(sender_id); let cur_client = self.client_stats_mut_for(sender_id); @@ -192,7 +192,7 @@ where impl PrometheusMonitor where - F: FnMut(String), + F: FnMut(&str), { pub fn new(listener: String, print_fn: F) -> Self { // Gauge's implementation of clone uses Arc @@ -352,7 +352,7 @@ pub async fn serve_metrics( #[derive(Clone, Hash, PartialEq, Eq, EncodeLabelSet, Debug)] pub struct Labels { client: u32, // sender_id: u32, to differentiate between clients when multiple are spawned. - stat: String, // for custom_stat filtering. + stat: Cow<'static, str>, // for custom_stat filtering. } #[derive(Clone)] diff --git a/libafl/src/monitors/tui/mod.rs b/libafl/src/monitors/tui/mod.rs index a5dcf514f7..149760aa69 100644 --- a/libafl/src/monitors/tui/mod.rs +++ b/libafl/src/monitors/tui/mod.rs @@ -1,6 +1,7 @@ //! Monitor based on ratatui -use alloc::{boxed::Box, string::ToString}; +use alloc::{borrow::Cow, boxed::Box, string::ToString}; +use core::cmp; use std::{ collections::VecDeque, fmt::Write as _, @@ -22,7 +23,7 @@ use crossterm::{ use hashbrown::HashMap; use libafl_bolts::{current_time, format_duration_hms, ClientId}; use ratatui::{backend::CrosstermBackend, Terminal}; -use serde_json::{self, Value}; +use serde_json::Value; #[cfg(feature = "introspection")] use super::{ClientPerfMonitor, PerfFeature}; @@ -210,7 +211,7 @@ pub struct ClientTuiContext { pub process_timing: ProcessTiming, pub item_geometry: ItemGeometry, - pub user_stats: HashMap, + pub user_stats: HashMap, UserStats>, } impl ClientTuiContext { @@ -336,12 +337,16 @@ pub struct TuiMonitor { } impl Monitor for TuiMonitor { - /// the client monitor, mutable + /// The client monitor, mutable + /// This also includes disabled "padding" clients. + /// Results should be filtered by `.enabled`. fn client_stats_mut(&mut self) -> &mut Vec { &mut self.client_stats } - /// the client monitor + /// The client monitor + /// This also includes disabled "padding" clients. + /// Results should be filtered by `.enabled`. fn client_stats(&self) -> &[ClientStats] { &self.client_stats } @@ -357,7 +362,7 @@ impl Monitor for TuiMonitor { } #[allow(clippy::cast_sign_loss)] - fn display(&mut self, event_msg: String, sender_id: ClientId) { + fn display(&mut self, event_msg: &str, sender_id: ClientId) { let cur_time = current_time(); { @@ -419,8 +424,8 @@ impl Monitor for TuiMonitor { #[cfg(feature = "introspection")] { - // Print the client performance monitor. Skip the Client 0 which is the broker - for (i, client) in self.client_stats.iter().skip(1).enumerate() { + // Print the client performance monitor. Skip the Client IDs that have never sent anything. + for (i, client) in self.client_stats.iter().filter(|x| x.enabled).enumerate() { self.context .write() .unwrap() @@ -484,25 +489,12 @@ impl TuiMonitor { } fn map_density(&self) -> String { - if self.client_stats.len() < 2 { - return "0%".to_string(); - } - let mut max_map_density = self - .client_stats() - .get(1) - .unwrap() - .get_user_stats("edges") - .map_or("0%".to_string(), ToString::to_string); - - for client in self.client_stats().iter().skip(2) { - let client_map_density = client - .get_user_stats("edges") - .map_or(String::new(), ToString::to_string); - if client_map_density > max_map_density { - max_map_density = client_map_density; - } - } - max_map_density + self.client_stats() + .iter() + .filter(|client| client.enabled) + .filter_map(|client| client.get_user_stats("edges")) + .map(ToString::to_string) + .fold("0%".to_string(), cmp::max) } fn item_geometry(&self) -> ItemGeometry { @@ -512,13 +504,13 @@ impl TuiMonitor { } let mut ratio_a: u64 = 0; let mut ratio_b: u64 = 0; - for client in self.client_stats().iter().skip(1) { + for client in self.client_stats().iter().filter(|client| client.enabled) { let afl_stats = client .get_user_stats("AflStats") .map_or("None".to_string(), ToString::to_string); let stability = client.get_user_stats("stability").map_or( UserStats::new(UserStatsValue::Ratio(0, 100), AggregatorOps::Avg), - core::clone::Clone::clone, + Clone::clone, ); if afl_stats != "None" { @@ -555,7 +547,7 @@ impl TuiMonitor { if self.client_stats.len() > 1 { let mut new_path_time = Duration::default(); let mut new_objectives_time = Duration::default(); - for client in self.client_stats().iter().skip(1) { + for client in self.client_stats().iter().filter(|client| client.enabled) { new_path_time = client.last_corpus_time.max(new_path_time); new_objectives_time = client.last_objective_time.max(new_objectives_time); } @@ -616,7 +608,7 @@ fn run_tui_thread( let timeout = tick_rate .checked_sub(last_tick.elapsed()) .unwrap_or_else(|| Duration::from_secs(0)); - if crossterm::event::poll(timeout)? { + if event::poll(timeout)? { if let Event::Key(key) = event::read()? { match key.code { KeyCode::Char(c) => ui.on_key(c), diff --git a/libafl/src/monitors/tui/ui.rs b/libafl/src/monitors/tui/ui.rs index a4ddf5bfb9..5790b63bf4 100644 --- a/libafl/src/monitors/tui/ui.rs +++ b/libafl/src/monitors/tui/ui.rs @@ -111,8 +111,8 @@ impl TuiUI { .as_ref() } else { [ - Constraint::Percentage(41), - Constraint::Percentage(27), + Constraint::Percentage(20), + Constraint::Percentage(48), Constraint::Percentage(32), ] .as_ref() @@ -264,16 +264,20 @@ impl TuiUI { .add_modifier(Modifier::BOLD), )) .borders(Borders::ALL); - let client_area = client_block.inner(area); + + #[allow(unused_mut)] + let mut client_area = client_block.inner(area); f.render_widget(client_block, area); #[cfg(feature = "introspection")] { - let introspection_layout = Layout::default() + let client_layout = Layout::default() .direction(Direction::Vertical) - .constraints([Constraint::Length(11), Constraint::Min(0)].as_ref()) - .split(client_area)[1]; - self.draw_introspection_text(f, app, introspection_layout); + .constraints([Constraint::Min(11), Constraint::Percentage(50)].as_ref()) + .split(client_area); + client_area = client_layout[0]; + let instrospection_layout = client_layout[1]; + self.draw_introspection_text(f, app, instrospection_layout); } let left_layout = Layout::default() @@ -579,33 +583,27 @@ impl TuiUI { ) where B: Backend, { - let items = vec![ - Row::new(vec![ - Cell::from(Span::raw("clients")), - Cell::from(Span::raw(format!("{}", self.clients))), - Cell::from(Span::raw("total execs")), - Cell::from(Span::raw(format!("{}", app.read().unwrap().total_execs))), - Cell::from(Span::raw("map density")), - Cell::from(Span::raw(app.read().unwrap().total_map_density.to_string())), - ]), - Row::new(vec![ - Cell::from(Span::raw("solutions")), - Cell::from(Span::raw(format!( - "{}", - app.read().unwrap().total_solutions - ))), - Cell::from(Span::raw("cycle done")), - Cell::from(Span::raw(format!( - "{}", - app.read().unwrap().total_cycles_done - ))), - Cell::from(Span::raw("corpus count")), - Cell::from(Span::raw(format!( - "{}", - app.read().unwrap().total_corpus_count - ))), - ]), - ]; + let items = { + let app = app.read().unwrap(); + vec![ + Row::new(vec![ + Cell::from(Span::raw("clients")), + Cell::from(Span::raw(format!("{}", self.clients))), + Cell::from(Span::raw("total execs")), + Cell::from(Span::raw(format!("{}", app.total_execs))), + Cell::from(Span::raw("map density")), + Cell::from(Span::raw(app.total_map_density.to_string())), + ]), + Row::new(vec![ + Cell::from(Span::raw("solutions")), + Cell::from(Span::raw(format!("{}", app.total_solutions))), + Cell::from(Span::raw("cycle done")), + Cell::from(Span::raw(format!("{}", app.total_cycles_done))), + Cell::from(Span::raw("corpus count")), + Cell::from(Span::raw(format!("{}", app.total_corpus_count))), + ]), + ] + }; let chunks = Layout::default() .constraints([Constraint::Percentage(100)].as_ref()) @@ -641,30 +639,29 @@ impl TuiUI { ) where B: Backend, { - let items = vec![ - Row::new(vec![ - Cell::from(Span::raw("cycles done")), - Cell::from(Span::raw(format!( - "{}", - app.read() - .unwrap() - .clients - .get(&self.clients_idx) - .map_or(0, |x| x.cycles_done) - ))), - ]), - Row::new(vec![ - Cell::from(Span::raw("solutions")), - Cell::from(Span::raw(format!( - "{}", - app.read() - .unwrap() - .clients - .get(&self.clients_idx) - .map_or(0, |x| x.objectives) - ))), - ]), - ]; + let items = { + let app = app.read().unwrap(); + vec![ + Row::new(vec![ + Cell::from(Span::raw("cycles done")), + Cell::from(Span::raw(format!( + "{}", + app.clients + .get(&self.clients_idx) + .map_or(0, |x| x.cycles_done) + ))), + ]), + Row::new(vec![ + Cell::from(Span::raw("solutions")), + Cell::from(Span::raw(format!( + "{}", + app.clients + .get(&self.clients_idx) + .map_or(0, |x| x.objectives) + ))), + ]), + ] + }; let chunks = Layout::default() .constraints([Constraint::Percentage(100)].as_ref()) @@ -693,40 +690,35 @@ impl TuiUI { ) where B: Backend, { - let items = vec![ - Row::new(vec![ - Cell::from(Span::raw("corpus count")), - Cell::from(Span::raw(format!( - "{}", - app.read() - .unwrap() - .clients - .get(&self.clients_idx) - .map_or(0, |x| x.corpus) - ))), - ]), - Row::new(vec![ - Cell::from(Span::raw("total execs")), - Cell::from(Span::raw(format!( - "{}", - app.read() - .unwrap() - .clients - .get(&self.clients_idx) - .map_or(0, |x| x.executions) - ))), - ]), - Row::new(vec![ - Cell::from(Span::raw("map density")), - Cell::from(Span::raw( - app.read() - .unwrap() - .clients - .get(&self.clients_idx) - .map_or("0%".to_string(), |x| x.map_density.to_string()), - )), - ]), - ]; + let items = { + let app = app.read().unwrap(); + vec![ + Row::new(vec![ + Cell::from(Span::raw("corpus count")), + Cell::from(Span::raw(format!( + "{}", + app.clients.get(&self.clients_idx).map_or(0, |x| x.corpus) + ))), + ]), + Row::new(vec![ + Cell::from(Span::raw("total execs")), + Cell::from(Span::raw(format!( + "{}", + app.clients + .get(&self.clients_idx) + .map_or(0, |x| x.executions) + ))), + ]), + Row::new(vec![ + Cell::from(Span::raw("map density")), + Cell::from(Span::raw( + app.clients + .get(&self.clients_idx) + .map_or("0%".to_string(), |x| x.map_density.to_string()), + )), + ]), + ] + }; let chunks = Layout::default() .constraints([Constraint::Percentage(100)].as_ref()) diff --git a/libafl/src/mutators/encoded_mutations.rs b/libafl/src/mutators/encoded_mutations.rs index 3c5b9215f0..8489698e41 100644 --- a/libafl/src/mutators/encoded_mutations.rs +++ b/libafl/src/mutators/encoded_mutations.rs @@ -1,6 +1,6 @@ //! Mutations for [`EncodedInput`]s //! -use alloc::vec::Vec; +use alloc::{borrow::Cow, vec::Vec}; use core::cmp::{max, min}; use libafl_bolts::{ @@ -15,7 +15,7 @@ use crate::{ mutations::{buffer_copy, buffer_self_copy, ARITH_MAX}, MutationResult, Mutator, Named, }, - random_corpus_id, + random_corpus_id_with_disabled, state::{HasCorpus, HasMaxSize, HasRand}, Error, }; @@ -25,12 +25,7 @@ use crate::{ pub struct EncodedRandMutator; impl Mutator for EncodedRandMutator { - fn mutate( - &mut self, - state: &mut S, - input: &mut EncodedInput, - _stage_idx: i32, - ) -> Result { + fn mutate(&mut self, state: &mut S, input: &mut EncodedInput) -> Result { if input.codes().is_empty() { Ok(MutationResult::Skipped) } else { @@ -42,8 +37,9 @@ impl Mutator for EncodedRandMutator { } impl Named for EncodedRandMutator { - fn name(&self) -> &str { - "EncodedRandMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("EncodedRandMutator"); + &NAME } } @@ -60,12 +56,7 @@ impl EncodedRandMutator { pub struct EncodedIncMutator; impl Mutator for EncodedIncMutator { - fn mutate( - &mut self, - state: &mut S, - input: &mut EncodedInput, - _stage_idx: i32, - ) -> Result { + fn mutate(&mut self, state: &mut S, input: &mut EncodedInput) -> Result { if input.codes().is_empty() { Ok(MutationResult::Skipped) } else { @@ -77,8 +68,9 @@ impl Mutator for EncodedIncMutator { } impl Named for EncodedIncMutator { - fn name(&self) -> &str { - "EncodedIncMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("EncodedIncMutator"); + &NAME } } @@ -95,12 +87,7 @@ impl EncodedIncMutator { pub struct EncodedDecMutator; impl Mutator for EncodedDecMutator { - fn mutate( - &mut self, - state: &mut S, - input: &mut EncodedInput, - _stage_idx: i32, - ) -> Result { + fn mutate(&mut self, state: &mut S, input: &mut EncodedInput) -> Result { if input.codes().is_empty() { Ok(MutationResult::Skipped) } else { @@ -112,8 +99,9 @@ impl Mutator for EncodedDecMutator { } impl Named for EncodedDecMutator { - fn name(&self) -> &str { - "EncodedDecMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("EncodedDecMutator"); + &NAME } } @@ -130,12 +118,7 @@ impl EncodedDecMutator { pub struct EncodedAddMutator; impl Mutator for EncodedAddMutator { - fn mutate( - &mut self, - state: &mut S, - input: &mut EncodedInput, - _stage_idx: i32, - ) -> Result { + fn mutate(&mut self, state: &mut S, input: &mut EncodedInput) -> Result { if input.codes().is_empty() { Ok(MutationResult::Skipped) } else { @@ -151,8 +134,9 @@ impl Mutator for EncodedAddMutator { } impl Named for EncodedAddMutator { - fn name(&self) -> &str { - "EncodedAddMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("EncodedAddMutator"); + &NAME } } @@ -169,19 +153,14 @@ impl EncodedAddMutator { pub struct EncodedDeleteMutator; impl Mutator for EncodedDeleteMutator { - fn mutate( - &mut self, - state: &mut S, - input: &mut EncodedInput, - _stage_idx: i32, - ) -> Result { + fn mutate(&mut self, state: &mut S, input: &mut EncodedInput) -> Result { let size = input.codes().len(); if size <= 2 { return Ok(MutationResult::Skipped); } - let off = state.rand_mut().below(size as u64) as usize; - let len = state.rand_mut().below((size - off) as u64) as usize; + let off = state.rand_mut().below(size); + let len = state.rand_mut().below(size - off); input.codes_mut().drain(off..off + len); Ok(MutationResult::Mutated) @@ -189,8 +168,9 @@ impl Mutator for EncodedDeleteMutator { } impl Named for EncodedDeleteMutator { - fn name(&self) -> &str { - "EncodedDeleteMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("EncodedDeleteMutator"); + &NAME } } @@ -212,19 +192,14 @@ impl Mutator for EncodedInsertCopyMutator where S: HasRand + HasMaxSize, { - fn mutate( - &mut self, - state: &mut S, - input: &mut EncodedInput, - _stage_idx: i32, - ) -> Result { + fn mutate(&mut self, state: &mut S, input: &mut EncodedInput) -> Result { let max_size = state.max_size(); let size = input.codes().len(); if size == 0 { return Ok(MutationResult::Skipped); } - let off = state.rand_mut().below((size + 1) as u64) as usize; - let mut len = 1 + state.rand_mut().below(min(16, size as u64)) as usize; + let off = state.rand_mut().below(size + 1); + let mut len = 1 + state.rand_mut().below(min(16, size)); if size + len > max_size { if max_size > size { @@ -237,7 +212,7 @@ where let from = if size == len { 0 } else { - state.rand_mut().below((size - len) as u64) as usize + state.rand_mut().below(size - len) }; input.codes_mut().resize(size + len, 0); @@ -254,8 +229,9 @@ where } impl Named for EncodedInsertCopyMutator { - fn name(&self) -> &str { - "EncodedInsertCopyMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("EncodedInsertCopyMutator"); + &NAME } } @@ -272,20 +248,15 @@ impl EncodedInsertCopyMutator { pub struct EncodedCopyMutator; impl Mutator for EncodedCopyMutator { - fn mutate( - &mut self, - state: &mut S, - input: &mut EncodedInput, - _stage_idx: i32, - ) -> Result { + fn mutate(&mut self, state: &mut S, input: &mut EncodedInput) -> Result { let size = input.codes().len(); if size <= 1 { return Ok(MutationResult::Skipped); } - let from = state.rand_mut().below(size as u64) as usize; - let to = state.rand_mut().below(size as u64) as usize; - let len = 1 + state.rand_mut().below((size - max(from, to)) as u64) as usize; + let from = state.rand_mut().below(size); + let to = state.rand_mut().below(size); + let len = 1 + state.rand_mut().below(size - max(from, to)); unsafe { buffer_self_copy(input.codes_mut(), from, to, len); @@ -296,8 +267,9 @@ impl Mutator for EncodedCopyMutator { } impl Named for EncodedCopyMutator { - fn name(&self) -> &str { - "EncodedCopyMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("EncodedCopyMutator"); + &NAME } } @@ -317,16 +289,11 @@ impl Mutator for EncodedCrossoverInsertMutator where S: UsesInput + HasRand + HasCorpus + HasMaxSize, { - fn mutate( - &mut self, - state: &mut S, - input: &mut EncodedInput, - _stage_idx: i32, - ) -> Result { + fn mutate(&mut self, state: &mut S, input: &mut EncodedInput) -> Result { let size = input.codes().len(); // We don't want to use the testcase we're already using for splicing - let idx = random_corpus_id!(state.corpus(), state.rand_mut()); + let idx = random_corpus_id_with_disabled!(state.corpus(), state.rand_mut()); if let Some(cur) = state.corpus().current() { if idx == *cur { return Ok(MutationResult::Skipped); @@ -334,7 +301,7 @@ where } let other_size = { - let mut other_testcase = state.corpus().get(idx)?.borrow_mut(); + let mut other_testcase = state.corpus().get_from_all(idx)?.borrow_mut(); other_testcase.load_input(state.corpus())?.codes().len() }; @@ -343,9 +310,9 @@ where } let max_size = state.max_size(); - let from = state.rand_mut().below(other_size as u64) as usize; - let to = state.rand_mut().below(size as u64) as usize; - let mut len = 1 + state.rand_mut().below((other_size - from) as u64) as usize; + let from = state.rand_mut().below(other_size); + let to = state.rand_mut().below(size); + let mut len = 1 + state.rand_mut().below(other_size - from); if size + len > max_size { if max_size > size { @@ -355,7 +322,7 @@ where } } - let other_testcase = state.corpus().get(idx)?.borrow_mut(); + let other_testcase = state.corpus().get_from_all(idx)?.borrow_mut(); // no need to `load_input` again - we did that above already. let other = other_testcase.input().as_ref().unwrap(); @@ -370,8 +337,9 @@ where } impl Named for EncodedCrossoverInsertMutator { - fn name(&self) -> &str { - "EncodedCrossoverInsertMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("EncodedCrossoverInsertMutator"); + &NAME } } @@ -391,19 +359,14 @@ impl Mutator for EncodedCrossoverReplaceMutator where S: UsesInput + HasRand + HasCorpus, { - fn mutate( - &mut self, - state: &mut S, - input: &mut EncodedInput, - _stage_idx: i32, - ) -> Result { + fn mutate(&mut self, state: &mut S, input: &mut EncodedInput) -> Result { let size = input.codes().len(); if size == 0 { return Ok(MutationResult::Skipped); } // We don't want to use the testcase we're already using for splicing - let idx = random_corpus_id!(state.corpus(), state.rand_mut()); + let idx = random_corpus_id_with_disabled!(state.corpus(), state.rand_mut()); if let Some(cur) = state.corpus().current() { if idx == *cur { return Ok(MutationResult::Skipped); @@ -412,7 +375,7 @@ where let other_size = { // new scope to make the borrow checker happy - let mut other_testcase = state.corpus().get(idx)?.borrow_mut(); + let mut other_testcase = state.corpus().get_from_all(idx)?.borrow_mut(); other_testcase.load_input(state.corpus())?.codes().len() }; @@ -420,11 +383,11 @@ where return Ok(MutationResult::Skipped); } - let from = state.rand_mut().below(other_size as u64) as usize; - let len = state.rand_mut().below(min(other_size - from, size) as u64) as usize; - let to = state.rand_mut().below((size - len) as u64) as usize; + let from = state.rand_mut().below(other_size); + let len = state.rand_mut().below(min(other_size - from, size)); + let to = state.rand_mut().below(size - len); - let other_testcase = state.corpus().get(idx)?.borrow_mut(); + let other_testcase = state.corpus().get_from_all(idx)?.borrow_mut(); // no need to load the input again, it'll already be present at this point. let other = other_testcase.input().as_ref().unwrap(); @@ -437,8 +400,9 @@ where } impl Named for EncodedCrossoverReplaceMutator { - fn name(&self) -> &str { - "EncodedCrossoverReplaceMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("EncodedCrossoverReplaceMutator"); + &NAME } } diff --git a/libafl/src/mutators/gramatron.rs b/libafl/src/mutators/gramatron.rs index 231523b3e7..4ac4b16afd 100644 --- a/libafl/src/mutators/gramatron.rs +++ b/libafl/src/mutators/gramatron.rs @@ -1,10 +1,13 @@ //! Gramatron is the rewritten gramatron fuzzer in rust. //! See the original gramatron repo [`Gramatron`](https://github.com/HexHive/Gramatron) for more details. -use alloc::vec::Vec; +use alloc::{borrow::Cow, vec::Vec}; use core::cmp::max; use hashbrown::HashMap; -use libafl_bolts::{rands::Rand, Named}; +use libafl_bolts::{ + rands::{choose, Rand}, + Named, +}; use serde::{Deserialize, Serialize}; use crate::{ @@ -13,11 +16,11 @@ use crate::{ inputs::{GramatronInput, Terminal}, mutators::{MutationResult, Mutator}, random_corpus_id, - state::{HasCorpus, HasMetadata, HasRand}, - Error, + state::{HasCorpus, HasRand}, + Error, HasMetadata, }; -const RECUR_THRESHOLD: u64 = 5; +const RECUR_THRESHOLD: usize = 5; /// A random mutator for grammar fuzzing #[derive(Debug)] @@ -36,10 +39,9 @@ where &mut self, state: &mut S, input: &mut GramatronInput, - _stage_idx: i32, ) -> Result { if !input.terminals().is_empty() { - let size = state.rand_mut().below(input.terminals().len() as u64 + 1) as usize; + let size = state.rand_mut().below(input.terminals().len() + 1); input.terminals_mut().truncate(size); } if self.generator.append_generated_terminals(input, state) > 0 { @@ -54,8 +56,9 @@ impl<'a, S> Named for GramatronRandomMutator<'a, S> where S: HasRand + HasMetadata, { - fn name(&self) -> &str { - "GramatronRandomMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("GramatronRandomMutator"); + &NAME } } @@ -109,7 +112,6 @@ where &mut self, state: &mut S, input: &mut GramatronInput, - _stage_idx: i32, ) -> Result { if input.terminals().is_empty() { return Ok(MutationResult::Skipped); @@ -117,9 +119,9 @@ where let idx = random_corpus_id!(state.corpus(), state.rand_mut()); - let insert_at = state.rand_mut().below(input.terminals().len() as u64) as usize; + let insert_at = state.rand_mut().below(input.terminals().len()); - let rand_num = state.rand_mut().next() as usize; + let rand_num = state.rand_mut().next(); let mut other_testcase = state.corpus().get(idx)?.borrow_mut(); @@ -136,7 +138,7 @@ where meta.map.get(&input.terminals()[insert_at].state).map_or( Ok(MutationResult::Skipped), |splice_points| { - let from = splice_points[rand_num % splice_points.len()]; + let from = *choose(splice_points, rand_num); input.terminals_mut().truncate(insert_at); input @@ -150,8 +152,9 @@ where } impl Named for GramatronSpliceMutator { - fn name(&self) -> &str { - "GramatronSpliceMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("GramatronSpliceMutator"); + &NAME } } @@ -180,7 +183,6 @@ where &mut self, state: &mut S, input: &mut GramatronInput, - _stage_idx: i32, ) -> Result { if input.terminals().is_empty() { return Ok(MutationResult::Skipped); @@ -210,11 +212,11 @@ where let chosen_nums = self.counters.get(&chosen).unwrap().0; #[allow(clippy::cast_sign_loss, clippy::pedantic)] - let mut first = state.rand_mut().below(chosen_nums as u64 - 1) as i64; + let mut first = state.rand_mut().below(chosen_nums - 1) as i64; #[allow(clippy::cast_sign_loss, clippy::pedantic)] let mut second = state .rand_mut() - .between(first as u64 + 1, chosen_nums as u64 - 1) as i64; + .between(first as usize + 1, chosen_nums - 1) as i64; let mut idx_1 = 0; let mut idx_2 = 0; @@ -253,8 +255,9 @@ where } impl Named for GramatronRecursionMutator { - fn name(&self) -> &str { - "GramatronRecursionMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("GramatronRecursionMutator"); + &NAME } } diff --git a/libafl/src/mutators/grimoire.rs b/libafl/src/mutators/grimoire.rs index c115b9db4d..821cbf2bf3 100644 --- a/libafl/src/mutators/grimoire.rs +++ b/libafl/src/mutators/grimoire.rs @@ -1,23 +1,26 @@ //! Grimoire is the rewritten grimoire mutator in rust. //! See the original repo [`Grimoire`](https://github.com/RUB-SysSec/grimoire) for more details. -use alloc::vec::Vec; +use alloc::{borrow::Cow, vec::Vec}; use core::cmp::{max, min}; -use libafl_bolts::{rands::Rand, Named}; +use libafl_bolts::{ + rands::{choose, fast_bound, Rand}, + Named, +}; use crate::{ corpus::Corpus, inputs::{GeneralizedInputMetadata, GeneralizedItem}, mutators::{token_mutations::Tokens, MutationResult, Mutator}, random_corpus_id, - state::{HasCorpus, HasMetadata, HasRand}, - Error, + state::{HasCorpus, HasRand}, + Error, HasMetadata, }; const RECURSIVE_REPLACEMENT_DEPTH: [usize; 6] = [2, 4, 8, 16, 32, 64]; const MAX_RECURSIVE_REPLACEMENT_LEN: usize = 64 << 10; -const CHOOSE_SUBINPUT_PROB: u64 = 50; +const CHOOSE_SUBINPUT_PROB: f64 = 0.5; fn extend_with_random_generalized( state: &mut S, @@ -29,10 +32,10 @@ where { let idx = random_corpus_id!(state.corpus(), state.rand_mut()); - if state.rand_mut().below(100) > CHOOSE_SUBINPUT_PROB { - if state.rand_mut().below(100) < 50 { - let rand1 = state.rand_mut().next() as usize; - let rand2 = state.rand_mut().next() as usize; + if state.rand_mut().coinflip(CHOOSE_SUBINPUT_PROB) { + if state.rand_mut().coinflip(0.5) { + let rand1 = state.rand_mut().next(); + let rand2 = state.rand_mut().next(); let other_testcase = state.corpus().get(idx)?.borrow(); if let Some(other) = other_testcase @@ -48,8 +51,8 @@ where { gap_indices.push(i); } - let min_idx = gap_indices[rand1 % gap_indices.len()]; - let max_idx = gap_indices[rand2 % gap_indices.len()]; + let min_idx = *choose(&*gap_indices, rand1); + let max_idx = *choose(&*gap_indices, rand2); let (mut min_idx, max_idx) = (min(min_idx, max_idx), max(min_idx, max_idx)); gap_indices.clear(); @@ -66,11 +69,11 @@ where } } - let rand1 = state.rand_mut().next() as usize; + let rand1 = state.rand_mut().next(); if let Some(meta) = state.metadata_map().get::() { if !meta.tokens().is_empty() { - let tok = &meta.tokens()[rand1 % meta.tokens().len()]; + let tok = choose(meta.tokens(), rand1); if items.last() != Some(&GeneralizedItem::Gap) { items.push(GeneralizedItem::Gap); } @@ -122,7 +125,6 @@ where &mut self, state: &mut S, generalised_meta: &mut GeneralizedInputMetadata, - _stage_idx: i32, ) -> Result { extend_with_random_generalized( state, @@ -133,8 +135,9 @@ where } impl Named for GrimoireExtensionMutator { - fn name(&self) -> &str { - "GrimoireExtensionMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("GrimoireExtensionMutator"); + &NAME } } @@ -163,7 +166,6 @@ where &mut self, state: &mut S, generalised_meta: &mut GeneralizedInputMetadata, - _stage_idx: i32, ) -> Result { let mut mutated = MutationResult::Skipped; @@ -208,8 +210,9 @@ where } impl Named for GrimoireRecursiveReplacementMutator { - fn name(&self) -> &str { - "GrimoireRecursiveReplacementMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("GrimoireRecursiveReplacementMutator"); + &NAME } } @@ -236,7 +239,6 @@ where &mut self, state: &mut S, generalised_meta: &mut GeneralizedInputMetadata, - _stage_idx: i32, ) -> Result { let tokens_len = { let meta = state.metadata_map().get::(); @@ -250,14 +252,14 @@ where } }; - let token_find = state.rand_mut().below(tokens_len as u64) as usize; - let mut token_replace = state.rand_mut().below(tokens_len as u64) as usize; + let token_find = state.rand_mut().below(tokens_len); + let mut token_replace = state.rand_mut().below(tokens_len); if token_find == token_replace { - token_replace = state.rand_mut().below(tokens_len as u64) as usize; + token_replace = state.rand_mut().below(tokens_len); } - let stop_at_first = state.rand_mut().below(100) > 50; - let mut rand_idx = state.rand_mut().next() as usize; + let stop_at_first = state.rand_mut().coinflip(0.5); + let rand_idx = state.rand_mut().next(); let meta = state.metadata_map().get::().unwrap(); let token_1 = &meta.tokens()[token_find]; @@ -266,7 +268,7 @@ where let mut mutated = MutationResult::Skipped; let gen = generalised_meta.generalized_mut(); - rand_idx %= gen.len(); + let rand_idx = fast_bound(rand_idx, gen.len()); 'first: for item in &mut gen[..rand_idx] { if let GeneralizedItem::Bytes(bytes) = item { @@ -320,8 +322,9 @@ where } impl Named for GrimoireStringReplacementMutator { - fn name(&self) -> &str { - "GrimoireStringReplacementMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("GrimoireStringReplacementMutator"); + &NAME } } @@ -347,7 +350,6 @@ where &mut self, state: &mut S, generalised_meta: &mut GeneralizedInputMetadata, - _stage_idx: i32, ) -> Result { let gen = generalised_meta.generalized_mut(); @@ -358,10 +360,8 @@ where { self.gap_indices.push(i); } - let min_idx = - self.gap_indices[state.rand_mut().below(self.gap_indices.len() as u64) as usize]; - let max_idx = - self.gap_indices[state.rand_mut().below(self.gap_indices.len() as u64) as usize]; + let min_idx = self.gap_indices[state.rand_mut().below(self.gap_indices.len())]; + let max_idx = self.gap_indices[state.rand_mut().below(self.gap_indices.len())]; let (min_idx, max_idx) = (min(min_idx, max_idx), max(min_idx, max_idx)); @@ -379,8 +379,9 @@ where } impl Named for GrimoireRandomDeleteMutator { - fn name(&self) -> &str { - "GrimoireRandomDeleteMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("GrimoireRandomDeleteMutator"); + &NAME } } diff --git a/libafl/src/mutators/mod.rs b/libafl/src/mutators/mod.rs index 7232b0e800..5024e67d48 100644 --- a/libafl/src/mutators/mod.rs +++ b/libafl/src/mutators/mod.rs @@ -33,11 +33,12 @@ pub use multi::*; #[cfg(feature = "nautilus")] pub mod nautilus; -use alloc::vec::Vec; +use alloc::{boxed::Box, vec::Vec}; -use libafl_bolts::{tuples::HasConstLen, Named}; +use libafl_bolts::{tuples::IntoVec, HasLen, Named}; #[cfg(feature = "nautilus")] pub use nautilus::*; +use tuple_list::NonEmptyTuple; use crate::{corpus::CorpusId, Error}; @@ -90,20 +91,15 @@ pub enum MutationResult { /// Simple as that. pub trait Mutator: Named { /// Mutate a given input - fn mutate( - &mut self, - state: &mut S, - input: &mut I, - stage_idx: i32, - ) -> Result; + fn mutate(&mut self, state: &mut S, input: &mut I) -> Result; /// Post-process given the outcome of the execution + /// `new_corpus_idx` will be `Some` if a new `Testcase` was created this execution. #[inline] fn post_exec( &mut self, _state: &mut S, - _stage_idx: i32, - _corpus_idx: Option, + _new_corpus_idx: Option, ) -> Result<(), Error> { Ok(()) } @@ -118,38 +114,32 @@ pub trait MultiMutator: Named { &mut self, state: &mut S, input: &I, - stage_idx: i32, max_count: Option, ) -> Result, Error>; /// Post-process given the outcome of the execution + /// `new_corpus_idx` will be `Some` if a new `Testcase` was created this execution. #[inline] fn multi_post_exec( &mut self, _state: &mut S, - _stage_idx: i32, - _corpus_idx: Option, + _new_corpus_idx: Option, ) -> Result<(), Error> { Ok(()) } } /// A `Tuple` of `Mutators` that can execute multiple `Mutators` in a row. -pub trait MutatorsTuple: HasConstLen { +pub trait MutatorsTuple: HasLen { /// Runs the `mutate` function on all `Mutators` in this `Tuple`. - fn mutate_all( - &mut self, - state: &mut S, - input: &mut I, - stage_idx: i32, - ) -> Result; + fn mutate_all(&mut self, state: &mut S, input: &mut I) -> Result; /// Runs the `post_exec` function on all `Mutators` in this `Tuple`. + /// `new_corpus_idx` will be `Some` if a new `Testcase` was created this execution. fn post_exec_all( &mut self, state: &mut S, - stage_idx: i32, - corpus_idx: Option, + new_corpus_idx: Option, ) -> Result<(), Error>; /// Gets the [`Mutator`] at the given index and runs the `mutate` function on it. @@ -158,30 +148,28 @@ pub trait MutatorsTuple: HasConstLen { index: MutationId, state: &mut S, input: &mut I, - stage_idx: i32, ) -> Result; /// Gets the [`Mutator`] at the given index and runs the `post_exec` function on it. + /// `new_corpus_idx` will be `Some` if a new `Testcase` was created this execution. fn get_and_post_exec( &mut self, index: usize, state: &mut S, - stage_idx: i32, + corpus_idx: Option, ) -> Result<(), Error>; + /// Gets all names of the wrapped [`Mutator`]`s`, reversed. + fn names_reversed(&self) -> Vec<&str>; + /// Gets all names of the wrapped [`Mutator`]`s`. fn names(&self) -> Vec<&str>; } impl MutatorsTuple for () { #[inline] - fn mutate_all( - &mut self, - _state: &mut S, - _input: &mut I, - _stage_idx: i32, - ) -> Result { + fn mutate_all(&mut self, _state: &mut S, _input: &mut I) -> Result { Ok(MutationResult::Skipped) } @@ -189,8 +177,7 @@ impl MutatorsTuple for () { fn post_exec_all( &mut self, _state: &mut S, - _stage_idx: i32, - _corpus_idx: Option, + _new_corpus_idx: Option, ) -> Result<(), Error> { Ok(()) } @@ -201,7 +188,6 @@ impl MutatorsTuple for () { _index: MutationId, _state: &mut S, _input: &mut I, - _stage_idx: i32, ) -> Result { Ok(MutationResult::Skipped) } @@ -211,12 +197,16 @@ impl MutatorsTuple for () { &mut self, _index: usize, _state: &mut S, - _stage_idx: i32, - _corpus_idx: Option, + _new_corpus_idx: Option, ) -> Result<(), Error> { Ok(()) } + #[inline] + fn names_reversed(&self) -> Vec<&str> { + Vec::new() + } + #[inline] fn names(&self) -> Vec<&str> { Vec::new() @@ -228,14 +218,9 @@ where Head: Mutator, Tail: MutatorsTuple, { - fn mutate_all( - &mut self, - state: &mut S, - input: &mut I, - stage_idx: i32, - ) -> Result { - let r = self.0.mutate(state, input, stage_idx)?; - if self.1.mutate_all(state, input, stage_idx)? == MutationResult::Mutated { + fn mutate_all(&mut self, state: &mut S, input: &mut I) -> Result { + let r = self.0.mutate(state, input)?; + if self.1.mutate_all(state, input)? == MutationResult::Mutated { Ok(MutationResult::Mutated) } else { Ok(r) @@ -245,11 +230,10 @@ where fn post_exec_all( &mut self, state: &mut S, - stage_idx: i32, - corpus_idx: Option, + new_corpus_idx: Option, ) -> Result<(), Error> { - self.0.post_exec(state, stage_idx, corpus_idx)?; - self.1.post_exec_all(state, stage_idx, corpus_idx) + self.0.post_exec(state, new_corpus_idx)?; + self.1.post_exec_all(state, new_corpus_idx) } fn get_and_mutate( @@ -257,13 +241,11 @@ where index: MutationId, state: &mut S, input: &mut I, - stage_idx: i32, ) -> Result { if index.0 == 0 { - self.0.mutate(state, input, stage_idx) + self.0.mutate(state, input) } else { - self.1 - .get_and_mutate((index.0 - 1).into(), state, input, stage_idx) + self.1.get_and_mutate((index.0 - 1).into(), state, input) } } @@ -271,195 +253,157 @@ where &mut self, index: usize, state: &mut S, - stage_idx: i32, - corpus_idx: Option, + new_corpus_idx: Option, ) -> Result<(), Error> { if index == 0 { - self.0.post_exec(state, stage_idx, corpus_idx) + self.0.post_exec(state, new_corpus_idx) } else { - self.1 - .get_and_post_exec(index - 1, state, stage_idx, corpus_idx) + self.1.get_and_post_exec(index - 1, state, new_corpus_idx) } } + fn names_reversed(&self) -> Vec<&str> { + let mut ret = self.1.names_reversed(); + ret.push(self.0.name()); + ret + } + fn names(&self) -> Vec<&str> { - let mut ret = self.1.names(); - ret.insert(0, self.0.name()); + let mut ret = self.names_reversed(); + ret.reverse(); ret } } -/// `Mutator` Python bindings -#[cfg(feature = "python")] -#[allow(missing_docs)] -pub mod pybind { - use core::ffi::CStr; - - use libafl_bolts::Named; - use pyo3::{prelude::*, AsPyPointer}; - - use super::{MutationResult, Mutator}; - use crate::{ - corpus::CorpusId, - inputs::{BytesInput, HasBytesVec}, - mutators::scheduled::pybind::PythonStdHavocMutator, - state::pybind::{PythonStdState, PythonStdStateWrapper}, - Error, - }; - - #[derive(Clone, Debug)] - pub struct PyObjectMutator { - inner: PyObject, - } - - impl PyObjectMutator { - #[must_use] - pub fn new(obj: PyObject) -> Self { - PyObjectMutator { inner: obj } - } +impl IntoVec>> for (Head, Tail) +where + Head: Mutator + 'static, + Tail: IntoVec>>, +{ + fn into_vec_reversed(self) -> Vec>> { + let (head, tail) = self.uncons(); + let mut ret = tail.into_vec_reversed(); + ret.push(Box::new(head)); + ret } - impl Named for PyObjectMutator { - fn name(&self) -> &str { - unsafe { CStr::from_ptr((*(*self.inner.as_ptr()).ob_type).tp_name) } - .to_str() - .unwrap() - } + fn into_vec(self) -> Vec>> { + let mut ret = self.into_vec_reversed(); + ret.reverse(); + ret } +} - impl Mutator for PyObjectMutator { - fn mutate( - &mut self, - state: &mut PythonStdState, - input: &mut BytesInput, - stage_idx: i32, - ) -> Result { - let mutated = Python::with_gil(|py| -> PyResult { - self.inner - .call_method1( - py, - "mutate", - (PythonStdStateWrapper::wrap(state), input.bytes(), stage_idx), - )? - .extract(py) - })?; - Ok(if mutated { - MutationResult::Mutated - } else { - MutationResult::Skipped - }) - } +impl MutatorsTuple for (Tail,) +where + Tail: MutatorsTuple, +{ + fn mutate_all(&mut self, state: &mut S, input: &mut I) -> Result { + self.0.mutate_all(state, input) + } - fn post_exec( - &mut self, - state: &mut PythonStdState, - stage_idx: i32, - corpus_idx: Option, - ) -> Result<(), Error> { - Python::with_gil(|py| -> PyResult<()> { - self.inner.call_method1( - py, - "post_exec", - ( - PythonStdStateWrapper::wrap(state), - stage_idx, - corpus_idx.map(|x| x.0), - ), - )?; - Ok(()) - })?; - Ok(()) - } + fn post_exec_all( + &mut self, + state: &mut S, + new_corpus_idx: Option, + ) -> Result<(), Error> { + self.0.post_exec_all(state, new_corpus_idx) + } + + fn get_and_mutate( + &mut self, + index: MutationId, + state: &mut S, + input: &mut I, + ) -> Result { + self.0.get_and_mutate(index, state, input) + } + + fn get_and_post_exec( + &mut self, + index: usize, + state: &mut S, + new_corpus_idx: Option, + ) -> Result<(), Error> { + self.0.get_and_post_exec(index, state, new_corpus_idx) + } + + fn names(&self) -> Vec<&str> { + self.0.names() } - #[derive(Debug, Clone)] - pub enum PythonMutatorWrapper { - StdHavoc(Py), - Python(PyObjectMutator), + fn names_reversed(&self) -> Vec<&str> { + self.0.names_reversed() } +} - /// Mutator Trait binding - #[pyclass(unsendable, name = "Mutator")] - #[derive(Debug, Clone)] - pub struct PythonMutator { - pub wrapper: PythonMutatorWrapper, +impl IntoVec>> for (Tail,) +where + Tail: IntoVec>>, +{ + fn into_vec(self) -> Vec>> { + self.0.into_vec() } +} - macro_rules! unwrap_me_mut { - ($wrapper:expr, $name:ident, $body:block) => { - libafl_bolts::unwrap_me_mut_body!($wrapper, $name, $body, PythonMutatorWrapper, { - StdHavoc - }, - { - Python(py_wrapper) => { - let $name = py_wrapper; - $body +impl MutatorsTuple for Vec>> { + fn mutate_all(&mut self, state: &mut S, input: &mut I) -> Result { + self.iter_mut() + .try_fold(MutationResult::Skipped, |ret, mutator| { + if mutator.mutate(state, input)? == MutationResult::Mutated { + Ok(MutationResult::Mutated) + } else { + Ok(ret) } }) - }; } - #[pymethods] - impl PythonMutator { - #[staticmethod] - #[must_use] - pub fn new_std_havoc(mgr: Py) -> Self { - Self { - wrapper: PythonMutatorWrapper::StdHavoc(mgr), - } - } - - #[staticmethod] - #[must_use] - pub fn new_py(obj: PyObject) -> Self { - Self { - wrapper: PythonMutatorWrapper::Python(PyObjectMutator::new(obj)), - } + fn post_exec_all( + &mut self, + state: &mut S, + new_corpus_idx: Option, + ) -> Result<(), Error> { + for mutator in self.iter_mut() { + mutator.post_exec(state, new_corpus_idx)?; } + Ok(()) + } - #[must_use] - pub fn unwrap_py(&self) -> Option { - match &self.wrapper { - PythonMutatorWrapper::Python(pyo) => Some(pyo.inner.clone()), - PythonMutatorWrapper::StdHavoc(_) => None, - } - } + fn get_and_mutate( + &mut self, + index: MutationId, + state: &mut S, + input: &mut I, + ) -> Result { + let mutator = self + .get_mut(index.0) + .ok_or_else(|| Error::key_not_found("Mutator with id {index:?} not found."))?; + mutator.mutate(state, input) } - impl Named for PythonMutator { - fn name(&self) -> &str { - match &self.wrapper { - PythonMutatorWrapper::Python(pyo) => pyo.name(), - PythonMutatorWrapper::StdHavoc(_) => "StdHavocPythonMutator", - } - } + fn get_and_post_exec( + &mut self, + index: usize, + state: &mut S, + new_corpus_idx: Option, + ) -> Result<(), Error> { + let mutator = self + .get_mut(index) + .ok_or_else(|| Error::key_not_found("Mutator with id {index:?} not found."))?; + mutator.post_exec(state, new_corpus_idx) } - impl Mutator for PythonMutator { - fn mutate( - &mut self, - state: &mut PythonStdState, - input: &mut BytesInput, - stage_idx: i32, - ) -> Result { - unwrap_me_mut!(self.wrapper, m, { m.mutate(state, input, stage_idx) }) - } + fn names_reversed(&self) -> Vec<&str> { + self.iter().rev().map(|x| x.name().as_ref()).collect() + } - fn post_exec( - &mut self, - state: &mut PythonStdState, - stage_idx: i32, - corpus_idx: Option, - ) -> Result<(), Error> { - unwrap_me_mut!(self.wrapper, m, { - m.post_exec(state, stage_idx, corpus_idx) - }) - } + fn names(&self) -> Vec<&str> { + self.iter().map(|x| x.name().as_ref()).collect() } +} - /// Register the classes to the python module - pub fn register(_py: Python, m: &PyModule) -> PyResult<()> { - m.add_class::()?; - Ok(()) +impl IntoVec>> for Vec>> { + fn into_vec(self) -> Vec>> { + self } } diff --git a/libafl/src/mutators/mopt_mutator.rs b/libafl/src/mutators/mopt_mutator.rs index 25c9647c67..583ceb41e4 100644 --- a/libafl/src/mutators/mopt_mutator.rs +++ b/libafl/src/mutators/mopt_mutator.rs @@ -1,8 +1,5 @@ //! The `MOpt` mutator scheduler, see and -use alloc::{ - string::{String, ToString}, - vec::Vec, -}; +use alloc::{borrow::Cow, string::ToString, vec::Vec}; use core::{ fmt::{self, Debug}, marker::PhantomData, @@ -18,8 +15,8 @@ use super::MutationId; use crate::{ corpus::{Corpus, CorpusId}, mutators::{ComposedByMutations, MutationResult, Mutator, MutatorsTuple, ScheduledMutator}, - state::{HasCorpus, HasMetadata, HasRand, HasSolutions}, - Error, + state::{HasCorpus, HasRand, HasSolutions}, + Error, HasMetadata, }; /// A Struct for managing MOpt-mutator parameters. @@ -53,9 +50,9 @@ pub struct MOpt { pub operator_num: usize, /// The number of swarms that we want to employ during the pilot fuzzing mode pub swarm_num: usize, - /// We'll generate inputs for `period_pilot` times before we call pso_update in pilot fuzzing module + /// We'll generate inputs for `period_pilot` times before we call `pso_update` in pilot fuzzing module pub period_pilot: usize, - /// We'll generate inputs for `period_core` times before we call pso_update in core fuzzing module + /// We'll generate inputs for `period_core` times before we call `pso_update` in core fuzzing module pub period_core: usize, /// The number of testcases generated during this pilot fuzzing mode pub pilot_time: usize, @@ -77,23 +74,23 @@ pub struct MOpt { probability_now: Vec>, /// The fitness for each swarm, we'll calculate the fitness in the pilot fuzzing mode and use the best one in the core fuzzing mode pub swarm_fitness: Vec, - /// (Pilot Mode) Finds by each operators. This vector is used in pso_update + /// (Pilot Mode) Finds by each operators. This vector is used in `pso_update` pub pilot_operator_finds: Vec>, /// (Pilot Mode) Finds by each operator till now. pub pilot_operator_finds_v2: Vec>, - /// (Pilot Mode) The number of mutation operator used. This vector is used in pso_update + /// (Pilot Mode) The number of mutation operator used. This vector is used in `pso_update` pub pilot_operator_cycles: Vec>, /// (Pilot Mode) The number of mutation operator used till now pub pilot_operator_cycles_v2: Vec>, /// (Pilot Mode) The number of mutation operator used till last execution pub pilot_operator_cycles_v3: Vec>, - /// Vector used in pso_update + /// Vector used in `pso_update` pub operator_finds_puppet: Vec, - /// (Core Mode) Finds by each operators. This vector is used in pso_update + /// (Core Mode) Finds by each operators. This vector is used in `pso_update` pub core_operator_finds: Vec, /// (Core Mode) Finds by each operator till now. pub core_operator_finds_v2: Vec, - /// (Core Mode) The number of mutation operator used. This vector is used in pso_update + /// (Core Mode) The number of mutation operator used. This vector is used in `pso_update` pub core_operator_cycles: Vec, /// (Core Mode) The number of mutation operator used till now pub core_operator_cycles_v2: Vec, @@ -202,7 +199,7 @@ impl MOpt { let mut total_x_now = 0.0; let mut x_sum = 0.0; for i in 0..self.operator_num { - self.x_now[swarm][i] = (self.rand.below(7000) as f64) * 0.0001 + 0.1; + self.x_now[swarm][i] = 0.7 * self.rand.next_float() + 0.1; total_x_now += self.x_now[swarm][i]; self.v_now[swarm][i] = 0.1; self.l_best[swarm][i] = 0.5; @@ -215,12 +212,8 @@ impl MOpt { for i in 0..self.operator_num { self.v_now[swarm][i] = self.w_now * self.v_now[swarm][i] - + (self.rand.below(1000) as f64) - * 0.001 - * (self.l_best[swarm][i] - self.x_now[swarm][i]) - + (self.rand.below(1000) as f64) - * 0.001 - * (self.g_best[i] - self.x_now[swarm][i]); + + self.rand.next_float() * (self.l_best[swarm][i] - self.x_now[swarm][i]) + + self.rand.next_float() * (self.g_best[i] - self.x_now[swarm][i]); self.x_now[swarm][i] += self.v_now[swarm][i]; self.x_now[swarm][i] = self.x_now[swarm][i].clamp(V_MIN, V_MAX); @@ -282,12 +275,8 @@ impl MOpt { for i in 0..self.operator_num { self.probability_now[swarm][i] = 0.0; self.v_now[swarm][i] = self.w_now * self.v_now[swarm][i] - + (self.rand.below(1000) as f64) - * 0.001 - * (self.l_best[swarm][i] - self.x_now[swarm][i]) - + (self.rand.below(1000) as f64) - * 0.001 - * (self.g_best[i] - self.x_now[swarm][i]); + + self.rand.next_float() * (self.l_best[swarm][i] - self.x_now[swarm][i]) + + self.rand.next_float() * (self.g_best[i] - self.x_now[swarm][i]); self.x_now[swarm][i] += self.v_now[swarm][i]; self.x_now[swarm][i] = self.x_now[swarm][i].clamp(V_MIN, V_MAX); @@ -328,8 +317,8 @@ impl MOpt { let operator_num = self.operator_num; // Fetch a random sele value - let select_prob: f64 = self.probability_now[self.swarm_now][operator_num - 1] - * ((self.rand.below(10000) as f64) * 0.0001); + let select_prob: f64 = + self.probability_now[self.swarm_now][operator_num - 1] * self.rand.next_float(); for i in 0..operator_num { if i == 0 { @@ -375,11 +364,11 @@ where MT: MutatorsTuple, S: HasRand + HasMetadata + HasCorpus + HasSolutions, { - name: String, + name: Cow<'static, str>, mode: MOptMode, finds_before: usize, mutations: MT, - max_stack_pow: u64, + max_stack_pow: usize, phantom: PhantomData<(I, S)>, } @@ -392,7 +381,7 @@ where write!( f, "StdMOptMutator with {} mutations for Input type {}", - MT::LEN, + self.mutations.len(), core::any::type_name::() ) } @@ -404,23 +393,13 @@ where S: HasRand + HasMetadata + HasCorpus + HasSolutions, { #[inline] - fn mutate( - &mut self, - state: &mut S, - input: &mut I, - stage_idx: i32, - ) -> Result { + fn mutate(&mut self, state: &mut S, input: &mut I) -> Result { self.finds_before = state.corpus().count() + state.solutions().count(); - self.scheduled_mutate(state, input, stage_idx) + self.scheduled_mutate(state, input) } #[allow(clippy::cast_precision_loss)] - fn post_exec( - &mut self, - state: &mut S, - _stage_idx: i32, - _corpus_idx: Option, - ) -> Result<(), Error> { + fn post_exec(&mut self, state: &mut S, _new_corpus_idx: Option) -> Result<(), Error> { let before = self.finds_before; let after = state.corpus().count() + state.solutions().count(); @@ -542,15 +521,15 @@ where pub fn new( state: &mut S, mutations: MT, - max_stack_pow: u64, + max_stack_pow: usize, swarm_num: usize, ) -> Result { if !state.has_metadata::() { let rand_seed = state.rand_mut().next(); - state.add_metadata::(MOpt::new(MT::LEN, swarm_num, rand_seed)?); + state.add_metadata::(MOpt::new(mutations.len(), swarm_num, rand_seed)?); } Ok(Self { - name: format!("StdMOptMutator[{}]", mutations.names().join(",")), + name: Cow::from(format!("StdMOptMutator[{}]", mutations.names().join(","))), mode: MOptMode::Pilotfuzzing, finds_before: 0, mutations, @@ -558,12 +537,7 @@ where phantom: PhantomData, }) } - fn core_mutate( - &mut self, - state: &mut S, - input: &mut I, - stage_idx: i32, - ) -> Result { + fn core_mutate(&mut self, state: &mut S, input: &mut I) -> Result { let mut r = MutationResult::Skipped; let mopt = state.metadata_map_mut().get_mut::().unwrap(); for i in 0..mopt.operator_num { @@ -572,9 +546,7 @@ where for _i in 0..self.iterations(state, input) { let idx = self.schedule(state, input); - let outcome = self - .mutations_mut() - .get_and_mutate(idx, state, input, stage_idx)?; + let outcome = self.mutations_mut().get_and_mutate(idx, state, input)?; if outcome == MutationResult::Mutated { r = MutationResult::Mutated; } @@ -588,12 +560,7 @@ where Ok(r) } - fn pilot_mutate( - &mut self, - state: &mut S, - input: &mut I, - stage_idx: i32, - ) -> Result { + fn pilot_mutate(&mut self, state: &mut S, input: &mut I) -> Result { let mut r = MutationResult::Skipped; let swarm_now; { @@ -608,9 +575,7 @@ where for _i in 0..self.iterations(state, input) { let idx = self.schedule(state, input); - let outcome = self - .mutations_mut() - .get_and_mutate(idx, state, input, stage_idx)?; + let outcome = self.mutations_mut().get_and_mutate(idx, state, input)?; if outcome == MutationResult::Mutated { r = MutationResult::Mutated; } @@ -649,7 +614,7 @@ where MT: MutatorsTuple, S: HasRand + HasMetadata + HasCorpus + HasSolutions, { - fn name(&self) -> &str { + fn name(&self) -> &Cow<'static, str> { &self.name } } @@ -675,16 +640,11 @@ where .unwrap() } - fn scheduled_mutate( - &mut self, - state: &mut S, - input: &mut I, - stage_idx: i32, - ) -> Result { + fn scheduled_mutate(&mut self, state: &mut S, input: &mut I) -> Result { let mode = self.mode; match mode { - MOptMode::Corefuzzing => self.core_mutate(state, input, stage_idx), - MOptMode::Pilotfuzzing => self.pilot_mutate(state, input, stage_idx), + MOptMode::Corefuzzing => self.core_mutate(state, input), + MOptMode::Pilotfuzzing => self.pilot_mutate(state, input), } } } diff --git a/libafl/src/mutators/multi.rs b/libafl/src/mutators/multi.rs index 838703a8f6..25f0643329 100644 --- a/libafl/src/mutators/multi.rs +++ b/libafl/src/mutators/multi.rs @@ -40,24 +40,18 @@ where &mut self, state: &mut S, input: &mut MultipartInput, - stage_idx: i32, ) -> Result { if input.parts().is_empty() { Ok(MutationResult::Skipped) } else { - let selected = state.rand_mut().below(input.parts().len() as u64) as usize; + let selected = state.rand_mut().below(input.parts().len()); let mutated = input.part_mut(selected).unwrap(); - self.mutate(state, mutated, stage_idx) + self.mutate(state, mutated) } } - fn post_exec( - &mut self, - state: &mut S, - stage_idx: i32, - corpus_idx: Option, - ) -> Result<(), Error> { - M::post_exec(self, state, stage_idx, corpus_idx) + fn post_exec(&mut self, state: &mut S, new_corpus_idx: Option) -> Result<(), Error> { + M::post_exec(self, state, new_corpus_idx) } } @@ -129,7 +123,6 @@ where &mut self, state: &mut S, input: &mut MultipartInput, - _stage_idx: i32, ) -> Result { // we can eat the slight bias; number of parts will be small let name_choice = state.rand_mut().next() as usize; @@ -160,7 +153,7 @@ where .map(|(idx, part)| (idx, part.bytes().len())); if let Some((part_idx, size)) = maybe_size { - let target = state.rand_mut().below(size as u64) as usize; + let target = state.rand_mut().below(size); let range = rand_range(state, other_size, min(other_size, size - target)); let [part, chosen] = match part_idx.cmp(&choice) { @@ -202,7 +195,7 @@ where drop(other_testcase); let size = part.bytes().len(); - let target = state.rand_mut().below(size as u64) as usize; + let target = state.rand_mut().below(size); let range = rand_range(state, other_size, min(other_size, size - target)); let other_testcase = state.corpus().get(idx)?.borrow_mut(); @@ -234,7 +227,6 @@ where &mut self, state: &mut S, input: &mut MultipartInput, - _stage_idx: i32, ) -> Result { // we can eat the slight bias; number of parts will be small let name_choice = state.rand_mut().next() as usize; @@ -265,7 +257,7 @@ where .map(|(idx, part)| (idx, part.bytes().len())); if let Some((part_idx, size)) = maybe_size { - let target = state.rand_mut().below(size as u64) as usize; + let target = state.rand_mut().below(size); let range = rand_range(state, other_size, min(other_size, size - target)); let [part, chosen] = match part_idx.cmp(&choice) { @@ -307,7 +299,7 @@ where drop(other_testcase); let size = part.bytes().len(); - let target = state.rand_mut().below(size as u64) as usize; + let target = state.rand_mut().below(size); let range = rand_range(state, other_size, min(other_size, size - target)); let other_testcase = state.corpus().get(idx)?.borrow_mut(); diff --git a/libafl/src/mutators/mutations.rs b/libafl/src/mutators/mutations.rs index b7ba0c489b..b3115c62eb 100644 --- a/libafl/src/mutators/mutations.rs +++ b/libafl/src/mutators/mutations.rs @@ -1,6 +1,9 @@ //! A wide variety of mutations used during fuzzing. -use alloc::{borrow::ToOwned, vec::Vec}; +use alloc::{ + borrow::{Cow, ToOwned}, + vec::Vec, +}; use core::{cmp::min, marker::PhantomData, mem::size_of, ops::Range}; use libafl_bolts::{rands::Rand, Named}; @@ -9,7 +12,7 @@ use crate::{ corpus::Corpus, inputs::{HasBytesVec, Input}, mutators::{MutationResult, Mutator}, - random_corpus_id, + random_corpus_id_with_disabled, state::{HasCorpus, HasMaxSize, HasRand}, Error, }; @@ -62,9 +65,9 @@ pub fn buffer_set(data: &mut [T], from: usize, len: usize, val: T) { /// This problem corresponds to: #[inline] pub fn rand_range(state: &mut S, upper: usize, max_len: usize) -> Range { - let len = 1 + state.rand_mut().below(max_len as u64) as usize; + let len = 1 + state.rand_mut().below(max_len); // sample from [1..upper + len] - let mut offset2 = 1 + state.rand_mut().below((upper + len - 1) as u64) as usize; + let mut offset2 = 1 + state.rand_mut().below(upper + len - 1); let offset1 = offset2.saturating_sub(len); if offset2 > upper { offset2 = upper; @@ -74,7 +77,7 @@ pub fn rand_range(state: &mut S, upper: usize, max_len: usize) -> Ra } /// The max value that will be added or subtracted during add mutations -pub const ARITH_MAX: u64 = 35; +pub const ARITH_MAX: usize = 35; /// Interesting 8-bit values from AFL pub const INTERESTING_8: [i8; 9] = [-128, -1, 0, 1, 16, 32, 64, 100, 127]; @@ -122,12 +125,7 @@ where S: HasRand, I: HasBytesVec, { - fn mutate( - &mut self, - state: &mut S, - input: &mut I, - _stage_idx: i32, - ) -> Result { + fn mutate(&mut self, state: &mut S, input: &mut I) -> Result { if input.bytes().is_empty() { Ok(MutationResult::Skipped) } else { @@ -140,8 +138,9 @@ where } impl Named for BitFlipMutator { - fn name(&self) -> &str { - "BitFlipMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("BitFlipMutator"); + &NAME } } @@ -162,12 +161,7 @@ where S: HasRand, I: HasBytesVec, { - fn mutate( - &mut self, - state: &mut S, - input: &mut I, - _stage_idx: i32, - ) -> Result { + fn mutate(&mut self, state: &mut S, input: &mut I) -> Result { if input.bytes().is_empty() { Ok(MutationResult::Skipped) } else { @@ -178,8 +172,9 @@ where } impl Named for ByteFlipMutator { - fn name(&self) -> &str { - "ByteFlipMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("ByteFlipMutator"); + &NAME } } @@ -200,12 +195,7 @@ where S: HasRand, I: HasBytesVec, { - fn mutate( - &mut self, - state: &mut S, - input: &mut I, - _stage_idx: i32, - ) -> Result { + fn mutate(&mut self, state: &mut S, input: &mut I) -> Result { if input.bytes().is_empty() { Ok(MutationResult::Skipped) } else { @@ -217,8 +207,9 @@ where } impl Named for ByteIncMutator { - fn name(&self) -> &str { - "ByteIncMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("ByteIncMutator"); + &NAME } } @@ -239,12 +230,7 @@ where S: HasRand, I: HasBytesVec, { - fn mutate( - &mut self, - state: &mut S, - input: &mut I, - _stage_idx: i32, - ) -> Result { + fn mutate(&mut self, state: &mut S, input: &mut I) -> Result { if input.bytes().is_empty() { Ok(MutationResult::Skipped) } else { @@ -256,8 +242,9 @@ where } impl Named for ByteDecMutator { - fn name(&self) -> &str { - "ByteDecMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("ByteDecMutator"); + &NAME } } @@ -278,12 +265,7 @@ where S: HasRand, I: HasBytesVec, { - fn mutate( - &mut self, - state: &mut S, - input: &mut I, - _stage_idx: i32, - ) -> Result { + fn mutate(&mut self, state: &mut S, input: &mut I) -> Result { if input.bytes().is_empty() { Ok(MutationResult::Skipped) } else { @@ -295,8 +277,9 @@ where } impl Named for ByteNegMutator { - fn name(&self) -> &str { - "ByteNegMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("ByteNegMutator"); + &NAME } } @@ -317,12 +300,7 @@ where S: HasRand, I: HasBytesVec, { - fn mutate( - &mut self, - state: &mut S, - input: &mut I, - _stage_idx: i32, - ) -> Result { + fn mutate(&mut self, state: &mut S, input: &mut I) -> Result { if input.bytes().is_empty() { Ok(MutationResult::Skipped) } else { @@ -334,8 +312,9 @@ where } impl Named for ByteRandMutator { - fn name(&self) -> &str { - "ByteRandMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("ByteRandMutator"); + &NAME } } @@ -365,7 +344,7 @@ macro_rules! add_mutator_impl { &mut self, state: &mut S, input: &mut I, - _stage_idx: i32, + ) -> Result { if input.bytes().len() < size_of::<$size>() { Ok(MutationResult::Skipped) @@ -394,8 +373,9 @@ macro_rules! add_mutator_impl { } impl Named for $name { - fn name(&self) -> &str { - stringify!($name) + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed(stringify!($name)); + &NAME } } @@ -428,18 +408,13 @@ macro_rules! interesting_mutator_impl { I: HasBytesVec, { #[allow(clippy::cast_sign_loss)] - fn mutate( - &mut self, - state: &mut S, - input: &mut I, - _stage_idx: i32, - ) -> Result { + fn mutate(&mut self, state: &mut S, input: &mut I) -> Result { if input.bytes().len() < size_of::<$size>() { Ok(MutationResult::Skipped) } else { let bytes = input.bytes_mut(); - let upper_bound = (bytes.len() + 1 - size_of::<$size>()) as u64; - let idx = state.rand_mut().below(upper_bound) as usize; + let upper_bound = (bytes.len() + 1 - size_of::<$size>()); + let idx = state.rand_mut().below(upper_bound); let val = *state.rand_mut().choose(&$interesting) as $size; let new_bytes = match state.rand_mut().choose(&[0, 1]) { 0 => val.to_be_bytes(), @@ -452,8 +427,9 @@ macro_rules! interesting_mutator_impl { } impl Named for $name { - fn name(&self) -> &str { - stringify!($name) + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed(stringify!($name)); + &NAME } } @@ -480,12 +456,7 @@ where S: HasRand, I: HasBytesVec, { - fn mutate( - &mut self, - state: &mut S, - input: &mut I, - _stage_idx: i32, - ) -> Result { + fn mutate(&mut self, state: &mut S, input: &mut I) -> Result { let size = input.bytes().len(); if size <= 2 { return Ok(MutationResult::Skipped); @@ -500,8 +471,9 @@ where } impl Named for BytesDeleteMutator { - fn name(&self) -> &str { - "BytesDeleteMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("BytesDeleteMutator"); + &NAME } } @@ -522,12 +494,7 @@ where S: HasRand + HasMaxSize, I: HasBytesVec, { - fn mutate( - &mut self, - state: &mut S, - input: &mut I, - _stage_idx: i32, - ) -> Result { + fn mutate(&mut self, state: &mut S, input: &mut I) -> Result { let max_size = state.max_size(); let size = input.bytes().len(); if size == 0 || size >= max_size { @@ -551,8 +518,9 @@ where } impl Named for BytesExpandMutator { - fn name(&self) -> &str { - "BytesExpandMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("BytesExpandMutator"); + &NAME } } @@ -573,20 +541,15 @@ where S: HasRand + HasMaxSize, I: HasBytesVec, { - fn mutate( - &mut self, - state: &mut S, - input: &mut I, - _stage_idx: i32, - ) -> Result { + fn mutate(&mut self, state: &mut S, input: &mut I) -> Result { let max_size = state.max_size(); let size = input.bytes().len(); if size == 0 || size >= max_size { return Ok(MutationResult::Skipped); } - let mut amount = 1 + state.rand_mut().below(16) as usize; - let offset = state.rand_mut().below(size as u64 + 1) as usize; + let mut amount = 1 + state.rand_mut().below(16); + let offset = state.rand_mut().below(size + 1); if size + amount > max_size { if max_size > size { @@ -596,7 +559,7 @@ where } } - let val = input.bytes()[state.rand_mut().below(size as u64) as usize]; + let val = input.bytes()[state.rand_mut().below(size)]; input.bytes_mut().resize(size + amount, 0); unsafe { @@ -609,8 +572,9 @@ where } impl Named for BytesInsertMutator { - fn name(&self) -> &str { - "BytesInsertMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("BytesInsertMutator"); + &NAME } } @@ -631,20 +595,15 @@ where S: HasRand + HasMaxSize, I: HasBytesVec, { - fn mutate( - &mut self, - state: &mut S, - input: &mut I, - _stage_idx: i32, - ) -> Result { + fn mutate(&mut self, state: &mut S, input: &mut I) -> Result { let max_size = state.max_size(); let size = input.bytes().len(); if size >= max_size { return Ok(MutationResult::Skipped); } - let mut amount = 1 + state.rand_mut().below(16) as usize; - let offset = state.rand_mut().below(size as u64 + 1) as usize; + let mut amount = 1 + state.rand_mut().below(16); + let offset = state.rand_mut().below(size + 1); if size + amount > max_size { if max_size > size { @@ -667,8 +626,9 @@ where } impl Named for BytesRandInsertMutator { - fn name(&self) -> &str { - "BytesRandInsertMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("BytesRandInsertMutator"); + &NAME } } @@ -689,12 +649,7 @@ where S: HasRand, I: HasBytesVec, { - fn mutate( - &mut self, - state: &mut S, - input: &mut I, - _stage_idx: i32, - ) -> Result { + fn mutate(&mut self, state: &mut S, input: &mut I) -> Result { let size = input.bytes().len(); if size == 0 { return Ok(MutationResult::Skipped); @@ -710,8 +665,9 @@ where } impl Named for BytesSetMutator { - fn name(&self) -> &str { - "BytesSetMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("BytesSetMutator"); + &NAME } } @@ -732,12 +688,7 @@ where S: HasRand, I: HasBytesVec, { - fn mutate( - &mut self, - state: &mut S, - input: &mut I, - _stage_idx: i32, - ) -> Result { + fn mutate(&mut self, state: &mut S, input: &mut I) -> Result { let size = input.bytes().len(); if size == 0 { return Ok(MutationResult::Skipped); @@ -753,8 +704,9 @@ where } impl Named for BytesRandSetMutator { - fn name(&self) -> &str { - "BytesRandSetMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("BytesRandSetMutator"); + &NAME } } @@ -775,18 +727,13 @@ where S: HasRand, I: HasBytesVec, { - fn mutate( - &mut self, - state: &mut S, - input: &mut I, - _stage_idx: i32, - ) -> Result { + fn mutate(&mut self, state: &mut S, input: &mut I) -> Result { let size = input.bytes().len(); if size <= 1 { return Ok(MutationResult::Skipped); } - let target = state.rand_mut().below(size as u64) as usize; + let target = state.rand_mut().below(size); let range = rand_range(state, size, size - target); unsafe { @@ -798,8 +745,9 @@ where } impl Named for BytesCopyMutator { - fn name(&self) -> &str { - "BytesCopyMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("BytesCopyMutator"); + &NAME } } @@ -822,18 +770,13 @@ where S: HasRand + HasMaxSize, I: HasBytesVec, { - fn mutate( - &mut self, - state: &mut S, - input: &mut I, - _stage_idx: i32, - ) -> Result { + fn mutate(&mut self, state: &mut S, input: &mut I) -> Result { let size = input.bytes().len(); if size <= 1 || size >= state.max_size() { return Ok(MutationResult::Skipped); } - let target = state.rand_mut().below(size as u64) as usize; + let target = state.rand_mut().below(size); // make sure that the sampled range is both in bounds and of an acceptable size let max_insert_len = min(size - target, state.max_size() - size); let range = rand_range(state, size, min(16, max_insert_len)); @@ -862,8 +805,9 @@ where } impl Named for BytesInsertCopyMutator { - fn name(&self) -> &str { - "BytesInsertCopyMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("BytesInsertCopyMutator"); + &NAME } } @@ -887,12 +831,7 @@ where S: HasRand, I: HasBytesVec, { - fn mutate( - &mut self, - state: &mut S, - input: &mut I, - _stage_idx: i32, - ) -> Result { + fn mutate(&mut self, state: &mut S, input: &mut I) -> Result { let size = input.bytes().len(); if size <= 1 { return Ok(MutationResult::Skipped); @@ -1070,8 +1009,9 @@ where } impl Named for BytesSwapMutator { - fn name(&self) -> &str { - "BytesSwapMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("BytesSwapMutator"); + &NAME } } @@ -1125,12 +1065,7 @@ where S: HasCorpus + HasRand + HasMaxSize, I: Input + HasBytesVec, { - fn mutate( - &mut self, - state: &mut S, - input: &mut S::Input, - _stage_idx: i32, - ) -> Result { + fn mutate(&mut self, state: &mut S, input: &mut S::Input) -> Result { let size = input.bytes().len(); let max_size = state.max_size(); if size >= max_size { @@ -1138,7 +1073,7 @@ where } // We don't want to use the testcase we're already using for splicing - let idx = random_corpus_id!(state.corpus(), state.rand_mut()); + let idx = random_corpus_id_with_disabled!(state.corpus(), state.rand_mut()); if let Some(cur) = state.corpus().current() { if idx == *cur { @@ -1147,7 +1082,7 @@ where } let other_size = { - let mut other_testcase = state.corpus().get(idx)?.borrow_mut(); + let mut other_testcase = state.corpus().get_from_all(idx)?.borrow_mut(); other_testcase.load_input(state.corpus())?.bytes().len() }; @@ -1156,9 +1091,9 @@ where } let range = rand_range(state, other_size, min(other_size, max_size - size)); - let target = state.rand_mut().below(size as u64) as usize; + let target = state.rand_mut().below(size); - let other_testcase = state.corpus().get(idx)?.borrow_mut(); + let other_testcase = state.corpus().get_from_all(idx)?.borrow_mut(); // No need to load the input again, it'll still be cached. let other = other_testcase.input().as_ref().unwrap(); @@ -1167,8 +1102,9 @@ where } impl Named for CrossoverInsertMutator { - fn name(&self) -> &str { - "CrossoverInsertMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("CrossoverInsertMutator"); + &NAME } } @@ -1213,19 +1149,14 @@ where S: HasCorpus + HasRand, I: Input + HasBytesVec, { - fn mutate( - &mut self, - state: &mut S, - input: &mut S::Input, - _stage_idx: i32, - ) -> Result { + fn mutate(&mut self, state: &mut S, input: &mut S::Input) -> Result { let size = input.bytes().len(); if size == 0 { return Ok(MutationResult::Skipped); } // We don't want to use the testcase we're already using for splicing - let idx = random_corpus_id!(state.corpus(), state.rand_mut()); + let idx = random_corpus_id_with_disabled!(state.corpus(), state.rand_mut()); if let Some(cur) = state.corpus().current() { if idx == *cur { return Ok(MutationResult::Skipped); @@ -1233,7 +1164,7 @@ where } let other_size = { - let mut testcase = state.corpus().get(idx)?.borrow_mut(); + let mut testcase = state.corpus().get_from_all(idx)?.borrow_mut(); testcase.load_input(state.corpus())?.bytes().len() }; @@ -1241,10 +1172,10 @@ where return Ok(MutationResult::Skipped); } - let target = state.rand_mut().below(size as u64) as usize; + let target = state.rand_mut().below(size); let range = rand_range(state, other_size, min(other_size, size - target)); - let other_testcase = state.corpus().get(idx)?.borrow_mut(); + let other_testcase = state.corpus().get_from_all(idx)?.borrow_mut(); // No need to load the input again, it'll still be cached. let other = other_testcase.input().as_ref().unwrap(); @@ -1253,8 +1184,9 @@ where } impl Named for CrossoverReplaceMutator { - fn name(&self) -> &str { - "CrossoverReplaceMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("CrossoverReplaceMutator"); + &NAME } } @@ -1295,14 +1227,9 @@ where S::Input: HasBytesVec, { #[allow(clippy::cast_sign_loss)] - fn mutate( - &mut self, - state: &mut S, - input: &mut S::Input, - _stage_idx: i32, - ) -> Result { + fn mutate(&mut self, state: &mut S, input: &mut S::Input) -> Result { // We don't want to use the testcase we're already using for splicing - let idx = random_corpus_id!(state.corpus(), state.rand_mut()); + let idx = random_corpus_id_with_disabled!(state.corpus(), state.rand_mut()); if let Some(cur) = state.corpus().current() { if idx == *cur { return Ok(MutationResult::Skipped); @@ -1310,21 +1237,21 @@ where } let (first_diff, last_diff) = { - let mut other_testcase = state.corpus().get(idx)?.borrow_mut(); + let mut other_testcase = state.corpus().get_from_all(idx)?.borrow_mut(); let other = other_testcase.load_input(state.corpus())?; let (f, l) = locate_diffs(input.bytes(), other.bytes()); if f != l && f >= 0 && l >= 2 { - (f as u64, l as u64) + (f as usize, l as usize) } else { return Ok(MutationResult::Skipped); } }; - let split_at = state.rand_mut().between(first_diff, last_diff) as usize; + let split_at = state.rand_mut().between(first_diff, last_diff); - let other_testcase = state.corpus().get(idx)?.borrow_mut(); + let other_testcase = state.corpus().get_from_all(idx)?.borrow_mut(); // Input will already be loaded. let other = other_testcase.input().as_ref().unwrap(); @@ -1337,8 +1264,9 @@ where } impl Named for SpliceMutator { - fn name(&self) -> &str { - "SpliceMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("SpliceMutator"); + &NAME } } @@ -1401,11 +1329,8 @@ mod tests { use super::*; use crate::{ - corpus::{Corpus, InMemoryCorpus}, - feedbacks::ConstFeedback, - inputs::BytesInput, - mutators::MutatorsTuple, - state::{HasMetadata, StdState}, + corpus::InMemoryCorpus, feedbacks::ConstFeedback, inputs::BytesInput, + mutators::MutatorsTuple, state::StdState, HasMetadata, }; type TestMutatorsTupleType = tuple_list_type!( @@ -1508,7 +1433,7 @@ mod tests { for input in &inputs { let mut mutant = input.clone(); match mutations - .get_and_mutate(idx.into(), &mut state, &mut mutant, 0) + .get_and_mutate(idx.into(), &mut state, &mut mutant) .unwrap() { MutationResult::Mutated => new_testcases.push(mutant), @@ -1534,7 +1459,7 @@ mod tests { for _ in 0..iters { let mut mutated = base.clone(); - if mutator.mutate(&mut state, &mut mutated, 0)? == MutationResult::Skipped { + if mutator.mutate(&mut state, &mut mutated)? == MutationResult::Skipped { continue; } let mut gaps = 0; @@ -1588,7 +1513,7 @@ mod tests { for _ in 0..iters { let mut mutated = base.clone(); - if mutator.mutate(&mut state, &mut mutated, 0)? == MutationResult::Skipped { + if mutator.mutate(&mut state, &mut mutated)? == MutationResult::Skipped { continue; } let mut expansion = 0; @@ -1637,7 +1562,7 @@ mod tests { for _ in 0..iters { let mut mutated = base.clone(); - if mutator.mutate(&mut state, &mut mutated, 0)? == MutationResult::Skipped { + if mutator.mutate(&mut state, &mut mutated)? == MutationResult::Skipped { continue; } let mut inserted = 0; @@ -1687,7 +1612,7 @@ mod tests { for _ in 0..iters { let mut mutated = base.clone(); - if mutator.mutate(&mut state, &mut mutated, 0)? == MutationResult::Skipped { + if mutator.mutate(&mut state, &mut mutated)? == MutationResult::Skipped { continue; } let mut inserted = 10; diff --git a/libafl/src/mutators/nautilus.rs b/libafl/src/mutators/nautilus.rs index d6b64808f9..c544b3e4ad 100644 --- a/libafl/src/mutators/nautilus.rs +++ b/libafl/src/mutators/nautilus.rs @@ -1,5 +1,6 @@ //! Mutators for the `Nautilus` grammmar fuzzer +use alloc::borrow::Cow; use core::fmt::Debug; use grammartec::{ @@ -14,8 +15,8 @@ use crate::{ generators::nautilus::NautilusContext, inputs::nautilus::NautilusInput, mutators::{MutationResult, Mutator}, - state::{HasCorpus, HasMetadata}, - Error, + state::HasCorpus, + Error, HasMetadata, }; /// The randomic mutator for `Nautilus` grammar. @@ -35,7 +36,6 @@ impl Mutator for NautilusRandomMutator<'_> { &mut self, _state: &mut S, input: &mut NautilusInput, - _stage_idx: i32, ) -> Result { // TODO get rid of tmp let mut tmp = vec![]; @@ -61,8 +61,9 @@ impl Mutator for NautilusRandomMutator<'_> { } impl Named for NautilusRandomMutator<'_> { - fn name(&self) -> &str { - "NautilusRandomMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("NautilusRandomMutator"); + &NAME } } @@ -96,7 +97,6 @@ impl Mutator for NautilusRecursionMutator<'_> { &mut self, _state: &mut S, input: &mut NautilusInput, - _stage_idx: i32, ) -> Result { // TODO don't calc recursions here if let Some(ref mut recursions) = input.tree.calc_recursions(self.ctx) { @@ -125,8 +125,9 @@ impl Mutator for NautilusRecursionMutator<'_> { } impl Named for NautilusRecursionMutator<'_> { - fn name(&self) -> &str { - "NautilusRecursionMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("NautilusRecursionMutator"); + &NAME } } @@ -162,7 +163,6 @@ where &mut self, state: &mut S, input: &mut NautilusInput, - _stage_idx: i32, ) -> Result { let meta = state .metadata_map() @@ -193,8 +193,9 @@ where } impl Named for NautilusSpliceMutator<'_> { - fn name(&self) -> &str { - "NautilusSpliceMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("NautilusSpliceMutator"); + &NAME } } diff --git a/libafl/src/mutators/scheduled.rs b/libafl/src/mutators/scheduled.rs index 4bbff8dd19..a4a5f095e5 100644 --- a/libafl/src/mutators/scheduled.rs +++ b/libafl/src/mutators/scheduled.rs @@ -1,15 +1,16 @@ //! The `ScheduledMutator` schedules multiple mutations internally. -use alloc::{string::String, vec::Vec}; +use alloc::{borrow::Cow, vec::Vec}; use core::{ fmt::{self, Debug}, marker::PhantomData, + ops::{Deref, DerefMut}, }; use libafl_bolts::{ rands::Rand, tuples::{tuple_list, tuple_list_type, Merge, NamedTuple}, - AsMutSlice, AsSlice, Named, + Named, }; use serde::{Deserialize, Serialize}; @@ -28,8 +29,8 @@ use crate::{ token_mutations::{TokenInsert, TokenReplace}, MutationResult, Mutator, MutatorsTuple, }, - state::{HasCorpus, HasMetadata, HasRand}, - Error, + state::{HasCorpus, HasRand}, + Error, HasMetadata, }; /// The metadata placed in a [`crate::corpus::Testcase`] by a [`LoggerScheduledMutator`]. @@ -40,30 +41,28 @@ use crate::{ )] // for SerdeAny pub struct LogMutationMetadata { /// A list of logs - pub list: Vec, + pub list: Vec>, } libafl_bolts::impl_serdeany!(LogMutationMetadata); -impl AsSlice for LogMutationMetadata { - type Entry = String; - #[must_use] - fn as_slice(&self) -> &[String] { - self.list.as_slice() +impl Deref for LogMutationMetadata { + type Target = [Cow<'static, str>]; + fn deref(&self) -> &[Cow<'static, str>] { + &self.list } } -impl AsMutSlice for LogMutationMetadata { - type Entry = String; +impl DerefMut for LogMutationMetadata { #[must_use] - fn as_mut_slice(&mut self) -> &mut [String] { - self.list.as_mut_slice() + fn deref_mut(&mut self) -> &mut [Cow<'static, str>] { + &mut self.list } } impl LogMutationMetadata { /// Creates new [`struct@LogMutationMetadata`]. #[must_use] - pub fn new(list: Vec) -> Self { + pub fn new(list: Vec>) -> Self { Self { list } } } @@ -93,19 +92,12 @@ where /// New default implementation for mutate. /// Implementations must forward `mutate()` to this method - fn scheduled_mutate( - &mut self, - state: &mut S, - input: &mut I, - stage_idx: i32, - ) -> Result { + fn scheduled_mutate(&mut self, state: &mut S, input: &mut I) -> Result { let mut r = MutationResult::Skipped; let num = self.iterations(state, input); for _ in 0..num { let idx = self.schedule(state, input); - let outcome = self - .mutations_mut() - .get_and_mutate(idx, state, input, stage_idx)?; + let outcome = self.mutations_mut().get_and_mutate(idx, state, input)?; if outcome == MutationResult::Mutated { r = MutationResult::Mutated; } @@ -120,9 +112,9 @@ where MT: MutatorsTuple, S: HasRand, { - name: String, + name: Cow<'static, str>, mutations: MT, - max_stack_pow: u64, + max_stack_pow: usize, phantom: PhantomData<(I, S)>, } @@ -135,7 +127,7 @@ where write!( f, "StdScheduledMutator with {} mutations for Input type {}", - MT::LEN, + self.mutations.len(), core::any::type_name::() ) } @@ -146,7 +138,7 @@ where MT: MutatorsTuple, S: HasRand, { - fn name(&self) -> &str { + fn name(&self) -> &Cow<'static, str> { &self.name } } @@ -157,13 +149,8 @@ where S: HasRand, { #[inline] - fn mutate( - &mut self, - state: &mut S, - input: &mut I, - stage_idx: i32, - ) -> Result { - self.scheduled_mutate(state, input, stage_idx) + fn mutate(&mut self, state: &mut S, input: &mut I) -> Result { + self.scheduled_mutate(state, input) } } @@ -197,8 +184,8 @@ where /// Get the next mutation to apply fn schedule(&self, state: &mut S, _: &I) -> MutationId { - debug_assert!(MT::LEN != 0); - state.rand_mut().below(MT::LEN as u64).into() + debug_assert!(self.mutations.len() != 0); + state.rand_mut().below(self.mutations.len()).into() } } @@ -210,7 +197,10 @@ where /// Create a new [`StdScheduledMutator`] instance specifying mutations pub fn new(mutations: MT) -> Self { StdScheduledMutator { - name: format!("StdScheduledMutator[{}]", mutations.names().join(", ")), + name: Cow::from(format!( + "StdScheduledMutator[{}]", + mutations.names().join(", ") + )), mutations, max_stack_pow: 7, phantom: PhantomData, @@ -218,9 +208,12 @@ where } /// Create a new [`StdScheduledMutator`] instance specifying mutations and the maximun number of iterations - pub fn with_max_stack_pow(mutations: MT, max_stack_pow: u64) -> Self { + pub fn with_max_stack_pow(mutations: MT, max_stack_pow: usize) -> Self { StdScheduledMutator { - name: format!("StdScheduledMutator[{}]", mutations.names().join(", ")), + name: Cow::from(format!( + "StdScheduledMutator[{}]", + mutations.names().join(", ") + )), mutations, max_stack_pow, phantom: PhantomData, @@ -352,7 +345,7 @@ where S: HasRand + HasCorpus, SM: ScheduledMutator, { - name: String, + name: Cow<'static, str>, scheduled: SM, mutation_log: Vec, phantom: PhantomData<(I, MT, S)>, @@ -380,7 +373,7 @@ where S: HasRand + HasCorpus, SM: ScheduledMutator, { - fn name(&self) -> &str { + fn name(&self) -> &Cow<'static, str> { &self.name } } @@ -391,26 +384,16 @@ where S: HasRand + HasCorpus, SM: ScheduledMutator, { - fn mutate( - &mut self, - state: &mut S, - input: &mut I, - stage_idx: i32, - ) -> Result { - self.scheduled_mutate(state, input, stage_idx) + fn mutate(&mut self, state: &mut S, input: &mut I) -> Result { + self.scheduled_mutate(state, input) } - fn post_exec( - &mut self, - state: &mut S, - _stage_idx: i32, - corpus_idx: Option, - ) -> Result<(), Error> { + fn post_exec(&mut self, state: &mut S, corpus_idx: Option) -> Result<(), Error> { if let Some(idx) = corpus_idx { let mut testcase = (*state.corpus_mut().get(idx)?).borrow_mut(); - let mut log = Vec::::new(); + let mut log = Vec::>::new(); while let Some(idx) = self.mutation_log.pop() { - let name = String::from(self.scheduled.mutations().name(idx.0).unwrap()); // TODO maybe return an Error on None + let name = self.scheduled.mutations().name(idx.0).unwrap().clone(); // TODO maybe return an Error on None log.push(name); } let meta = LogMutationMetadata::new(log); @@ -453,24 +436,17 @@ where /// Get the next mutation to apply fn schedule(&self, state: &mut S, _: &I) -> MutationId { debug_assert!(MT::LEN != 0); - state.rand_mut().below(MT::LEN as u64).into() + state.rand_mut().below(MT::LEN).into() } - fn scheduled_mutate( - &mut self, - state: &mut S, - input: &mut I, - stage_idx: i32, - ) -> Result { + fn scheduled_mutate(&mut self, state: &mut S, input: &mut I) -> Result { let mut r = MutationResult::Skipped; let num = self.iterations(state, input); self.mutation_log.clear(); for _ in 0..num { let idx = self.schedule(state, input); self.mutation_log.push(idx); - let outcome = self - .mutations_mut() - .get_and_mutate(idx, state, input, stage_idx)?; + let outcome = self.mutations_mut().get_and_mutate(idx, state, input)?; if outcome == MutationResult::Mutated { r = MutationResult::Mutated; } @@ -489,7 +465,7 @@ where /// This mutator logs all mutators. pub fn new(scheduled: SM) -> Self { Self { - name: format!("LoggerScheduledMutator[{}]", scheduled.name()), + name: Cow::from(format!("LoggerScheduledMutator[{}]", scheduled.name())), scheduled, mutation_log: vec![], phantom: PhantomData, @@ -499,7 +475,7 @@ where #[cfg(test)] mod tests { - use libafl_bolts::rands::{Rand, StdRand, XkcdRand}; + use libafl_bolts::rands::{StdRand, XkcdRand}; use crate::{ corpus::{Corpus, InMemoryCorpus, Testcase}, @@ -515,8 +491,7 @@ mod tests { #[test] fn test_mut_scheduled() { - // With the current impl, seed of 1 will result in a split at pos 2. - let mut rand = XkcdRand::with_seed(5); + let rand = XkcdRand::with_seed(0); let mut corpus: InMemoryCorpus = InMemoryCorpus::new(); corpus .add(Testcase::new(vec![b'a', b'b', b'c'].into())) @@ -539,21 +514,17 @@ mod tests { ) .unwrap(); - rand.set_seed(5); - let mut splice = SpliceMutator::new(); - splice.mutate(&mut state, &mut input, 0).unwrap(); + splice.mutate(&mut state, &mut input).unwrap(); log::trace!("{:?}", input.bytes()); // The pre-seeded rand should have spliced at position 2. - // TODO: Maybe have a fixed rand for this purpose? assert_eq!(input.bytes(), &[b'a', b'b', b'f']); } #[test] fn test_havoc() { - // With the current impl, seed of 1 will result in a split at pos 2. let rand = StdRand::with_seed(0x1337); let mut corpus: InMemoryCorpus = InMemoryCorpus::new(); corpus @@ -584,8 +555,8 @@ mod tests { let mut equal_in_a_row = 0; - for i in 0..42 { - havoc.mutate(&mut state, &mut input, i).unwrap(); + for _ in 0..42 { + havoc.mutate(&mut state, &mut input).unwrap(); // Make sure we actually mutate something, at least sometimes equal_in_a_row = if input == input_prior { @@ -597,44 +568,3 @@ mod tests { } } } - -/// `SchedulerMutator` Python bindings -#[cfg(feature = "python")] -#[allow(missing_docs)] -#[allow(clippy::unnecessary_fallible_conversions)] -pub mod pybind { - use pyo3::prelude::*; - - use super::{havoc_mutations, Debug, HavocMutationsType, StdScheduledMutator}; - use crate::{ - inputs::BytesInput, mutators::pybind::PythonMutator, state::pybind::PythonStdState, - }; - - #[pyclass(unsendable, name = "StdHavocMutator")] - #[derive(Debug)] - /// Python class for StdHavocMutator - pub struct PythonStdHavocMutator { - /// Rust wrapped StdHavocMutator object - pub inner: StdScheduledMutator, PythonStdState>, - } - - #[pymethods] - impl PythonStdHavocMutator { - #[new] - fn new() -> Self { - Self { - inner: StdScheduledMutator::new(havoc_mutations()), - } - } - - fn as_mutator(slf: Py) -> PythonMutator { - PythonMutator::new_std_havoc(slf) - } - } - - /// Register the classes to the python module - pub fn register(_py: Python, m: &PyModule) -> PyResult<()> { - m.add_class::()?; - Ok(()) - } -} diff --git a/libafl/src/mutators/string.rs b/libafl/src/mutators/string.rs index 76a820aaae..75665f1741 100644 --- a/libafl/src/mutators/string.rs +++ b/libafl/src/mutators/string.rs @@ -1,5 +1,5 @@ //! Mutators for preserving string categories, which may be useful for certain targets which are primarily string-oriented. -use alloc::vec::Vec; +use alloc::{borrow::Cow, vec::Vec}; use core::{ cmp::{Ordering, Reverse}, ops::Range, @@ -16,7 +16,8 @@ use crate::{ mutational::{MutatedTransform, MutatedTransformPost}, StringIdentificationMetadata, }, - state::{HasCorpus, HasMaxSize, HasMetadata, HasRand}, + state::{HasCorpus, HasMaxSize, HasRand}, + HasMetadata, }; /// Unicode category data, as used by string analysis and mutators. @@ -34,11 +35,7 @@ where { type Post = StringIdentificationMetadata; - fn try_transform_from( - base: &mut Testcase, - state: &S, - _corpus_idx: CorpusId, - ) -> Result { + fn try_transform_from(base: &mut Testcase, state: &S) -> Result { let input = base.load_input(state.corpus())?.clone(); let metadata = base.metadata::().cloned()?; Ok((input, metadata)) @@ -53,12 +50,7 @@ impl MutatedTransformPost for StringIdentificationMetadata where S: HasTestcase, { - fn post_exec( - self, - state: &mut S, - _stage_idx: i32, - corpus_idx: Option, - ) -> Result<(), Error> { + fn post_exec(self, state: &mut S, corpus_idx: Option) -> Result<(), Error> { if let Some(corpus_idx) = corpus_idx { let mut tc = state.testcase_mut(corpus_idx)?; tc.add_metadata(self); @@ -74,7 +66,7 @@ fn choose_start( bytes: &[u8], meta: &StringIdentificationMetadata, ) -> Option<(usize, usize)> { - let idx = rand.below(bytes.len() as u64) as usize; + let idx = rand.below(bytes.len()); let mut options = Vec::new(); for (start, range) in meta.ranges() { if idx @@ -91,9 +83,9 @@ fn choose_start( _ => { // bias towards longer strings options.sort_by_cached_key(|(_, entries)| entries.count_ones()); - let selected = libafl_bolts::math::integer_sqrt( - rand.below((options.len() * options.len()) as u64), - ) as usize; + let selected = + libafl_bolts::math::integer_sqrt(rand.below(options.len() * options.len()) as u64) + as usize; Some((options[selected].0, options[selected].1.len())) } } @@ -141,7 +133,7 @@ fn choose_category_range( string: &str, ) -> (Range, &'static [(u32, u32)]) { let chars = string.char_indices().collect::>(); - let idx = rand.below(chars.len() as u64) as usize; + let idx = rand.below(chars.len()); let c = chars[idx].1; // figure out the categories for this char @@ -168,7 +160,7 @@ fn choose_category_range( ) }); let options = categories.len() * categories.len(); - let selected_idx = libafl_bolts::math::integer_sqrt(rand.below(options as u64)) as usize; + let selected_idx = libafl_bolts::math::integer_sqrt(rand.below(options) as u64) as usize; let selected = categories[selected_idx]; @@ -185,7 +177,7 @@ fn choose_category_range( fn choose_subcategory_range(rand: &mut R, string: &str) -> (Range, (u32, u32)) { let chars = string.char_indices().collect::>(); - let idx = rand.below(chars.len() as u64) as usize; + let idx = rand.below(chars.len()); let c = chars[idx].1; // figure out the categories for this char @@ -205,7 +197,7 @@ fn choose_subcategory_range(rand: &mut R, string: &str) -> (Range char>( return MutationResult::Skipped; } - let replace_len = state.rand_mut().below(MAX_CHARS as u64) as usize; + let replace_len = state.rand_mut().below(MAX_CHARS); let orig_len = range.end - range.start; if input.0.len() - orig_len + replace_len > state.max_size() { return MutationResult::Skipped; @@ -279,8 +271,9 @@ fn rand_replace_range char>( pub struct StringCategoryRandMutator; impl Named for StringCategoryRandMutator { - fn name(&self) -> &str { - "string-category-rand" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("string-category-rand"); + &NAME } } @@ -288,12 +281,7 @@ impl Mutator for StringCategoryRandMutator where S: HasRand + HasMaxSize, { - fn mutate( - &mut self, - state: &mut S, - input: &mut UnicodeInput, - _stage_idx: i32, - ) -> Result { + fn mutate(&mut self, state: &mut S, input: &mut UnicodeInput) -> Result { if input.0.bytes().is_empty() { return Ok(MutationResult::Skipped); } @@ -310,15 +298,15 @@ where core::str::from_utf8(&bytes[range.clone()]) ); - let options: u64 = category + let options: usize = category .iter() - .map(|&(start, end)| u64::from(end) - u64::from(start) + 1) + .map(|&(start, end)| end as usize - start as usize + 1) .sum(); let char_gen = |state: &mut S| loop { let mut selected = state.rand_mut().below(options); for &(min, max) in category { if let Some(next_selected) = - selected.checked_sub(u64::from(max) - u64::from(min) + 1) + selected.checked_sub(max as usize - min as usize + 1) { selected = next_selected; } else if let Some(new_c) = char::from_u32(selected as u32 + min) { @@ -342,8 +330,9 @@ where pub struct StringSubcategoryRandMutator; impl Named for StringSubcategoryRandMutator { - fn name(&self) -> &str { - "string-subcategory-rand" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("string-subcategory-rand"); + &NAME } } @@ -351,12 +340,7 @@ impl Mutator for StringSubcategoryRandMutator where S: HasRand + HasMaxSize, { - fn mutate( - &mut self, - state: &mut S, - input: &mut UnicodeInput, - _stage_idx: i32, - ) -> Result { + fn mutate(&mut self, state: &mut S, input: &mut UnicodeInput) -> Result { if input.0.bytes().is_empty() { return Ok(MutationResult::Skipped); } @@ -373,7 +357,7 @@ where core::str::from_utf8(&bytes[range.clone()]) ); - let options: u64 = u64::from(subcategory.1) - u64::from(subcategory.0) + 1; + let options = subcategory.1 as usize - subcategory.0 as usize + 1; let char_gen = |state: &mut S| loop { let selected = state.rand_mut().below(options); if let Some(new_c) = char::from_u32(selected as u32 + subcategory.0) { @@ -393,8 +377,9 @@ where pub struct StringCategoryTokenReplaceMutator; impl Named for StringCategoryTokenReplaceMutator { - fn name(&self) -> &str { - "string-category-token-replace" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("string-category-token-replace"); + &NAME } } @@ -402,27 +387,21 @@ impl Mutator for StringCategoryTokenReplaceMutator where S: HasRand + HasMaxSize + HasMetadata, { - fn mutate( - &mut self, - state: &mut S, - input: &mut UnicodeInput, - _stage_idx: i32, - ) -> Result { + fn mutate(&mut self, state: &mut S, input: &mut UnicodeInput) -> Result { if input.0.bytes().is_empty() { return Ok(MutationResult::Skipped); } let tokens_len = { - let meta = state.metadata_map().get::(); - if meta.is_none() { + let Some(meta) = state.metadata_map().get::() else { return Ok(MutationResult::Skipped); - } - if meta.unwrap().tokens().is_empty() { + }; + if meta.tokens().is_empty() { return Ok(MutationResult::Skipped); } - meta.unwrap().tokens().len() + meta.tokens().len() }; - let token_idx = state.rand_mut().below(tokens_len as u64) as usize; + let token_idx = state.rand_mut().below(tokens_len); let bytes = input.0.bytes(); let meta = &input.1; @@ -458,8 +437,9 @@ where pub struct StringSubcategoryTokenReplaceMutator; impl Named for StringSubcategoryTokenReplaceMutator { - fn name(&self) -> &str { - "string-subcategory-replace" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("string-subcategory-replace"); + &NAME } } @@ -467,27 +447,21 @@ impl Mutator for StringSubcategoryTokenReplaceMutator where S: HasRand + HasMaxSize + HasMetadata, { - fn mutate( - &mut self, - state: &mut S, - input: &mut UnicodeInput, - _stage_idx: i32, - ) -> Result { + fn mutate(&mut self, state: &mut S, input: &mut UnicodeInput) -> Result { if input.0.bytes().is_empty() { return Ok(MutationResult::Skipped); } let tokens_len = { - let meta = state.metadata_map().get::(); - if meta.is_none() { + let Some(meta) = state.metadata_map().get::() else { return Ok(MutationResult::Skipped); - } - if meta.unwrap().tokens().is_empty() { + }; + if meta.tokens().is_empty() { return Ok(MutationResult::Skipped); } - meta.unwrap().tokens().len() + meta.tokens().len() }; - let token_idx = state.rand_mut().below(tokens_len as u64) as usize; + let token_idx = state.rand_mut().below(tokens_len); let bytes = input.0.bytes(); let meta = &input.1; @@ -520,10 +494,15 @@ where #[cfg(test)] mod test { - use libafl_bolts::rands::StdRand; - - use super::*; - use crate::{corpus::NopCorpus, stages::extract_metadata, state::StdState}; + use libafl_bolts::{rands::StdRand, Error}; + + use crate::{ + corpus::NopCorpus, + inputs::{BytesInput, HasBytesVec}, + mutators::{Mutator, StringCategoryRandMutator, StringSubcategoryRandMutator}, + stages::extract_metadata, + state::StdState, + }; // a not-so-useful test for this #[test] @@ -545,7 +524,7 @@ mod test { for _ in 0..(1 << 12) { let metadata = extract_metadata(bytes.bytes()); let mut input = (bytes, metadata); - let _ = mutator.mutate(&mut state, &mut input, 0); + let _ = mutator.mutate(&mut state, &mut input); println!("{:?}", core::str::from_utf8(input.0.bytes()).unwrap()); bytes = input.0; } @@ -577,7 +556,7 @@ mod test { for _ in 0..(1 << 12) { let metadata = extract_metadata(bytes.bytes()); let mut input = (bytes, metadata); - let _ = mutator.mutate(&mut state, &mut input, 0); + let _ = mutator.mutate(&mut state, &mut input); println!("{:?}", core::str::from_utf8(input.0.bytes()).unwrap()); bytes = input.0; } diff --git a/libafl/src/mutators/token_mutations.rs b/libafl/src/mutators/token_mutations.rs index 6f229d2154..03d221a64a 100644 --- a/libafl/src/mutators/token_mutations.rs +++ b/libafl/src/mutators/token_mutations.rs @@ -1,12 +1,12 @@ //! Tokens are what AFL calls extras or dictionaries. //! They may be inserted as part of mutations during fuzzing. -use alloc::vec::Vec; +use alloc::{borrow::Cow, vec::Vec}; #[cfg(any(target_os = "linux", target_vendor = "apple"))] use core::slice::from_raw_parts; use core::{ fmt::Debug, mem::size_of, - ops::{Add, AddAssign}, + ops::{Add, AddAssign, Deref}, slice::Iter, }; #[cfg(feature = "std")] @@ -23,14 +23,15 @@ use serde::{Deserialize, Serialize}; #[cfg(feature = "std")] use crate::mutators::str_decode; use crate::{ + corpus::{CorpusId, HasCurrentCorpusIdx}, inputs::{HasBytesVec, UsesInput}, mutators::{ buffer_self_copy, mutations::buffer_copy, MultiMutator, MutationResult, Mutator, Named, }, observers::cmp::{AFLppCmpValuesMetadata, CmpValues, CmpValuesMetadata}, stages::TaintMetadata, - state::{HasCorpus, HasMaxSize, HasMetadata, HasRand}, - Error, + state::{HasCorpus, HasMaxSize, HasRand}, + Error, HasMetadata, }; /// A state metadata holding a list of tokens @@ -271,9 +272,9 @@ where } } -impl AsSlice for Tokens { - type Entry = Vec; - fn as_slice(&self) -> &[Vec] { +impl Deref for Tokens { + type Target = [Vec]; + fn deref(&self) -> &[Vec] { self.tokens() } } @@ -306,27 +307,21 @@ where S: HasMetadata + HasRand + HasMaxSize, I: HasBytesVec, { - fn mutate( - &mut self, - state: &mut S, - input: &mut I, - _stage_idx: i32, - ) -> Result { + fn mutate(&mut self, state: &mut S, input: &mut I) -> Result { let max_size = state.max_size(); let tokens_len = { - let meta = state.metadata_map().get::(); - if meta.is_none() { + let Some(meta) = state.metadata_map().get::() else { return Ok(MutationResult::Skipped); - } - if meta.unwrap().tokens().is_empty() { + }; + if meta.tokens().is_empty() { return Ok(MutationResult::Skipped); } - meta.unwrap().tokens().len() + meta.tokens().len() }; - let token_idx = state.rand_mut().below(tokens_len as u64) as usize; + let token_idx = state.rand_mut().below(tokens_len); let size = input.bytes().len(); - let off = state.rand_mut().below((size + 1) as u64) as usize; + let off = state.rand_mut().below(size + 1); let meta = state.metadata_map().get::().unwrap(); let token = &meta.tokens()[token_idx]; @@ -351,8 +346,9 @@ where } impl Named for TokenInsert { - fn name(&self) -> &str { - "TokenInsert" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("TokenInsert"); + &NAME } } @@ -374,30 +370,24 @@ where S: UsesInput + HasMetadata + HasRand + HasMaxSize, I: HasBytesVec, { - fn mutate( - &mut self, - state: &mut S, - input: &mut I, - _stage_idx: i32, - ) -> Result { + fn mutate(&mut self, state: &mut S, input: &mut I) -> Result { let size = input.bytes().len(); if size == 0 { return Ok(MutationResult::Skipped); } let tokens_len = { - let meta = state.metadata_map().get::(); - if meta.is_none() { + let Some(meta) = state.metadata_map().get::() else { return Ok(MutationResult::Skipped); - } - if meta.unwrap().tokens().is_empty() { + }; + if meta.tokens().is_empty() { return Ok(MutationResult::Skipped); } - meta.unwrap().tokens().len() + meta.tokens().len() }; - let token_idx = state.rand_mut().below(tokens_len as u64) as usize; + let token_idx = state.rand_mut().below(tokens_len); - let off = state.rand_mut().below(size as u64) as usize; + let off = state.rand_mut().below(size); let meta = state.metadata_map().get::().unwrap(); let token = &meta.tokens()[token_idx]; @@ -415,8 +405,9 @@ where } impl Named for TokenReplace { - fn name(&self) -> &str { - "TokenReplace" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("TokenReplace"); + &NAME } } @@ -439,31 +430,25 @@ where I: HasBytesVec, { #[allow(clippy::too_many_lines)] - fn mutate( - &mut self, - state: &mut S, - input: &mut I, - _stage_idx: i32, - ) -> Result { + fn mutate(&mut self, state: &mut S, input: &mut I) -> Result { let size = input.bytes().len(); if size == 0 { return Ok(MutationResult::Skipped); } let cmps_len = { - let meta = state.metadata_map().get::(); - log::trace!("meta: {:x?}", meta); - if meta.is_none() { + let Some(meta) = state.metadata_map().get::() else { return Ok(MutationResult::Skipped); - } - if meta.unwrap().list.is_empty() { + }; + log::trace!("meta: {:x?}", meta); + if meta.list.is_empty() { return Ok(MutationResult::Skipped); } - meta.unwrap().list.len() + meta.list.len() }; - let idx = state.rand_mut().below(cmps_len as u64) as usize; + let idx = state.rand_mut().below(cmps_len); - let off = state.rand_mut().below(size as u64) as usize; + let off = state.rand_mut().below(size); let len = input.bytes().len(); let bytes = input.bytes_mut(); @@ -605,8 +590,9 @@ where } impl Named for I2SRandReplace { - fn name(&self) -> &str { - "I2SRandReplace" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("I2SRandReplace"); + &NAME } } @@ -632,6 +618,9 @@ pub struct AFLppRedQueen { enable_transform: bool, enable_arith: bool, text_type: TextType, + /// We use this variable to check if we scheduled a new `corpus_idx` + /// - and, hence, need to recalculate `text_type` + last_corpus_idx: Option, } impl AFLppRedQueen { @@ -1098,7 +1087,7 @@ impl AFLppRedQueen { impl MultiMutator for AFLppRedQueen where - S: UsesInput + HasMetadata + HasRand + HasMaxSize + HasCorpus, + S: UsesInput + HasMetadata + HasRand + HasMaxSize + HasCorpus + HasCurrentCorpusIdx, I: HasBytesVec + From>, { #[allow(clippy::needless_range_loop)] @@ -1107,7 +1096,6 @@ where &mut self, state: &mut S, input: &I, - stage_idx: i32, max_count: Option, ) -> Result, Error> { // TODO @@ -1118,17 +1106,18 @@ where } let (cmp_len, cmp_meta, taint_meta) = { - let cmp_meta = state.metadata_map().get::(); - let taint_meta = state.metadata_map().get::(); - if cmp_meta.is_none() || taint_meta.is_none() { + let (Some(cmp_meta), Some(taint_meta)) = ( + state.metadata_map().get::(), + state.metadata_map().get::(), + ) else { return Ok(vec![]); - } + }; - let cmp_len = cmp_meta.unwrap().headers().len(); + let cmp_len = cmp_meta.headers().len(); if cmp_len == 0 { return Ok(vec![]); } - (cmp_len, cmp_meta.unwrap(), taint_meta.unwrap()) + (cmp_len, cmp_meta, taint_meta) }; // These idxes must saved in this mutator itself! @@ -1146,8 +1135,10 @@ where // println!("orig: {:#?} new: {:#?}", orig_cmpvals, new_cmpvals); // Compute when mutating it for the 1st time. - if stage_idx == 0 { + let current_corpus_idx = state.current_corpus_idx()?.ok_or_else(|| Error::key_not_found("No corpus-idx is currently being fuzzed, but called AFLppRedQueen::multi_mutated()."))?; + if self.last_corpus_idx.is_none() || self.last_corpus_idx.unwrap() != current_corpus_idx { self.text_type = check_if_text(orig_bytes, orig_bytes.len()); + self.last_corpus_idx = Some(current_corpus_idx); } // println!("approximate size: {cmp_len} x {input_len}"); for cmp_idx in 0..cmp_len { @@ -1685,8 +1676,9 @@ where } impl Named for AFLppRedQueen { - fn name(&self) -> &str { - "AFLppRedQueen" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("AFLppRedQueen"); + &NAME } } @@ -1698,6 +1690,7 @@ impl AFLppRedQueen { enable_transform: false, enable_arith: false, text_type: TextType::None, + last_corpus_idx: None, } } @@ -1708,6 +1701,7 @@ impl AFLppRedQueen { enable_transform: transform, enable_arith: arith, text_type: TextType::None, + last_corpus_idx: None, } } diff --git a/libafl/src/mutators/tuneable.rs b/libafl/src/mutators/tuneable.rs index 2ce0686c1c..4ef1dec2c9 100644 --- a/libafl/src/mutators/tuneable.rs +++ b/libafl/src/mutators/tuneable.rs @@ -2,7 +2,7 @@ //! Instead of a random mutator for a random amount of iterations, we can run //! a specific mutator for a specified amount of iterations -use alloc::{string::String, vec::Vec}; +use alloc::{borrow::Cow, vec::Vec}; use core::{ fmt::{self, Debug}, marker::PhantomData, @@ -18,8 +18,8 @@ use crate::{ mutators::{ ComposedByMutations, MutationId, MutationResult, Mutator, MutatorsTuple, ScheduledMutator, }, - state::{HasMetadata, HasRand}, - Error, + state::HasRand, + Error, HasMetadata, }; /// Metadata in the state, that controls the behavior of the [`TuneableScheduledMutator`] at runtime @@ -85,9 +85,9 @@ where MT: MutatorsTuple, S: HasRand, { - name: String, + name: Cow<'static, str>, mutations: MT, - max_stack_pow: u64, + max_stack_pow: usize, phantom: PhantomData<(I, S)>, } @@ -100,7 +100,7 @@ where write!( f, "TuneableScheduledMutator with {} mutations for Input type {}", - MT::LEN, + self.mutations.len(), core::any::type_name::() ) } @@ -112,13 +112,8 @@ where S: HasRand + HasMetadata, { #[inline] - fn mutate( - &mut self, - state: &mut S, - input: &mut I, - stage_id: i32, - ) -> Result { - self.scheduled_mutate(state, input, stage_id) + fn mutate(&mut self, state: &mut S, input: &mut I) -> Result { + self.scheduled_mutate(state, input) } } @@ -145,7 +140,7 @@ where MT: MutatorsTuple, S: HasRand, { - fn name(&self) -> &str { + fn name(&self) -> &Cow<'static, str> { &self.name } } @@ -170,8 +165,7 @@ where // We will sample using the mutation probabilities. // Doing this outside of the original if branch to make the borrow checker happy. #[allow(clippy::cast_precision_loss)] - let coin = state.rand_mut().next() as f32 / u64::MAX as f32; - debug_assert!(coin <= 1.0_f32); + let coin = state.rand_mut().next_float() as f32; let metadata = TuneableScheduledMutatorMetadata::get_mut(state).unwrap(); let power = metadata @@ -186,7 +180,7 @@ where /// Get the next mutation to apply fn schedule(&self, state: &mut S, _: &I) -> MutationId { - debug_assert!(MT::LEN != 0); + debug_assert!(self.mutations.len() != 0); // Assumption: we can not reach this code path without previously adding this metadatum. let metadata = TuneableScheduledMutatorMetadata::get_mut(state).unwrap(); @@ -199,7 +193,7 @@ where metadata.next_id = 0.into(); } debug_assert!( - MT::LEN > ret.0, + self.mutations.len() > ret.0, "TuneableScheduler: next vec may not contain id larger than number of mutations!" ); return ret; @@ -209,12 +203,11 @@ where // We will sample using the mutation probabilities. // Doing this outside of the original if branch to make the borrow checker happy. #[allow(clippy::cast_precision_loss)] - let coin = state.rand_mut().next() as f32 / u64::MAX as f32; - debug_assert!(coin <= 1.0_f32); + let coin = state.rand_mut().next_float() as f32; let metadata = TuneableScheduledMutatorMetadata::get_mut(state).unwrap(); debug_assert_eq!( - MT::LEN, + self.mutations.len(), metadata.mutation_probabilities_cumulative.len(), "TuneableScheduler: mutation probabilities do not match with number of mutations" ); @@ -230,7 +223,7 @@ where } // fall back to random if no entries in either vec, the scheduling is not tuned. - state.rand_mut().below(MT::LEN as u64).into() + state.rand_mut().below(self.mutations.len()).into() } } @@ -245,7 +238,7 @@ where state.add_metadata(TuneableScheduledMutatorMetadata::default()); } TuneableScheduledMutator { - name: format!("TuneableMutator[{}]", mutations.names().join(", ")), + name: Cow::from(format!("TuneableMutator[{}]", mutations.names().join(", "))), mutations, max_stack_pow: 7, phantom: PhantomData, @@ -376,11 +369,13 @@ mod test { use crate::{ inputs::BytesInput, mutators::{ByteRandMutator, ScheduledMutator}, - state::test::NopState, + state::NopState, }; #[test] fn test_tuning() { + // # Safety + // No concurrency per testcase #[cfg(any(not(feature = "serdeany_autoreg"), miri))] unsafe { TuneableScheduledMutatorMetadata::register(); @@ -404,6 +399,8 @@ mod test { #[test] fn test_mutation_distribution() { + // # Safety + // No concurrency per testcase #[cfg(any(not(feature = "serdeany_autoreg"), miri))] unsafe { TuneableScheduledMutatorMetadata::register(); diff --git a/libafl/src/observers/cmp.rs b/libafl/src/observers/cmp.rs index 0c77f9aaa6..1dcd8a9f3d 100644 --- a/libafl/src/observers/cmp.rs +++ b/libafl/src/observers/cmp.rs @@ -1,19 +1,18 @@ //! The `CmpObserver` provides access to the logged values of CMP instructions -use alloc::{ - string::{String, ToString}, - vec::Vec, +use alloc::{borrow::Cow, vec::Vec}; +use core::{ + fmt::Debug, + marker::PhantomData, + ops::{Deref, DerefMut}, }; -use core::{fmt::Debug, marker::PhantomData}; use c2rust_bitfields::BitfieldStruct; use hashbrown::HashMap; -use libafl_bolts::{ownedref::OwnedRefMut, serdeany::SerdeAny, AsMutSlice, AsSlice, Named}; +use libafl_bolts::{ownedref::OwnedRefMut, serdeany::SerdeAny, Named}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; -use crate::{ - executors::ExitKind, inputs::UsesInput, observers::Observer, state::HasMetadata, Error, -}; +use crate::{executors::ExitKind, inputs::UsesInput, observers::Observer, Error, HasMetadata}; /// Generic metadata trait for use in a `CmpObserver`, which adds comparisons from a `CmpObserver` /// primarily intended for use with `AFLppCmpValuesMetadata` or `CmpValuesMetadata` @@ -88,21 +87,16 @@ pub struct CmpValuesMetadata { libafl_bolts::impl_serdeany!(CmpValuesMetadata); -impl AsSlice for CmpValuesMetadata { - type Entry = CmpValues; - /// Convert to a slice - #[must_use] - fn as_slice(&self) -> &[CmpValues] { - self.list.as_slice() +impl Deref for CmpValuesMetadata { + type Target = [CmpValues]; + fn deref(&self) -> &[CmpValues] { + &self.list } } -impl AsMutSlice for CmpValuesMetadata { - type Entry = CmpValues; - /// Convert to a slice - #[must_use] - fn as_mut_slice(&mut self) -> &mut [CmpValues] { - self.list.as_mut_slice() +impl DerefMut for CmpValuesMetadata { + fn deref_mut(&mut self) -> &mut [CmpValues] { + &mut self.list } } @@ -232,12 +226,7 @@ where S: HasMetadata, { #[allow(clippy::option_if_let_else)] // we can't mutate state in a closure - let meta = if let Some(meta) = state.metadata_map_mut().get_mut::() { - meta - } else { - state.add_metadata(M::new_metadata()); - state.metadata_map_mut().get_mut::().unwrap() - }; + let meta = state.metadata_or_insert_with(|| M::new_metadata()); let usable_count = self.usable_count(); let cmp_observer_data = self.cmp_observer_data(); @@ -257,7 +246,7 @@ where { cmp_map: OwnedRefMut<'a, CM>, size: Option>, - name: String, + name: Cow<'static, str>, add_meta: bool, data: M::Data, phantom: PhantomData, @@ -320,7 +309,7 @@ where S: UsesInput + HasMetadata, M: CmpObserverMetadata<'a, CM>, { - fn name(&self) -> &str { + fn name(&self) -> &Cow<'static, str> { &self.name } } @@ -335,7 +324,7 @@ where #[must_use] pub fn new(name: &'static str, map: OwnedRefMut<'a, CM>, add_meta: bool) -> Self { Self { - name: name.to_string(), + name: Cow::from(name), size: None, cmp_map: map, add_meta, @@ -354,7 +343,7 @@ where data: M::Data, ) -> Self { Self { - name: name.to_string(), + name: Cow::from(name), size: None, cmp_map, add_meta, @@ -372,7 +361,7 @@ where size: OwnedRefMut<'a, usize>, ) -> Self { Self { - name: name.to_string(), + name: Cow::from(name), size: Some(size), cmp_map, add_meta, @@ -392,7 +381,7 @@ where size: OwnedRefMut<'a, usize>, ) -> Self { Self { - name: name.to_string(), + name: Cow::from(name), size: Some(size), cmp_map, add_meta, @@ -401,7 +390,7 @@ where } } - /// Reference the stored auxiliary data associated with the [`CmpObserverMetadata`] + /// Handle the stored auxiliary data associated with the [`CmpObserverMetadata`] pub fn data(&self) -> &M::Data { &self.data } @@ -468,10 +457,10 @@ struct cmp_map { allow(clippy::unsafe_derive_deserialize) )] // for SerdeAny pub struct AFLppCmpValuesMetadata { - /// The first map of AFLppCmpLogVals retrieved by running the un-mutated input + /// The first map of `AFLppCmpLogVals` retrieved by running the un-mutated input #[serde(skip)] pub orig_cmpvals: HashMap>, - /// The second map of AFLppCmpLogVals retrieved by runnning the mutated input + /// The second map of `AFLppCmpLogVals` retrieved by runnning the mutated input #[serde(skip)] pub new_cmpvals: HashMap>, /// The list of logged idx and headers retrieved by runnning the mutated input diff --git a/libafl/src/observers/concolic/mod.rs b/libafl/src/observers/concolic/mod.rs index 1f2eea9396..c05ca9317c 100644 --- a/libafl/src/observers/concolic/mod.rs +++ b/libafl/src/observers/concolic/mod.rs @@ -28,13 +28,13 @@ pub struct Location(usize); impl Debug for Location { fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error> { - core::fmt::Debug::fmt(&self.0, f) + Debug::fmt(&self.0, f) } } impl Display for Location { fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error> { - core::fmt::Display::fmt(&self.0, f) + Display::fmt(&self.0, f) } } @@ -60,7 +60,6 @@ pub enum SymExpr { offset: usize, value: u8, }, - Integer { value: u64, bits: u8, @@ -69,6 +68,7 @@ pub enum SymExpr { high: u64, low: u64, }, + IntegerFromBuffer {}, Float { value: f64, is_double: bool, diff --git a/libafl/src/observers/concolic/observer.rs b/libafl/src/observers/concolic/observer.rs index c84da0651f..a3e47ed1d3 100644 --- a/libafl/src/observers/concolic/observer.rs +++ b/libafl/src/observers/concolic/observer.rs @@ -1,4 +1,4 @@ -use alloc::string::String; +use alloc::borrow::Cow; use libafl_bolts::Named; use serde::{Deserialize, Serialize}; @@ -16,7 +16,7 @@ use crate::{ pub struct ConcolicObserver<'map> { #[serde(skip)] map: &'map [u8], - name: String, + name: Cow<'static, str>, } impl<'map, S> Observer for ConcolicObserver<'map> where S: UsesInput {} @@ -32,7 +32,7 @@ impl<'map> ConcolicObserver<'map> { } impl<'map> Named for ConcolicObserver<'map> { - fn name(&self) -> &str { + fn name(&self) -> &Cow<'static, str> { &self.name } } @@ -40,7 +40,10 @@ impl<'map> Named for ConcolicObserver<'map> { impl<'map> ConcolicObserver<'map> { /// Creates a new [`ConcolicObserver`] with the given name and memory buffer. #[must_use] - pub fn new(name: String, map: &'map [u8]) -> Self { - Self { map, name } + pub fn new(name: &'static str, map: &'map [u8]) -> Self { + Self { + map, + name: Cow::from(name), + } } } diff --git a/libafl/src/observers/concolic/serialization_format.rs b/libafl/src/observers/concolic/serialization_format.rs index e30ab5f5cd..bac1063843 100644 --- a/libafl/src/observers/concolic/serialization_format.rs +++ b/libafl/src/observers/concolic/serialization_format.rs @@ -41,8 +41,6 @@ //! //! ... making for a total of 5 bytes. -#![cfg(feature = "std")] - use std::{ fmt::{self, Debug, Formatter}, io::{self, Cursor, Read, Seek, SeekFrom, Write}, @@ -93,7 +91,7 @@ impl MessageFileReader { Some(Ok((message_id, message))) } Err(e) => match *e { - bincode::ErrorKind::Io(ref io_err) => match io_err.kind() { + ErrorKind::Io(ref io_err) => match io_err.kind() { io::ErrorKind::UnexpectedEof => None, _ => Some(Err(e)), }, @@ -117,6 +115,7 @@ impl MessageFileReader { SymExpr::InputByte { .. } | SymExpr::Integer { .. } | SymExpr::Integer128 { .. } + | SymExpr::IntegerFromBuffer { .. } | SymExpr::Float { .. } | SymExpr::NullPointer | SymExpr::True @@ -290,6 +289,7 @@ impl MessageFileWriter { SymExpr::InputByte { .. } | SymExpr::Integer { .. } | SymExpr::Integer128 { .. } + | SymExpr::IntegerFromBuffer { .. } | SymExpr::Float { .. } | SymExpr::NullPointer | SymExpr::True diff --git a/libafl/src/observers/list.rs b/libafl/src/observers/list.rs new file mode 100644 index 0000000000..99a50e96f5 --- /dev/null +++ b/libafl/src/observers/list.rs @@ -0,0 +1,67 @@ +use alloc::{borrow::Cow, vec::Vec}; +use core::fmt::Debug; + +use libafl_bolts::{ownedref::OwnedMutPtr, Error, Named}; +use serde::{Deserialize, Serialize}; + +use crate::{inputs::UsesInput, observers::Observer}; + +/// A simple observer with a list of things. +#[derive(Serialize, Deserialize, Debug)] +#[serde(bound = "T: serde::de::DeserializeOwned + serde::Serialize")] +#[allow(clippy::unsafe_derive_deserialize)] +pub struct ListObserver { + name: Cow<'static, str>, + /// The list + list: OwnedMutPtr>, +} + +impl ListObserver +where + T: Debug + Serialize + serde::de::DeserializeOwned, +{ + /// Creates a new [`ListObserver`] with the given name. + /// + /// # Safety + /// Will dereference the list. + /// The list may not move in memory. + #[must_use] + pub fn new(name: &'static str, list: OwnedMutPtr>) -> Self { + Self { + name: Cow::from(name), + list, + } + } + + /// Get a list ref + #[must_use] + pub fn list(&self) -> &Vec { + self.list.as_ref() + } + + /// Get a list mut + #[must_use] + pub fn list_mut(&mut self) -> &mut Vec { + self.list.as_mut() + } +} + +impl Observer for ListObserver +where + S: UsesInput, + T: Debug + Serialize + serde::de::DeserializeOwned, +{ + fn pre_exec(&mut self, _state: &mut S, _input: &S::Input) -> Result<(), Error> { + self.list.as_mut().clear(); + Ok(()) + } +} + +impl Named for ListObserver +where + T: Debug + Serialize + serde::de::DeserializeOwned, +{ + fn name(&self) -> &Cow<'static, str> { + &self.name + } +} diff --git a/libafl/src/observers/map.rs b/libafl/src/observers/map.rs deleted file mode 100644 index 652c2b5aac..0000000000 --- a/libafl/src/observers/map.rs +++ /dev/null @@ -1,2833 +0,0 @@ -//! The `MapObserver` provides access a map, usually injected into the target - -use alloc::{ - string::{String, ToString}, - vec::Vec, -}; -use core::{ - fmt::Debug, - hash::{BuildHasher, Hasher}, - iter::Flatten, - marker::PhantomData, - mem::size_of, - slice::{self, Iter, IterMut}, -}; - -use ahash::RandomState; -use libafl_bolts::{ - ownedref::{OwnedMutPtr, OwnedMutSlice}, - AsIter, AsIterMut, AsMutSlice, AsSlice, HasLen, Named, Truncate, -}; -use meminterval::IntervalTree; -use num_traits::Bounded; -use serde::{Deserialize, Serialize}; - -use crate::{ - executors::ExitKind, - inputs::UsesInput, - observers::{DifferentialObserver, Observer, ObserversTuple}, - Error, -}; - -/// Hitcounts class lookup -static COUNT_CLASS_LOOKUP: [u8; 256] = [ - 0, 1, 2, 4, 8, 8, 8, 8, 16, 16, 16, 16, 16, 16, 16, 16, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, -]; - -/// Hitcounts class lookup for 16-byte values -static mut COUNT_CLASS_LOOKUP_16: Vec = vec![]; - -/// Initialize the 16-byte hitcounts map -/// -/// # Safety -/// -/// Calling this from multiple threads may be racey and hence leak 65k mem -fn init_count_class_16() { - unsafe { - if !COUNT_CLASS_LOOKUP_16.is_empty() { - return; - } - - COUNT_CLASS_LOOKUP_16 = vec![0; 65536]; - for i in 0..256 { - for j in 0..256 { - COUNT_CLASS_LOOKUP_16[(i << 8) + j] = - (u16::from(COUNT_CLASS_LOOKUP[i]) << 8) | u16::from(COUNT_CLASS_LOOKUP[j]); - } - } - } -} - -/// Compute the hash of a slice -fn hash_slice(slice: &[T]) -> u64 { - let mut hasher = RandomState::with_seeds(0, 0, 0, 0).build_hasher(); - let ptr = slice.as_ptr() as *const u8; - let map_size = slice.len() / size_of::(); - unsafe { - hasher.write(slice::from_raw_parts(ptr, map_size)); - } - hasher.finish() -} - -/// A [`MapObserver`] observes the static map, as oftentimes used for AFL-like coverage information -/// -/// TODO: enforce `iter() -> AssociatedTypeIter` when generic associated types stabilize -pub trait MapObserver: HasLen + Named + Serialize + serde::de::DeserializeOwned -// where -// for<'it> &'it Self: IntoIterator -{ - /// Type of each entry in this map - type Entry: Bounded + PartialEq + Default + Copy + Debug + 'static; - - /// Get the value at `idx` - fn get(&self, idx: usize) -> &Self::Entry; - - /// Get the value at `idx` (mutable) - fn get_mut(&mut self, idx: usize) -> &mut Self::Entry; - - /// Get the number of usable entries in the map (all by default) - fn usable_count(&self) -> usize; - - /// Count the set bytes in the map - fn count_bytes(&self) -> u64; - - /// Compute the hash of the map - fn hash(&self) -> u64; - - /// Get the initial value for `reset()` - fn initial(&self) -> Self::Entry; - - /// Reset the map - fn reset_map(&mut self) -> Result<(), Error>; - - /// Get these observer's contents as [`Vec`] - fn to_vec(&self) -> Vec; - - /// Get the number of set entries with the specified indexes - fn how_many_set(&self, indexes: &[usize]) -> usize; -} - -/// A Simple iterator calling `MapObserver::get` -#[derive(Debug)] -pub struct MapObserverSimpleIterator<'a, O> -where - O: 'a + MapObserver, -{ - index: usize, - observer: *const O, - phantom: PhantomData<&'a u8>, -} - -impl<'a, O> Iterator for MapObserverSimpleIterator<'a, O> -where - O: 'a + MapObserver, -{ - type Item = &'a O::Entry; - fn next(&mut self) -> Option { - unsafe { - if self.index >= self.observer.as_ref().unwrap().usable_count() { - None - } else { - let i = self.index; - self.index += 1; - Some(self.observer.as_ref().unwrap().get(i)) - } - } - } -} - -/// A Simple iterator calling `MapObserver::get_mut` -#[derive(Debug)] -pub struct MapObserverSimpleIteratorMut<'a, O> -where - O: 'a + MapObserver, -{ - index: usize, - observer: *mut O, - phantom: PhantomData<&'a u8>, -} - -impl<'a, O> Iterator for MapObserverSimpleIteratorMut<'a, O> -where - O: 'a + MapObserver, -{ - type Item = &'a O::Entry; - fn next(&mut self) -> Option { - unsafe { - if self.index >= self.observer.as_ref().unwrap().usable_count() { - None - } else { - let i = self.index; - self.index += 1; - Some(self.observer.as_mut().unwrap().get_mut(i)) - } - } - } -} - -/// The Map Observer retrieves the state of a map, -/// that will get updated by the target. -/// A well-known example is the AFL-Style coverage map. -#[derive(Clone, Serialize, Deserialize, Debug)] -#[serde(bound = "T: serde::de::DeserializeOwned")] -#[allow(clippy::unsafe_derive_deserialize)] -pub struct StdMapObserver<'a, T, const DIFFERENTIAL: bool> -where - T: Default + Copy + 'static + Serialize, -{ - map: OwnedMutSlice<'a, T>, - initial: T, - name: String, -} - -impl<'a, S, T> Observer for StdMapObserver<'a, T, false> -where - S: UsesInput, - T: Bounded - + PartialEq - + Default - + Copy - + 'static - + Serialize - + serde::de::DeserializeOwned - + Debug, -{ - #[inline] - fn pre_exec(&mut self, _state: &mut S, _input: &S::Input) -> Result<(), Error> { - self.reset_map() - } -} - -impl<'a, S, T> Observer for StdMapObserver<'a, T, true> -where - S: UsesInput, - T: Bounded - + PartialEq - + Default - + Copy - + 'static - + Serialize - + serde::de::DeserializeOwned - + Debug, -{ -} - -impl<'a, T, const DIFFERENTIAL: bool> Named for StdMapObserver<'a, T, DIFFERENTIAL> -where - T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned, -{ - #[inline] - fn name(&self) -> &str { - self.name.as_str() - } -} - -impl<'a, T, const DIFFERENTIAL: bool> HasLen for StdMapObserver<'a, T, DIFFERENTIAL> -where - T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned, -{ - #[inline] - fn len(&self) -> usize { - self.map.as_slice().len() - } -} - -impl<'a, 'it, T, const DIFFERENTIAL: bool> AsIter<'it> for StdMapObserver<'a, T, DIFFERENTIAL> -where - T: Bounded - + PartialEq - + Default - + Copy - + 'static - + Serialize - + serde::de::DeserializeOwned - + Debug, -{ - type Item = T; - type IntoIter = Iter<'it, T>; - - fn as_iter(&'it self) -> Self::IntoIter { - let cnt = self.usable_count(); - self.as_slice()[..cnt].iter() - } -} - -impl<'a, 'it, T, const DIFFERENTIAL: bool> AsIterMut<'it> for StdMapObserver<'a, T, DIFFERENTIAL> -where - T: Bounded - + PartialEq - + Default - + Copy - + 'static - + Serialize - + serde::de::DeserializeOwned - + Debug, -{ - type Item = T; - type IntoIter = IterMut<'it, T>; - - fn as_iter_mut(&'it mut self) -> Self::IntoIter { - let cnt = self.usable_count(); - self.as_mut_slice()[..cnt].iter_mut() - } -} - -impl<'a, 'it, T, const DIFFERENTIAL: bool> IntoIterator for &'it StdMapObserver<'a, T, DIFFERENTIAL> -where - T: Bounded - + PartialEq - + Default - + Copy - + 'static - + Serialize - + serde::de::DeserializeOwned - + Debug, -{ - type Item = as Iterator>::Item; - type IntoIter = Iter<'it, T>; - - fn into_iter(self) -> Self::IntoIter { - let cnt = self.usable_count(); - self.as_slice()[..cnt].iter() - } -} - -impl<'a, 'it, T, const DIFFERENTIAL: bool> IntoIterator - for &'it mut StdMapObserver<'a, T, DIFFERENTIAL> -where - T: Bounded - + PartialEq - + Default - + Copy - + 'static - + Serialize - + serde::de::DeserializeOwned - + Debug, -{ - type Item = as Iterator>::Item; - type IntoIter = IterMut<'it, T>; - - fn into_iter(self) -> Self::IntoIter { - let cnt = self.usable_count(); - self.as_mut_slice()[..cnt].iter_mut() - } -} - -impl<'a, T, const DIFFERENTIAL: bool> StdMapObserver<'a, T, DIFFERENTIAL> -where - T: Bounded - + PartialEq - + Default - + Copy - + 'static - + Serialize - + serde::de::DeserializeOwned - + Debug, -{ - /// Returns an iterator over the map. - pub fn iter(&self) -> Iter<'_, T> { - <&Self as IntoIterator>::into_iter(self) - } - - /// Returns a mutable iterator over the map. - pub fn iter_mut(&mut self) -> IterMut<'_, T> { - <&mut Self as IntoIterator>::into_iter(self) - } -} - -impl<'a, T, const DIFFERENTIAL: bool> MapObserver for StdMapObserver<'a, T, DIFFERENTIAL> -where - T: Bounded - + PartialEq - + Default - + Copy - + 'static - + Serialize - + serde::de::DeserializeOwned - + Debug, -{ - type Entry = T; - - #[inline] - fn get(&self, pos: usize) -> &T { - &self.as_slice()[pos] - } - - #[inline] - fn get_mut(&mut self, idx: usize) -> &mut T { - &mut self.as_mut_slice()[idx] - } - - /// Count the set bytes in the map - fn count_bytes(&self) -> u64 { - let initial = self.initial(); - let cnt = self.usable_count(); - let map = self.as_slice(); - let mut res = 0; - for x in &map[0..cnt] { - if *x != initial { - res += 1; - } - } - res - } - - #[inline] - fn usable_count(&self) -> usize { - self.as_slice().len() - } - - fn hash(&self) -> u64 { - hash_slice(self.as_slice()) - } - - #[inline] - fn initial(&self) -> T { - self.initial - } - - fn to_vec(&self) -> Vec { - self.as_slice().to_vec() - } - - /// Reset the map - #[inline] - fn reset_map(&mut self) -> Result<(), Error> { - // Normal memset, see https://rust.godbolt.org/z/Trs5hv - let initial = self.initial(); - let cnt = self.usable_count(); - let map = self.as_mut_slice(); - for x in &mut map[0..cnt] { - *x = initial; - } - Ok(()) - } - - fn how_many_set(&self, indexes: &[usize]) -> usize { - let initial = self.initial(); - let cnt = self.usable_count(); - let map = self.as_slice(); - let mut res = 0; - for i in indexes { - if *i < cnt && map[*i] != initial { - res += 1; - } - } - res - } -} - -impl<'a, T, const DIFFERENTIAL: bool> Truncate for StdMapObserver<'a, T, DIFFERENTIAL> -where - T: Bounded - + PartialEq - + Default - + Copy - + 'static - + Serialize - + serde::de::DeserializeOwned - + Debug, -{ - fn truncate(&mut self, new_len: usize) { - self.map.truncate(new_len); - } -} - -impl<'a, T, const DIFFERENTIAL: bool> AsSlice for StdMapObserver<'a, T, DIFFERENTIAL> -where - T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned + Debug, -{ - type Entry = T; - #[must_use] - #[inline] - fn as_slice(&self) -> &[T] { - self.map.as_slice() - } -} -impl<'a, T, const DIFFERENTIAL: bool> AsMutSlice for StdMapObserver<'a, T, DIFFERENTIAL> -where - T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned + Debug, -{ - type Entry = T; - #[must_use] - #[inline] - fn as_mut_slice(&mut self) -> &mut [T] { - self.map.as_mut_slice() - } -} - -impl<'a, T, const DIFFERENTIAL: bool> StdMapObserver<'a, T, DIFFERENTIAL> -where - T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned, -{ - /// Creates a new [`MapObserver`] - /// - /// # Safety - /// Will get a pointer to the map and dereference it at any point in time. - /// The map must not move in memory! - #[must_use] - unsafe fn maybe_differential(name: S, map: &'a mut [T]) -> Self - where - S: Into, - { - let len = map.len(); - let ptr = map.as_mut_ptr(); - Self::maybe_differential_from_mut_ptr(name, ptr, len) - } - - /// Creates a new [`MapObserver`] from an [`OwnedMutSlice`] - #[must_use] - fn maybe_differential_from_mut_slice(name: S, map: OwnedMutSlice<'a, T>) -> Self - where - S: Into, - { - StdMapObserver { - name: name.into(), - map, - initial: T::default(), - } - } - - /// Creates a new [`MapObserver`] with an owned map - #[must_use] - fn maybe_differential_owned(name: S, map: Vec) -> Self - where - S: Into, - { - Self { - map: OwnedMutSlice::from(map), - name: name.into(), - initial: T::default(), - } - } - - /// Creates a new [`MapObserver`] from an [`OwnedMutSlice`] map. - /// - /// # Safety - /// Will dereference the owned slice with up to len elements. - #[must_use] - fn maybe_differential_from_ownedref(name: S, map: OwnedMutSlice<'a, T>) -> Self - where - S: Into, - { - Self { - map, - name: name.into(), - initial: T::default(), - } - } - - /// Creates a new [`MapObserver`] from a raw pointer - /// - /// # Safety - /// Will dereference the `map_ptr` with up to len elements. - unsafe fn maybe_differential_from_mut_ptr(name: S, map_ptr: *mut T, len: usize) -> Self - where - S: Into, - { - Self::maybe_differential_from_mut_slice( - name, - OwnedMutSlice::from_raw_parts_mut(map_ptr, len), - ) - } - - /// Gets the initial value for this map, mutably - pub fn initial_mut(&mut self) -> &mut T { - &mut self.initial - } - - /// Gets the backing for this map - pub fn map(&self) -> &OwnedMutSlice<'a, T> { - &self.map - } - - /// Gets the backing for this map mutably - pub fn map_mut(&mut self) -> &mut OwnedMutSlice<'a, T> { - &mut self.map - } -} - -impl<'a, T> StdMapObserver<'a, T, false> -where - T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned, -{ - /// Creates a new [`MapObserver`] - /// - /// # Safety - /// The observer will keep a pointer to the map. - /// Hence, the map may never move in memory. - #[must_use] - pub unsafe fn new(name: S, map: &'a mut [T]) -> Self - where - S: Into, - { - Self::maybe_differential(name, map) - } - - /// Creates a new [`MapObserver`] from an [`OwnedMutSlice`] - pub fn from_mut_slice(name: S, map: OwnedMutSlice<'a, T>) -> Self - where - S: Into, - { - Self::maybe_differential_from_mut_slice(name, map) - } - - /// Creates a new [`MapObserver`] with an owned map - #[must_use] - pub fn owned(name: S, map: Vec) -> Self - where - S: Into, - { - Self::maybe_differential_owned(name, map) - } - - /// Creates a new [`MapObserver`] from an [`OwnedMutSlice`] map. - /// - /// # Note - /// Will dereference the owned slice with up to len elements. - #[must_use] - pub fn from_ownedref(name: S, map: OwnedMutSlice<'a, T>) -> Self - where - S: Into, - { - Self::maybe_differential_from_ownedref(name, map) - } - - /// Creates a new [`MapObserver`] from a raw pointer - /// - /// # Safety - /// Will dereference the `map_ptr` with up to len elements. - pub unsafe fn from_mut_ptr(name: S, map_ptr: *mut T, len: usize) -> Self - where - S: Into, - { - Self::maybe_differential_from_mut_ptr(name, map_ptr, len) - } -} - -impl<'a, T> StdMapObserver<'a, T, true> -where - T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned, -{ - /// Creates a new [`MapObserver`] in differential mode - /// - /// # Safety - /// Will get a pointer to the map and dereference it at any point in time. - /// The map must not move in memory! - #[must_use] - pub unsafe fn differential(name: S, map: &'a mut [T]) -> Self - where - S: Into, - { - Self::maybe_differential(name, map) - } - - /// Creates a new [`MapObserver`] with an owned map in differential mode - #[must_use] - pub fn differential_owned(name: S, map: Vec) -> Self - where - S: Into, - { - Self::maybe_differential_owned(name, map) - } - - /// Creates a new [`MapObserver`] from an [`OwnedMutSlice`] map in differential mode. - /// - /// # Note - /// Will dereference the owned slice with up to len elements. - #[must_use] - pub fn differential_from_ownedref(name: S, map: OwnedMutSlice<'a, T>) -> Self - where - S: Into, - { - Self::maybe_differential_from_ownedref(name, map) - } - - /// Creates a new [`MapObserver`] from a raw pointer in differential mode - /// - /// # Safety - /// Will dereference the `map_ptr` with up to len elements. - pub unsafe fn differential_from_mut_ptr(name: S, map_ptr: *mut T, len: usize) -> Self - where - S: Into, - { - Self::maybe_differential_from_mut_ptr(name, map_ptr, len) - } -} - -impl<'a, OTA, OTB, S, T> DifferentialObserver for StdMapObserver<'a, T, true> -where - OTA: ObserversTuple, - OTB: ObserversTuple, - S: UsesInput, - T: Bounded - + PartialEq - + Default - + Copy - + 'static - + Serialize - + serde::de::DeserializeOwned - + Debug, -{ -} - -/// Use a const size to speedup `Feedback::is_interesting` when the user can -/// know the size of the map at compile time. -#[derive(Serialize, Deserialize, Debug)] -#[serde(bound = "T: serde::de::DeserializeOwned")] -#[allow(clippy::unsafe_derive_deserialize)] -pub struct ConstMapObserver<'a, T, const N: usize> -where - T: Default + Copy + 'static + Serialize, -{ - map: OwnedMutSlice<'a, T>, - initial: T, - name: String, -} - -impl<'a, S, T, const N: usize> Observer for ConstMapObserver<'a, T, N> -where - S: UsesInput, - T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned + Debug, - Self: MapObserver, -{ - #[inline] - fn pre_exec(&mut self, _state: &mut S, _input: &S::Input) -> Result<(), Error> { - self.reset_map() - } -} - -impl<'a, T, const N: usize> Named for ConstMapObserver<'a, T, N> -where - T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned, -{ - #[inline] - fn name(&self) -> &str { - self.name.as_str() - } -} - -impl<'a, T, const N: usize> HasLen for ConstMapObserver<'a, T, N> -where - T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned, -{ - #[inline] - fn len(&self) -> usize { - N - } -} - -impl<'a, 'it, T, const N: usize> AsIter<'it> for ConstMapObserver<'a, T, N> -where - T: Bounded - + PartialEq - + Default - + Copy - + 'static - + Serialize - + serde::de::DeserializeOwned - + Debug, -{ - type Item = T; - type IntoIter = Iter<'it, T>; - - fn as_iter(&'it self) -> Self::IntoIter { - let cnt = self.usable_count(); - self.as_slice()[..cnt].iter() - } -} - -impl<'a, 'it, T, const N: usize> AsIterMut<'it> for ConstMapObserver<'a, T, N> -where - T: Bounded - + PartialEq - + Default - + Copy - + 'static - + Serialize - + serde::de::DeserializeOwned - + Debug, -{ - type Item = T; - type IntoIter = IterMut<'it, T>; - - fn as_iter_mut(&'it mut self) -> Self::IntoIter { - let cnt = self.usable_count(); - self.as_mut_slice()[..cnt].iter_mut() - } -} - -impl<'a, 'it, T, const N: usize> IntoIterator for &'it ConstMapObserver<'a, T, N> -where - T: Bounded - + PartialEq - + Default - + Copy - + 'static - + Serialize - + serde::de::DeserializeOwned - + Debug, -{ - type Item = as Iterator>::Item; - type IntoIter = Iter<'it, T>; - - fn into_iter(self) -> Self::IntoIter { - let cnt = self.usable_count(); - self.as_slice()[..cnt].iter() - } -} - -impl<'a, 'it, T, const N: usize> IntoIterator for &'it mut ConstMapObserver<'a, T, N> -where - T: Bounded - + PartialEq - + Default - + Copy - + 'static - + Serialize - + serde::de::DeserializeOwned - + Debug, -{ - type Item = as Iterator>::Item; - type IntoIter = IterMut<'it, T>; - - fn into_iter(self) -> Self::IntoIter { - let cnt = self.usable_count(); - self.as_mut_slice()[..cnt].iter_mut() - } -} - -impl<'a, T, const N: usize> ConstMapObserver<'a, T, N> -where - T: Bounded - + PartialEq - + Default - + Copy - + 'static - + Serialize - + serde::de::DeserializeOwned - + Debug, -{ - /// Returns an iterator over the map. - pub fn iter(&self) -> Iter<'_, T> { - <&Self as IntoIterator>::into_iter(self) - } - - /// Returns a mutable iterator over the map. - pub fn iter_mut(&mut self) -> IterMut<'_, T> { - <&mut Self as IntoIterator>::into_iter(self) - } -} - -impl<'a, T, const N: usize> MapObserver for ConstMapObserver<'a, T, N> -where - T: Bounded - + PartialEq - + Default - + Copy - + 'static - + Serialize - + serde::de::DeserializeOwned - + Debug, -{ - type Entry = T; - - #[inline] - fn initial(&self) -> T { - self.initial - } - - #[inline] - fn get(&self, idx: usize) -> &T { - &self.as_slice()[idx] - } - - #[inline] - fn get_mut(&mut self, idx: usize) -> &mut T { - &mut self.as_mut_slice()[idx] - } - - /// Count the set bytes in the map - fn count_bytes(&self) -> u64 { - let initial = self.initial(); - let cnt = self.usable_count(); - let map = self.as_slice(); - let mut res = 0; - for x in &map[0..cnt] { - if *x != initial { - res += 1; - } - } - res - } - - fn usable_count(&self) -> usize { - self.as_slice().len() - } - - fn hash(&self) -> u64 { - hash_slice(self.as_slice()) - } - - /// Reset the map - #[inline] - fn reset_map(&mut self) -> Result<(), Error> { - // Normal memset, see https://rust.godbolt.org/z/Trs5hv - let initial = self.initial(); - let cnt = self.usable_count(); - let map = self.as_mut_slice(); - for x in &mut map[0..cnt] { - *x = initial; - } - Ok(()) - } - - fn to_vec(&self) -> Vec { - self.as_slice().to_vec() - } - - /// Get the number of set entries with the specified indexes - fn how_many_set(&self, indexes: &[usize]) -> usize { - let initial = self.initial(); - let cnt = self.usable_count(); - let map = self.as_slice(); - let mut res = 0; - for i in indexes { - if *i < cnt && map[*i] != initial { - res += 1; - } - } - res - } -} - -impl<'a, T, const N: usize> AsSlice for ConstMapObserver<'a, T, N> -where - T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned + Debug, -{ - type Entry = T; - #[inline] - fn as_slice(&self) -> &[T] { - self.map.as_slice() - } -} -impl<'a, T, const N: usize> AsMutSlice for ConstMapObserver<'a, T, N> -where - T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned + Debug, -{ - type Entry = T; - #[inline] - fn as_mut_slice(&mut self) -> &mut [T] { - self.map.as_mut_slice() - } -} - -impl<'a, T, const N: usize> ConstMapObserver<'a, T, N> -where - T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned, -{ - /// Creates a new [`MapObserver`] - /// - /// # Note - /// Will get a pointer to the map and dereference it at any point in time. - /// The map must not move in memory! - #[must_use] - pub fn new(name: &'static str, map: &'a mut [T]) -> Self { - assert!(map.len() >= N); - Self { - map: OwnedMutSlice::from(map), - name: name.to_string(), - initial: T::default(), - } - } - - /// Creates a new [`MapObserver`] with an owned map - #[must_use] - pub fn owned(name: &'static str, map: Vec) -> Self { - assert!(map.len() >= N); - let initial = if map.is_empty() { T::default() } else { map[0] }; - Self { - map: OwnedMutSlice::from(map), - name: name.to_string(), - initial, - } - } - - /// Creates a new [`MapObserver`] from a raw pointer - /// - /// # Safety - /// Will dereference the `map_ptr` with up to len elements. - pub unsafe fn from_mut_ptr(name: &'static str, map_ptr: *mut T) -> Self { - ConstMapObserver { - map: OwnedMutSlice::from_raw_parts_mut(map_ptr, N), - name: name.to_string(), - initial: T::default(), - } - } -} - -/// Overlooking a variable bitmap -#[derive(Serialize, Deserialize, Debug)] -#[serde(bound = "T: serde::de::DeserializeOwned")] -#[allow(clippy::unsafe_derive_deserialize)] -pub struct VariableMapObserver<'a, T> -where - T: Default + Copy + 'static + Serialize + PartialEq + Bounded, -{ - map: OwnedMutSlice<'a, T>, - size: OwnedMutPtr, - initial: T, - name: String, -} - -impl<'a, S, T> Observer for VariableMapObserver<'a, T> -where - S: UsesInput, - T: Default - + Copy - + 'static - + Serialize - + serde::de::DeserializeOwned - + Debug - + Bounded - + PartialEq, - Self: MapObserver, -{ - #[inline] - fn pre_exec(&mut self, _state: &mut S, _input: &S::Input) -> Result<(), Error> { - self.reset_map() - } -} - -impl<'a, T> Named for VariableMapObserver<'a, T> -where - T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned + Bounded + PartialEq, -{ - #[inline] - fn name(&self) -> &str { - self.name.as_str() - } -} - -impl<'a, T> HasLen for VariableMapObserver<'a, T> -where - T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned + PartialEq + Bounded, -{ - #[inline] - fn len(&self) -> usize { - *self.size.as_ref() - } -} - -impl<'a, 'it, T> AsIter<'it> for VariableMapObserver<'a, T> -where - T: Bounded - + PartialEq - + Default - + Copy - + 'static - + Serialize - + serde::de::DeserializeOwned - + Debug - + PartialEq - + Bounded, -{ - type Item = T; - type IntoIter = Iter<'it, T>; - - fn as_iter(&'it self) -> Self::IntoIter { - let cnt = self.usable_count(); - self.as_slice()[..cnt].iter() - } -} - -impl<'a, 'it, T> AsIterMut<'it> for VariableMapObserver<'a, T> -where - T: Bounded - + PartialEq - + Default - + Copy - + 'static - + Serialize - + serde::de::DeserializeOwned - + Debug - + PartialEq - + Bounded, -{ - type Item = T; - type IntoIter = IterMut<'it, T>; - - fn as_iter_mut(&'it mut self) -> Self::IntoIter { - let cnt = self.usable_count(); - self.as_mut_slice()[..cnt].iter_mut() - } -} - -impl<'a, 'it, T> IntoIterator for &'it VariableMapObserver<'a, T> -where - T: Bounded - + PartialEq - + Default - + Copy - + 'static - + Serialize - + serde::de::DeserializeOwned - + Debug - + PartialEq - + Bounded, -{ - type Item = as Iterator>::Item; - type IntoIter = Iter<'it, T>; - - fn into_iter(self) -> Self::IntoIter { - let cnt = self.usable_count(); - self.as_slice()[..cnt].iter() - } -} - -impl<'a, 'it, T> IntoIterator for &'it mut VariableMapObserver<'a, T> -where - T: Bounded - + PartialEq - + Default - + Copy - + 'static - + Serialize - + serde::de::DeserializeOwned - + Debug - + PartialEq - + Bounded, -{ - type Item = as Iterator>::Item; - type IntoIter = IterMut<'it, T>; - - fn into_iter(self) -> Self::IntoIter { - let cnt = self.usable_count(); - self.as_mut_slice()[..cnt].iter_mut() - } -} - -impl<'a, T> VariableMapObserver<'a, T> -where - T: Bounded - + PartialEq - + Default - + Copy - + 'static - + Serialize - + serde::de::DeserializeOwned - + Debug - + PartialEq - + Bounded, -{ - /// Returns an iterator over the map. - pub fn iter(&self) -> Iter<'_, T> { - <&Self as IntoIterator>::into_iter(self) - } - - /// Returns a mutable iterator over the map. - pub fn iter_mut(&mut self) -> IterMut<'_, T> { - <&mut Self as IntoIterator>::into_iter(self) - } -} - -impl<'a, T> MapObserver for VariableMapObserver<'a, T> -where - T: Bounded - + PartialEq - + Default - + Copy - + 'static - + Serialize - + serde::de::DeserializeOwned - + Debug - + PartialEq - + Bounded, -{ - type Entry = T; - - #[inline] - fn initial(&self) -> T { - self.initial - } - - #[inline] - fn usable_count(&self) -> usize { - *self.size.as_ref() - } - - fn get(&self, idx: usize) -> &T { - &self.map.as_slice()[idx] - } - - fn get_mut(&mut self, idx: usize) -> &mut T { - &mut self.map.as_mut_slice()[idx] - } - - /// Count the set bytes in the map - fn count_bytes(&self) -> u64 { - let initial = self.initial(); - let cnt = self.usable_count(); - let map = self.as_slice(); - let mut res = 0; - for x in &map[0..cnt] { - if *x != initial { - res += 1; - } - } - res - } - fn hash(&self) -> u64 { - hash_slice(self.as_slice()) - } - - /// Reset the map - #[inline] - fn reset_map(&mut self) -> Result<(), Error> { - // Normal memset, see https://rust.godbolt.org/z/Trs5hv - let initial = self.initial(); - let cnt = self.usable_count(); - let map = self.as_mut_slice(); - for x in &mut map[0..cnt] { - *x = initial; - } - Ok(()) - } - - fn to_vec(&self) -> Vec { - self.as_slice().to_vec() - } - - fn how_many_set(&self, indexes: &[usize]) -> usize { - let initial = self.initial(); - let cnt = self.usable_count(); - let map = self.as_slice(); - let mut res = 0; - for i in indexes { - if *i < cnt && map[*i] != initial { - res += 1; - } - } - res - } -} - -impl<'a, T> AsSlice for VariableMapObserver<'a, T> -where - T: Bounded - + PartialEq - + Default - + Copy - + 'static - + Serialize - + serde::de::DeserializeOwned - + Debug - + PartialEq - + Bounded, -{ - type Entry = T; - #[inline] - fn as_slice(&self) -> &[T] { - let cnt = self.usable_count(); - &self.map.as_slice()[..cnt] - } -} -impl<'a, T> AsMutSlice for VariableMapObserver<'a, T> -where - T: 'static - + Default - + Copy - + Serialize - + serde::de::DeserializeOwned - + Debug - + PartialEq - + Bounded, -{ - type Entry = T; - #[inline] - fn as_mut_slice(&mut self) -> &mut [T] { - let cnt = self.usable_count(); - &mut self.map.as_mut_slice()[..cnt] - } -} - -impl<'a, T> VariableMapObserver<'a, T> -where - T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + PartialEq + Bounded, -{ - /// Creates a new [`MapObserver`] from an [`OwnedMutSlice`] - /// - /// # Safety - /// The observer will dereference the owned slice, as well as the `map_ptr`. - /// Dereferences `map_ptr` with up to `max_len` elements of size. - pub unsafe fn from_mut_slice( - name: &'static str, - map_slice: OwnedMutSlice<'a, T>, - size: *mut usize, - ) -> Self { - VariableMapObserver { - name: name.into(), - map: map_slice, - size: OwnedMutPtr::Ptr(size), - initial: T::default(), - } - } - - /// Creates a new [`MapObserver`] from a raw pointer - /// - /// # Safety - /// The observer will dereference the `size` ptr, as well as the `map_ptr`. - /// Dereferences `map_ptr` with up to `max_len` elements of size. - pub unsafe fn from_mut_ptr( - name: &'static str, - map_ptr: *mut T, - max_len: usize, - size: *mut usize, - ) -> Self { - Self::from_mut_slice( - name, - OwnedMutSlice::from_raw_parts_mut(map_ptr, max_len), - size, - ) - } -} - -/// Map observer with AFL-like hitcounts postprocessing -/// -/// [`MapObserver`]s that are not slice-backed, -/// such as [`MultiMapObserver`], can use [`HitcountsIterableMapObserver`] instead. -#[derive(Serialize, Deserialize, Clone, Debug)] -#[serde(bound = "M: serde::de::DeserializeOwned")] -pub struct HitcountsMapObserver -where - M: Serialize, -{ - base: M, -} - -impl Observer for HitcountsMapObserver -where - M: MapObserver + Observer + AsMutSlice, - S: UsesInput, -{ - #[inline] - fn pre_exec(&mut self, state: &mut S, input: &S::Input) -> Result<(), Error> { - self.base.pre_exec(state, input) - } - - #[inline] - #[allow(clippy::cast_ptr_alignment)] - fn post_exec( - &mut self, - state: &mut S, - input: &S::Input, - exit_kind: &ExitKind, - ) -> Result<(), Error> { - let map = self.as_mut_slice(); - let mut len = map.len(); - let align_offset = map.as_ptr().align_offset(size_of::()); - - // if len == 1, the next branch will already do this lookup - if len > 1 && align_offset != 0 { - debug_assert_eq!( - align_offset, 1, - "Aligning u8 to u16 should always be offset of 1?" - ); - unsafe { - *map.get_unchecked_mut(0) = - *COUNT_CLASS_LOOKUP.get_unchecked(*map.get_unchecked(0) as usize); - } - len -= 1; - } - - // Fix the last element - if (len & 1) != 0 { - unsafe { - *map.get_unchecked_mut(len - 1) = - *COUNT_CLASS_LOOKUP.get_unchecked(*map.get_unchecked(len - 1) as usize); - } - } - - let cnt = len / 2; - - let map16 = unsafe { - slice::from_raw_parts_mut(map.as_mut_ptr().add(align_offset) as *mut u16, cnt) - }; - // 2022-07: Adding `enumerate` here increases execution speed/register allocation on x86_64. - #[allow(clippy::unused_enumerate_index)] - for (_i, item) in map16[0..cnt].iter_mut().enumerate() { - unsafe { - *item = *COUNT_CLASS_LOOKUP_16.get_unchecked(*item as usize); - } - } - - self.base.post_exec(state, input, exit_kind) - } -} - -impl Named for HitcountsMapObserver -where - M: Named + Serialize + serde::de::DeserializeOwned, -{ - #[inline] - fn name(&self) -> &str { - self.base.name() - } -} - -impl HasLen for HitcountsMapObserver -where - M: MapObserver, -{ - #[inline] - fn len(&self) -> usize { - self.base.len() - } -} - -impl MapObserver for HitcountsMapObserver -where - M: MapObserver, -{ - type Entry = u8; - - #[inline] - fn initial(&self) -> u8 { - self.base.initial() - } - - #[inline] - fn usable_count(&self) -> usize { - self.base.usable_count() - } - - #[inline] - fn get(&self, idx: usize) -> &u8 { - self.base.get(idx) - } - - #[inline] - fn get_mut(&mut self, idx: usize) -> &mut u8 { - self.base.get_mut(idx) - } - - /// Count the set bytes in the map - fn count_bytes(&self) -> u64 { - self.base.count_bytes() - } - - /// Reset the map - #[inline] - fn reset_map(&mut self) -> Result<(), Error> { - self.base.reset_map() - } - - fn hash(&self) -> u64 { - self.base.hash() - } - fn to_vec(&self) -> Vec { - self.base.to_vec() - } - - fn how_many_set(&self, indexes: &[usize]) -> usize { - self.base.how_many_set(indexes) - } -} - -impl Truncate for HitcountsMapObserver -where - M: Named + Serialize + serde::de::DeserializeOwned + Truncate, -{ - fn truncate(&mut self, new_len: usize) { - self.base.truncate(new_len); - } -} - -impl AsSlice for HitcountsMapObserver -where - M: MapObserver + AsSlice, -{ - type Entry = ::Entry; - #[inline] - fn as_slice(&self) -> &[Self::Entry] { - self.base.as_slice() - } -} - -impl AsMutSlice for HitcountsMapObserver -where - M: MapObserver + AsMutSlice, -{ - type Entry = ::Entry; - #[inline] - fn as_mut_slice(&mut self) -> &mut [Self::Entry] { - self.base.as_mut_slice() - } -} - -impl HitcountsMapObserver -where - M: Serialize + serde::de::DeserializeOwned, -{ - /// Creates a new [`MapObserver`] - pub fn new(base: M) -> Self { - init_count_class_16(); - Self { base } - } -} - -impl<'it, M> AsIter<'it> for HitcountsMapObserver -where - M: Named + Serialize + serde::de::DeserializeOwned + AsIter<'it, Item = u8>, -{ - type Item = u8; - type IntoIter = >::IntoIter; - - fn as_iter(&'it self) -> Self::IntoIter { - self.base.as_iter() - } -} - -impl<'it, M> AsIterMut<'it> for HitcountsMapObserver -where - M: Named + Serialize + serde::de::DeserializeOwned + AsIterMut<'it, Item = u8>, -{ - type Item = u8; - type IntoIter = >::IntoIter; - - fn as_iter_mut(&'it mut self) -> Self::IntoIter { - self.base.as_iter_mut() - } -} - -impl<'it, M> IntoIterator for &'it HitcountsMapObserver -where - M: Serialize + serde::de::DeserializeOwned, - &'it M: IntoIterator, -{ - type Item = &'it u8; - type IntoIter = <&'it M as IntoIterator>::IntoIter; - - fn into_iter(self) -> Self::IntoIter { - self.base.into_iter() - } -} - -impl<'it, M> IntoIterator for &'it mut HitcountsMapObserver -where - M: Serialize + serde::de::DeserializeOwned, - &'it mut M: IntoIterator, -{ - type Item = &'it mut u8; - type IntoIter = <&'it mut M as IntoIterator>::IntoIter; - - fn into_iter(self) -> Self::IntoIter { - self.base.into_iter() - } -} - -impl HitcountsMapObserver -where - M: Serialize + serde::de::DeserializeOwned, - for<'it> &'it M: IntoIterator, -{ - /// Returns an iterator over the map. - pub fn iter(&self) -> <&M as IntoIterator>::IntoIter { - <&Self as IntoIterator>::into_iter(self) - } -} - -impl HitcountsMapObserver -where - M: Serialize + serde::de::DeserializeOwned, - for<'it> &'it mut M: IntoIterator, -{ - /// Returns a mutable iterator over the map. - pub fn iter_mut(&mut self) -> <&mut M as IntoIterator>::IntoIter { - <&mut Self as IntoIterator>::into_iter(self) - } -} - -impl DifferentialObserver for HitcountsMapObserver -where - M: DifferentialObserver - + MapObserver - + Serialize - + AsMutSlice, - OTA: ObserversTuple, - OTB: ObserversTuple, - S: UsesInput, -{ - fn pre_observe_first(&mut self, observers: &mut OTA) -> Result<(), Error> { - self.base.pre_observe_first(observers) - } - - fn post_observe_first(&mut self, observers: &mut OTA) -> Result<(), Error> { - self.base.post_observe_first(observers) - } - - fn pre_observe_second(&mut self, observers: &mut OTB) -> Result<(), Error> { - self.base.pre_observe_second(observers) - } - - fn post_observe_second(&mut self, observers: &mut OTB) -> Result<(), Error> { - self.base.post_observe_second(observers) - } -} - -/// Map observer with hitcounts postprocessing -/// Less optimized version for non-slice iterators. -/// Slice-backed observers should use a [`HitcountsMapObserver`]. -#[derive(Serialize, Deserialize, Clone, Debug)] -#[serde(bound = "M: serde::de::DeserializeOwned")] -pub struct HitcountsIterableMapObserver -where - M: Serialize, -{ - base: M, -} - -impl Observer for HitcountsIterableMapObserver -where - M: MapObserver + Observer, - for<'it> M: AsIterMut<'it, Item = u8>, - S: UsesInput, -{ - #[inline] - fn pre_exec(&mut self, state: &mut S, input: &S::Input) -> Result<(), Error> { - self.base.pre_exec(state, input) - } - - #[inline] - #[allow(clippy::cast_ptr_alignment)] - fn post_exec( - &mut self, - state: &mut S, - input: &S::Input, - exit_kind: &ExitKind, - ) -> Result<(), Error> { - for item in self.as_iter_mut() { - *item = unsafe { *COUNT_CLASS_LOOKUP.get_unchecked((*item) as usize) }; - } - - self.base.post_exec(state, input, exit_kind) - } -} - -impl Named for HitcountsIterableMapObserver -where - M: Named + Serialize + serde::de::DeserializeOwned, -{ - #[inline] - fn name(&self) -> &str { - self.base.name() - } -} - -impl HasLen for HitcountsIterableMapObserver -where - M: MapObserver, -{ - #[inline] - fn len(&self) -> usize { - self.base.len() - } -} - -impl MapObserver for HitcountsIterableMapObserver -where - M: MapObserver, - for<'it> M: AsIterMut<'it, Item = u8>, -{ - type Entry = u8; - - #[inline] - fn initial(&self) -> u8 { - self.base.initial() - } - - #[inline] - fn usable_count(&self) -> usize { - self.base.usable_count() - } - - #[inline] - fn get(&self, idx: usize) -> &u8 { - self.base.get(idx) - } - - #[inline] - fn get_mut(&mut self, idx: usize) -> &mut u8 { - self.base.get_mut(idx) - } - - /// Count the set bytes in the map - fn count_bytes(&self) -> u64 { - self.base.count_bytes() - } - - /// Reset the map - #[inline] - fn reset_map(&mut self) -> Result<(), Error> { - self.base.reset_map() - } - - fn hash(&self) -> u64 { - self.base.hash() - } - fn to_vec(&self) -> Vec { - self.base.to_vec() - } - - fn how_many_set(&self, indexes: &[usize]) -> usize { - self.base.how_many_set(indexes) - } -} - -impl Truncate for HitcountsIterableMapObserver -where - M: Named + Serialize + serde::de::DeserializeOwned + Truncate, -{ - fn truncate(&mut self, new_len: usize) { - self.base.truncate(new_len); - } -} - -impl AsSlice for HitcountsIterableMapObserver -where - M: MapObserver + AsSlice, -{ - type Entry = ::Entry; - #[inline] - fn as_slice(&self) -> &[Self::Entry] { - self.base.as_slice() - } -} -impl AsMutSlice for HitcountsIterableMapObserver -where - M: MapObserver + AsMutSlice, -{ - type Entry = ::Entry; - #[inline] - fn as_mut_slice(&mut self) -> &mut [Self::Entry] { - self.base.as_mut_slice() - } -} - -impl HitcountsIterableMapObserver -where - M: Serialize + serde::de::DeserializeOwned, -{ - /// Creates a new [`MapObserver`] - pub fn new(base: M) -> Self { - init_count_class_16(); - Self { base } - } -} - -impl<'it, M> AsIter<'it> for HitcountsIterableMapObserver -where - M: Named + Serialize + serde::de::DeserializeOwned + AsIter<'it, Item = u8>, -{ - type Item = u8; - type IntoIter = >::IntoIter; - - fn as_iter(&'it self) -> Self::IntoIter { - self.base.as_iter() - } -} - -impl<'it, M> AsIterMut<'it> for HitcountsIterableMapObserver -where - M: Named + Serialize + serde::de::DeserializeOwned + AsIterMut<'it, Item = u8>, -{ - type Item = u8; - type IntoIter = >::IntoIter; - - fn as_iter_mut(&'it mut self) -> Self::IntoIter { - self.base.as_iter_mut() - } -} - -impl<'it, M> IntoIterator for &'it HitcountsIterableMapObserver -where - M: Serialize + serde::de::DeserializeOwned, - &'it M: IntoIterator, -{ - type Item = &'it u8; - type IntoIter = <&'it M as IntoIterator>::IntoIter; - - fn into_iter(self) -> Self::IntoIter { - self.base.into_iter() - } -} - -impl<'it, M> IntoIterator for &'it mut HitcountsIterableMapObserver -where - M: Serialize + serde::de::DeserializeOwned, - &'it mut M: IntoIterator, -{ - type Item = &'it mut u8; - type IntoIter = <&'it mut M as IntoIterator>::IntoIter; - - fn into_iter(self) -> Self::IntoIter { - self.base.into_iter() - } -} - -impl HitcountsIterableMapObserver -where - M: Serialize + serde::de::DeserializeOwned, - for<'it> &'it M: IntoIterator, -{ - /// Returns an iterator over the map. - pub fn iter(&self) -> <&M as IntoIterator>::IntoIter { - <&Self as IntoIterator>::into_iter(self) - } -} - -impl HitcountsIterableMapObserver -where - M: Serialize + serde::de::DeserializeOwned, - for<'it> &'it mut M: IntoIterator, -{ - /// Returns a mutable iterator over the map. - pub fn iter_mut(&mut self) -> <&mut M as IntoIterator>::IntoIter { - <&mut Self as IntoIterator>::into_iter(self) - } -} - -impl DifferentialObserver for HitcountsIterableMapObserver -where - M: MapObserver + Observer + DifferentialObserver, - for<'it> M: AsIterMut<'it, Item = u8>, - OTA: ObserversTuple, - OTB: ObserversTuple, - S: UsesInput, -{ - fn pre_observe_first(&mut self, observers: &mut OTA) -> Result<(), Error> { - self.base.pre_observe_first(observers) - } - - fn post_observe_first(&mut self, observers: &mut OTA) -> Result<(), Error> { - self.base.post_observe_first(observers) - } - - fn pre_observe_second(&mut self, observers: &mut OTB) -> Result<(), Error> { - self.base.pre_observe_second(observers) - } - - fn post_observe_second(&mut self, observers: &mut OTB) -> Result<(), Error> { - self.base.post_observe_second(observers) - } -} - -/// The Multi Map Observer merge different maps into one observer -#[derive(Serialize, Deserialize, Debug)] -#[serde(bound = "T: serde::de::DeserializeOwned")] -#[allow(clippy::unsafe_derive_deserialize)] -pub struct MultiMapObserver<'a, T, const DIFFERENTIAL: bool> -where - T: 'static + Default + Copy + Serialize + Debug, -{ - maps: Vec>, - intervals: IntervalTree, - len: usize, - initial: T, - name: String, - iter_idx: usize, -} - -impl<'a, S, T> Observer for MultiMapObserver<'a, T, false> -where - S: UsesInput, - T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, - Self: MapObserver, -{ - #[inline] - fn pre_exec(&mut self, _state: &mut S, _input: &S::Input) -> Result<(), Error> { - self.reset_map() - } -} - -impl<'a, S, T> Observer for MultiMapObserver<'a, T, true> -where - S: UsesInput, - T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, - Self: MapObserver, -{ - // in differential mode, we are *not* responsible for resetting the map! -} - -impl<'a, T, const DIFFERENTIAL: bool> Named for MultiMapObserver<'a, T, DIFFERENTIAL> -where - T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, -{ - #[inline] - fn name(&self) -> &str { - self.name.as_str() - } -} - -impl<'a, T, const DIFFERENTIAL: bool> HasLen for MultiMapObserver<'a, T, DIFFERENTIAL> -where - T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, -{ - #[inline] - fn len(&self) -> usize { - self.len - } -} - -impl<'a, T, const DIFFERENTIAL: bool> MapObserver for MultiMapObserver<'a, T, DIFFERENTIAL> -where - T: 'static - + Bounded - + PartialEq - + Default - + Copy - + Serialize - + serde::de::DeserializeOwned - + Debug, -{ - type Entry = T; - - #[inline] - fn get(&self, idx: usize) -> &T { - let elem = self.intervals.query(idx..=idx).next().unwrap(); - let i = *elem.value; - let j = idx - elem.interval.start; - &self.maps[i].as_slice()[j] - } - - #[inline] - fn get_mut(&mut self, idx: usize) -> &mut T { - let elem = self.intervals.query(idx..=idx).next().unwrap(); - let i = *elem.value; - let j = idx - elem.interval.start; - &mut self.maps[i].as_mut_slice()[j] - } - - #[inline] - fn initial(&self) -> T { - self.initial - } - - fn count_bytes(&self) -> u64 { - let initial = self.initial(); - let mut res = 0; - for map in &self.maps { - for x in map.as_slice() { - if *x != initial { - res += 1; - } - } - } - res - } - - fn hash(&self) -> u64 { - let mut hasher = RandomState::with_seeds(0, 0, 0, 0).build_hasher(); - for map in &self.maps { - let slice = map.as_slice(); - let ptr = slice.as_ptr() as *const u8; - let map_size = slice.len() / size_of::(); - unsafe { - hasher.write(slice::from_raw_parts(ptr, map_size)); - } - } - hasher.finish() - } - - fn reset_map(&mut self) -> Result<(), Error> { - let initial = self.initial(); - for map in &mut self.maps { - for x in map.as_mut_slice() { - *x = initial; - } - } - Ok(()) - } - - fn usable_count(&self) -> usize { - self.len() - } - - fn to_vec(&self) -> Vec { - let cnt = self.usable_count(); - let mut res = Vec::with_capacity(cnt); - for i in 0..cnt { - res.push(*self.get(i)); - } - res - } - - /// Get the number of set entries with the specified indexes - fn how_many_set(&self, indexes: &[usize]) -> usize { - let initial = self.initial(); - let cnt = self.usable_count(); - let mut res = 0; - for i in indexes { - if *i < cnt && *self.get(*i) != initial { - res += 1; - } - } - res - } -} - -impl<'a, T, const DIFFERENTIAL: bool> MultiMapObserver<'a, T, DIFFERENTIAL> -where - T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, -{ - /// Creates a new [`MultiMapObserver`], maybe in differential mode - #[must_use] - fn maybe_differential(name: &'static str, maps: Vec>) -> Self { - let mut idx = 0; - let mut intervals = IntervalTree::new(); - for (v, x) in maps.iter().enumerate() { - let l = x.as_slice().len(); - intervals.insert(idx..(idx + l), v); - idx += l; - } - Self { - maps, - intervals, - len: idx, - name: name.to_string(), - initial: T::default(), - iter_idx: 0, - } - } -} - -impl<'a, T> MultiMapObserver<'a, T, true> -where - T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, -{ - /// Creates a new [`MultiMapObserver`] in differential mode - #[must_use] - pub fn differential(name: &'static str, maps: Vec>) -> Self { - Self::maybe_differential(name, maps) - } -} - -impl<'a, T> MultiMapObserver<'a, T, false> -where - T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, -{ - /// Creates a new [`MultiMapObserver`] - #[must_use] - pub fn new(name: &'static str, maps: Vec>) -> Self { - Self::maybe_differential(name, maps) - } - - /// Creates a new [`MultiMapObserver`] with an owned map - #[must_use] - pub fn owned(name: &'static str, maps: Vec>) -> Self { - let mut idx = 0; - let mut v = 0; - let mut intervals = IntervalTree::new(); - let maps: Vec<_> = maps - .into_iter() - .map(|x| { - let l = x.len(); - intervals.insert(idx..(idx + l), v); - idx += l; - v += 1; - OwnedMutSlice::from(x) - }) - .collect(); - Self { - maps, - intervals, - len: idx, - name: name.to_string(), - initial: T::default(), - iter_idx: 0, - } - } -} - -impl<'a, 'it, T, const DIFFERENTIAL: bool> AsIter<'it> for MultiMapObserver<'a, T, DIFFERENTIAL> -where - T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, - 'a: 'it, -{ - type Item = T; - type IntoIter = Flatten>>; - - fn as_iter(&'it self) -> Self::IntoIter { - self.maps.iter().flatten() - } -} - -impl<'a, 'it, T, const DIFFERENTIAL: bool> AsIterMut<'it> for MultiMapObserver<'a, T, DIFFERENTIAL> -where - T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, - 'a: 'it, -{ - type Item = T; - type IntoIter = Flatten>>; - - fn as_iter_mut(&'it mut self) -> Self::IntoIter { - self.maps.iter_mut().flatten() - } -} - -impl<'a, 'it, T, const DIFFERENTIAL: bool> IntoIterator - for &'it MultiMapObserver<'a, T, DIFFERENTIAL> -where - T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, -{ - type Item = as Iterator>::Item; - type IntoIter = Flatten>>; - - fn into_iter(self) -> Self::IntoIter { - self.maps.iter().flatten() - } -} - -impl<'a, 'it, T, const DIFFERENTIAL: bool> IntoIterator - for &'it mut MultiMapObserver<'a, T, DIFFERENTIAL> -where - T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, -{ - type Item = as Iterator>::Item; - type IntoIter = Flatten>>; - - fn into_iter(self) -> Self::IntoIter { - self.maps.iter_mut().flatten() - } -} - -impl<'a, T, const DIFFERENTIAL: bool> MultiMapObserver<'a, T, DIFFERENTIAL> -where - T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, -{ - /// Returns an iterator over the map. - pub fn iter(&self) -> <&Self as IntoIterator>::IntoIter { - <&Self as IntoIterator>::into_iter(self) - } - - /// Returns a mutable iterator over the map. - pub fn iter_mut(&mut self) -> <&mut Self as IntoIterator>::IntoIter { - <&mut Self as IntoIterator>::into_iter(self) - } -} - -impl<'a, T, OTA, OTB, S> DifferentialObserver for MultiMapObserver<'a, T, true> -where - T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, - Self: MapObserver, - OTA: ObserversTuple, - OTB: ObserversTuple, - S: UsesInput, -{ -} - -/// Exact copy of `StdMapObserver` that owns its map -/// Used for python bindings -#[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(bound = "T: serde::de::DeserializeOwned")] -#[allow(clippy::unsafe_derive_deserialize)] -pub struct OwnedMapObserver -where - T: 'static + Default + Copy + Serialize, -{ - map: Vec, - initial: T, - name: String, -} - -impl Observer for OwnedMapObserver -where - S: UsesInput, - T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, - Self: MapObserver, -{ - #[inline] - fn pre_exec(&mut self, _state: &mut S, _input: &S::Input) -> Result<(), Error> { - self.reset_map() - } -} - -impl Named for OwnedMapObserver -where - T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned, -{ - #[inline] - fn name(&self) -> &str { - self.name.as_str() - } -} - -impl HasLen for OwnedMapObserver -where - T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned, -{ - #[inline] - fn len(&self) -> usize { - self.map.as_slice().len() - } -} - -impl<'it, T> AsIter<'it> for OwnedMapObserver -where - T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, -{ - type Item = T; - type IntoIter = Iter<'it, T>; - - fn as_iter(&'it self) -> Self::IntoIter { - self.as_slice().iter() - } -} - -impl<'it, T> AsIterMut<'it> for OwnedMapObserver -where - T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, -{ - type Item = T; - type IntoIter = IterMut<'it, T>; - - fn as_iter_mut(&'it mut self) -> Self::IntoIter { - self.as_mut_slice().iter_mut() - } -} - -impl<'it, T> IntoIterator for &'it OwnedMapObserver -where - T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, -{ - type Item = as Iterator>::Item; - type IntoIter = Iter<'it, T>; - - fn into_iter(self) -> Self::IntoIter { - self.as_slice().iter() - } -} - -impl<'it, T> IntoIterator for &'it mut OwnedMapObserver -where - T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, -{ - type Item = as Iterator>::Item; - type IntoIter = IterMut<'it, T>; - - fn into_iter(self) -> Self::IntoIter { - self.as_mut_slice().iter_mut() - } -} - -impl OwnedMapObserver -where - T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, -{ - /// Returns an iterator over the map. - pub fn iter(&self) -> Iter<'_, T> { - <&Self as IntoIterator>::into_iter(self) - } - - /// Returns a mutable iterator over the map. - pub fn iter_mut(&mut self) -> IterMut<'_, T> { - <&mut Self as IntoIterator>::into_iter(self) - } -} - -impl MapObserver for OwnedMapObserver -where - T: 'static - + Bounded - + PartialEq - + Default - + Copy - + Serialize - + serde::de::DeserializeOwned - + Debug, -{ - type Entry = T; - - #[inline] - fn get(&self, pos: usize) -> &T { - &self.as_slice()[pos] - } - - #[inline] - fn get_mut(&mut self, idx: usize) -> &mut T { - &mut self.as_mut_slice()[idx] - } - - /// Count the set bytes in the map - fn count_bytes(&self) -> u64 { - let initial = self.initial(); - let cnt = self.usable_count(); - let map = self.as_slice(); - let mut res = 0; - for x in &map[0..cnt] { - if *x != initial { - res += 1; - } - } - res - } - - #[inline] - fn usable_count(&self) -> usize { - self.as_slice().len() - } - - fn hash(&self) -> u64 { - hash_slice(self.as_slice()) - } - - #[inline] - fn initial(&self) -> T { - self.initial - } - - /// Reset the map - #[inline] - fn reset_map(&mut self) -> Result<(), Error> { - // Normal memset, see https://rust.godbolt.org/z/Trs5hv - let initial = self.initial(); - let cnt = self.usable_count(); - let map = self.as_mut_slice(); - for x in &mut map[0..cnt] { - *x = initial; - } - Ok(()) - } - fn to_vec(&self) -> Vec { - self.as_slice().to_vec() - } - - fn how_many_set(&self, indexes: &[usize]) -> usize { - let initial = self.initial(); - let cnt = self.usable_count(); - let map = self.as_slice(); - let mut res = 0; - for i in indexes { - if *i < cnt && map[*i] != initial { - res += 1; - } - } - res - } -} - -impl AsSlice for OwnedMapObserver -where - T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, -{ - type Entry = T; - #[must_use] - #[inline] - fn as_slice(&self) -> &[T] { - self.map.as_slice() - } -} - -impl AsMutSlice for OwnedMapObserver -where - T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, -{ - type Entry = T; - #[must_use] - #[inline] - fn as_mut_slice(&mut self) -> &mut [T] { - self.map.as_mut_slice() - } -} - -impl OwnedMapObserver -where - T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned, -{ - /// Creates a new [`MapObserver`] with an owned map - #[must_use] - pub fn new(name: &'static str, map: Vec) -> Self { - let initial = if map.is_empty() { T::default() } else { map[0] }; - Self { - map, - name: name.to_string(), - initial, - } - } -} - -/// `MapObserver` Python bindings -#[cfg(feature = "python")] -#[allow(missing_docs)] -pub mod pybind { - use concat_idents::concat_idents; - use pyo3::prelude::*; - use serde::{Deserialize, Serialize}; - - use super::{ - AsIter, AsIterMut, AsMutSlice, AsSlice, Debug, Error, HasLen, Iter, IterMut, MapObserver, - Named, Observer, OwnedMapObserver, StdMapObserver, String, Vec, - }; - use crate::{inputs::UsesInput, observers::pybind::PythonObserver}; - - #[macro_export] - macro_rules! mapob_unwrap_me { - ($wrapper_name:ident, $wrapper:expr, $name:ident, $body:block) => { - match &$wrapper { - $wrapper_name::Std(py_wrapper) => Python::with_gil(|py| -> PyResult<_> { - let borrowed = py_wrapper.borrow(py); - let $name = &borrowed.inner; - Ok($body) - }) - .unwrap(), - $wrapper_name::Owned(py_wrapper) => Python::with_gil(|py| -> PyResult<_> { - let borrowed = py_wrapper.borrow(py); - let $name = &borrowed.inner; - Ok($body) - }) - .unwrap(), - $wrapper_name::None => panic!("Serde is not supported ATM"), - } - }; - } - - #[macro_export] - macro_rules! mapob_unwrap_me_mut { - ($wrapper_name:ident, $wrapper:expr, $name:ident, $body:block) => { - match &mut $wrapper { - $wrapper_name::Std(py_wrapper) => Python::with_gil(|py| -> PyResult<_> { - let mut borrowed = py_wrapper.borrow_mut(py); - let $name = &mut borrowed.inner; - Ok($body) - }) - .unwrap(), - $wrapper_name::Owned(py_wrapper) => Python::with_gil(|py| -> PyResult<_> { - let mut borrowed = py_wrapper.borrow_mut(py); - let $name = &mut borrowed.inner; - Ok($body) - }) - .unwrap(), - $wrapper_name::None => panic!("Serde is not supported ATM"), - } - }; - } - - macro_rules! define_python_map_observer { - ($struct_name1:ident, $py_name1:tt, $struct_name2:ident, $py_name2:tt, $struct_name_trait:ident, $py_name_trait:tt, $datatype:ty, $wrapper_name: ident) => { - - #[pyclass(unsendable, name = $py_name1)] - #[allow(clippy::unsafe_derive_deserialize)] - #[derive(Serialize, Deserialize, Debug, Clone)] - /// Python class for StdMapObserver - pub struct $struct_name1 { - /// Rust wrapped StdMapObserver object - pub inner: StdMapObserver<'static, $datatype, false>, - } - - #[pymethods] - impl $struct_name1 { - #[new] - fn new(name: String, ptr: usize, size: usize) -> Self { - Self { - inner: unsafe { StdMapObserver::from_mut_ptr(name, ptr as *mut $datatype, size) } - } - } - - #[must_use] - pub fn as_map_observer(slf: Py) -> $struct_name_trait { - $struct_name_trait::new_std(slf) - } - - #[must_use] - pub fn as_observer(slf: Py) -> PythonObserver { - let m = Self::as_map_observer(slf); - Python::with_gil(|py| -> PyResult { - let p: Py<_> = Py::new(py, m)?; - Ok($struct_name_trait::as_observer(p)) - }).unwrap() - } - - fn __getitem__(&self, idx: usize) -> $datatype { - *self.inner.get(idx) - } - - fn __setitem__(&mut self, idx: usize, val: $datatype) { - *self.inner.get_mut(idx) = val; - } - - #[pyo3(name = "usable_count")] - fn pyusable_count(&self) -> usize { - self.inner.usable_count() - } - - #[pyo3(name = "len")] - fn pylen(&self) -> usize { - self.inner.len() - } - - #[pyo3(name = "name")] - fn pyname(&self) -> &str { - self.inner.name() - } - - } - - #[pyclass(unsendable, name = $py_name2)] - #[allow(clippy::unsafe_derive_deserialize)] - #[derive(Serialize, Deserialize, Debug, Clone)] - /// Python class for OwnedMapObserver (i.e. StdMapObserver with owned map) - pub struct $struct_name2 { - /// Rust wrapped OwnedMapObserver object - pub inner: OwnedMapObserver<$datatype>, - } - - #[pymethods] - impl $struct_name2 { - #[new] - fn new(name: String, map: Vec<$datatype>) -> Self { - Self { - //TODO: Not leak memory - inner: OwnedMapObserver::new(alloc::boxed::Box::leak(name.into_boxed_str()), map), - } - } - - #[must_use] - pub fn as_map_observer(slf: Py) -> $struct_name_trait { - $struct_name_trait::new_owned(slf) - } - - #[must_use] - pub fn as_observer(slf: Py) -> PythonObserver { - let m = Self::as_map_observer(slf); - Python::with_gil(|py| -> PyResult { - let p: Py<_> = Py::new(py, m)?; - Ok($struct_name_trait::as_observer(p)) - }).unwrap() - } - - fn __getitem__(&self, idx: usize) -> $datatype { - *self.inner.get(idx) - } - - fn __setitem__(&mut self, idx: usize, val: $datatype) { - *self.inner.get_mut(idx) = val; - } - - #[pyo3(name = "usable_count")] - fn pyusable_count(&self) -> usize { - self.inner.usable_count() - } - - #[pyo3(name = "len")] - fn pylen(&self) -> usize { - self.inner.len() - } - - #[pyo3(name = "name")] - fn pyname(&self) -> &str { - self.inner.name() - } - } - - #[derive(Debug, Clone)] - pub enum $wrapper_name { - Std(Py<$struct_name1>), - Owned(Py<$struct_name2>), - None - } - - impl Default for $wrapper_name { - fn default() -> Self { - $wrapper_name::None - } - } - - // Should not be exposed to user - #[pyclass(unsendable, name = $py_name_trait)] - #[allow(clippy::unsafe_derive_deserialize)] - #[derive(Serialize, Deserialize, Debug, Clone)] - /// MapObserver + Observer Trait binding - pub struct $struct_name_trait { - #[serde(skip)] - pub wrapper: $wrapper_name, - } - - #[pymethods] - impl $struct_name_trait { - #[staticmethod] - fn new_std(std_map: Py<$struct_name1>) -> Self { - Self { - wrapper: $wrapper_name::Std(std_map), - } - } - - #[staticmethod] - fn new_owned(owned_map: Py<$struct_name2>) -> Self { - Self { - wrapper: $wrapper_name::Owned(owned_map), - } - } - - #[must_use] - pub fn as_observer(slf: Py) -> PythonObserver { - concat_idents!(func = new_map_,$datatype { - PythonObserver::func(slf) - }) - } - - fn __getitem__(&self, idx: usize) -> $datatype { - *self.get(idx) - } - - fn __setitem__(&mut self, idx: usize, val: $datatype) { - *self.get_mut(idx) = val; - } - - #[pyo3(name = "usable_count")] - fn pyusable_count(&self) -> usize { - self.usable_count() - } - - #[pyo3(name = "len")] - fn pylen(&self) -> usize { - self.len() - } - - #[pyo3(name = "name")] - fn pyname(&self) -> &str { - self.name() - } - } - - impl<'it> AsIter<'it> for $struct_name_trait { - type Item = $datatype; - type IntoIter = Iter<'it, $datatype>; - - fn as_iter(&'it self) -> Self::IntoIter { - mapob_unwrap_me!($wrapper_name, self.wrapper, m, { unsafe { std::mem::transmute::<_, Self::IntoIter>(m.as_iter()) } }) - } - } - - impl<'it> AsIterMut<'it> for $struct_name_trait { - type Item = $datatype; - type IntoIter = IterMut<'it, $datatype>; - - fn as_iter_mut(&'it mut self) -> Self::IntoIter { - mapob_unwrap_me_mut!($wrapper_name, self.wrapper, m, { unsafe { std::mem::transmute::<_, Self::IntoIter>(m.as_iter_mut()) } }) - } - } - - impl AsSlice for $struct_name_trait { - type Entry = $datatype; - fn as_slice(&self) -> &[$datatype] { - mapob_unwrap_me!($wrapper_name, self.wrapper, m, { unsafe { std::mem::transmute(m.as_slice()) }} ) - } - } - - impl AsMutSlice for $struct_name_trait { - type Entry = $datatype; - fn as_mut_slice(&mut self) -> &mut [$datatype] { - mapob_unwrap_me_mut!($wrapper_name, self.wrapper, m, { unsafe { std::mem::transmute(m.as_mut_slice()) }} ) - } - } - - impl MapObserver for $struct_name_trait { - type Entry = $datatype; - - #[inline] - fn get(&self, idx: usize) -> &$datatype { - let ptr = mapob_unwrap_me!($wrapper_name, self.wrapper, m, { m.get(idx) as *const $datatype }); - unsafe { ptr.as_ref().unwrap() } - } - - #[inline] - fn get_mut(&mut self, idx: usize) -> &mut $datatype { - let ptr = mapob_unwrap_me_mut!($wrapper_name, self.wrapper, m, { m.get_mut(idx) as *mut $datatype }); - unsafe { ptr.as_mut().unwrap() } - } - - #[inline] - fn count_bytes(&self) -> u64 { - mapob_unwrap_me!($wrapper_name, self.wrapper, m, { m.count_bytes() }) - } - #[inline] - fn usable_count(&self) -> usize { - mapob_unwrap_me!($wrapper_name, self.wrapper, m, { m.usable_count() }) - } - - fn hash(&self) -> u64 { - mapob_unwrap_me!($wrapper_name, self.wrapper, m, { m.hash() }) - } - - #[inline] - fn initial(&self) -> $datatype { - mapob_unwrap_me!($wrapper_name, self.wrapper, m, { m.initial() }) - } - - #[inline] - fn reset_map(&mut self) -> Result<(), Error> { - mapob_unwrap_me_mut!($wrapper_name, self.wrapper, m, { m.reset_map() }) - } - - #[inline] - fn to_vec(&self) -> Vec<$datatype> { - mapob_unwrap_me!($wrapper_name, self.wrapper, m, { m.to_vec() }) - } - - #[inline] - fn how_many_set(&self, indexes: &[usize]) -> usize { - mapob_unwrap_me!($wrapper_name, self.wrapper, m, { m.how_many_set(indexes) }) - } - } - - impl Named for $struct_name_trait { - #[inline] - fn name(&self) -> &str { - let ptr = mapob_unwrap_me!($wrapper_name, self.wrapper, m, { m.name() as *const str }); - unsafe { ptr.as_ref().unwrap() } - } - } - - impl HasLen for $struct_name_trait { - #[inline] - fn len(&self) -> usize { - mapob_unwrap_me!($wrapper_name, self.wrapper, m, { m.len() }) - } - } - - impl Observer for $struct_name_trait - where - Self: MapObserver, - S: UsesInput, - { - #[inline] - fn pre_exec(&mut self, state: &mut S, input: &S::Input) -> Result<(), Error> { - mapob_unwrap_me_mut!($wrapper_name, self.wrapper, m, { m.pre_exec(state, input) }) - } - } - }; - } - - define_python_map_observer!( - PythonStdMapObserverI8, - "StdMapObserverI8", - PythonOwnedMapObserverI8, - "OwnedMapObserverI8", - PythonMapObserverI8, - "MapObserverI8", - i8, - PythonMapObserverWrapperI8 - ); - define_python_map_observer!( - PythonStdMapObserverI16, - "StdMapObserverI16", - PythonOwnedMapObserverI16, - "OwnedMapObserverI16", - PythonMapObserverI16, - "MapObserverI16", - i16, - PythonMapObserverWrapperI16 - ); - define_python_map_observer!( - PythonStdMapObserverI32, - "StdMapObserverI32", - PythonOwnedMapObserverI32, - "OwnedMapObserverI32", - PythonMapObserverI32, - "MapObserverI32", - i32, - PythonMapObserverWrapperI32 - ); - define_python_map_observer!( - PythonStdMapObserverI64, - "StdMapObserverI64", - PythonOwnedMapObserverI64, - "OwnedMapObserverI64", - PythonMapObserverI64, - "MapObserverI64", - i64, - PythonMapObserverWrapperI64 - ); - - define_python_map_observer!( - PythonStdMapObserverU8, - "StdMapObserverU8", - PythonOwnedMapObserverU8, - "OwnedMapObserverU8", - PythonMapObserverU8, - "MapObserverU8", - u8, - PythonMapObserverWrapperU8 - ); - define_python_map_observer!( - PythonStdMapObserverU16, - "StdMapObserverU16", - PythonOwnedMapObserverU16, - "OwnedMapObserverU16", - PythonMapObserverU16, - "MapObserverU16", - u16, - PythonMapObserverWrapperU16 - ); - define_python_map_observer!( - PythonStdMapObserverU32, - "StdMapObserverU32", - PythonOwnedMapObserverU32, - "OwnedMapObserverU32", - PythonMapObserverU32, - "MapObserverU32", - u32, - PythonMapObserverWrapperU32 - ); - define_python_map_observer!( - PythonStdMapObserverU64, - "StdMapObserverU64", - PythonOwnedMapObserverU64, - "OwnedMapObserverU64", - PythonMapObserverU64, - "MapObserverU64", - u64, - PythonMapObserverWrapperU64 - ); - - /// Register the classes to the python module - pub fn register(_py: Python, m: &PyModule) -> PyResult<()> { - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - Ok(()) - } -} diff --git a/libafl/src/observers/map/const_map.rs b/libafl/src/observers/map/const_map.rs new file mode 100644 index 0000000000..533b7ded55 --- /dev/null +++ b/libafl/src/observers/map/const_map.rs @@ -0,0 +1,313 @@ +//! Map observer with a const size + +use alloc::{borrow::Cow, vec::Vec}; +use core::{ + fmt::Debug, + hash::{Hash, Hasher}, + ops::{Deref, DerefMut}, + slice::{Iter, IterMut}, +}; + +use ahash::RandomState; +use libafl_bolts::{ownedref::OwnedMutSlice, AsSlice, AsSliceMut, HasLen, Named}; +use num_traits::Bounded; +use serde::{Deserialize, Serialize}; + +use crate::{ + inputs::UsesInput, + observers::{map::MapObserver, Observer}, + Error, +}; + +/// Use a const size to speedup `Feedback::is_interesting` when the user can +/// know the size of the map at compile time. +#[derive(Serialize, Deserialize, Debug)] +#[serde(bound = "T: serde::de::DeserializeOwned")] +#[allow(clippy::unsafe_derive_deserialize)] +pub struct ConstMapObserver<'a, T, const N: usize> +where + T: Default + Copy + 'static + Serialize, +{ + map: OwnedMutSlice<'a, T>, + initial: T, + name: Cow<'static, str>, +} + +impl<'a, S, T, const N: usize> Observer for ConstMapObserver<'a, T, N> +where + S: UsesInput, + T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned + Debug, + Self: MapObserver, +{ + #[inline] + fn pre_exec(&mut self, _state: &mut S, _input: &S::Input) -> Result<(), Error> { + self.reset_map() + } +} + +impl<'a, T, const N: usize> Named for ConstMapObserver<'a, T, N> +where + T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned, +{ + #[inline] + fn name(&self) -> &Cow<'static, str> { + &self.name + } +} + +impl<'a, T, const N: usize> HasLen for ConstMapObserver<'a, T, N> +where + T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned, +{ + #[inline] + fn len(&self) -> usize { + N + } +} + +impl<'a, 'it, T, const N: usize> IntoIterator for &'it ConstMapObserver<'a, T, N> +where + T: Bounded + + PartialEq + + Default + + Copy + + Hash + + 'static + + Serialize + + serde::de::DeserializeOwned + + Debug, +{ + type Item = as Iterator>::Item; + type IntoIter = Iter<'it, T>; + + fn into_iter(self) -> Self::IntoIter { + let cnt = self.usable_count(); + self.as_slice()[..cnt].iter() + } +} + +impl<'a, 'it, T, const N: usize> IntoIterator for &'it mut ConstMapObserver<'a, T, N> +where + T: Bounded + + PartialEq + + Default + + Copy + + Hash + + 'static + + Serialize + + serde::de::DeserializeOwned + + Debug, +{ + type Item = as Iterator>::Item; + type IntoIter = IterMut<'it, T>; + + fn into_iter(self) -> Self::IntoIter { + let cnt = self.usable_count(); + self.as_slice_mut()[..cnt].iter_mut() + } +} + +impl<'a, T, const N: usize> ConstMapObserver<'a, T, N> +where + T: Bounded + + PartialEq + + Default + + Copy + + Hash + + 'static + + Serialize + + serde::de::DeserializeOwned + + Debug, +{ + /// Returns an iterator over the map. + pub fn iter(&self) -> Iter<'_, T> { + <&Self as IntoIterator>::into_iter(self) + } + + /// Returns a mutable iterator over the map. + pub fn iter_mut(&mut self) -> IterMut<'_, T> { + <&mut Self as IntoIterator>::into_iter(self) + } +} + +impl<'a, T, const N: usize> Hash for ConstMapObserver<'a, T, N> +where + T: Bounded + + PartialEq + + Default + + Copy + + Hash + + 'static + + Serialize + + serde::de::DeserializeOwned + + Debug, +{ + #[inline] + fn hash(&self, hasher: &mut H) { + self.as_slice().hash(hasher); + } +} +impl<'a, T, const N: usize> AsRef for ConstMapObserver<'a, T, N> +where + T: Default + Copy + 'static + Serialize, +{ + fn as_ref(&self) -> &Self { + self + } +} + +impl<'a, T, const N: usize> AsMut for ConstMapObserver<'a, T, N> +where + T: Default + Copy + 'static + Serialize, +{ + fn as_mut(&mut self) -> &mut Self { + self + } +} + +impl<'a, T, const N: usize> MapObserver for ConstMapObserver<'a, T, N> +where + T: Bounded + + PartialEq + + Default + + Copy + + Hash + + 'static + + Serialize + + serde::de::DeserializeOwned + + Debug, +{ + type Entry = T; + + #[inline] + fn initial(&self) -> T { + self.initial + } + + #[inline] + fn get(&self, idx: usize) -> T { + self.as_slice()[idx] + } + + #[inline] + fn set(&mut self, idx: usize, val: T) { + self.map.as_slice_mut()[idx] = val; + } + + /// Count the set bytes in the map + fn count_bytes(&self) -> u64 { + let initial = self.initial(); + let cnt = self.usable_count(); + let map = self.as_slice(); + let mut res = 0; + for x in &map[0..cnt] { + if *x != initial { + res += 1; + } + } + res + } + + fn usable_count(&self) -> usize { + self.as_slice().len() + } + + #[inline] + fn hash_simple(&self) -> u64 { + RandomState::with_seeds(0, 0, 0, 0).hash_one(self) + } + + /// Reset the map + #[inline] + fn reset_map(&mut self) -> Result<(), Error> { + // Normal memset, see https://rust.godbolt.org/z/Trs5hv + let initial = self.initial(); + let cnt = self.usable_count(); + let map = self.as_slice_mut(); + for x in &mut map[0..cnt] { + *x = initial; + } + Ok(()) + } + + fn to_vec(&self) -> Vec { + self.as_slice().to_vec() + } + + /// Get the number of set entries with the specified indexes + fn how_many_set(&self, indexes: &[usize]) -> usize { + let initial = self.initial(); + let cnt = self.usable_count(); + let map = self.as_slice(); + let mut res = 0; + for i in indexes { + if *i < cnt && map[*i] != initial { + res += 1; + } + } + res + } +} + +impl<'a, T, const N: usize> Deref for ConstMapObserver<'a, T, N> +where + T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned + Debug, +{ + type Target = [T]; + fn deref(&self) -> &[T] { + &self.map + } +} + +impl<'a, T, const N: usize> DerefMut for ConstMapObserver<'a, T, N> +where + T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned + Debug, +{ + fn deref_mut(&mut self) -> &mut [T] { + &mut self.map + } +} + +impl<'a, T, const N: usize> ConstMapObserver<'a, T, N> +where + T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned, +{ + /// Creates a new [`MapObserver`] + /// + /// # Note + /// Will get a pointer to the map and dereference it at any point in time. + /// The map must not move in memory! + #[must_use] + pub fn new(name: &'static str, map: &'a mut [T]) -> Self { + assert!(map.len() >= N); + Self { + map: OwnedMutSlice::from(map), + name: Cow::from(name), + initial: T::default(), + } + } + + /// Creates a new [`MapObserver`] with an owned map + #[must_use] + pub fn owned(name: &'static str, map: Vec) -> Self { + assert!(map.len() >= N); + let initial = if map.is_empty() { T::default() } else { map[0] }; + Self { + map: OwnedMutSlice::from(map), + name: Cow::from(name), + initial, + } + } + + /// Creates a new [`MapObserver`] from a raw pointer + /// + /// # Safety + /// Will dereference the `map_ptr` with up to len elements. + pub unsafe fn from_mut_ptr(name: &'static str, map_ptr: *mut T) -> Self { + ConstMapObserver { + map: OwnedMutSlice::from_raw_parts_mut(map_ptr, N), + name: Cow::from(name), + initial: T::default(), + } + } +} diff --git a/libafl/src/observers/map/hitcount_map.rs b/libafl/src/observers/map/hitcount_map.rs new file mode 100644 index 0000000000..cd8f1f7193 --- /dev/null +++ b/libafl/src/observers/map/hitcount_map.rs @@ -0,0 +1,583 @@ +//! Hitcount map observer is for implementing AFL's hit count bucket +use alloc::{borrow::Cow, vec::Vec}; +use core::{fmt::Debug, hash::Hash, mem::size_of, slice}; + +use libafl_bolts::{AsIter, AsIterMut, AsSlice, AsSliceMut, HasLen, Named, Truncate}; +use serde::{Deserialize, Serialize}; + +use crate::{ + executors::ExitKind, + inputs::UsesInput, + observers::{map::MapObserver, DifferentialObserver, Observer, ObserversTuple}, + Error, +}; + +/// Hitcounts class lookup +static COUNT_CLASS_LOOKUP: [u8; 256] = [ + 0, 1, 2, 4, 8, 8, 8, 8, 16, 16, 16, 16, 16, 16, 16, 16, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 32, 32, 32, 32, 32, 32, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, +]; + +/// Hitcounts class lookup for 16-byte values +static mut COUNT_CLASS_LOOKUP_16: Vec = vec![]; + +/// Initialize the 16-byte hitcounts map +/// +/// # Safety +/// +/// Calling this from multiple threads may be racey and hence leak 65k mem +fn init_count_class_16() { + unsafe { + if !COUNT_CLASS_LOOKUP_16.is_empty() { + return; + } + + COUNT_CLASS_LOOKUP_16 = vec![0; 65536]; + for i in 0..256 { + for j in 0..256 { + COUNT_CLASS_LOOKUP_16[(i << 8) + j] = + (u16::from(COUNT_CLASS_LOOKUP[i]) << 8) | u16::from(COUNT_CLASS_LOOKUP[j]); + } + } + } +} + +/// Map observer with AFL-like hitcounts postprocessing +/// +/// [`MapObserver`]s that are not slice-backed, such as `MultiMapObserver`, can use +/// [`HitcountsIterableMapObserver`] instead. +#[derive(Serialize, Deserialize, Clone, Debug, Hash)] +#[serde(bound = "M: serde::de::DeserializeOwned")] +pub struct HitcountsMapObserver +where + M: Serialize, +{ + base: M, +} + +impl Observer for HitcountsMapObserver +where + M: MapObserver + Observer + for<'a> AsSliceMut<'a, Entry = u8>, + S: UsesInput, +{ + #[inline] + fn pre_exec(&mut self, state: &mut S, input: &S::Input) -> Result<(), Error> { + self.base.pre_exec(state, input) + } + + #[inline] + #[allow(clippy::cast_ptr_alignment)] + fn post_exec( + &mut self, + state: &mut S, + input: &S::Input, + exit_kind: &ExitKind, + ) -> Result<(), Error> { + let mut map = self.as_slice_mut(); + let mut len = map.len(); + let align_offset = map.as_ptr().align_offset(size_of::()); + + // if len == 1, the next branch will already do this lookup + if len > 1 && align_offset != 0 { + debug_assert_eq!( + align_offset, 1, + "Aligning u8 to u16 should always be offset of 1?" + ); + unsafe { + *map.get_unchecked_mut(0) = + *COUNT_CLASS_LOOKUP.get_unchecked(*map.get_unchecked(0) as usize); + } + len -= 1; + } + + // Fix the last element + if (len & 1) != 0 { + unsafe { + *map.get_unchecked_mut(len - 1) = + *COUNT_CLASS_LOOKUP.get_unchecked(*map.get_unchecked(len - 1) as usize); + } + } + + let cnt = len / 2; + + let map16 = unsafe { + slice::from_raw_parts_mut(map.as_mut_ptr().add(align_offset) as *mut u16, cnt) + }; + // 2022-07: Adding `enumerate` here increases execution speed/register allocation on x86_64. + #[allow(clippy::unused_enumerate_index)] + for (_i, item) in map16[0..cnt].iter_mut().enumerate() { + unsafe { + *item = *COUNT_CLASS_LOOKUP_16.get_unchecked(*item as usize); + } + } + + drop(map); + + self.base.post_exec(state, input, exit_kind) + } +} + +impl Named for HitcountsMapObserver +where + M: Named + Serialize + serde::de::DeserializeOwned, +{ + #[inline] + fn name(&self) -> &Cow<'static, str> { + self.base.name() + } +} + +impl HasLen for HitcountsMapObserver +where + M: MapObserver, +{ + #[inline] + fn len(&self) -> usize { + self.base.len() + } +} + +impl AsRef for HitcountsMapObserver +where + M: MapObserver, +{ + fn as_ref(&self) -> &Self { + self + } +} + +impl AsMut for HitcountsMapObserver +where + M: MapObserver, +{ + fn as_mut(&mut self) -> &mut Self { + self + } +} + +impl MapObserver for HitcountsMapObserver +where + M: MapObserver, +{ + type Entry = u8; + + #[inline] + fn initial(&self) -> u8 { + self.base.initial() + } + + #[inline] + fn usable_count(&self) -> usize { + self.base.usable_count() + } + + #[inline] + fn get(&self, idx: usize) -> u8 { + self.base.get(idx) + } + + #[inline] + fn set(&mut self, idx: usize, val: u8) { + self.base.set(idx, val); + } + + /// Count the set bytes in the map + fn count_bytes(&self) -> u64 { + self.base.count_bytes() + } + + /// Reset the map + #[inline] + fn reset_map(&mut self) -> Result<(), Error> { + self.base.reset_map() + } + + #[inline] + fn hash_simple(&self) -> u64 { + self.base.hash_simple() + } + fn to_vec(&self) -> Vec { + self.base.to_vec() + } + + fn how_many_set(&self, indexes: &[usize]) -> usize { + self.base.how_many_set(indexes) + } +} + +impl Truncate for HitcountsMapObserver +where + M: Named + Serialize + serde::de::DeserializeOwned + Truncate, +{ + fn truncate(&mut self, new_len: usize) { + self.base.truncate(new_len); + } +} + +impl<'a, M> AsSlice<'a> for HitcountsMapObserver +where + M: MapObserver + AsSlice<'a>, +{ + type Entry = >::Entry; + type SliceRef = >::SliceRef; + + #[inline] + fn as_slice(&'a self) -> Self::SliceRef { + self.base.as_slice() + } +} + +impl<'a, M> AsSliceMut<'a> for HitcountsMapObserver +where + M: MapObserver + AsSliceMut<'a>, +{ + type SliceRefMut = >::SliceRefMut; + #[inline] + fn as_slice_mut(&'a mut self) -> Self::SliceRefMut { + self.base.as_slice_mut() + } +} + +impl HitcountsMapObserver +where + M: MapObserver, +{ + /// Creates a new [`MapObserver`] + pub fn new(base: M) -> Self { + init_count_class_16(); + Self { base } + } +} + +impl<'it, M> IntoIterator for &'it HitcountsMapObserver +where + M: Serialize + serde::de::DeserializeOwned, + &'it M: IntoIterator, +{ + type Item = &'it u8; + type IntoIter = <&'it M as IntoIterator>::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.base.into_iter() + } +} + +impl<'it, M> IntoIterator for &'it mut HitcountsMapObserver +where + M: Serialize + serde::de::DeserializeOwned, + &'it mut M: IntoIterator, +{ + type Item = &'it mut u8; + type IntoIter = <&'it mut M as IntoIterator>::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.base.into_iter() + } +} + +impl HitcountsMapObserver +where + M: Serialize + serde::de::DeserializeOwned, + for<'it> &'it M: IntoIterator, +{ + /// Returns an iterator over the map. + pub fn iter(&self) -> <&M as IntoIterator>::IntoIter { + <&Self as IntoIterator>::into_iter(self) + } +} + +impl HitcountsMapObserver +where + M: Serialize + serde::de::DeserializeOwned, + for<'it> &'it mut M: IntoIterator, +{ + /// Returns a mutable iterator over the map. + pub fn iter_mut(&mut self) -> <&mut M as IntoIterator>::IntoIter { + <&mut Self as IntoIterator>::into_iter(self) + } +} + +impl DifferentialObserver for HitcountsMapObserver +where + M: DifferentialObserver + + MapObserver + + Serialize + + for<'a> AsSliceMut<'a, Entry = u8>, + OTA: ObserversTuple, + OTB: ObserversTuple, + S: UsesInput, +{ + fn pre_observe_first(&mut self, observers: &mut OTA) -> Result<(), Error> { + self.base.pre_observe_first(observers) + } + + fn post_observe_first(&mut self, observers: &mut OTA) -> Result<(), Error> { + self.base.post_observe_first(observers) + } + + fn pre_observe_second(&mut self, observers: &mut OTB) -> Result<(), Error> { + self.base.pre_observe_second(observers) + } + + fn post_observe_second(&mut self, observers: &mut OTB) -> Result<(), Error> { + self.base.post_observe_second(observers) + } +} + +/// Map observer with hitcounts postprocessing +/// Less optimized version for non-slice iterators. +/// Slice-backed observers should use a [`HitcountsMapObserver`]. +#[derive(Serialize, Deserialize, Clone, Debug, Hash)] +#[serde(bound = "M: serde::de::DeserializeOwned")] +pub struct HitcountsIterableMapObserver +where + M: Serialize, +{ + base: M, +} + +impl Observer for HitcountsIterableMapObserver +where + M: MapObserver + Observer, + for<'it> M: AsIterMut<'it, Item = u8>, + S: UsesInput, +{ + #[inline] + fn pre_exec(&mut self, state: &mut S, input: &S::Input) -> Result<(), Error> { + self.base.pre_exec(state, input) + } + + #[inline] + #[allow(clippy::cast_ptr_alignment)] + fn post_exec( + &mut self, + state: &mut S, + input: &S::Input, + exit_kind: &ExitKind, + ) -> Result<(), Error> { + for mut item in self.as_iter_mut() { + *item = unsafe { *COUNT_CLASS_LOOKUP.get_unchecked((*item) as usize) }; + } + + self.base.post_exec(state, input, exit_kind) + } +} + +impl Named for HitcountsIterableMapObserver +where + M: Named + Serialize + serde::de::DeserializeOwned, +{ + #[inline] + fn name(&self) -> &Cow<'static, str> { + self.base.name() + } +} + +impl HasLen for HitcountsIterableMapObserver +where + M: MapObserver, +{ + #[inline] + fn len(&self) -> usize { + self.base.len() + } +} + +impl AsRef for HitcountsIterableMapObserver +where + M: MapObserver, + for<'it> M: AsIterMut<'it, Item = u8>, +{ + fn as_ref(&self) -> &Self { + self + } +} + +impl AsMut for HitcountsIterableMapObserver +where + M: MapObserver, + for<'it> M: AsIterMut<'it, Item = u8>, +{ + fn as_mut(&mut self) -> &mut Self { + self + } +} + +impl MapObserver for HitcountsIterableMapObserver +where + M: MapObserver, + for<'it> M: AsIterMut<'it, Item = u8>, +{ + type Entry = u8; + + #[inline] + fn initial(&self) -> u8 { + self.base.initial() + } + + #[inline] + fn usable_count(&self) -> usize { + self.base.usable_count() + } + + #[inline] + fn get(&self, idx: usize) -> u8 { + self.base.get(idx) + } + + #[inline] + fn set(&mut self, idx: usize, val: u8) { + self.base.set(idx, val); + } + + /// Count the set bytes in the map + fn count_bytes(&self) -> u64 { + self.base.count_bytes() + } + + /// Reset the map + #[inline] + fn reset_map(&mut self) -> Result<(), Error> { + self.base.reset_map() + } + + #[inline] + fn hash_simple(&self) -> u64 { + self.base.hash_simple() + } + fn to_vec(&self) -> Vec { + self.base.to_vec() + } + + fn how_many_set(&self, indexes: &[usize]) -> usize { + self.base.how_many_set(indexes) + } +} + +impl Truncate for HitcountsIterableMapObserver +where + M: Named + Serialize + serde::de::DeserializeOwned + Truncate, +{ + fn truncate(&mut self, new_len: usize) { + self.base.truncate(new_len); + } +} + +impl HitcountsIterableMapObserver +where + M: Serialize + serde::de::DeserializeOwned, +{ + /// Creates a new [`MapObserver`] + pub fn new(base: M) -> Self { + init_count_class_16(); + Self { base } + } +} + +impl<'it, M> AsIter<'it> for HitcountsIterableMapObserver +where + M: Named + Serialize + serde::de::DeserializeOwned + AsIter<'it, Item = u8>, +{ + type Item = u8; + type Ref = >::Ref; + type IntoIter = >::IntoIter; + + fn as_iter(&'it self) -> Self::IntoIter { + self.base.as_iter() + } +} + +impl<'it, M> AsIterMut<'it> for HitcountsIterableMapObserver +where + M: Named + Serialize + serde::de::DeserializeOwned + AsIterMut<'it, Item = u8>, +{ + type RefMut = >::RefMut; + type IntoIterMut = >::IntoIterMut; + + fn as_iter_mut(&'it mut self) -> Self::IntoIterMut { + self.base.as_iter_mut() + } +} + +impl<'it, M> IntoIterator for &'it HitcountsIterableMapObserver +where + M: Serialize + serde::de::DeserializeOwned, + &'it M: IntoIterator, +{ + type Item = &'it u8; + type IntoIter = <&'it M as IntoIterator>::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.base.into_iter() + } +} + +impl<'it, M> IntoIterator for &'it mut HitcountsIterableMapObserver +where + M: Serialize + serde::de::DeserializeOwned, + &'it mut M: IntoIterator, +{ + type Item = &'it mut u8; + type IntoIter = <&'it mut M as IntoIterator>::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.base.into_iter() + } +} + +impl HitcountsIterableMapObserver +where + M: Serialize + serde::de::DeserializeOwned, + for<'it> &'it M: IntoIterator, +{ + /// Returns an iterator over the map. + pub fn iter(&self) -> <&M as IntoIterator>::IntoIter { + <&Self as IntoIterator>::into_iter(self) + } +} + +impl HitcountsIterableMapObserver +where + M: Serialize + serde::de::DeserializeOwned, + for<'it> &'it mut M: IntoIterator, +{ + /// Returns a mutable iterator over the map. + pub fn iter_mut(&mut self) -> <&mut M as IntoIterator>::IntoIter { + <&mut Self as IntoIterator>::into_iter(self) + } +} + +impl DifferentialObserver for HitcountsIterableMapObserver +where + M: MapObserver + Observer + DifferentialObserver, + for<'it> M: AsIterMut<'it, Item = u8>, + OTA: ObserversTuple, + OTB: ObserversTuple, + S: UsesInput, +{ + fn pre_observe_first(&mut self, observers: &mut OTA) -> Result<(), Error> { + self.base.pre_observe_first(observers) + } + + fn post_observe_first(&mut self, observers: &mut OTA) -> Result<(), Error> { + self.base.post_observe_first(observers) + } + + fn pre_observe_second(&mut self, observers: &mut OTB) -> Result<(), Error> { + self.base.pre_observe_second(observers) + } + + fn post_observe_second(&mut self, observers: &mut OTB) -> Result<(), Error> { + self.base.post_observe_second(observers) + } +} diff --git a/libafl/src/observers/map/mod.rs b/libafl/src/observers/map/mod.rs new file mode 100644 index 0000000000..a5fb8d3e26 --- /dev/null +++ b/libafl/src/observers/map/mod.rs @@ -0,0 +1,932 @@ +//! All the map observer variants + +use alloc::{borrow::Cow, vec::Vec}; +use core::{ + fmt::Debug, + hash::{Hash, Hasher}, + ops::{Deref, DerefMut}, + slice::{Iter, IterMut}, +}; + +use ahash::RandomState; +use libafl_bolts::{ownedref::OwnedMutSlice, AsSlice, AsSliceMut, HasLen, Named, Truncate}; +use num_traits::Bounded; +use serde::{Deserialize, Serialize}; + +use crate::{ + executors::ExitKind, + inputs::UsesInput, + observers::{DifferentialObserver, Observer, ObserversTuple}, + Error, +}; + +pub mod const_map; +pub use const_map::*; + +pub mod variable_map; +pub use variable_map::*; + +pub mod hitcount_map; +pub use hitcount_map::*; + +pub mod multi_map; +pub use multi_map::*; + +pub mod owned_map; +pub use owned_map::*; + +/// Trait marker which indicates that this [`MapObserver`] is tracked for indices or novelties. +/// Implementors of feedbacks similar to [`crate::feedbacks::MapFeedback`] may wish to use this to +/// ensure that edge metadata is recorded as is appropriate for the provided observer. +/// +/// If you get a type constraint failure for your map due to this type being unfulfilled, you must +/// call [`CanTrack::track_indices`] or [`CanTrack::track_novelties`] **at +/// the initialisation site of your map**. +/// +/// This trait allows various components which interact with map metadata to ensure that the +/// information they need is actually recorded by the map feedback. +/// For example, if you are using [`crate::schedulers::MinimizerScheduler`]: +/// ``` +/// # use libafl::corpus::InMemoryCorpus; +/// # use libafl::feedbacks::{Feedback, MapFeedbackMetadata}; +/// use libafl::feedbacks::MaxMapFeedback; +/// # use libafl::inputs::BytesInput; +/// use libafl::observers::{StdMapObserver, CanTrack}; +/// use libafl::schedulers::{IndexesLenTimeMinimizerScheduler, QueueScheduler}; +/// # use libafl::state::StdState; +/// # use libafl_bolts::serdeany::RegistryBuilder; +/// # +/// # #[cfg(any(not(feature = "serdeany_autoreg"), miri))] +/// # unsafe { MapFeedbackMetadata::::register() } +/// # #[cfg(not(feature = "std"))] +/// # #[no_mangle] +/// # pub extern "C" fn external_current_millis() -> u64 { 0 } +/// +/// use libafl_bolts::ownedref::OwnedMutSlice; +/// # use libafl_bolts::rands::StdRand; +/// +/// // initialise your map as necessary +/// let edges_observer = StdMapObserver::from_ownedref("edges", OwnedMutSlice::from(vec![0u8; 16])); +/// // inform the feedback to track indices (required by IndexesLenTimeMinimizerScheduler), but not novelties +/// // this *MUST* be done before it is passed to MaxMapFeedback! +/// let edges_observer = edges_observer.track_indices(); +/// +/// // init the feedback +/// let mut feedback = MaxMapFeedback::new(&edges_observer); +/// # +/// # // init the state +/// # let mut state = StdState::new( +/// # StdRand::with_seed(0), +/// # InMemoryCorpus::::new(), +/// # InMemoryCorpus::new(), +/// # &mut feedback, +/// # &mut () +/// # ).unwrap(); +/// # feedback.init_state(&mut state).unwrap(); +/// +/// let scheduler = IndexesLenTimeMinimizerScheduler::new(&edges_observer, QueueScheduler::new()); +/// # scheduler.cull(&state).unwrap(); +/// ``` +/// +/// [`MapObserver`] implementors: see [`StdMapObserver`] for an example implementation. +pub trait CanTrack { + /// The resulting type of enabling index tracking. + type WithIndexTracking: CanTrack; + /// The resulting type of enabling novelty tracking. + type WithNoveltiesTracking: CanTrack; + + /// Whether indices should be tracked for this [`MapObserver`]. + const INDICES: bool; + /// Whether novelties should be tracked for this [`MapObserver`]. + const NOVELTIES: bool; + + /// Convert this map observer into one that tracks indices. + fn track_indices(self) -> Self::WithIndexTracking; + /// Convert this map observer into one that tracks novelties. + fn track_novelties(self) -> Self::WithNoveltiesTracking; +} + +/// Struct which wraps [`MapObserver`] instances to explicitly give them tracking data. +/// +/// # Safety +/// +/// This is a bit of a magic structure. We pass it to the observer tuple as itself, but when its +/// referred to with `match_name`, there is a cast from this type to its inner type. This is +/// *guaranteed to be safe* by `#[repr(transparent)]`. +#[derive(Copy, Clone, Debug, Deserialize, Serialize)] +pub struct ExplicitTracking(T); + +impl CanTrack for ExplicitTracking { + type WithIndexTracking = ExplicitTracking; + type WithNoveltiesTracking = ExplicitTracking; + const INDICES: bool = ITH; + const NOVELTIES: bool = NTH; + + fn track_indices(self) -> Self::WithIndexTracking { + ExplicitTracking::(self.0) + } + + fn track_novelties(self) -> Self::WithNoveltiesTracking { + ExplicitTracking::(self.0) + } +} + +impl AsRef for ExplicitTracking { + fn as_ref(&self) -> &T { + &self.0 + } +} + +impl AsMut for ExplicitTracking { + fn as_mut(&mut self) -> &mut T { + &mut self.0 + } +} + +impl Named for ExplicitTracking +where + T: Named, +{ + fn name(&self) -> &Cow<'static, str> { + self.0.name() + } +} + +impl Observer for ExplicitTracking +where + S: UsesInput, + T: Observer, +{ + fn flush(&mut self) -> Result<(), Error> { + self.0.flush() + } + + fn pre_exec(&mut self, state: &mut S, input: &S::Input) -> Result<(), Error> { + self.0.pre_exec(state, input) + } + + fn post_exec( + &mut self, + state: &mut S, + input: &S::Input, + exit_kind: &ExitKind, + ) -> Result<(), Error> { + self.0.post_exec(state, input, exit_kind) + } + + fn pre_exec_child(&mut self, state: &mut S, input: &S::Input) -> Result<(), Error> { + self.0.pre_exec_child(state, input) + } + + fn post_exec_child( + &mut self, + state: &mut S, + input: &S::Input, + exit_kind: &ExitKind, + ) -> Result<(), Error> { + self.0.post_exec_child(state, input, exit_kind) + } + + fn observes_stdout(&self) -> bool { + self.0.observes_stdout() + } + + fn observes_stderr(&self) -> bool { + self.0.observes_stderr() + } + + fn observe_stdout(&mut self, stdout: &[u8]) { + self.0.observe_stdout(stdout); + } + + fn observe_stderr(&mut self, stderr: &[u8]) { + self.0.observe_stderr(stderr); + } +} + +impl DifferentialObserver + for ExplicitTracking +where + OTA: ObserversTuple, + OTB: ObserversTuple, + S: UsesInput, + T: DifferentialObserver, +{ + fn pre_observe_first(&mut self, observers: &mut OTA) -> Result<(), Error> { + self.as_mut().pre_observe_first(observers) + } + + fn post_observe_first(&mut self, observers: &mut OTA) -> Result<(), Error> { + self.as_mut().post_observe_first(observers) + } + + fn pre_observe_second(&mut self, observers: &mut OTB) -> Result<(), Error> { + self.as_mut().pre_observe_second(observers) + } + + fn post_observe_second(&mut self, observers: &mut OTB) -> Result<(), Error> { + self.as_mut().post_observe_second(observers) + } +} + +/// Module which holds the necessary functions and types for map-relevant macros, namely +/// [`crate::require_index_tracking`] and [`crate::require_novelties_tracking`]. +pub mod macros { + pub use const_format::{concatcp, str_repeat}; + pub use const_panic::{concat_panic, FmtArg}; + + /// Use in the constructor of your component which requires index tracking of a + /// [`super::MapObserver`]. See [`super::CanTrack`] for details. + /// + /// As an example, if you are developing the type `MyCustomScheduler` which requires novelty + /// tracking, use this in your constructor: + /// ``` + /// # use libafl::observers::{MapObserver, CanTrack}; + /// # use libafl::require_index_tracking; + /// # use core::marker::PhantomData; + /// # + /// # struct MyCustomScheduler { + /// # phantom: PhantomData<(C, O)>, + /// # } + /// # + /// impl MyCustomScheduler where O: MapObserver, C: CanTrack + AsRef { + /// pub fn new(obs: &C) -> Self { + /// require_index_tracking!("MyCustomScheduler", C); + /// todo!("Construct your type") + /// } + /// } + /// ``` + #[macro_export] + macro_rules! require_index_tracking { + ($name: literal, $obs: ident) => { + struct SanityCheck { + phantom: ::core::marker::PhantomData, + } + + impl SanityCheck { + #[rustfmt::skip] + const MESSAGE: &'static str = { + const LINE_OFFSET: usize = line!().ilog10() as usize + 2; + const SPACING: &str = $crate::observers::map::macros::str_repeat!(" ", LINE_OFFSET); + $crate::observers::map::macros::concatcp!( + "\n", + SPACING, "|\n", + SPACING, "= note: index tracking is required by ", $name, "\n", + SPACING, "= note: see the documentation of CanTrack for details\n", + SPACING, "|\n", + SPACING, "= hint: call `.track_indices()` on the map observer passed to ", $name, " at the point where it is defined\n", + SPACING, "|\n", + SPACING, "| ", + ) + }; + const TRACKING_SANITY: bool = { + if !O::INDICES { + panic!("{}", Self::MESSAGE) + } else { + true + } + }; + + #[inline(always)] + fn check_sanity() { + if !Self::TRACKING_SANITY { + unreachable!("{}", Self::MESSAGE); + } + } + } + SanityCheck::<$obs>::check_sanity(); // check that tracking is enabled for this map + }; + } + + /// Use in the constructor of your component which requires novelties tracking of a + /// [`super::MapObserver`]. See [`super::CanTrack`] for details on the concept. + /// + /// As an example, if you are developing the type `MyCustomScheduler` which requires novelty + /// tracking, use this in your constructor: + /// ``` + /// # use libafl::observers::{MapObserver, CanTrack}; + /// # use libafl::require_novelties_tracking; + /// # use core::marker::PhantomData; + /// # + /// # struct MyCustomScheduler { + /// # phantom: PhantomData<(C, O)>, + /// # } + /// # + /// impl MyCustomScheduler where O: MapObserver, C: CanTrack + AsRef { + /// pub fn new(obs: &C) -> Self { + /// require_novelties_tracking!("MyCustomScheduler", C); + /// todo!("Construct your type") + /// } + /// } + /// ``` + #[macro_export] + macro_rules! require_novelties_tracking { + ($name: literal, $obs: ident) => { + struct SanityCheck { + phantom: ::core::marker::PhantomData, + } + + impl SanityCheck { + #[rustfmt::skip] + const MESSAGE: &'static str = { + const LINE_OFFSET: usize = line!().ilog10() as usize + 2; + const SPACING: &str = + $crate::observers::map::macros::str_repeat!(" ", LINE_OFFSET); + $crate::observers::map::macros::concatcp!( + "\n", + SPACING, "|\n", + SPACING, "= note: novelty tracking is required by ", $name, "\n", + SPACING, "= note: see the documentation of CanTrack for details\n", + SPACING, "|\n", + SPACING, "= hint: call `.track_novelties()` on the map observer passed to ", $name, " at the point where it is defined\n", + SPACING, "|\n", + SPACING, "| ", + ) + }; + const TRACKING_SANITY: bool = { + if !O::NOVELTIES { + panic!("{}", Self::MESSAGE) + } else { + true + } + }; + + #[inline(always)] + fn check_sanity() { + if !Self::TRACKING_SANITY { + unreachable!("{}", Self::MESSAGE); + } + } + } + SanityCheck::<$obs>::check_sanity(); // check that tracking is enabled for this map + }; + } +} + +/// A [`MapObserver`] observes the static map, as oftentimes used for AFL-like coverage information +/// +/// When referring to this type in a constraint (e.g. `O: MapObserver`), ensure that you only refer +/// to instances of a second type, e.g. `C: AsRef` or `A: AsMut`. Map observer instances are +/// passed around in a way that may be potentially wrapped by e.g. [`ExplicitTracking`] as a way to +/// encode metadata into the type. This is an unfortunate additional requirement that we can't get +/// around without specialization. +/// +/// See [`crate::require_index_tracking`] for an example of how to do so. +/// +/// TODO: enforce `iter() -> AssociatedTypeIter` when generic associated types stabilize +pub trait MapObserver: + HasLen + Named + Serialize + serde::de::DeserializeOwned + AsRef + AsMut + Hash +// where +// for<'it> &'it Self: IntoIterator +{ + /// Type of each entry in this map + type Entry: Bounded + PartialEq + Default + Copy + Debug + Hash + 'static; + + /// Get the value at `idx` + fn get(&self, idx: usize) -> Self::Entry; + + /// Set the value at `idx` + fn set(&mut self, idx: usize, val: Self::Entry); + + /// Get the number of usable entries in the map (all by default) + fn usable_count(&self) -> usize; + + /// Count the set bytes in the map + fn count_bytes(&self) -> u64; + + /// Compute the hash of the map without needing to provide a hasher + fn hash_simple(&self) -> u64; + + /// Get the initial value for `reset()` + fn initial(&self) -> Self::Entry; + + /// Reset the map + fn reset_map(&mut self) -> Result<(), Error>; + + /// Get these observer's contents as [`Vec`] + fn to_vec(&self) -> Vec; + + /// Get the number of set entries with the specified indexes + fn how_many_set(&self, indexes: &[usize]) -> usize; +} + +impl CanTrack for M +where + M: MapObserver, +{ + type WithIndexTracking = ExplicitTracking; + type WithNoveltiesTracking = ExplicitTracking; + const INDICES: bool = false; + const NOVELTIES: bool = false; + + fn track_indices(self) -> Self::WithIndexTracking { + ExplicitTracking::(self) + } + + fn track_novelties(self) -> Self::WithNoveltiesTracking { + ExplicitTracking::(self) + } +} + +/// The Map Observer retrieves the state of a map, +/// that will get updated by the target. +/// A well-known example is the AFL-Style coverage map. +#[derive(Clone, Serialize, Deserialize, Debug)] +#[serde(bound = "T: serde::de::DeserializeOwned")] +#[allow(clippy::unsafe_derive_deserialize)] +pub struct StdMapObserver<'a, T, const DIFFERENTIAL: bool> +where + T: Default + Copy + 'static + Serialize, +{ + map: OwnedMutSlice<'a, T>, + initial: T, + name: Cow<'static, str>, +} + +impl<'a, S, T> Observer for StdMapObserver<'a, T, false> +where + S: UsesInput, + T: Bounded + + PartialEq + + Default + + Copy + + Hash + + 'static + + Serialize + + serde::de::DeserializeOwned + + Debug, +{ + #[inline] + fn pre_exec(&mut self, _state: &mut S, _input: &S::Input) -> Result<(), Error> { + self.reset_map() + } +} + +impl<'a, S, T> Observer for StdMapObserver<'a, T, true> +where + S: UsesInput, + T: Bounded + + PartialEq + + Default + + Copy + + 'static + + Serialize + + serde::de::DeserializeOwned + + Debug, +{ +} + +impl<'a, T, const DIFFERENTIAL: bool> Named for StdMapObserver<'a, T, DIFFERENTIAL> +where + T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned, +{ + #[inline] + fn name(&self) -> &Cow<'static, str> { + &self.name + } +} + +impl<'a, T, const DIFFERENTIAL: bool> HasLen for StdMapObserver<'a, T, DIFFERENTIAL> +where + T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned, +{ + #[inline] + fn len(&self) -> usize { + self.map.as_slice().len() + } +} + +impl<'a, 'it, T, const DIFFERENTIAL: bool> IntoIterator for &'it StdMapObserver<'a, T, DIFFERENTIAL> +where + T: Bounded + + PartialEq + + Default + + Copy + + Hash + + 'static + + Serialize + + serde::de::DeserializeOwned + + Debug, +{ + type Item = as Iterator>::Item; + type IntoIter = Iter<'it, T>; + + fn into_iter(self) -> Self::IntoIter { + let cnt = self.usable_count(); + self.as_slice()[..cnt].iter() + } +} + +impl<'a, 'it, T, const DIFFERENTIAL: bool> IntoIterator + for &'it mut StdMapObserver<'a, T, DIFFERENTIAL> +where + T: Bounded + + PartialEq + + Default + + Copy + + Hash + + 'static + + Serialize + + serde::de::DeserializeOwned + + Debug, +{ + type Item = as Iterator>::Item; + type IntoIter = IterMut<'it, T>; + + fn into_iter(self) -> Self::IntoIter { + let cnt = self.usable_count(); + self.as_slice_mut()[..cnt].iter_mut() + } +} + +impl<'a, T, const DIFFERENTIAL: bool> StdMapObserver<'a, T, DIFFERENTIAL> +where + T: Bounded + + PartialEq + + Default + + Copy + + Hash + + 'static + + Serialize + + serde::de::DeserializeOwned + + Debug, +{ + /// Returns an iterator over the map. + pub fn iter(&self) -> Iter<'_, T> { + <&Self as IntoIterator>::into_iter(self) + } + + /// Returns a mutable iterator over the map. + pub fn iter_mut(&mut self) -> IterMut<'_, T> { + <&mut Self as IntoIterator>::into_iter(self) + } +} + +impl<'a, T, const DIFFERENTIAL: bool> Hash for StdMapObserver<'a, T, DIFFERENTIAL> +where + T: Bounded + + PartialEq + + Default + + Copy + + Hash + + 'static + + Serialize + + serde::de::DeserializeOwned + + Debug, +{ + #[inline] + fn hash(&self, hasher: &mut H) { + self.as_slice().hash(hasher); + } +} + +impl<'a, T, const DIFFERENTIAL: bool> AsRef for StdMapObserver<'a, T, DIFFERENTIAL> +where + T: Default + Copy + 'static + Serialize, +{ + fn as_ref(&self) -> &Self { + self + } +} + +impl<'a, T, const DIFFERENTIAL: bool> AsMut for StdMapObserver<'a, T, DIFFERENTIAL> +where + T: Default + Copy + 'static + Serialize, +{ + fn as_mut(&mut self) -> &mut Self { + self + } +} + +impl<'a, T, const DIFFERENTIAL: bool> MapObserver for StdMapObserver<'a, T, DIFFERENTIAL> +where + T: Bounded + + PartialEq + + Default + + Copy + + Hash + + 'static + + Serialize + + serde::de::DeserializeOwned + + Debug, +{ + type Entry = T; + + #[inline] + fn get(&self, pos: usize) -> T { + self.as_slice()[pos] + } + + fn set(&mut self, pos: usize, val: T) { + self.map.as_slice_mut()[pos] = val; + } + + /// Count the set bytes in the map + fn count_bytes(&self) -> u64 { + let initial = self.initial(); + let cnt = self.usable_count(); + let map = self.as_slice(); + let mut res = 0; + for x in &map[0..cnt] { + if *x != initial { + res += 1; + } + } + res + } + + #[inline] + fn usable_count(&self) -> usize { + self.as_slice().len() + } + + #[inline] + fn hash_simple(&self) -> u64 { + RandomState::with_seeds(0, 0, 0, 0).hash_one(self) + } + + #[inline] + fn initial(&self) -> T { + self.initial + } + + fn to_vec(&self) -> Vec { + self.as_slice().to_vec() + } + + /// Reset the map + #[inline] + fn reset_map(&mut self) -> Result<(), Error> { + // Normal memset, see https://rust.godbolt.org/z/Trs5hv + let initial = self.initial(); + let cnt = self.usable_count(); + let map = self.as_slice_mut(); + for x in &mut map[0..cnt] { + *x = initial; + } + Ok(()) + } + + fn how_many_set(&self, indexes: &[usize]) -> usize { + let initial = self.initial(); + let cnt = self.usable_count(); + let map = self.as_slice(); + let mut res = 0; + for i in indexes { + if *i < cnt && map[*i] != initial { + res += 1; + } + } + res + } +} + +impl<'a, T, const DIFFERENTIAL: bool> Truncate for StdMapObserver<'a, T, DIFFERENTIAL> +where + T: Bounded + + PartialEq + + Default + + Copy + + 'static + + Serialize + + serde::de::DeserializeOwned + + Debug, +{ + fn truncate(&mut self, new_len: usize) { + self.map.truncate(new_len); + } +} + +impl<'a, T, const DIFFERENTIAL: bool> Deref for StdMapObserver<'a, T, DIFFERENTIAL> +where + T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned + Debug, +{ + type Target = [T]; + fn deref(&self) -> &[T] { + &self.map + } +} + +impl<'a, T, const DIFFERENTIAL: bool> DerefMut for StdMapObserver<'a, T, DIFFERENTIAL> +where + T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned + Debug, +{ + fn deref_mut(&mut self) -> &mut [T] { + &mut self.map + } +} + +impl<'a, T, const DIFFERENTIAL: bool> StdMapObserver<'a, T, DIFFERENTIAL> +where + T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned, +{ + /// Creates a new [`MapObserver`] + /// + /// # Safety + /// Will get a pointer to the map and dereference it at any point in time. + /// The map must not move in memory! + #[must_use] + unsafe fn maybe_differential(name: S, map: &'a mut [T]) -> Self + where + S: Into>, + { + let len = map.len(); + let ptr = map.as_mut_ptr(); + Self::maybe_differential_from_mut_ptr(name, ptr, len) + } + + /// Creates a new [`MapObserver`] from an [`OwnedMutSlice`] + #[must_use] + fn maybe_differential_from_mut_slice(name: S, map: OwnedMutSlice<'a, T>) -> Self + where + S: Into>, + { + StdMapObserver { + name: name.into(), + map, + initial: T::default(), + } + } + + /// Creates a new [`MapObserver`] with an owned map + #[must_use] + fn maybe_differential_owned(name: S, map: Vec) -> Self + where + S: Into>, + { + Self { + map: OwnedMutSlice::from(map), + name: name.into(), + initial: T::default(), + } + } + + /// Creates a new [`MapObserver`] from an [`OwnedMutSlice`] map. + /// + /// # Safety + /// Will dereference the owned slice with up to len elements. + #[must_use] + fn maybe_differential_from_ownedref(name: S, map: OwnedMutSlice<'a, T>) -> Self + where + S: Into>, + { + Self { + map, + name: name.into(), + initial: T::default(), + } + } + + /// Creates a new [`MapObserver`] from a raw pointer + /// + /// # Safety + /// Will dereference the `map_ptr` with up to len elements. + unsafe fn maybe_differential_from_mut_ptr(name: S, map_ptr: *mut T, len: usize) -> Self + where + S: Into>, + { + Self::maybe_differential_from_mut_slice( + name, + OwnedMutSlice::from_raw_parts_mut(map_ptr, len), + ) + } + + /// Gets the initial value for this map, mutably + pub fn initial_mut(&mut self) -> &mut T { + &mut self.initial + } + + /// Gets the backing for this map + pub fn map(&self) -> &OwnedMutSlice<'a, T> { + &self.map + } + + /// Gets the backing for this map mutably + pub fn map_mut(&mut self) -> &mut OwnedMutSlice<'a, T> { + &mut self.map + } +} + +impl<'a, T> StdMapObserver<'a, T, false> +where + T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned, +{ + /// Creates a new [`MapObserver`] + /// + /// # Safety + /// The observer will keep a pointer to the map. + /// Hence, the map may never move in memory. + #[must_use] + pub unsafe fn new(name: S, map: &'a mut [T]) -> Self + where + S: Into>, + { + Self::maybe_differential(name, map) + } + + /// Creates a new [`MapObserver`] from an [`OwnedMutSlice`] + pub fn from_mut_slice(name: S, map: OwnedMutSlice<'a, T>) -> Self + where + S: Into>, + { + Self::maybe_differential_from_mut_slice(name, map) + } + + /// Creates a new [`MapObserver`] with an owned map + #[must_use] + pub fn owned(name: S, map: Vec) -> Self + where + S: Into>, + { + Self::maybe_differential_owned(name, map) + } + + /// Creates a new [`MapObserver`] from an [`OwnedMutSlice`] map. + /// + /// # Note + /// Will dereference the owned slice with up to len elements. + #[must_use] + pub fn from_ownedref(name: S, map: OwnedMutSlice<'a, T>) -> Self + where + S: Into>, + { + Self::maybe_differential_from_ownedref(name, map) + } + + /// Creates a new [`MapObserver`] from a raw pointer + /// + /// # Safety + /// Will dereference the `map_ptr` with up to len elements. + pub unsafe fn from_mut_ptr(name: S, map_ptr: *mut T, len: usize) -> Self + where + S: Into>, + { + Self::maybe_differential_from_mut_ptr(name, map_ptr, len) + } +} + +impl<'a, T> StdMapObserver<'a, T, true> +where + T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned, +{ + /// Creates a new [`MapObserver`] in differential mode + /// + /// # Safety + /// Will get a pointer to the map and dereference it at any point in time. + /// The map must not move in memory! + #[must_use] + pub unsafe fn differential(name: S, map: &'a mut [T]) -> Self + where + S: Into>, + { + Self::maybe_differential(name, map) + } + + /// Creates a new [`MapObserver`] with an owned map in differential mode + #[must_use] + pub fn differential_owned(name: S, map: Vec) -> Self + where + S: Into>, + { + Self::maybe_differential_owned(name, map) + } + + /// Creates a new [`MapObserver`] from an [`OwnedMutSlice`] map in differential mode. + /// + /// # Note + /// Will dereference the owned slice with up to len elements. + #[must_use] + pub fn differential_from_ownedref(name: S, map: OwnedMutSlice<'a, T>) -> Self + where + S: Into>, + { + Self::maybe_differential_from_ownedref(name, map) + } + + /// Creates a new [`MapObserver`] from a raw pointer in differential mode + /// + /// # Safety + /// Will dereference the `map_ptr` with up to len elements. + pub unsafe fn differential_from_mut_ptr(name: S, map_ptr: *mut T, len: usize) -> Self + where + S: Into>, + { + Self::maybe_differential_from_mut_ptr(name, map_ptr, len) + } +} + +impl<'a, OTA, OTB, S, T> DifferentialObserver for StdMapObserver<'a, T, true> +where + OTA: ObserversTuple, + OTB: ObserversTuple, + S: UsesInput, + T: Bounded + + PartialEq + + Default + + Copy + + 'static + + Serialize + + serde::de::DeserializeOwned + + Debug, +{ +} diff --git a/libafl/src/observers/map/multi_map.rs b/libafl/src/observers/map/multi_map.rs new file mode 100644 index 0000000000..0a835e4969 --- /dev/null +++ b/libafl/src/observers/map/multi_map.rs @@ -0,0 +1,356 @@ +//! An observer that takes multiple pointers or slices to observe + +use alloc::{borrow::Cow, vec::Vec}; +use core::{ + fmt::Debug, + hash::{Hash, Hasher}, + iter::Flatten, + mem::size_of, + slice::{self, Iter, IterMut}, +}; + +use ahash::RandomState; +use libafl_bolts::{ + ownedref::OwnedMutSlice, AsIter, AsIterMut, AsSlice, AsSliceMut, HasLen, Named, +}; +use meminterval::IntervalTree; +use num_traits::Bounded; +use serde::{Deserialize, Serialize}; + +use crate::{ + inputs::UsesInput, + observers::{map::MapObserver, DifferentialObserver, Observer, ObserversTuple}, + Error, +}; + +/// The Multi Map Observer merge different maps into one observer +#[derive(Serialize, Deserialize, Debug)] +#[serde(bound = "T: serde::de::DeserializeOwned")] +#[allow(clippy::unsafe_derive_deserialize)] +pub struct MultiMapObserver<'a, T, const DIFFERENTIAL: bool> +where + T: 'static + Default + Copy + Serialize + Debug, +{ + maps: Vec>, + intervals: IntervalTree, + len: usize, + initial: T, + name: Cow<'static, str>, + iter_idx: usize, +} + +impl<'a, S, T> Observer for MultiMapObserver<'a, T, false> +where + S: UsesInput, + T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, + Self: MapObserver, +{ + #[inline] + fn pre_exec(&mut self, _state: &mut S, _input: &S::Input) -> Result<(), Error> { + self.reset_map() + } +} + +impl<'a, S, T> Observer for MultiMapObserver<'a, T, true> +where + S: UsesInput, + T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, + Self: MapObserver, +{ + // in differential mode, we are *not* responsible for resetting the map! +} + +impl<'a, T, const DIFFERENTIAL: bool> Named for MultiMapObserver<'a, T, DIFFERENTIAL> +where + T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, +{ + #[inline] + fn name(&self) -> &Cow<'static, str> { + &self.name + } +} + +impl<'a, T, const DIFFERENTIAL: bool> HasLen for MultiMapObserver<'a, T, DIFFERENTIAL> +where + T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, +{ + #[inline] + fn len(&self) -> usize { + self.len + } +} + +impl<'a, T, const DIFFERENTIAL: bool> Hash for MultiMapObserver<'a, T, DIFFERENTIAL> +where + T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, +{ + fn hash(&self, hasher: &mut H) { + for map in &self.maps { + let slice = map.as_slice(); + let ptr = slice.as_ptr() as *const u8; + let map_size = slice.len() / size_of::(); + unsafe { + hasher.write(slice::from_raw_parts(ptr, map_size)); + } + } + } +} + +impl<'a, T, const DIFFERENTIAL: bool> AsRef for MultiMapObserver<'a, T, DIFFERENTIAL> +where + T: 'static + Default + Copy + Serialize + Debug, +{ + fn as_ref(&self) -> &Self { + self + } +} + +impl<'a, T, const DIFFERENTIAL: bool> AsMut for MultiMapObserver<'a, T, DIFFERENTIAL> +where + T: 'static + Default + Copy + Serialize + Debug, +{ + fn as_mut(&mut self) -> &mut Self { + self + } +} + +impl<'a, T, const DIFFERENTIAL: bool> MapObserver for MultiMapObserver<'a, T, DIFFERENTIAL> +where + T: 'static + + Bounded + + PartialEq + + Default + + Copy + + Hash + + Serialize + + serde::de::DeserializeOwned + + Debug, +{ + type Entry = T; + + #[inline] + fn get(&self, idx: usize) -> T { + let elem = self.intervals.query(idx..=idx).next().unwrap(); + let i = *elem.value; + let j = idx - elem.interval.start; + self.maps[i].as_slice()[j] + } + + #[inline] + fn set(&mut self, idx: usize, val: Self::Entry) { + let elem = self.intervals.query(idx..=idx).next().unwrap(); + let i = *elem.value; + let j = idx - elem.interval.start; + self.maps[i].as_slice_mut()[j] = val; + } + + #[inline] + fn initial(&self) -> T { + self.initial + } + + fn count_bytes(&self) -> u64 { + let initial = self.initial(); + let mut res = 0; + for map in &self.maps { + for x in map.as_slice() { + if *x != initial { + res += 1; + } + } + } + res + } + + #[inline] + fn hash_simple(&self) -> u64 { + RandomState::with_seeds(0, 0, 0, 0).hash_one(self) + } + + fn reset_map(&mut self) -> Result<(), Error> { + let initial = self.initial(); + for map in &mut self.maps { + for x in map.as_slice_mut() { + *x = initial; + } + } + Ok(()) + } + + fn usable_count(&self) -> usize { + self.len() + } + + fn to_vec(&self) -> Vec { + let cnt = self.usable_count(); + let mut res = Vec::with_capacity(cnt); + for i in 0..cnt { + res.push(self.get(i)); + } + res + } + + /// Get the number of set entries with the specified indexes + fn how_many_set(&self, indexes: &[usize]) -> usize { + let initial = self.initial(); + let cnt = self.usable_count(); + let mut res = 0; + for i in indexes { + if *i < cnt && self.get(*i) != initial { + res += 1; + } + } + res + } +} + +impl<'a, T, const DIFFERENTIAL: bool> MultiMapObserver<'a, T, DIFFERENTIAL> +where + T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, +{ + /// Creates a new [`MultiMapObserver`], maybe in differential mode + #[must_use] + fn maybe_differential(name: &'static str, maps: Vec>) -> Self { + let mut idx = 0; + let mut intervals = IntervalTree::new(); + for (v, x) in maps.iter().enumerate() { + let l = x.as_slice().len(); + intervals.insert(idx..(idx + l), v); + idx += l; + } + Self { + maps, + intervals, + len: idx, + name: Cow::from(name), + initial: T::default(), + iter_idx: 0, + } + } +} + +impl<'a, T> MultiMapObserver<'a, T, true> +where + T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, +{ + /// Creates a new [`MultiMapObserver`] in differential mode + #[must_use] + pub fn differential(name: &'static str, maps: Vec>) -> Self { + Self::maybe_differential(name, maps) + } +} + +impl<'a, T> MultiMapObserver<'a, T, false> +where + T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, +{ + /// Creates a new [`MultiMapObserver`] + #[must_use] + pub fn new(name: &'static str, maps: Vec>) -> Self { + Self::maybe_differential(name, maps) + } + + /// Creates a new [`MultiMapObserver`] with an owned map + #[must_use] + pub fn owned(name: &'static str, maps: Vec>) -> Self { + let mut idx = 0; + let mut v = 0; + let mut intervals = IntervalTree::new(); + let maps: Vec<_> = maps + .into_iter() + .map(|x| { + let l = x.len(); + intervals.insert(idx..(idx + l), v); + idx += l; + v += 1; + OwnedMutSlice::from(x) + }) + .collect(); + Self { + maps, + intervals, + len: idx, + name: Cow::from(name), + initial: T::default(), + iter_idx: 0, + } + } +} + +impl<'a, 'it, T, const DIFFERENTIAL: bool> AsIter<'it> for MultiMapObserver<'a, T, DIFFERENTIAL> +where + T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, + 'a: 'it, +{ + type Item = T; + type Ref = &'it T; + type IntoIter = Flatten>>; + + fn as_iter(&'it self) -> Self::IntoIter { + self.maps.iter().flatten() + } +} + +impl<'a, 'it, T, const DIFFERENTIAL: bool> AsIterMut<'it> for MultiMapObserver<'a, T, DIFFERENTIAL> +where + T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, + 'a: 'it, +{ + type RefMut = &'it mut T; + type IntoIterMut = Flatten>>; + + fn as_iter_mut(&'it mut self) -> Self::IntoIterMut { + self.maps.iter_mut().flatten() + } +} + +impl<'a, 'it, T, const DIFFERENTIAL: bool> IntoIterator + for &'it MultiMapObserver<'a, T, DIFFERENTIAL> +where + T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, +{ + type Item = as Iterator>::Item; + type IntoIter = Flatten>>; + + fn into_iter(self) -> Self::IntoIter { + self.maps.iter().flatten() + } +} + +impl<'a, 'it, T, const DIFFERENTIAL: bool> IntoIterator + for &'it mut MultiMapObserver<'a, T, DIFFERENTIAL> +where + T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, +{ + type Item = as Iterator>::Item; + type IntoIter = Flatten>>; + + fn into_iter(self) -> Self::IntoIter { + self.maps.iter_mut().flatten() + } +} + +impl<'a, T, const DIFFERENTIAL: bool> MultiMapObserver<'a, T, DIFFERENTIAL> +where + T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, +{ + /// Returns an iterator over the map. + pub fn iter(&self) -> <&Self as IntoIterator>::IntoIter { + <&Self as IntoIterator>::into_iter(self) + } + + /// Returns a mutable iterator over the map. + pub fn iter_mut(&mut self) -> <&mut Self as IntoIterator>::IntoIter { + <&mut Self as IntoIterator>::into_iter(self) + } +} + +impl<'a, T, OTA, OTB, S> DifferentialObserver for MultiMapObserver<'a, T, true> +where + T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, + Self: MapObserver, + OTA: ObserversTuple, + OTB: ObserversTuple, + S: UsesInput, +{ +} diff --git a/libafl/src/observers/map/owned_map.rs b/libafl/src/observers/map/owned_map.rs new file mode 100644 index 0000000000..518eb9d2da --- /dev/null +++ b/libafl/src/observers/map/owned_map.rs @@ -0,0 +1,251 @@ +//! An observer that owns its map + +use alloc::{borrow::Cow, vec::Vec}; +use core::{ + fmt::Debug, + hash::{Hash, Hasher}, + ops::{Deref, DerefMut}, + slice::{Iter, IterMut}, +}; + +use ahash::RandomState; +use libafl_bolts::{AsSlice, AsSliceMut, HasLen, Named}; +use num_traits::Bounded; +use serde::{Deserialize, Serialize}; + +use crate::{ + inputs::UsesInput, + observers::{map::MapObserver, Observer}, + Error, +}; + +/// Exact copy of `StdMapObserver` that owns its map +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(bound = "T: serde::de::DeserializeOwned")] +#[allow(clippy::unsafe_derive_deserialize)] +pub struct OwnedMapObserver +where + T: 'static + Default + Copy + Serialize, +{ + map: Vec, + initial: T, + name: Cow<'static, str>, +} + +impl Observer for OwnedMapObserver +where + S: UsesInput, + T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, + Self: MapObserver, +{ + #[inline] + fn pre_exec(&mut self, _state: &mut S, _input: &S::Input) -> Result<(), Error> { + self.reset_map() + } +} + +impl Named for OwnedMapObserver +where + T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned, +{ + #[inline] + fn name(&self) -> &Cow<'static, str> { + &self.name + } +} + +impl HasLen for OwnedMapObserver +where + T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned, +{ + #[inline] + fn len(&self) -> usize { + self.map.as_slice().len() + } +} + +impl<'it, T> IntoIterator for &'it OwnedMapObserver +where + T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, +{ + type Item = as Iterator>::Item; + type IntoIter = Iter<'it, T>; + + fn into_iter(self) -> Self::IntoIter { + self.as_slice().iter() + } +} + +impl<'it, T> IntoIterator for &'it mut OwnedMapObserver +where + T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, +{ + type Item = as Iterator>::Item; + type IntoIter = IterMut<'it, T>; + + fn into_iter(self) -> Self::IntoIter { + self.as_slice_mut().iter_mut() + } +} + +impl OwnedMapObserver +where + T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, +{ + /// Returns an iterator over the map. + pub fn iter(&self) -> Iter<'_, T> { + <&Self as IntoIterator>::into_iter(self) + } + + /// Returns a mutable iterator over the map. + pub fn iter_mut(&mut self) -> IterMut<'_, T> { + <&mut Self as IntoIterator>::into_iter(self) + } +} + +impl Hash for OwnedMapObserver +where + T: 'static + Hash + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, +{ + #[inline] + fn hash(&self, hasher: &mut H) { + self.as_slice().hash(hasher); + } +} + +impl AsRef for OwnedMapObserver +where + T: 'static + Default + Copy + Serialize, +{ + fn as_ref(&self) -> &Self { + self + } +} + +impl AsMut for OwnedMapObserver +where + T: 'static + Default + Copy + Serialize, +{ + fn as_mut(&mut self) -> &mut Self { + self + } +} + +impl MapObserver for OwnedMapObserver +where + T: 'static + + Bounded + + PartialEq + + Default + + Copy + + Hash + + Serialize + + serde::de::DeserializeOwned + + Debug, +{ + type Entry = T; + + #[inline] + fn get(&self, pos: usize) -> T { + self.as_slice()[pos] + } + + #[inline] + fn set(&mut self, pos: usize, val: Self::Entry) { + self.as_slice_mut()[pos] = val; + } + + /// Count the set bytes in the map + fn count_bytes(&self) -> u64 { + let initial = self.initial(); + let cnt = self.usable_count(); + let map = self.as_slice(); + let mut res = 0; + for x in &map[0..cnt] { + if *x != initial { + res += 1; + } + } + res + } + + #[inline] + fn usable_count(&self) -> usize { + self.as_slice().len() + } + + #[inline] + fn hash_simple(&self) -> u64 { + RandomState::with_seeds(0, 0, 0, 0).hash_one(self) + } + + #[inline] + fn initial(&self) -> T { + self.initial + } + + /// Reset the map + #[inline] + fn reset_map(&mut self) -> Result<(), Error> { + // Normal memset, see https://rust.godbolt.org/z/Trs5hv + let initial = self.initial(); + let cnt = self.usable_count(); + let map = self.as_slice_mut(); + for x in &mut map[0..cnt] { + *x = initial; + } + Ok(()) + } + fn to_vec(&self) -> Vec { + self.as_slice().to_vec() + } + + fn how_many_set(&self, indexes: &[usize]) -> usize { + let initial = self.initial(); + let cnt = self.usable_count(); + let map = self.as_slice(); + let mut res = 0; + for i in indexes { + if *i < cnt && map[*i] != initial { + res += 1; + } + } + res + } +} + +impl Deref for OwnedMapObserver +where + T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, +{ + type Target = [T]; + + fn deref(&self) -> &[T] { + &self.map + } +} + +impl DerefMut for OwnedMapObserver +where + T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + Debug, +{ + fn deref_mut(&mut self) -> &mut [T] { + &mut self.map + } +} + +impl OwnedMapObserver +where + T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned, +{ + /// Creates a new [`MapObserver`] with an owned map + #[must_use] + pub fn new(name: &'static str, map: Vec) -> Self { + let initial = if map.is_empty() { T::default() } else { map[0] }; + Self { + map, + name: Cow::from(name), + initial, + } + } +} diff --git a/libafl/src/observers/map/variable_map.rs b/libafl/src/observers/map/variable_map.rs new file mode 100644 index 0000000000..381b837d0f --- /dev/null +++ b/libafl/src/observers/map/variable_map.rs @@ -0,0 +1,349 @@ +//! Map observer with a shrinkable size + +use alloc::{borrow::Cow, vec::Vec}; +use core::{ + fmt::Debug, + hash::{Hash, Hasher}, + ops::{Deref, DerefMut}, + slice::{Iter, IterMut}, +}; + +use ahash::RandomState; +use libafl_bolts::{ + ownedref::{OwnedMutPtr, OwnedMutSlice}, + AsSlice, AsSliceMut, HasLen, Named, +}; +use num_traits::Bounded; +use serde::{Deserialize, Serialize}; + +use crate::{ + inputs::UsesInput, + observers::{map::MapObserver, Observer}, + Error, +}; + +/// Overlooking a variable bitmap +#[derive(Serialize, Deserialize, Debug)] +#[serde(bound = "T: serde::de::DeserializeOwned")] +#[allow(clippy::unsafe_derive_deserialize)] +pub struct VariableMapObserver<'a, T> +where + T: Default + Copy + 'static + Serialize + PartialEq + Bounded, +{ + map: OwnedMutSlice<'a, T>, + size: OwnedMutPtr, + initial: T, + name: Cow<'static, str>, +} + +impl<'a, S, T> Observer for VariableMapObserver<'a, T> +where + S: UsesInput, + T: Default + + Copy + + 'static + + Serialize + + serde::de::DeserializeOwned + + Debug + + Bounded + + PartialEq, + Self: MapObserver, +{ + #[inline] + fn pre_exec(&mut self, _state: &mut S, _input: &S::Input) -> Result<(), Error> { + self.reset_map() + } +} + +impl<'a, T> Named for VariableMapObserver<'a, T> +where + T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned + Bounded + PartialEq, +{ + #[inline] + fn name(&self) -> &Cow<'static, str> { + &self.name + } +} + +impl<'a, T> HasLen for VariableMapObserver<'a, T> +where + T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned + PartialEq + Bounded, +{ + #[inline] + fn len(&self) -> usize { + *self.size.as_ref() + } +} + +impl<'a, 'it, T> IntoIterator for &'it VariableMapObserver<'a, T> +where + T: Bounded + + PartialEq + + Default + + Copy + + Hash + + 'static + + Serialize + + serde::de::DeserializeOwned + + Debug + + PartialEq + + Bounded, +{ + type Item = as Iterator>::Item; + type IntoIter = Iter<'it, T>; + + fn into_iter(self) -> Self::IntoIter { + let cnt = self.usable_count(); + self.as_slice()[..cnt].iter() + } +} + +impl<'a, 'it, T> IntoIterator for &'it mut VariableMapObserver<'a, T> +where + T: Bounded + + PartialEq + + Default + + Copy + + Hash + + 'static + + Serialize + + serde::de::DeserializeOwned + + Debug + + PartialEq + + Bounded, +{ + type Item = as Iterator>::Item; + type IntoIter = IterMut<'it, T>; + + fn into_iter(self) -> Self::IntoIter { + let cnt = self.usable_count(); + self.as_slice_mut()[..cnt].iter_mut() + } +} + +impl<'a, T> VariableMapObserver<'a, T> +where + T: Bounded + + PartialEq + + Default + + Copy + + Hash + + 'static + + Serialize + + serde::de::DeserializeOwned + + Debug + + PartialEq + + Bounded, +{ + /// Returns an iterator over the map. + pub fn iter(&self) -> Iter<'_, T> { + <&Self as IntoIterator>::into_iter(self) + } + + /// Returns a mutable iterator over the map. + pub fn iter_mut(&mut self) -> IterMut<'_, T> { + <&mut Self as IntoIterator>::into_iter(self) + } +} + +impl<'a, T> Hash for VariableMapObserver<'a, T> +where + T: Bounded + + PartialEq + + Default + + Copy + + Hash + + 'static + + Serialize + + serde::de::DeserializeOwned + + Debug + + PartialEq + + Bounded, +{ + #[inline] + fn hash(&self, hasher: &mut H) { + self.as_slice().hash(hasher); + } +} + +impl<'a, T> AsRef for VariableMapObserver<'a, T> +where + T: Default + Copy + 'static + Serialize + PartialEq + Bounded, +{ + fn as_ref(&self) -> &Self { + self + } +} + +impl<'a, T> AsMut for VariableMapObserver<'a, T> +where + T: Default + Copy + 'static + Serialize + PartialEq + Bounded, +{ + fn as_mut(&mut self) -> &mut Self { + self + } +} + +impl<'a, T> MapObserver for VariableMapObserver<'a, T> +where + T: Bounded + + PartialEq + + Default + + Copy + + Hash + + 'static + + Serialize + + serde::de::DeserializeOwned + + Debug + + PartialEq + + Bounded, +{ + type Entry = T; + + #[inline] + fn initial(&self) -> T { + self.initial + } + + #[inline] + fn usable_count(&self) -> usize { + *self.size.as_ref() + } + + fn get(&self, idx: usize) -> T { + self.map.as_slice()[idx] + } + + fn set(&mut self, idx: usize, val: T) { + self.map.as_slice_mut()[idx] = val; + } + + /// Count the set bytes in the map + fn count_bytes(&self) -> u64 { + let initial = self.initial(); + let cnt = self.usable_count(); + let map = self.as_slice(); + let mut res = 0; + for x in &map[0..cnt] { + if *x != initial { + res += 1; + } + } + res + } + + #[inline] + fn hash_simple(&self) -> u64 { + RandomState::with_seeds(0, 0, 0, 0).hash_one(self) + } + + /// Reset the map + #[inline] + fn reset_map(&mut self) -> Result<(), Error> { + // Normal memset, see https://rust.godbolt.org/z/Trs5hv + let initial = self.initial(); + let cnt = self.usable_count(); + let map = self.as_slice_mut(); + for x in &mut map[0..cnt] { + *x = initial; + } + Ok(()) + } + + fn to_vec(&self) -> Vec { + self.as_slice().to_vec() + } + + fn how_many_set(&self, indexes: &[usize]) -> usize { + let initial = self.initial(); + let cnt = self.usable_count(); + let map = self.as_slice(); + let mut res = 0; + for i in indexes { + if *i < cnt && map[*i] != initial { + res += 1; + } + } + res + } +} + +impl<'a, T> Deref for VariableMapObserver<'a, T> +where + T: Bounded + + PartialEq + + Default + + Copy + + Hash + + 'static + + Serialize + + serde::de::DeserializeOwned + + Debug + + PartialEq + + Bounded, +{ + type Target = [T]; + fn deref(&self) -> &[T] { + let cnt = self.usable_count(); + &self.map[..cnt] + } +} + +impl<'a, T> DerefMut for VariableMapObserver<'a, T> +where + T: 'static + + Default + + Copy + + Hash + + Serialize + + serde::de::DeserializeOwned + + Debug + + PartialEq + + Bounded, +{ + fn deref_mut(&mut self) -> &mut [T] { + let cnt = self.usable_count(); + &mut self.map[..cnt] + } +} + +impl<'a, T> VariableMapObserver<'a, T> +where + T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + PartialEq + Bounded, +{ + /// Creates a new [`MapObserver`] from an [`OwnedMutSlice`] + /// + /// # Safety + /// The observer will dereference the owned slice, as well as the `map_ptr`. + /// Dereferences `map_ptr` with up to `max_len` elements of size. + pub unsafe fn from_mut_slice( + name: &'static str, + map_slice: OwnedMutSlice<'a, T>, + size: *mut usize, + ) -> Self { + VariableMapObserver { + name: name.into(), + map: map_slice, + size: OwnedMutPtr::Ptr(size), + initial: T::default(), + } + } + + /// Creates a new [`MapObserver`] from a raw pointer + /// + /// # Safety + /// The observer will dereference the `size` ptr, as well as the `map_ptr`. + /// Dereferences `map_ptr` with up to `max_len` elements of size. + pub unsafe fn from_mut_ptr( + name: &'static str, + map_ptr: *mut T, + max_len: usize, + size: *mut usize, + ) -> Self { + Self::from_mut_slice( + name, + OwnedMutSlice::from_raw_parts_mut(map_ptr, max_len), + size, + ) + } +} diff --git a/libafl/src/observers/mod.rs b/libafl/src/observers/mod.rs index ec4a0e3974..093f45f422 100644 --- a/libafl/src/observers/mod.rs +++ b/libafl/src/observers/mod.rs @@ -1,7 +1,5 @@ //! Observers give insights about runs of a target, such as coverage, timing, stack depth, and more. - -pub mod map; -pub use map::*; +use alloc::borrow::Cow; pub mod cmp; pub use cmp::*; @@ -17,34 +15,26 @@ pub mod stacktrace; pub use stacktrace::*; pub mod concolic; +pub mod map; +pub use map::*; pub mod value; -use alloc::{ - string::{String, ToString}, - vec::Vec, -}; +/// List observer +pub mod list; use core::{fmt::Debug, time::Duration}; #[cfg(feature = "std")] use std::time::Instant; #[cfg(feature = "no_std")] use libafl_bolts::current_time; -use libafl_bolts::{ownedref::OwnedMutPtr, tuples::MatchName, Named}; +use libafl_bolts::{tuples::MatchName, Named}; +pub use list::*; use serde::{Deserialize, Serialize}; pub use value::*; use crate::{executors::ExitKind, inputs::UsesInput, state::UsesState, Error}; -/// Something that uses observer like mapfeedbacks -pub trait UsesObserver -where - S: UsesInput, -{ - /// The observer type used - type Observer: Observer; -} - /// Observers observe different information about the target. /// They can then be used by various sorts of feedback. pub trait Observer: Named @@ -410,7 +400,7 @@ where /// A simple observer, just overlooking the runtime of the target. #[derive(Serialize, Deserialize, Debug, Clone)] pub struct TimeObserver { - name: String, + name: Cow<'static, str>, #[cfg(feature = "std")] #[serde(with = "instant_serializer")] @@ -453,7 +443,7 @@ impl TimeObserver { #[must_use] pub fn new(name: &'static str) -> Self { Self { - name: name.to_string(), + name: Cow::from(name), #[cfg(feature = "std")] start_time: Instant::now(), @@ -513,7 +503,7 @@ where } impl Named for TimeObserver { - fn name(&self) -> &str { + fn name(&self) -> &Cow<'static, str> { &self.name } } @@ -526,860 +516,6 @@ where { } -/// A simple observer with a list of things. -#[derive(Serialize, Deserialize, Debug)] -#[serde(bound = "T: serde::de::DeserializeOwned")] -#[allow(clippy::unsafe_derive_deserialize)] -pub struct ListObserver -where - T: Debug + Serialize, -{ - name: String, - /// The list - list: OwnedMutPtr>, -} - -impl ListObserver -where - T: Debug + Serialize + serde::de::DeserializeOwned, -{ - /// Creates a new [`ListObserver`] with the given name. - /// - /// # Safety - /// Will dereference the list. - /// The list may not move in memory. - #[must_use] - pub unsafe fn new(name: &'static str, list: *mut Vec) -> Self { - Self { - name: name.to_string(), - list: OwnedMutPtr::Ptr(list), - } - } - - /// Get a list ref - #[must_use] - pub fn list(&self) -> &Vec { - self.list.as_ref() - } - - /// Get a list mut - #[must_use] - pub fn list_mut(&mut self) -> &mut Vec { - self.list.as_mut() - } -} - -impl Observer for ListObserver -where - S: UsesInput, - T: Debug + Serialize + serde::de::DeserializeOwned, -{ - fn pre_exec(&mut self, _state: &mut S, _input: &S::Input) -> Result<(), Error> { - self.list.as_mut().clear(); - Ok(()) - } -} - -impl Named for ListObserver -where - T: Debug + Serialize + serde::de::DeserializeOwned, -{ - fn name(&self) -> &str { - &self.name - } -} - -/// `Observer` Python bindings -#[cfg(feature = "python")] -#[allow(missing_docs)] -pub mod pybind { - use std::cell::UnsafeCell; - - use libafl_bolts::{ - tuples::{type_eq, MatchName}, - Named, - }; - use pyo3::prelude::*; - use serde::{Deserialize, Serialize}; - - use super::{Debug, Observer, ObserversTuple, String, Vec}; - use crate::{ - executors::{pybind::PythonExitKind, ExitKind}, - inputs::{BytesInput, HasBytesVec}, - observers::map::pybind::{ - PythonMapObserverI16, PythonMapObserverI32, PythonMapObserverI64, PythonMapObserverI8, - PythonMapObserverU16, PythonMapObserverU32, PythonMapObserverU64, PythonMapObserverU8, - PythonMapObserverWrapperI16, PythonMapObserverWrapperI32, PythonMapObserverWrapperI64, - PythonMapObserverWrapperI8, PythonMapObserverWrapperU16, PythonMapObserverWrapperU32, - PythonMapObserverWrapperU64, PythonMapObserverWrapperU8, - }, - state::pybind::{PythonStdState, PythonStdStateWrapper}, - Error, - }; - - #[derive(Debug)] - pub struct PyObjectObserver { - inner: PyObject, - name: UnsafeCell, - } - - impl Clone for PyObjectObserver { - fn clone(&self) -> PyObjectObserver { - PyObjectObserver { - inner: self.inner.clone(), - name: UnsafeCell::new(String::new()), - } - } - } - - impl PyObjectObserver { - #[must_use] - pub fn new(obj: PyObject) -> Self { - PyObjectObserver { - inner: obj, - name: UnsafeCell::new(String::new()), - } - } - } - - libafl_bolts::impl_serde_pyobjectwrapper!(PyObjectObserver, inner); - - impl Named for PyObjectObserver { - fn name(&self) -> &str { - let s = Python::with_gil(|py| -> PyResult { - let s: String = self.inner.call_method0(py, "name")?.extract(py)?; - Ok(s) - }) - .unwrap(); - unsafe { - *self.name.get() = s; - &*self.name.get() - } - } - } - - impl Observer for PyObjectObserver { - fn flush(&mut self) -> Result<(), Error> { - Python::with_gil(|py| -> PyResult<()> { - self.inner.call_method0(py, "flush")?; - Ok(()) - }) - .unwrap(); - Ok(()) - } - - fn pre_exec( - &mut self, - state: &mut PythonStdState, - input: &BytesInput, - ) -> Result<(), Error> { - Python::with_gil(|py| -> PyResult<()> { - self.inner.call_method1( - py, - "pre_exec", - (PythonStdStateWrapper::wrap(state), input.bytes()), - )?; - Ok(()) - })?; - Ok(()) - } - - fn post_exec( - &mut self, - state: &mut PythonStdState, - input: &BytesInput, - exit_kind: &ExitKind, - ) -> Result<(), Error> { - Python::with_gil(|py| -> PyResult<()> { - self.inner.call_method1( - py, - "post_exec", - ( - PythonStdStateWrapper::wrap(state), - input.bytes(), - PythonExitKind::from(*exit_kind), - ), - )?; - Ok(()) - })?; - Ok(()) - } - - fn pre_exec_child( - &mut self, - state: &mut PythonStdState, - input: &BytesInput, - ) -> Result<(), Error> { - Python::with_gil(|py| -> PyResult<()> { - self.inner.call_method1( - py, - "pre_exec_child", - (PythonStdStateWrapper::wrap(state), input.bytes()), - )?; - Ok(()) - })?; - Ok(()) - } - - fn post_exec_child( - &mut self, - state: &mut PythonStdState, - input: &BytesInput, - exit_kind: &ExitKind, - ) -> Result<(), Error> { - Python::with_gil(|py| -> PyResult<()> { - self.inner.call_method1( - py, - "post_exec_child", - ( - PythonStdStateWrapper::wrap(state), - input.bytes(), - PythonExitKind::from(*exit_kind), - ), - )?; - Ok(()) - })?; - Ok(()) - } - } - - #[derive(Serialize, Deserialize, Clone, Debug)] - pub enum PythonObserverWrapper { - MapI8(Py), - MapI16(Py), - MapI32(Py), - MapI64(Py), - MapU8(Py), - MapU16(Py), - MapU32(Py), - MapU64(Py), - Python(PyObjectObserver), - } - - #[pyclass(unsendable, name = "Observer")] - #[allow(clippy::unsafe_derive_deserialize)] - #[derive(Serialize, Deserialize, Clone, Debug)] - /// Observer Trait binding - pub struct PythonObserver { - pub wrapper: PythonObserverWrapper, - } - - macro_rules! unwrap_me { - ($wrapper:expr, $name:ident, $body:block) => { - match &$wrapper { - PythonObserverWrapper::MapI8(py_wrapper) => Python::with_gil(|py| -> PyResult<_> { - let borrowed = py_wrapper.borrow(py); - Ok(crate::mapob_unwrap_me!( - PythonMapObserverWrapperI8, - borrowed.wrapper, - $name, - $body - )) - }) - .unwrap(), - PythonObserverWrapper::MapI16(py_wrapper) => { - Python::with_gil(|py| -> PyResult<_> { - let borrowed = py_wrapper.borrow(py); - Ok(crate::mapob_unwrap_me!( - PythonMapObserverWrapperI16, - borrowed.wrapper, - $name, - $body - )) - }) - .unwrap() - } - PythonObserverWrapper::MapI32(py_wrapper) => { - Python::with_gil(|py| -> PyResult<_> { - let borrowed = py_wrapper.borrow(py); - Ok(crate::mapob_unwrap_me!( - PythonMapObserverWrapperI32, - borrowed.wrapper, - $name, - $body - )) - }) - .unwrap() - } - PythonObserverWrapper::MapI64(py_wrapper) => { - Python::with_gil(|py| -> PyResult<_> { - let borrowed = py_wrapper.borrow(py); - Ok(crate::mapob_unwrap_me!( - PythonMapObserverWrapperI64, - borrowed.wrapper, - $name, - $body - )) - }) - .unwrap() - } - PythonObserverWrapper::MapU8(py_wrapper) => Python::with_gil(|py| -> PyResult<_> { - let borrowed = py_wrapper.borrow(py); - Ok(crate::mapob_unwrap_me!( - PythonMapObserverWrapperU8, - borrowed.wrapper, - $name, - $body - )) - }) - .unwrap(), - PythonObserverWrapper::MapU16(py_wrapper) => { - Python::with_gil(|py| -> PyResult<_> { - let borrowed = py_wrapper.borrow(py); - Ok(crate::mapob_unwrap_me!( - PythonMapObserverWrapperU16, - borrowed.wrapper, - $name, - $body - )) - }) - .unwrap() - } - PythonObserverWrapper::MapU32(py_wrapper) => { - Python::with_gil(|py| -> PyResult<_> { - let borrowed = py_wrapper.borrow(py); - Ok(crate::mapob_unwrap_me!( - PythonMapObserverWrapperU32, - borrowed.wrapper, - $name, - $body - )) - }) - .unwrap() - } - PythonObserverWrapper::MapU64(py_wrapper) => { - Python::with_gil(|py| -> PyResult<_> { - let borrowed = py_wrapper.borrow(py); - Ok(crate::mapob_unwrap_me!( - PythonMapObserverWrapperU64, - borrowed.wrapper, - $name, - $body - )) - }) - .unwrap() - } - PythonObserverWrapper::Python(py_wrapper) => { - let $name = py_wrapper; - $body - } - } - }; - } - - macro_rules! unwrap_me_mut { - ($wrapper:expr, $name:ident, $body:block) => { - match &mut $wrapper { - PythonObserverWrapper::MapI8(py_wrapper) => Python::with_gil(|py| -> PyResult<_> { - let mut borrowed = py_wrapper.borrow_mut(py); - Ok(crate::mapob_unwrap_me_mut!( - PythonMapObserverWrapperI8, - borrowed.wrapper, - $name, - $body - )) - }) - .unwrap(), - PythonObserverWrapper::MapI16(py_wrapper) => { - Python::with_gil(|py| -> PyResult<_> { - let mut borrowed = py_wrapper.borrow_mut(py); - Ok(crate::mapob_unwrap_me_mut!( - PythonMapObserverWrapperI16, - borrowed.wrapper, - $name, - $body - )) - }) - .unwrap() - } - PythonObserverWrapper::MapI32(py_wrapper) => { - Python::with_gil(|py| -> PyResult<_> { - let mut borrowed = py_wrapper.borrow_mut(py); - Ok(crate::mapob_unwrap_me_mut!( - PythonMapObserverWrapperI32, - borrowed.wrapper, - $name, - $body - )) - }) - .unwrap() - } - PythonObserverWrapper::MapI64(py_wrapper) => { - Python::with_gil(|py| -> PyResult<_> { - let mut borrowed = py_wrapper.borrow_mut(py); - Ok(crate::mapob_unwrap_me_mut!( - PythonMapObserverWrapperI64, - borrowed.wrapper, - $name, - $body - )) - }) - .unwrap() - } - PythonObserverWrapper::MapU8(py_wrapper) => Python::with_gil(|py| -> PyResult<_> { - let mut borrowed = py_wrapper.borrow_mut(py); - Ok(crate::mapob_unwrap_me_mut!( - PythonMapObserverWrapperU8, - borrowed.wrapper, - $name, - $body - )) - }) - .unwrap(), - PythonObserverWrapper::MapU16(py_wrapper) => { - Python::with_gil(|py| -> PyResult<_> { - let mut borrowed = py_wrapper.borrow_mut(py); - Ok(crate::mapob_unwrap_me_mut!( - PythonMapObserverWrapperU16, - borrowed.wrapper, - $name, - $body - )) - }) - .unwrap() - } - PythonObserverWrapper::MapU32(py_wrapper) => { - Python::with_gil(|py| -> PyResult<_> { - let mut borrowed = py_wrapper.borrow_mut(py); - Ok(crate::mapob_unwrap_me_mut!( - PythonMapObserverWrapperU32, - borrowed.wrapper, - $name, - $body - )) - }) - .unwrap() - } - PythonObserverWrapper::MapU64(py_wrapper) => { - Python::with_gil(|py| -> PyResult<_> { - let mut borrowed = py_wrapper.borrow_mut(py); - Ok(crate::mapob_unwrap_me_mut!( - PythonMapObserverWrapperU64, - borrowed.wrapper, - $name, - $body - )) - }) - .unwrap() - } - PythonObserverWrapper::Python(py_wrapper) => { - let $name = py_wrapper; - $body - } - } - }; - } - - #[pymethods] - impl PythonObserver { - #[staticmethod] - #[must_use] - pub fn new_map_i8(map_observer: Py) -> Self { - Self { - wrapper: PythonObserverWrapper::MapI8(map_observer), - } - } - #[staticmethod] - #[must_use] - pub fn new_map_i16(map_observer: Py) -> Self { - Self { - wrapper: PythonObserverWrapper::MapI16(map_observer), - } - } - #[staticmethod] - #[must_use] - pub fn new_map_i32(map_observer: Py) -> Self { - Self { - wrapper: PythonObserverWrapper::MapI32(map_observer), - } - } - #[staticmethod] - #[must_use] - pub fn new_map_i64(map_observer: Py) -> Self { - Self { - wrapper: PythonObserverWrapper::MapI64(map_observer), - } - } - - #[staticmethod] - #[must_use] - pub fn new_map_u8(map_observer: Py) -> Self { - Self { - wrapper: PythonObserverWrapper::MapU8(map_observer), - } - } - #[staticmethod] - #[must_use] - pub fn new_map_u16(map_observer: Py) -> Self { - Self { - wrapper: PythonObserverWrapper::MapU16(map_observer), - } - } - #[staticmethod] - #[must_use] - pub fn new_map_u32(map_observer: Py) -> Self { - Self { - wrapper: PythonObserverWrapper::MapU32(map_observer), - } - } - #[staticmethod] - #[must_use] - pub fn new_map_u64(map_observer: Py) -> Self { - Self { - wrapper: PythonObserverWrapper::MapU64(map_observer), - } - } - #[staticmethod] - #[must_use] - pub fn new_py(py_observer: PyObject) -> Self { - Self { - wrapper: PythonObserverWrapper::Python(PyObjectObserver::new(py_observer)), - } - } - - pub fn unwrap_py(&self) -> Option { - match &self.wrapper { - PythonObserverWrapper::Python(pyo) => Some(pyo.inner.clone()), - _ => None, - } - } - } - - impl Named for PythonObserver { - fn name(&self) -> &str { - let ptr = unwrap_me!(self.wrapper, o, { core::ptr::from_ref::(o.name()) }); - unsafe { ptr.as_ref().unwrap() } - } - } - - impl Observer for PythonObserver { - fn flush(&mut self) -> Result<(), Error> { - unwrap_me_mut!(self.wrapper, o, { Observer::::flush(o) }) - } - - fn pre_exec( - &mut self, - state: &mut PythonStdState, - input: &BytesInput, - ) -> Result<(), Error> { - unwrap_me_mut!(self.wrapper, o, { o.pre_exec(state, input) }) - } - - fn post_exec( - &mut self, - state: &mut PythonStdState, - input: &BytesInput, - exit_kind: &ExitKind, - ) -> Result<(), Error> { - unwrap_me_mut!(self.wrapper, o, { o.post_exec(state, input, exit_kind) }) - } - - fn pre_exec_child( - &mut self, - state: &mut PythonStdState, - input: &BytesInput, - ) -> Result<(), Error> { - unwrap_me_mut!(self.wrapper, o, { o.pre_exec_child(state, input) }) - } - - fn post_exec_child( - &mut self, - state: &mut PythonStdState, - input: &BytesInput, - exit_kind: &ExitKind, - ) -> Result<(), Error> { - unwrap_me_mut!(self.wrapper, o, { - o.post_exec_child(state, input, exit_kind) - }) - } - } - - #[derive(Serialize, Deserialize, Clone, Debug)] - #[allow(clippy::unsafe_derive_deserialize)] - #[pyclass(unsendable, name = "ObserversTuple")] - pub struct PythonObserversTuple { - list: Vec, - } - - #[pymethods] - impl PythonObserversTuple { - #[new] - fn new(list: Vec) -> Self { - Self { list } - } - - fn len(&self) -> usize { - self.list.len() - } - - fn __getitem__(&self, idx: usize) -> PythonObserver { - self.list[idx].clone() - } - - #[pyo3(name = "match_name")] - fn pymatch_name(&self, name: &str) -> Option { - for ob in &self.list { - if *ob.name() == *name { - return Some(ob.clone()); - } - } - None - } - } - - impl ObserversTuple for PythonObserversTuple { - fn pre_exec_all( - &mut self, - state: &mut PythonStdState, - input: &BytesInput, - ) -> Result<(), Error> { - for ob in &mut self.list { - ob.pre_exec(state, input)?; - } - Ok(()) - } - - fn post_exec_all( - &mut self, - state: &mut PythonStdState, - input: &BytesInput, - exit_kind: &ExitKind, - ) -> Result<(), Error> { - for ob in &mut self.list { - ob.post_exec(state, input, exit_kind)?; - } - Ok(()) - } - - fn pre_exec_child_all( - &mut self, - state: &mut PythonStdState, - input: &BytesInput, - ) -> Result<(), Error> { - for ob in &mut self.list { - ob.pre_exec_child(state, input)?; - } - Ok(()) - } - - fn post_exec_child_all( - &mut self, - state: &mut PythonStdState, - input: &BytesInput, - exit_kind: &ExitKind, - ) -> Result<(), Error> { - for ob in &mut self.list { - ob.post_exec_child(state, input, exit_kind)?; - } - Ok(()) - } - - // TODO: expose stdout/stderr to python - #[inline] - fn observes_stdout(&self) -> bool { - false - } - - #[inline] - fn observes_stderr(&self) -> bool { - false - } - - #[inline] - fn observe_stderr(&mut self, _: &[u8]) {} - - #[inline] - fn observe_stdout(&mut self, _: &[u8]) {} - } - - impl MatchName for PythonObserversTuple { - fn match_name(&self, name: &str) -> Option<&T> { - unsafe { - let mut r = None; - for ob in &self.list { - Python::with_gil(|py| -> PyResult<_> { - match &ob.wrapper { - PythonObserverWrapper::MapI8(py_wrapper) => { - if type_eq::() - && py_wrapper.borrow(py).name() == name - { - r = (std::ptr::addr_of!(*(*py_wrapper).borrow(py)) as *const T) - .as_ref(); - } - } - PythonObserverWrapper::MapI16(py_wrapper) => { - if type_eq::() - && py_wrapper.borrow(py).name() == name - { - r = (std::ptr::addr_of!(*(*py_wrapper).borrow(py)) as *const T) - .as_ref(); - } - } - PythonObserverWrapper::MapI32(py_wrapper) => { - if type_eq::() - && py_wrapper.borrow(py).name() == name - { - r = (std::ptr::addr_of!(*(*py_wrapper).borrow(py)) as *const T) - .as_ref(); - } - } - PythonObserverWrapper::MapI64(py_wrapper) => { - if type_eq::() - && py_wrapper.borrow(py).name() == name - { - r = (std::ptr::addr_of!(*(*py_wrapper).borrow(py)) as *const T) - .as_ref(); - } - } - - PythonObserverWrapper::MapU8(py_wrapper) => { - if type_eq::() - && py_wrapper.borrow(py).name() == name - { - r = (std::ptr::addr_of!(*(*py_wrapper).borrow(py)) as *const T) - .as_ref(); - } - } - PythonObserverWrapper::MapU16(py_wrapper) => { - if type_eq::() - && py_wrapper.borrow(py).name() == name - { - r = (std::ptr::addr_of!(*(*py_wrapper).borrow(py)) as *const T) - .as_ref(); - } - } - PythonObserverWrapper::MapU32(py_wrapper) => { - if type_eq::() - && py_wrapper.borrow(py).name() == name - { - r = (std::ptr::addr_of!(*(*py_wrapper).borrow(py)) as *const T) - .as_ref(); - } - } - PythonObserverWrapper::MapU64(py_wrapper) => { - if type_eq::() - && py_wrapper.borrow(py).name() == name - { - r = (std::ptr::addr_of!(*(*py_wrapper).borrow(py)) as *const T) - .as_ref(); - } - } - PythonObserverWrapper::Python(py_wrapper) => { - if type_eq::() && py_wrapper.name() == name { - r = (core::ptr::from_ref(py_wrapper) as *const T).as_ref(); - } - } - } - Ok(()) - }) - .unwrap(); - } - r - } - } - - fn match_name_mut(&mut self, name: &str) -> Option<&mut T> { - unsafe { - let mut r = None; - for ob in &mut self.list { - Python::with_gil(|py| -> PyResult<_> { - match &mut ob.wrapper { - PythonObserverWrapper::MapI8(py_wrapper) => { - if type_eq::() - && py_wrapper.borrow(py).name() == name - { - r = (std::ptr::addr_of!(*(*py_wrapper).borrow_mut(py)) - as *mut T) - .as_mut(); - } - } - PythonObserverWrapper::MapI16(py_wrapper) => { - if type_eq::() - && py_wrapper.borrow(py).name() == name - { - r = (std::ptr::addr_of!(*(*py_wrapper).borrow_mut(py)) - as *mut T) - .as_mut(); - } - } - PythonObserverWrapper::MapI32(py_wrapper) => { - if type_eq::() - && py_wrapper.borrow(py).name() == name - { - r = (std::ptr::addr_of!(*(*py_wrapper).borrow_mut(py)) - as *mut T) - .as_mut(); - } - } - PythonObserverWrapper::MapI64(py_wrapper) => { - if type_eq::() - && py_wrapper.borrow(py).name() == name - { - r = (std::ptr::addr_of!(*(*py_wrapper).borrow_mut(py)) - as *mut T) - .as_mut(); - } - } - - PythonObserverWrapper::MapU8(py_wrapper) => { - if type_eq::() - && py_wrapper.borrow(py).name() == name - { - r = (std::ptr::addr_of!(*(*py_wrapper).borrow_mut(py)) - as *mut T) - .as_mut(); - } - } - PythonObserverWrapper::MapU16(py_wrapper) => { - if type_eq::() - && py_wrapper.borrow(py).name() == name - { - r = (std::ptr::addr_of!(*(*py_wrapper).borrow_mut(py)) - as *mut T) - .as_mut(); - } - } - PythonObserverWrapper::MapU32(py_wrapper) => { - if type_eq::() - && py_wrapper.borrow(py).name() == name - { - r = (std::ptr::addr_of!(*(*py_wrapper).borrow_mut(py)) - as *mut T) - .as_mut(); - } - } - PythonObserverWrapper::MapU64(py_wrapper) => { - if type_eq::() - && py_wrapper.borrow(py).name() == name - { - r = (std::ptr::addr_of!(*(*py_wrapper).borrow_mut(py)) - as *mut T) - .as_mut(); - } - } - PythonObserverWrapper::Python(py_wrapper) => { - if type_eq::() && py_wrapper.name() == name { - r = (core::ptr::from_mut(py_wrapper) as *mut T).as_mut(); - } - } - } - Ok(()) - }) - .unwrap(); - } - r - } - } - } - - /// Register the classes to the python module - pub fn register(_py: Python, m: &PyModule) -> PyResult<()> { - m.add_class::()?; - m.add_class::()?; - Ok(()) - } -} - #[cfg(feature = "std")] #[cfg(test)] mod tests { diff --git a/libafl/src/observers/stacktrace.rs b/libafl/src/observers/stacktrace.rs index b1d2ee1719..9f082afa43 100644 --- a/libafl/src/observers/stacktrace.rs +++ b/libafl/src/observers/stacktrace.rs @@ -1,13 +1,11 @@ //! the ``StacktraceObserver`` looks up the stacktrace on the execution thread and computes a hash for it for dedupe -use alloc::{ - string::{String, ToString}, - vec::Vec, -}; +use alloc::{borrow::Cow, string::String, vec::Vec}; #[cfg(feature = "casr")] use std::{ collections::hash_map::DefaultHasher, hash::{Hash, Hasher}, + string::ToString, }; use std::{ fmt::Debug, @@ -83,10 +81,10 @@ pub fn collect_backtrace() -> u64 { if symbols.len() > 1 { let symbol = &symbols[0]; if let Some(name) = symbol.name() { - strace_entry.function = name.as_str().unwrap_or("").to_string(); + strace_entry.function = name.as_str().map_or_else(String::new, str::to_string); } if let Some(file) = symbol.filename() { - strace_entry.debug.file = file.to_str().unwrap_or("").to_string(); + strace_entry.debug.file = file.to_string_lossy().to_string(); } strace_entry.debug.line = u64::from(symbol.lineno().unwrap_or(0)); strace_entry.debug.column = u64::from(symbol.colno().unwrap_or(0)); @@ -116,7 +114,7 @@ pub enum HarnessType { #[allow(clippy::unsafe_derive_deserialize)] #[derive(Serialize, Deserialize, Debug)] pub struct BacktraceObserver<'a> { - observer_name: String, + observer_name: Cow<'static, str>, hash: OwnedRefMut<'a, Option>, harness_type: HarnessType, } @@ -125,13 +123,16 @@ impl<'a> BacktraceObserver<'a> { #[cfg(not(feature = "casr"))] /// Creates a new [`BacktraceObserver`] with the given name. #[must_use] - pub fn new( - observer_name: &str, + pub fn new( + observer_name: S, backtrace_hash: OwnedRefMut<'a, Option>, harness_type: HarnessType, - ) -> Self { + ) -> Self + where + S: Into>, + { Self { - observer_name: observer_name.to_string(), + observer_name: observer_name.into(), hash: backtrace_hash, harness_type, } @@ -140,14 +141,17 @@ impl<'a> BacktraceObserver<'a> { #[cfg(feature = "casr")] /// Creates a new [`BacktraceObserver`] with the given name. #[must_use] - pub fn new( - observer_name: &str, + pub fn new( + observer_name: S, backtrace_hash: OwnedRefMut<'a, Option>, harness_type: HarnessType, - ) -> Self { + ) -> Self + where + S: Into>, + { init_ignored_frames!("rust", "cpp", "go"); Self { - observer_name: observer_name.to_string(), + observer_name: observer_name.into(), hash: backtrace_hash, harness_type, } @@ -155,7 +159,10 @@ impl<'a> BacktraceObserver<'a> { /// Creates a new [`BacktraceObserver`] with the given name, owning a new `backtrace_hash` variable. #[must_use] - pub fn owned(observer_name: &str, harness_type: HarnessType) -> Self { + pub fn owned(observer_name: S, harness_type: HarnessType) -> Self + where + S: Into>, + { Self::new(observer_name, OwnedRefMut::owned(None), harness_type) } @@ -227,7 +234,7 @@ where } impl<'a> Named for BacktraceObserver<'a> { - fn name(&self) -> &str { + fn name(&self) -> &Cow<'static, str> { &self.observer_name } } @@ -263,7 +270,7 @@ pub fn get_asan_runtime_flags() -> String { /// An observer looking at the backtrace of target command using ASAN output #[derive(Serialize, Deserialize, Debug, Clone)] pub struct AsanBacktraceObserver { - observer_name: String, + observer_name: Cow<'static, str>, hash: Option, } @@ -271,9 +278,12 @@ impl AsanBacktraceObserver { #[cfg(not(feature = "casr"))] /// Creates a new [`BacktraceObserver`] with the given name. #[must_use] - pub fn new(observer_name: &str) -> Self { + pub fn new(observer_name: S) -> Self + where + S: Into>, + { Self { - observer_name: observer_name.to_string(), + observer_name: observer_name.into(), hash: None, } } @@ -281,10 +291,13 @@ impl AsanBacktraceObserver { #[cfg(feature = "casr")] /// Creates a new [`BacktraceObserver`] with the given name. #[must_use] - pub fn new(observer_name: &str) -> Self { + pub fn new(observer_name: S) -> Self + where + S: Into>, + { init_ignored_frames!("rust", "cpp", "go"); Self { - observer_name: observer_name.to_string(), + observer_name: observer_name.into(), hash: None, } } @@ -390,7 +403,7 @@ where } impl Named for AsanBacktraceObserver { - fn name(&self) -> &str { + fn name(&self) -> &Cow<'static, str> { &self.observer_name } } diff --git a/libafl/src/observers/stdio.rs b/libafl/src/observers/stdio.rs index 5e0ddc98ad..bc921c95f2 100644 --- a/libafl/src/observers/stdio.rs +++ b/libafl/src/observers/stdio.rs @@ -2,7 +2,7 @@ //! The executor must explicitly support these observers. //! For example, they are supported on the [`crate::executors::CommandExecutor`]. -use alloc::string::String; +use alloc::borrow::Cow; use std::vec::Vec; use libafl_bolts::Named; @@ -15,7 +15,7 @@ use crate::{inputs::UsesInput, observers::Observer}; #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct StdOutObserver { /// The name of the observer. - pub name: String, + pub name: Cow<'static, str>, /// The stdout of the target during its last execution. pub stdout: Option>, } @@ -24,8 +24,11 @@ pub struct StdOutObserver { impl StdOutObserver { /// Create a new [`StdOutObserver`] with the given name. #[must_use] - pub fn new(name: String) -> Self { - Self { name, stdout: None } + pub fn new(name: &'static str) -> Self { + Self { + name: Cow::from(name), + stdout: None, + } } } @@ -45,7 +48,7 @@ where } impl Named for StdOutObserver { - fn name(&self) -> &str { + fn name(&self) -> &Cow<'static, str> { &self.name } } @@ -55,7 +58,7 @@ impl Named for StdOutObserver { #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct StdErrObserver { /// The name of the observer. - pub name: String, + pub name: Cow<'static, str>, /// The stderr of the target during its last execution. pub stderr: Option>, } @@ -64,8 +67,11 @@ pub struct StdErrObserver { impl StdErrObserver { /// Create a new [`StdErrObserver`] with the given name. #[must_use] - pub fn new(name: String) -> Self { - Self { name, stderr: None } + pub fn new(name: &'static str) -> Self { + Self { + name: Cow::from(name), + stderr: None, + } } } @@ -85,7 +91,7 @@ where } impl Named for StdErrObserver { - fn name(&self) -> &str { + fn name(&self) -> &Cow<'static, str> { &self.name } } diff --git a/libafl/src/observers/value.rs b/libafl/src/observers/value.rs index d8610486c2..d23e3856d8 100644 --- a/libafl/src/observers/value.rs +++ b/libafl/src/observers/value.rs @@ -1,18 +1,15 @@ //! A simple observer with a single value. -use alloc::{ - boxed::Box, - string::{String, ToString}, -}; +use alloc::{borrow::Cow, boxed::Box, vec::Vec}; use core::{ - cell::{Ref, RefCell}, + cell::{Ref, RefCell, RefMut}, fmt::Debug, - hash::Hash, - ops::Deref, + hash::{Hash, Hasher}, + ops::{Deref, DerefMut}, }; use ahash::RandomState; -use libafl_bolts::{ownedref::OwnedRef, Named}; +use libafl_bolts::{ownedref::OwnedRef, AsIter, AsIterMut, AsSlice, AsSliceMut, Named}; use serde::{Deserialize, Serialize}; use super::Observer; @@ -29,7 +26,7 @@ where T: Debug + Serialize, { /// The name of this observer. - name: String, + name: Cow<'static, str>, /// The value. pub value: OwnedRef<'a, T>, } @@ -42,7 +39,7 @@ where #[must_use] pub fn new(name: &'static str, value: OwnedRef<'a, T>) -> Self { Self { - name: name.to_string(), + name: Cow::from(name), value, } } @@ -87,7 +84,7 @@ impl<'a, T> Named for ValueObserver<'a, T> where T: Debug + Serialize + serde::de::DeserializeOwned, { - fn name(&self) -> &str { + fn name(&self) -> &Cow<'static, str> { &self.name } } @@ -103,31 +100,27 @@ where /// A simple observer with a single [`RefCell`]'d value. #[derive(Serialize, Deserialize, Debug)] -#[serde(bound = "T: serde::de::DeserializeOwned")] -pub struct RefCellValueObserver<'a, T> -where - T: Debug + Serialize, -{ +#[serde(bound = "T: serde::de::DeserializeOwned + serde::Serialize")] +pub struct RefCellValueObserver<'a, T> { /// The name of this observer. - name: String, + name: Cow<'static, str>, /// The value. pub value: OwnedRef<'a, RefCell>, } -impl<'a, T> RefCellValueObserver<'a, T> -where - T: Debug + Serialize + serde::de::DeserializeOwned, -{ +impl<'a, T> RefCellValueObserver<'a, T> { /// Creates a new [`RefCellValueObserver`] with the given name. #[must_use] pub fn new(name: &'static str, value: OwnedRef<'a, RefCell>) -> Self { Self { - name: name.to_string(), + name: Cow::from(name), value, } } /// Get a reference to the underlying value. + /// + /// Panics if it can't borrow. #[must_use] pub fn get_ref<'b>(&'b self) -> Ref<'a, T> where @@ -136,6 +129,17 @@ where self.value.as_ref().borrow() } + /// Get a mutable reference to the underlying value. + /// + /// Panics if it can't borrow. + #[must_use] + pub fn get_ref_mut<'b>(&'b self) -> RefMut<'a, T> + where + 'b: 'a, + { + self.value.as_ref().borrow_mut() + } + /// Set the value. pub fn set(&mut self, new_value: T) { self.value.as_ref().replace(new_value); @@ -159,27 +163,246 @@ where impl<'a, S, T> Observer for RefCellValueObserver<'a, T> where S: UsesInput, - T: Debug + Serialize + serde::de::DeserializeOwned, { fn pre_exec(&mut self, _state: &mut S, _input: &S::Input) -> Result<(), Error> { Ok(()) } } -impl<'a, T> Named for RefCellValueObserver<'a, T> -where - T: Debug + Serialize + serde::de::DeserializeOwned, -{ - fn name(&self) -> &str { +impl<'a, T> Named for RefCellValueObserver<'a, T> { + fn name(&self) -> &Cow<'static, str> { &self.name } } -impl<'a, T: Hash> ObserverWithHashField for RefCellValueObserver<'a, T> +impl<'a, T> ObserverWithHashField for RefCellValueObserver<'a, T> where - T: Debug + Serialize + serde::de::DeserializeOwned, + T: Hash, { fn hash(&self) -> Option { Some(RandomState::with_seeds(1, 2, 3, 4).hash_one(&*self.value.as_ref().borrow())) } } + +/// [`Iterator`] over [`RefCellValueObserver`] of a [`Deref`] to `[T]`. +#[derive(Debug)] +pub struct RefCellValueObserverIter<'it, T> { + v: Ref<'it, [T]>, +} + +impl<'it, T: 'it, A: Deref> AsSlice<'it> for RefCellValueObserver<'_, A> { + type Entry = T; + type SliceRef = Ref<'it, [T]>; + + fn as_slice(&'it self) -> Self::SliceRef { + Ref::map(self.value.as_ref().borrow(), |s| &**s) + } +} + +impl<'it, T: 'it, A: DerefMut> AsSliceMut<'it> for RefCellValueObserver<'_, A> { + type SliceRefMut = RefMut<'it, [T]>; + + fn as_slice_mut(&'it mut self) -> Self::SliceRefMut { + RefMut::map(self.value.as_ref().borrow_mut(), |s| &mut **s) + } +} + +impl<'it, T: 'it, A: Deref> AsIter<'it> for RefCellValueObserver<'_, A> { + type Item = T; + type Ref = Ref<'it, Self::Item>; + type IntoIter = RefCellValueObserverIter<'it, T>; + + fn as_iter(&'it self) -> Self::IntoIter { + Self::IntoIter { + v: Ref::map(self.value.as_ref().borrow(), Deref::deref), + } + } +} + +impl<'it, T: 'it> Iterator for RefCellValueObserverIter<'it, T> { + type Item = Ref<'it, T>; + + fn next(&mut self) -> Option { + if self.v.is_empty() { + return None; + } + let cloned = Ref::clone(&self.v); + let (next, remainder) = Ref::map_split(cloned, |v| (&v[0], &v[1..])); + self.v = remainder; + Some(next) + } +} + +/// [`Iterator`] over [`RefCellValueObserver`] of a [`DerefMut`] to `[T]`. +#[derive(Debug)] +pub struct RefCellValueObserverIterMut<'it, T> { + v: Option>, +} + +impl<'it, T: 'it, A: Debug + DerefMut + Serialize> AsIterMut<'it> + for RefCellValueObserver<'_, A> +{ + type RefMut = RefMut<'it, T>; + type IntoIterMut = RefCellValueObserverIterMut<'it, T>; + + fn as_iter_mut(&'it mut self) -> Self::IntoIterMut { + Self::IntoIterMut { + v: Some(RefMut::map( + self.value.as_ref().borrow_mut(), + DerefMut::deref_mut, + )), + } + } +} + +impl<'it, T: 'it> Iterator for RefCellValueObserverIterMut<'it, T> { + type Item = RefMut<'it, T>; + + fn next(&mut self) -> Option { + if let Some(v) = self.v.take() { + let next_slice = if v.len() > 1 { + let (next, remainder) = RefMut::map_split(v, |v| v.split_at_mut(1)); + self.v = Some(remainder); + next + } else { + v + }; + Some(RefMut::map(next_slice, |v| &mut v[0])) + } else { + None + } + } +} + +impl<'a, T: Hash, A> Hash for RefCellValueObserver<'a, A> +where + T: Debug, + A: Debug + Deref + Serialize + serde::de::DeserializeOwned, +{ + /// Panics if the contained value is already mutably borrowed (calls + /// [`RefCell::borrow`]). + #[inline] + fn hash(&self, hasher: &mut H) { + (*self.get_ref()).hash(hasher); + } +} + +/// Panics if the contained value is already mutably borrowed (calls +/// [`RefCell::borrow`]). +impl libafl_bolts::HasLen for RefCellValueObserver<'_, A> +where + T: Debug, + A: Debug + Deref + Serialize + serde::de::DeserializeOwned, +{ + /// Panics if the contained value is already mutably borrowed (calls + /// [`RefCell::borrow`]). + fn len(&self) -> usize { + self.get_ref().len() + } + + /// Panics if the contained value is already mutably borrowed (calls + /// [`RefCell::borrow`]). + fn is_empty(&self) -> bool { + self.get_ref().is_empty() + } +} + +impl AsMut + for RefCellValueObserver<'_, T> +{ + fn as_mut(&mut self) -> &mut Self { + self + } +} + +impl AsRef + for RefCellValueObserver<'_, T> +{ + fn as_ref(&self) -> &Self { + self + } +} + +impl crate::observers::MapObserver for RefCellValueObserver<'_, A> +where + T: Copy + Debug + Default + Eq + Hash + num_traits::bounds::Bounded + 'static, + A: Debug + + Default + + Deref + + DerefMut + + serde::de::DeserializeOwned + + Serialize + + 'static, +{ + type Entry = T; + + /// Panics if the contained value is already mutably borrowed (calls + /// [`RefCell::borrow`]). + fn get(&self, idx: usize) -> Self::Entry { + self.get_ref()[idx] + } + + /// Panics if the contained value is already borrowed (calls + /// [`RefCell::borrow_mut`]). + fn set(&mut self, idx: usize, val: Self::Entry) { + self.get_ref_mut()[idx] = val; + } + + /// Panics if the contained value is already mutably borrowed (calls + /// [`RefCell::borrow`]). + fn hash_simple(&self) -> u64 { + RandomState::with_seeds(0, 0, 0, 0).hash_one(self) + } + + /// Panics if the contained value is already mutably borrowed (calls + /// [`RefCell::borrow`]). + fn usable_count(&self) -> usize { + self.get_ref().len() + } + + /// Panics if the contained value is already mutably borrowed (calls + /// [`RefCell::borrow`]). + fn count_bytes(&self) -> u64 { + let default = Self::Entry::default(); + let mut count = 0; + for entry in self.get_ref().iter() { + if entry != &default { + count += 1; + } + } + count + } + + /// Panics if the contained value is already borrowed (calls + /// [`RefCell::borrow_mut`]). + fn reset_map(&mut self) -> Result<(), Error> { + // This is less than optimal for `Vec`, for which we could use + // `.clear()`. However, it makes the `impl` more general. Worth it? + *self.get_ref_mut() = A::default(); + Ok(()) + } + + /// Panics if the contained value is already mutably borrowed (calls + /// [`RefCell::borrow`]). + fn to_vec(&self) -> Vec { + (*self.get_ref()).to_vec() + } + + /// Panics if the contained value is already mutably borrowed (calls + /// [`RefCell::borrow`]). + fn how_many_set(&self, indexes: &[usize]) -> usize { + let default = Self::Entry::default(); + let mut count = 0; + let arr = self.get_ref(); + for idx in indexes { + if arr[*idx] != default { + count += 1; + } + } + count + } + + fn initial(&self) -> Self::Entry { + Self::Entry::default() + } +} diff --git a/libafl/src/schedulers/accounting.rs b/libafl/src/schedulers/accounting.rs index e6778cbce0..e28e71cb82 100644 --- a/libafl/src/schedulers/accounting.rs +++ b/libafl/src/schedulers/accounting.rs @@ -1,23 +1,26 @@ //! Coverage accounting corpus scheduler, more details at use alloc::vec::Vec; -use core::fmt::Debug; +use core::{ + fmt::Debug, + ops::{Deref, DerefMut}, +}; use hashbrown::HashMap; -use libafl_bolts::{rands::Rand, AsMutSlice, AsSlice, HasLen, HasRefCnt}; +use libafl_bolts::{rands::Rand, HasLen, HasRefCnt}; use serde::{Deserialize, Serialize}; use crate::{ corpus::{Corpus, CorpusId}, feedbacks::MapIndexesMetadata, inputs::UsesInput, - observers::ObserversTuple, + observers::{CanTrack, ObserversTuple}, schedulers::{ minimizer::{IsFavoredMetadata, MinimizerScheduler, DEFAULT_SKIP_NON_FAVORED_PROB}, LenTimeMulTestcaseScore, Scheduler, }, - state::{HasCorpus, HasMetadata, HasRand, UsesState}, - Error, + state::{HasCorpus, HasRand, UsesState}, + Error, HasMetadata, }; /// A testcase metadata holding a list of indexes of a map @@ -35,19 +38,15 @@ pub struct AccountingIndexesMetadata { libafl_bolts::impl_serdeany!(AccountingIndexesMetadata); -impl AsSlice for AccountingIndexesMetadata { - type Entry = usize; - /// Convert to a slice - fn as_slice(&self) -> &[usize] { - self.list.as_slice() +impl Deref for AccountingIndexesMetadata { + type Target = [usize]; + fn deref(&self) -> &[usize] { + &self.list } } -impl AsMutSlice for AccountingIndexesMetadata { - type Entry = usize; - - /// Convert to a slice - fn as_mut_slice(&mut self) -> &mut [usize] { - self.list.as_mut_slice() +impl DerefMut for AccountingIndexesMetadata { + fn deref_mut(&mut self) -> &mut [usize] { + &mut self.list } } @@ -106,21 +105,22 @@ impl TopAccountingMetadata { /// A minimizer scheduler using coverage accounting #[derive(Debug)] -pub struct CoverageAccountingScheduler<'a, CS> +pub struct CoverageAccountingScheduler<'a, CS, O> where CS: UsesState, CS::State: Debug, { accounting_map: &'a [u32], - skip_non_favored_prob: u64, + skip_non_favored_prob: f64, inner: MinimizerScheduler< CS, LenTimeMulTestcaseScore<::State>, MapIndexesMetadata, + O, >, } -impl<'a, CS> UsesState for CoverageAccountingScheduler<'a, CS> +impl<'a, CS, O> UsesState for CoverageAccountingScheduler<'a, CS, O> where CS: UsesState, CS::State: Debug, @@ -128,11 +128,12 @@ where type State = CS::State; } -impl<'a, CS> Scheduler for CoverageAccountingScheduler<'a, CS> +impl<'a, CS, O> Scheduler for CoverageAccountingScheduler<'a, CS, O> where CS: Scheduler, CS::State: HasCorpus + HasMetadata + HasRand + Debug, ::Input: HasLen, + O: CanTrack, { fn on_add(&mut self, state: &mut Self::State, idx: CorpusId) -> Result<(), Error> { self.update_accounting_score(state, idx)?; @@ -169,7 +170,7 @@ where .borrow() .has_metadata::(); has - } && state.rand_mut().below(100) < self.skip_non_favored_prob + } && state.rand_mut().coinflip(self.skip_non_favored_prob) { idx = self.inner.base_mut().next(state)?; } @@ -190,11 +191,12 @@ where } } -impl<'a, CS> CoverageAccountingScheduler<'a, CS> +impl<'a, CS, O> CoverageAccountingScheduler<'a, CS, O> where CS: Scheduler, CS::State: HasCorpus + HasMetadata + HasRand + Debug, ::Input: HasLen, + O: CanTrack, { /// Update the `Corpus` score #[allow(clippy::unused_self)] @@ -307,7 +309,9 @@ where /// Creates a new [`CoverageAccountingScheduler`] that wraps a `base` [`Scheduler`] /// and has a default probability to skip non-faved Testcases of [`DEFAULT_SKIP_NON_FAVORED_PROB`]. - pub fn new(state: &mut CS::State, base: CS, accounting_map: &'a [u32]) -> Self { + /// + /// Provide the observer responsible for determining new indexes. + pub fn new(observer: &O, state: &mut CS::State, base: CS, accounting_map: &'a [u32]) -> Self { match state.metadata_map().get::() { Some(meta) => { if meta.max_accounting.len() != accounting_map.len() { @@ -320,17 +324,20 @@ where } Self { accounting_map, - inner: MinimizerScheduler::new(base), + inner: MinimizerScheduler::new(observer, base), skip_non_favored_prob: DEFAULT_SKIP_NON_FAVORED_PROB, } } /// Creates a new [`CoverageAccountingScheduler`] that wraps a `base` [`Scheduler`] /// and has a non-default probability to skip non-faved Testcases using (`skip_non_favored_prob`). + /// + /// Provide the observer responsible for determining new indexes. pub fn with_skip_prob( + observer: &O, state: &mut CS::State, base: CS, - skip_non_favored_prob: u64, + skip_non_favored_prob: f64, accounting_map: &'a [u32], ) -> Self { match state.metadata_map().get::() { @@ -345,7 +352,7 @@ where } Self { accounting_map, - inner: MinimizerScheduler::with_skip_prob(base, skip_non_favored_prob), + inner: MinimizerScheduler::with_skip_prob(observer, base, skip_non_favored_prob), skip_non_favored_prob, } } diff --git a/libafl/src/schedulers/minimizer.rs b/libafl/src/schedulers/minimizer.rs index 62d249b6d9..ddf871185b 100644 --- a/libafl/src/schedulers/minimizer.rs +++ b/libafl/src/schedulers/minimizer.rs @@ -5,21 +5,22 @@ use alloc::vec::Vec; use core::{any::type_name, cmp::Ordering, marker::PhantomData}; use hashbrown::{HashMap, HashSet}; -use libafl_bolts::{rands::Rand, serdeany::SerdeAny, AsSlice, HasRefCnt}; +use libafl_bolts::{rands::Rand, serdeany::SerdeAny, AsIter, HasRefCnt}; use serde::{Deserialize, Serialize}; use crate::{ corpus::{Corpus, CorpusId, Testcase}, feedbacks::MapIndexesMetadata, inputs::UsesInput, - observers::ObserversTuple, + observers::{CanTrack, ObserversTuple}, + require_index_tracking, schedulers::{LenTimeMulTestcaseScore, RemovableScheduler, Scheduler, TestcaseScore}, - state::{HasCorpus, HasMetadata, HasRand, UsesState}, - Error, + state::{HasCorpus, HasRand, UsesState}, + Error, HasMetadata, }; /// Default probability to skip the non-favored values -pub const DEFAULT_SKIP_NON_FAVORED_PROB: u64 = 95; +pub const DEFAULT_SKIP_NON_FAVORED_PROB: f64 = 0.95; /// A testcase metadata saying if a testcase is favored #[derive(Debug, Serialize, Deserialize)] @@ -70,26 +71,27 @@ impl Default for TopRatedsMetadata { /// corpus that exercise all the requested features (e.g. all the coverage seen so far) /// prioritizing [`Testcase`]`s` using [`TestcaseScore`] #[derive(Debug, Clone)] -pub struct MinimizerScheduler { +pub struct MinimizerScheduler { base: CS, - skip_non_favored_prob: u64, + skip_non_favored_prob: f64, remove_metadata: bool, - phantom: PhantomData<(F, M)>, + phantom: PhantomData<(F, M, O)>, } -impl UsesState for MinimizerScheduler +impl UsesState for MinimizerScheduler where CS: UsesState, { type State = CS::State; } -impl RemovableScheduler for MinimizerScheduler +impl RemovableScheduler for MinimizerScheduler where CS: RemovableScheduler, F: TestcaseScore, - M: AsSlice + SerdeAny + HasRefCnt, + M: for<'a> AsIter<'a, Item = usize> + SerdeAny + HasRefCnt, CS::State: HasCorpus + HasMetadata + HasRand, + O: CanTrack, { /// Replaces the testcase at the given idx fn on_replace( @@ -128,13 +130,13 @@ where let factor = F::compute(state, &mut *old)?; if let Some(old_map) = old.metadata_map_mut().get_mut::() { let mut e_iter = entries.iter(); - let mut map_iter = old_map.as_slice().iter(); // ASSERTION: guaranteed to be in order? + let mut map_iter = old_map.as_iter(); // ASSERTION: guaranteed to be in order? // manual set intersection let mut entry = e_iter.next(); let mut map_entry = map_iter.next(); while let Some(e) = entry { - if let Some(me) = map_entry { + if let Some(ref me) = map_entry { match e.cmp(me) { Ordering::Less => { entry = e_iter.next(); @@ -191,12 +193,13 @@ where } } -impl Scheduler for MinimizerScheduler +impl Scheduler for MinimizerScheduler where CS: Scheduler, F: TestcaseScore, - M: AsSlice + SerdeAny + HasRefCnt, + M: for<'a> AsIter<'a, Item = usize> + SerdeAny + HasRefCnt, CS::State: HasCorpus + HasMetadata + HasRand, + O: CanTrack, { /// Called when a [`Testcase`] is added to the corpus fn on_add(&mut self, state: &mut CS::State, idx: CorpusId) -> Result<(), Error> { @@ -228,7 +231,7 @@ where .borrow() .has_metadata::(); has - } && state.rand_mut().below(100) < self.skip_non_favored_prob + } && state.rand_mut().coinflip(self.skip_non_favored_prob) { idx = self.base.next(state)?; } @@ -246,12 +249,13 @@ where } } -impl MinimizerScheduler +impl MinimizerScheduler where CS: Scheduler, F: TestcaseScore, - M: AsSlice + SerdeAny + HasRefCnt, + M: for<'a> AsIter<'a, Item = usize> + SerdeAny + HasRefCnt, CS::State: HasCorpus + HasMetadata + HasRand, + O: CanTrack, { /// Update the [`Corpus`] score using the [`MinimizerScheduler`] #[allow(clippy::unused_self)] @@ -272,8 +276,8 @@ where )) })?; let top_rateds = state.metadata_map().get::().unwrap(); - for elem in meta.as_slice() { - if let Some(old_idx) = top_rateds.map.get(elem) { + for elem in meta.as_iter() { + if let Some(old_idx) = top_rateds.map.get(&*elem) { if *old_idx == idx { new_favoreds.push(*elem); // always retain current; we'll drop it later otherwise continue; @@ -346,7 +350,7 @@ where type_name::() )) })?; - for elem in meta.as_slice() { + for elem in meta.as_iter() { acc.insert(*elem); } @@ -371,7 +375,10 @@ where /// and has a default probability to skip non-faved [`Testcase`]s of [`DEFAULT_SKIP_NON_FAVORED_PROB`]. /// This will remove the metadata `M` when it is no longer needed, after consumption. This might /// for example be a `MapIndexesMetadata`. - pub fn new(base: CS) -> Self { + /// + /// When calling, pass the edges observer which will provided the indexes to minimize over. + pub fn new(_observer: &O, base: CS) -> Self { + require_index_tracking!("MinimizerScheduler", O); Self { base, skip_non_favored_prob: DEFAULT_SKIP_NON_FAVORED_PROB, @@ -383,7 +390,10 @@ where /// Creates a new [`MinimizerScheduler`] that wraps a `base` [`Scheduler`] /// and has a default probability to skip non-faved [`Testcase`]s of [`DEFAULT_SKIP_NON_FAVORED_PROB`]. /// This method will prevent the metadata `M` from being removed at the end of scoring. - pub fn non_metadata_removing(base: CS) -> Self { + /// + /// When calling, pass the edges observer which will provided the indexes to minimize over. + pub fn non_metadata_removing(_observer: &O, base: CS) -> Self { + require_index_tracking!("MinimizerScheduler", O); Self { base, skip_non_favored_prob: DEFAULT_SKIP_NON_FAVORED_PROB, @@ -394,7 +404,10 @@ where /// Creates a new [`MinimizerScheduler`] that wraps a `base` [`Scheduler`] /// and has a non-default probability to skip non-faved [`Testcase`]s using (`skip_non_favored_prob`). - pub fn with_skip_prob(base: CS, skip_non_favored_prob: u64) -> Self { + /// + /// When calling, pass the edges observer which will provided the indexes to minimize over. + pub fn with_skip_prob(_observer: &O, base: CS, skip_non_favored_prob: f64) -> Self { + require_index_tracking!("MinimizerScheduler", O); Self { base, skip_non_favored_prob, @@ -405,10 +418,14 @@ where } /// A [`MinimizerScheduler`] with [`LenTimeMulTestcaseScore`] to prioritize quick and small [`Testcase`]`s`. -pub type LenTimeMinimizerScheduler = - MinimizerScheduler::State>, M>; +pub type LenTimeMinimizerScheduler = + MinimizerScheduler::State>, M, O>; /// A [`MinimizerScheduler`] with [`LenTimeMulTestcaseScore`] to prioritize quick and small [`Testcase`]`s` /// that exercise all the entries registered in the [`MapIndexesMetadata`]. -pub type IndexesLenTimeMinimizerScheduler = - MinimizerScheduler::State>, MapIndexesMetadata>; +pub type IndexesLenTimeMinimizerScheduler = MinimizerScheduler< + CS, + LenTimeMulTestcaseScore<::State>, + MapIndexesMetadata, + O, +>; diff --git a/libafl/src/schedulers/mod.rs b/libafl/src/schedulers/mod.rs index 5e6782394d..212f97a495 100644 --- a/libafl/src/schedulers/mod.rs +++ b/libafl/src/schedulers/mod.rs @@ -1,9 +1,6 @@ //! Schedule the access to the Corpus. -use alloc::{ - borrow::ToOwned, - string::{String, ToString}, -}; +use alloc::{borrow::ToOwned, string::ToString}; use core::marker::PhantomData; pub mod testcase_score; @@ -30,7 +27,10 @@ pub mod weighted; pub use weighted::{StdWeightedScheduler, WeightedScheduler}; pub mod tuneable; -use libafl_bolts::rands::Rand; +use libafl_bolts::{ + rands::Rand, + tuples::{Handle, MatchNameRef}, +}; pub use tuneable::*; use crate::{ @@ -38,8 +38,8 @@ use crate::{ inputs::UsesInput, observers::{MapObserver, ObserversTuple}, random_corpus_id, - state::{HasCorpus, HasMetadata, HasRand, State, UsesState}, - Error, + state::{HasCorpus, HasRand, State, UsesState}, + Error, HasMetadata, }; /// The scheduler also implements `on_remove` and `on_replace` if it implements this stage. @@ -68,85 +68,12 @@ where } } -/// Define the metadata operations when removing testcase from AFL-style scheduler -pub trait HasAFLRemovableScheduler: RemovableScheduler -where - Self::State: HasCorpus + HasMetadata + HasTestcase, -{ - #[allow(clippy::cast_precision_loss)] - #[allow(clippy::cast_precision_loss)] - /// Adjusting metadata when removing the testcase - fn on_remove_metadata( - &mut self, - state: &mut Self::State, - _idx: CorpusId, - prev: &Option::Input>>, - ) -> Result<(), Error> { - let prev = prev.as_ref().ok_or_else(|| { - Error::illegal_argument( - "Power schedulers must be aware of the removed corpus entry for reweighting.", - ) - })?; - - let prev_meta = prev.metadata::()?; - - // Use these to adjust `SchedulerMetadata` - let (prev_total_time, prev_cycles) = prev_meta.cycle_and_time(); - let prev_bitmap_size = prev_meta.bitmap_size(); - let prev_bitmap_size_log = libm::log2(prev_bitmap_size as f64); - - let psmeta = state.metadata_mut::()?; - - psmeta.set_exec_time(psmeta.exec_time() - prev_total_time); - psmeta.set_cycles(psmeta.cycles() - (prev_cycles as u64)); - psmeta.set_bitmap_size(psmeta.bitmap_size() - prev_bitmap_size); - psmeta.set_bitmap_size_log(psmeta.bitmap_size_log() - prev_bitmap_size_log); - psmeta.set_bitmap_entries(psmeta.bitmap_entries() - 1); - - Ok(()) - } - - #[allow(clippy::cast_precision_loss)] - /// Adjusting metadata when replacing the corpus - fn on_replace_metadata( - &mut self, - state: &mut Self::State, - idx: CorpusId, - prev: &Testcase<::Input>, - ) -> Result<(), Error> { - let prev_meta = prev.metadata::()?; - - // Next depth is + 1 - let prev_depth = prev_meta.depth() + 1; - - // Use these to adjust `SchedulerMetadata` - let (prev_total_time, prev_cycles) = prev_meta.cycle_and_time(); - let prev_bitmap_size = prev_meta.bitmap_size(); - let prev_bitmap_size_log = libm::log2(prev_bitmap_size as f64); - - let psmeta = state.metadata_mut::()?; - - // We won't add new one because it'll get added when it gets executed in calirbation next time. - psmeta.set_exec_time(psmeta.exec_time() - prev_total_time); - psmeta.set_cycles(psmeta.cycles() - (prev_cycles as u64)); - psmeta.set_bitmap_size(psmeta.bitmap_size() - prev_bitmap_size); - psmeta.set_bitmap_size_log(psmeta.bitmap_size_log() - prev_bitmap_size_log); - psmeta.set_bitmap_entries(psmeta.bitmap_entries() - 1); - - state - .corpus() - .get(idx)? - .borrow_mut() - .add_metadata(SchedulerTestcaseMetadata::new(prev_depth)); - Ok(()) - } -} - /// Defines the common metadata operations for the AFL-style schedulers -pub trait HasAFLSchedulerMetadata: Scheduler +pub trait AflScheduler: Scheduler where Self::State: HasCorpus + HasMetadata + HasTestcase, O: MapObserver, + C: AsRef, { /// Return the last hash fn last_hash(&self) -> usize; @@ -155,7 +82,7 @@ where fn set_last_hash(&mut self, value: usize); /// Get the observer map observer name - fn map_observer_name(&self) -> &String; + fn map_observer_ref(&self) -> &Handle; /// Called when a [`Testcase`] is added to the corpus fn on_add_metadata(&self, state: &mut Self::State, idx: CorpusId) -> Result<(), Error> { @@ -194,10 +121,11 @@ where OT: ObserversTuple, { let observer = observers - .match_name::(self.map_observer_name()) - .ok_or_else(|| Error::key_not_found("MapObserver not found".to_string()))?; + .get(self.map_observer_ref()) + .ok_or_else(|| Error::key_not_found("MapObserver not found".to_string()))? + .as_ref(); - let mut hash = observer.hash() as usize; + let mut hash = observer.hash_simple() as usize; let psmeta = state.metadata_mut::()?; @@ -303,7 +231,10 @@ where /// Gets the next entry at random fn next(&mut self, state: &mut Self::State) -> Result { if state.corpus().count() == 0 { - Err(Error::empty("No entries in corpus".to_owned())) + Err(Error::empty( + "No entries in corpus. This often implies the target is not properly instrumented." + .to_owned(), + )) } else { let id = random_corpus_id!(state.corpus(), state.rand_mut()); self.set_current_scheduled(state, Some(id))?; diff --git a/libafl/src/schedulers/powersched.rs b/libafl/src/schedulers/powersched.rs index 38051e27a9..d131a66d50 100644 --- a/libafl/src/schedulers/powersched.rs +++ b/libafl/src/schedulers/powersched.rs @@ -1,22 +1,21 @@ //! The queue corpus scheduler for power schedules. -use alloc::{ - string::{String, ToString}, - vec::Vec, -}; +use alloc::vec::Vec; use core::{marker::PhantomData, time::Duration}; +use libafl_bolts::{ + tuples::{Handle, Handler}, + Named, +}; use serde::{Deserialize, Serialize}; use crate::{ corpus::{Corpus, CorpusId, HasTestcase, Testcase}, inputs::UsesInput, observers::{MapObserver, ObserversTuple}, - schedulers::{ - HasAFLRemovableScheduler, HasAFLSchedulerMetadata, RemovableScheduler, Scheduler, - }, - state::{HasCorpus, HasMetadata, State, UsesState}, - Error, + schedulers::{AflScheduler, RemovableScheduler, Scheduler}, + state::{HasCorpus, State, UsesState}, + Error, HasMetadata, }; /// The n fuzz size @@ -39,7 +38,7 @@ pub struct SchedulerMetadata { cycles: u64, /// Size of the observer map bitmap_size: u64, - /// Sum of log(bitmap_size) + /// Sum of `log(bitmap_size`) bitmap_size_log: f64, /// Number of filled map entries bitmap_entries: u64, @@ -172,55 +171,52 @@ pub enum PowerSchedule { /// Note that this corpus is merely holding the metadata necessary for the power calculation /// and here we DON'T actually calculate the power (we do it in the stage) #[derive(Clone, Debug)] -pub struct PowerQueueScheduler { +pub struct PowerQueueScheduler { strat: PowerSchedule, - map_observer_name: String, + map_observer_ref: Handle, last_hash: usize, phantom: PhantomData<(O, S)>, } -impl UsesState for PowerQueueScheduler +impl UsesState for PowerQueueScheduler where S: State, { type State = S; } -impl HasAFLRemovableScheduler for PowerQueueScheduler +impl RemovableScheduler for PowerQueueScheduler where S: State + HasTestcase + HasMetadata + HasCorpus, O: MapObserver, + C: AsRef, { -} - -impl RemovableScheduler for PowerQueueScheduler -where - S: HasCorpus + HasMetadata + HasTestcase + State, - O: MapObserver, -{ + /// This will *NOT* neutralize the effect of this removed testcase from the global data such as `SchedulerMetadata` fn on_remove( &mut self, - state: &mut Self::State, - idx: CorpusId, - prev: &Option::Input>>, + _state: &mut Self::State, + _idx: CorpusId, + _prev: &Option::Input>>, ) -> Result<(), Error> { - self.on_remove_metadata(state, idx, prev) + Ok(()) } + /// This will *NOT* neutralize the effect of this removed testcase from the global data such as `SchedulerMetadata` fn on_replace( &mut self, - state: &mut Self::State, - idx: CorpusId, - prev: &Testcase<::Input>, + _state: &mut Self::State, + _idx: CorpusId, + _prev: &Testcase<::Input>, ) -> Result<(), Error> { - self.on_replace_metadata(state, idx, prev) + Ok(()) } } -impl HasAFLSchedulerMetadata for PowerQueueScheduler +impl AflScheduler for PowerQueueScheduler where S: HasCorpus + HasMetadata + HasTestcase + State, O: MapObserver, + C: AsRef, { fn last_hash(&self) -> usize { self.last_hash @@ -230,15 +226,16 @@ where self.last_hash = hash; } - fn map_observer_name(&self) -> &String { - &self.map_observer_name + fn map_observer_ref(&self) -> &Handle { + &self.map_observer_ref } } -impl Scheduler for PowerQueueScheduler +impl Scheduler for PowerQueueScheduler where S: HasCorpus + HasMetadata + HasTestcase + State, O: MapObserver, + C: AsRef, { /// Called when a [`Testcase`] is added to the corpus fn on_add(&mut self, state: &mut Self::State, idx: CorpusId) -> Result<(), Error> { @@ -259,7 +256,9 @@ where fn next(&mut self, state: &mut Self::State) -> Result { if state.corpus().count() == 0 { - Err(Error::empty(String::from("No entries in corpus"))) + Err(Error::empty( + "No entries in corpus. This often implies the target is not properly instrumented.", + )) } else { let id = match state.corpus().current() { Some(cur) => { @@ -292,20 +291,21 @@ where } } -impl PowerQueueScheduler +impl PowerQueueScheduler where S: HasMetadata, O: MapObserver, + C: AsRef + Named, { /// Create a new [`PowerQueueScheduler`] #[must_use] - pub fn new(state: &mut S, map_observer: &O, strat: PowerSchedule) -> Self { + pub fn new(state: &mut S, map_observer: &C, strat: PowerSchedule) -> Self { if !state.has_metadata::() { state.add_metadata::(SchedulerMetadata::new(Some(strat))); } PowerQueueScheduler { strat, - map_observer_name: map_observer.name().to_string(), + map_observer_ref: map_observer.handle(), last_hash: 0, phantom: PhantomData, } diff --git a/libafl/src/schedulers/probabilistic_sampling.rs b/libafl/src/schedulers/probabilistic_sampling.rs index a5fe0cb644..4d528e75c6 100644 --- a/libafl/src/schedulers/probabilistic_sampling.rs +++ b/libafl/src/schedulers/probabilistic_sampling.rs @@ -9,11 +9,11 @@ use libafl_bolts::rands::Rand; use serde::{Deserialize, Serialize}; use crate::{ - corpus::{Corpus, CorpusId, HasTestcase}, + corpus::{Corpus, CorpusId, HasTestcase, Testcase}, inputs::UsesInput, - schedulers::{Scheduler, TestcaseScore}, - state::{HasCorpus, HasMetadata, HasRand, State, UsesState}, - Error, + schedulers::{RemovableScheduler, Scheduler, TestcaseScore}, + state::{HasCorpus, HasRand, State, UsesState}, + Error, HasMetadata, }; /// Conduct reservoir sampling (probabilistic sampling) over all corpus elements. @@ -74,23 +74,60 @@ where #[allow(clippy::cast_precision_loss)] #[allow(clippy::unused_self)] pub fn store_probability(&self, state: &mut S, idx: CorpusId) -> Result<(), Error> { - let factor = F::compute(state, &mut *state.corpus().get(idx)?.borrow_mut())?; - if factor == 0.0 { - return Err(Error::illegal_state( - "Infinity probability calculated for probabilistic sampling scheduler", - )); - } + let prob = F::compute(state, &mut *state.corpus().get(idx)?.borrow_mut())?; + debug_assert!( + prob >= 0.0 && prob.is_finite(), + "scheduler probability is {prob}; to work correctly it must be >= 0.0 and finite" + ); let meta = state .metadata_map_mut() .get_mut::() .unwrap(); - let prob = 1.0 / factor; meta.map.insert(idx, prob); meta.total_probability += prob; Ok(()) } } +impl RemovableScheduler for ProbabilitySamplingScheduler +where + F: TestcaseScore, + S: HasCorpus + HasMetadata + HasRand + HasTestcase + State, +{ + fn on_remove( + &mut self, + state: &mut Self::State, + idx: CorpusId, + _testcase: &Option::Input>>, + ) -> Result<(), Error> { + let meta = state + .metadata_map_mut() + .get_mut::() + .unwrap(); + if let Some(prob) = meta.map.remove(&idx) { + meta.total_probability -= prob; + } + Ok(()) + } + + fn on_replace( + &mut self, + state: &mut Self::State, + idx: CorpusId, + _prev: &Testcase<::Input>, + ) -> Result<(), Error> { + let meta = state + .metadata_map_mut() + .get_mut::() + .unwrap(); + if let Some(prob) = meta.map.remove(&idx) { + meta.total_probability -= prob; + } + + self.store_probability(state, idx) + } +} + impl UsesState for ProbabilitySamplingScheduler where S: State + HasTestcase, @@ -121,9 +158,11 @@ where #[allow(clippy::cast_precision_loss)] fn next(&mut self, state: &mut Self::State) -> Result { if state.corpus().count() == 0 { - Err(Error::empty(String::from("No entries in corpus"))) + Err(Error::empty(String::from( + "No entries in corpus. This often implies the target is not properly instrumented.", + ))) } else { - let rand_prob: f64 = (state.rand_mut().below(100) as f64) / 100.0; + let rand_prob: f64 = state.rand_mut().next_float(); let meta = state.metadata_map().get::().unwrap(); let threshold = meta.total_probability * rand_prob; let mut k: f64 = 0.0; @@ -163,8 +202,8 @@ mod tests { feedbacks::ConstFeedback, inputs::{bytes::BytesInput, Input, UsesInput}, schedulers::{ProbabilitySamplingScheduler, Scheduler, TestcaseScore}, - state::{HasCorpus, HasMetadata, StdState}, - Error, + state::{HasCorpus, StdState}, + Error, HasMetadata, }; const FACTOR: f64 = 1337.0; @@ -191,13 +230,15 @@ mod tests { #[test] fn test_prob_sampling() { + // # Safety + // No concurrency per testcase #[cfg(any(not(feature = "serdeany_autoreg"), miri))] unsafe { super::ProbabilityMetadata::register(); } - // the first 3 probabilities will be .69, .86, .44 - let rand = StdRand::with_seed(12); + // the first 3 probabilities will be .76, .86, .36 + let rand = StdRand::with_seed(2); let mut scheduler = UniformProbabilitySamplingScheduler::new(); diff --git a/libafl/src/schedulers/queue.rs b/libafl/src/schedulers/queue.rs index 12276f4487..0608f453c1 100644 --- a/libafl/src/schedulers/queue.rs +++ b/libafl/src/schedulers/queue.rs @@ -44,7 +44,10 @@ where /// Gets the next entry in the queue fn next(&mut self, state: &mut Self::State) -> Result { if state.corpus().count() == 0 { - Err(Error::empty("No entries in corpus".to_owned())) + Err(Error::empty( + "No entries in corpus. This often implies the target is not properly instrumented." + .to_owned(), + )) } else { let id = state .corpus() diff --git a/libafl/src/schedulers/testcase_score.rs b/libafl/src/schedulers/testcase_score.rs index 17dc86feaa..4e57549712 100644 --- a/libafl/src/schedulers/testcase_score.rs +++ b/libafl/src/schedulers/testcase_score.rs @@ -11,8 +11,8 @@ use crate::{ minimizer::{IsFavoredMetadata, TopRatedsMetadata}, powersched::{PowerSchedule, SchedulerMetadata}, }, - state::{HasCorpus, HasMetadata}, - Error, + state::HasCorpus, + Error, HasMetadata, }; /// Compute the favor factor of a [`Testcase`]. Higher is better. @@ -303,19 +303,13 @@ where let q_bitmap_size = tcmeta.bitmap_size() as f64; - if let Some(strat) = psmeta.strat() { - match strat { - PowerSchedule::FAST - | PowerSchedule::COE - | PowerSchedule::LIN - | PowerSchedule::QUAD => { - let hits = psmeta.n_fuzz()[tcmeta.n_fuzz_entry()]; - if hits > 0 { - weight /= libm::log10(f64::from(hits)) + 1.0; - } - } - // EXPLORE and EXPLOIT fall into this - _ => {} + if let Some( + PowerSchedule::FAST | PowerSchedule::COE | PowerSchedule::LIN | PowerSchedule::QUAD, + ) = psmeta.strat() + { + let hits = psmeta.n_fuzz()[tcmeta.n_fuzz_entry()]; + if hits > 0 { + weight /= libm::log10(f64::from(hits)) + 1.0; } } diff --git a/libafl/src/schedulers/tuneable.rs b/libafl/src/schedulers/tuneable.rs index eb6577aaf3..de4fb44332 100644 --- a/libafl/src/schedulers/tuneable.rs +++ b/libafl/src/schedulers/tuneable.rs @@ -12,8 +12,8 @@ use super::RemovableScheduler; use crate::{ corpus::{Corpus, CorpusId, HasTestcase}, schedulers::Scheduler, - state::{HasCorpus, HasMetadata, State, UsesState}, - Error, + state::{HasCorpus, State, UsesState}, + Error, HasMetadata, }; #[derive(Default, Clone, Copy, Eq, PartialEq, Debug, Serialize, Deserialize)] @@ -119,7 +119,10 @@ where /// Gets the next entry in the queue fn next(&mut self, state: &mut Self::State) -> Result { if state.corpus().count() == 0 { - return Err(Error::empty("No entries in corpus".to_owned())); + return Err(Error::empty( + "No entries in corpus. This often implies the target is not properly instrumented." + .to_owned(), + )); } let id = if let Some(next) = Self::get_next(state) { // next was set diff --git a/libafl/src/schedulers/weighted.rs b/libafl/src/schedulers/weighted.rs index 0057678ee9..b6c7719f42 100644 --- a/libafl/src/schedulers/weighted.rs +++ b/libafl/src/schedulers/weighted.rs @@ -1,11 +1,14 @@ -//! The queue corpus scheduler with weighted queue item selection from aflpp (`https://github.com/AFLplusplus/AFLplusplus/blob/1d4f1e48797c064ee71441ba555b29fc3f467983/src/afl-fuzz-queue.c#L32`) +//! The queue corpus scheduler with weighted queue item selection [from AFL++](https://github.com/AFLplusplus/AFLplusplus/blob/1d4f1e48797c064ee71441ba555b29fc3f467983/src/afl-fuzz-queue.c#L32). //! This queue corpus scheduler needs calibration stage. -use alloc::string::{String, ToString}; use core::marker::PhantomData; use hashbrown::HashMap; -use libafl_bolts::rands::Rand; +use libafl_bolts::{ + rands::Rand, + tuples::{Handle, Handler}, + Named, +}; use serde::{Deserialize, Serialize}; use crate::{ @@ -16,10 +19,10 @@ use crate::{ schedulers::{ powersched::{PowerSchedule, SchedulerMetadata}, testcase_score::{CorpusWeightTestcaseScore, TestcaseScore}, - HasAFLRemovableScheduler, HasAFLSchedulerMetadata, RemovableScheduler, Scheduler, + AflScheduler, RemovableScheduler, Scheduler, }, - state::{HasCorpus, HasMetadata, HasRand, State, UsesState}, - Error, + state::{HasCorpus, HasRand, State, UsesState}, + Error, HasMetadata, }; /// The Metadata for `WeightedScheduler` @@ -92,39 +95,38 @@ libafl_bolts::impl_serdeany!(WeightedScheduleMetadata); /// A corpus scheduler using power schedules with weighted queue item selection algo. #[derive(Clone, Debug)] -pub struct WeightedScheduler { +pub struct WeightedScheduler { + table_invalidated: bool, strat: Option, - map_observer_name: String, + map_observer_ref: Handle, last_hash: usize, phantom: PhantomData<(F, O, S)>, } -impl WeightedScheduler +impl WeightedScheduler where F: TestcaseScore, O: MapObserver, S: HasCorpus + HasMetadata + HasRand, + C: AsRef + Named, { /// Create a new [`WeightedScheduler`] without any power schedule #[must_use] - pub fn new(state: &mut S, map_observer: &O) -> Self { + pub fn new(state: &mut S, map_observer: &C) -> Self { Self::with_schedule(state, map_observer, None) } /// Create a new [`WeightedScheduler`] #[must_use] - pub fn with_schedule(state: &mut S, map_observer: &O, strat: Option) -> Self { - if !state.has_metadata::() { - state.add_metadata(SchedulerMetadata::new(strat)); - } + pub fn with_schedule(state: &mut S, map_observer: &C, strat: Option) -> Self { + let _ = state.metadata_or_insert_with(|| SchedulerMetadata::new(strat)); + let _ = state.metadata_or_insert_with(WeightedScheduleMetadata::new); - if !state.has_metadata::() { - state.add_metadata(WeightedScheduleMetadata::new()); - } Self { strat, - map_observer_name: map_observer.name().to_string(), + map_observer_ref: map_observer.handle(), last_hash: 0, + table_invalidated: true, phantom: PhantomData, } } @@ -220,51 +222,49 @@ where } } -impl UsesState for WeightedScheduler +impl UsesState for WeightedScheduler where S: State, { type State = S; } -impl HasAFLRemovableScheduler for WeightedScheduler -where - F: TestcaseScore, - S: State + HasTestcase + HasMetadata + HasCorpus + HasRand, - O: MapObserver, -{ -} - -impl RemovableScheduler for WeightedScheduler +impl RemovableScheduler for WeightedScheduler where F: TestcaseScore, O: MapObserver, S: HasCorpus + HasMetadata + HasRand + HasTestcase + State, + C: AsRef + Named, { + /// This will *NOT* neutralize the effect of this removed testcase from the global data such as `SchedulerMetadata` fn on_remove( &mut self, - state: &mut Self::State, - idx: CorpusId, - prev: &Option::Input>>, + _state: &mut Self::State, + _idx: CorpusId, + _prev: &Option::Input>>, ) -> Result<(), Error> { - self.on_remove_metadata(state, idx, prev) + self.table_invalidated = true; + Ok(()) } + /// This will *NOT* neutralize the effect of this removed testcase from the global data such as `SchedulerMetadata` fn on_replace( &mut self, - state: &mut Self::State, - idx: CorpusId, - prev: &Testcase<::Input>, + _state: &mut Self::State, + _idx: CorpusId, + _prev: &Testcase<::Input>, ) -> Result<(), Error> { - self.on_replace_metadata(state, idx, prev) + self.table_invalidated = true; + Ok(()) } } -impl HasAFLSchedulerMetadata for WeightedScheduler +impl AflScheduler for WeightedScheduler where F: TestcaseScore, - S: HasCorpus + HasMetadata + HasTestcase + HasRand + State, O: MapObserver, + S: HasCorpus + HasMetadata + HasTestcase + HasRand + State, + C: AsRef + Named, { fn last_hash(&self) -> usize { self.last_hash @@ -274,21 +274,23 @@ where self.last_hash = hash; } - fn map_observer_name(&self) -> &String { - &self.map_observer_name + fn map_observer_ref(&self) -> &Handle { + &self.map_observer_ref } } -impl Scheduler for WeightedScheduler +impl Scheduler for WeightedScheduler where F: TestcaseScore, O: MapObserver, S: HasCorpus + HasMetadata + HasRand + HasTestcase + State, + C: AsRef + Named, { /// Called when a [`Testcase`] is added to the corpus fn on_add(&mut self, state: &mut S, idx: CorpusId) -> Result<(), Error> { self.on_add_metadata(state, idx)?; - self.create_alias_table(state) + self.table_invalidated = true; + Ok(()) } fn on_evaluation( @@ -305,14 +307,20 @@ where #[allow(clippy::similar_names, clippy::cast_precision_loss)] fn next(&mut self, state: &mut S) -> Result { + if self.table_invalidated { + self.create_alias_table(state)?; + self.table_invalidated = false; + } let corpus_counts = state.corpus().count(); if corpus_counts == 0 { - Err(Error::empty(String::from("No entries in corpus"))) + Err(Error::empty( + "No entries in corpus. This often implies the target is not properly instrumented.", + )) } else { let s = random_corpus_id!(state.corpus(), state.rand_mut()); - // Choose a random value between 0.000000000 and 1.000000000 - let probability = state.rand_mut().between(0, 1000000000) as f64 / 1000000000_f64; + // Choose a random value between 0.0 and 1.0 + let probability = state.rand_mut().next_float(); let wsmeta = state.metadata_mut::()?; @@ -355,5 +363,5 @@ where } } -/// The standard corpus weight, same as aflpp -pub type StdWeightedScheduler = WeightedScheduler, O, S>; +/// The standard corpus weight, same as in `AFL++` +pub type StdWeightedScheduler = WeightedScheduler, O, S>; diff --git a/libafl/src/stages/calibrate.rs b/libafl/src/stages/calibrate.rs index 6e08ff9952..1a4c796a93 100644 --- a/libafl/src/stages/calibrate.rs +++ b/libafl/src/stages/calibrate.rs @@ -1,28 +1,26 @@ //! The calibration stage. The fuzzer measures the average exec time and the bitmap size. -use alloc::{ - string::{String, ToString}, - vec::Vec, -}; +use alloc::{borrow::Cow, vec::Vec}; use core::{fmt::Debug, marker::PhantomData, time::Duration}; use hashbrown::HashSet; -use libafl_bolts::{current_time, impl_serdeany, AsIter, Named}; +use libafl_bolts::{current_time, impl_serdeany, tuples::Handle, AsIter, Named}; use num_traits::Bounded; use serde::{Deserialize, Serialize}; use crate::{ - corpus::{Corpus, HasCurrentCorpusIdx, SchedulerTestcaseMetadata}, + corpus::{Corpus, SchedulerTestcaseMetadata}, events::{Event, EventFirer, LogSeverity}, executors::{Executor, ExitKind, HasObservers}, - feedbacks::{map::MapFeedbackMetadata, HasObserverName}, + feedbacks::{map::MapFeedbackMetadata, HasObserverReference}, fuzzer::Evaluator, + inputs::UsesInput, monitors::{AggregatorOps, UserStats, UserStatsValue}, - observers::{MapObserver, ObserversTuple, UsesObserver}, + observers::{MapObserver, ObserversTuple}, schedulers::powersched::SchedulerMetadata, - stages::Stage, - state::{HasCorpus, HasExecutions, HasMetadata, HasNamedMetadata, State, UsesState}, - Error, + stages::{ExecutionCountRestartHelper, Stage}, + state::{HasCorpus, HasCurrentTestcase, HasExecutions, State, UsesState}, + Error, HasMetadata, HasNamedMetadata, }; /// The metadata to keep unstable entries @@ -64,36 +62,37 @@ impl UnstableEntriesMetadata { /// The calibration stage will measure the average exec time and the target's stability for this input. #[derive(Clone, Debug)] -pub struct CalibrationStage { - map_observer_name: String, - map_name: String, +pub struct CalibrationStage { + map_observer_ref: Handle, + map_name: Cow<'static, str>, stage_max: usize, + /// If we should track stability track_stability: bool, + restart_helper: ExecutionCountRestartHelper, phantom: PhantomData<(O, OT, S)>, } const CAL_STAGE_START: usize = 4; // AFL++'s CAL_CYCLES_FAST + 1 const CAL_STAGE_MAX: usize = 8; // AFL++'s CAL_CYCLES + 1 -impl UsesState for CalibrationStage +impl UsesState for CalibrationStage where S: State, { type State = S; } -impl Stage for CalibrationStage +impl Stage for CalibrationStage where E: Executor + HasObservers, EM: EventFirer, O: MapObserver, + C: AsRef, for<'de> ::Entry: Serialize + Deserialize<'de> + 'static, OT: ObserversTuple, E::State: HasCorpus + HasMetadata + HasNamedMetadata + HasExecutions, Z: Evaluator, { - type Progress = (); // TODO stage may be resumed, but how? - #[inline] #[allow( clippy::let_and_return, @@ -107,25 +106,21 @@ where state: &mut E::State, mgr: &mut EM, ) -> Result<(), Error> { - let Some(corpus_idx) = state.current_corpus_idx()? else { - return Err(Error::illegal_state( - "state is not currently processing a corpus index", - )); - }; - // Run this stage only once for each corpus entry and only if we haven't already inspected it { - let corpus = state.corpus().get(corpus_idx)?.borrow(); + let testcase = state.current_testcase()?; // println!("calibration; corpus.scheduled_count() : {}", corpus.scheduled_count()); - if corpus.scheduled_count() > 0 { + if testcase.scheduled_count() > 0 { return Ok(()); } } let mut iter = self.stage_max; + // If we restarted after a timeout or crash, do less iterations. + iter -= usize::try_from(self.restart_helper.execs_since_progress_start(state)?)?; - let input = state.corpus().cloned_input_for_id(corpus_idx)?; + let input = state.current_input_cloned()?; // Run once to get the initial calibration map executor.observers_mut().pre_exec_all(state, &input)?; @@ -149,10 +144,8 @@ where .observers_mut() .post_exec_all(state, &input, &exit_kind)?; - let map_first = &executor - .observers() - .match_name::(&self.map_observer_name) - .ok_or_else(|| Error::key_not_found("MapObserver not found".to_string()))? + let map_first = &executor.observers()[&self.map_observer_ref] + .as_ref() .to_vec(); let mut unstable_entries: Vec = vec![]; @@ -163,7 +156,7 @@ where let mut has_errors = false; while i < iter { - let input = state.corpus().cloned_input_for_id(corpus_idx)?; + let input = state.current_input_cloned()?; executor.observers_mut().pre_exec_all(state, &input)?; start = current_time(); @@ -192,10 +185,8 @@ where .post_exec_all(state, &input, &exit_kind)?; if self.track_stability { - let map = &executor - .observers() - .match_name::(&self.map_observer_name) - .ok_or_else(|| Error::key_not_found("MapObserver not found".to_string()))? + let map = &executor.observers()[&self.map_observer_ref] + .as_ref() .to_vec(); let history_map = &mut state @@ -248,13 +239,12 @@ where // If weighted scheduler or powerscheduler is used, update it if state.has_metadata::() { - let map = executor - .observers() - .match_name::(&self.map_observer_name) - .ok_or_else(|| Error::key_not_found("MapObserver not found".to_string()))?; - - let bitmap_size = map.count_bytes(); + let observers = executor.observers(); + let map = observers[&self.map_observer_ref].as_ref(); + let mut bitmap_size = map.count_bytes(); + assert!(bitmap_size != 0); + bitmap_size = bitmap_size.max(1); // just don't make it 0 because we take log2 of it later. let psmeta = state .metadata_map_mut() .get_mut::() @@ -267,7 +257,7 @@ where psmeta.set_bitmap_size_log(psmeta.bitmap_size_log() + libm::log2(bitmap_size as f64)); psmeta.set_bitmap_entries(psmeta.bitmap_entries() + 1); - let mut testcase = state.corpus().get(corpus_idx)?.borrow_mut(); + let mut testcase = state.current_testcase_mut()?; testcase.set_exec_time(total_time / (iter as u32)); // log::trace!("time: {:#?}", testcase.exec_time()); @@ -300,7 +290,7 @@ where data.set_handicap(handicap); } - *state.executions_mut() += i; + *state.executions_mut() += u64::try_from(i).unwrap(); // Send the stability event to the broker if unstable_found { @@ -310,7 +300,7 @@ where mgr.fire( state, Event::UpdateUserStats { - name: "stability".to_string(), + name: Cow::from("stability"), value: UserStats::new( UserStatsValue::Ratio( (map_len - unstable_entries) as u64, @@ -326,26 +316,38 @@ where Ok(()) } + + fn restart_progress_should_run(&mut self, state: &mut Self::State) -> Result { + // TODO: Make sure this is the correct way / there may be a better way? + self.restart_helper.restart_progress_should_run(state) + } + + fn clear_restart_progress(&mut self, state: &mut Self::State) -> Result<(), Error> { + // TODO: Make sure this is the correct way / there may be a better way? + self.restart_helper.clear_restart_progress(state) + } } -impl CalibrationStage +impl CalibrationStage where O: MapObserver, + for<'it> O: AsIter<'it, Item = O::Entry>, + C: AsRef, OT: ObserversTuple, - S: HasCorpus + HasMetadata + HasNamedMetadata, + S: UsesInput + HasNamedMetadata, { /// Create a new [`CalibrationStage`]. #[must_use] pub fn new(map_feedback: &F) -> Self where - F: HasObserverName + Named + UsesObserver, - for<'it> O: AsIter<'it, Item = O::Entry>, + F: HasObserverReference + Named, { Self { - map_observer_name: map_feedback.observer_name().to_string(), - map_name: map_feedback.name().to_string(), + map_observer_ref: map_feedback.observer_ref().clone(), + map_name: map_feedback.name().clone(), stage_max: CAL_STAGE_START, track_stability: true, + restart_helper: ExecutionCountRestartHelper::default(), phantom: PhantomData, } } @@ -354,14 +356,14 @@ where #[must_use] pub fn ignore_stability(map_feedback: &F) -> Self where - F: HasObserverName + Named + UsesObserver, - for<'it> O: AsIter<'it, Item = O::Entry>, + F: HasObserverReference + Named, { Self { - map_observer_name: map_feedback.observer_name().to_string(), - map_name: map_feedback.name().to_string(), + map_observer_ref: map_feedback.observer_ref().clone(), + map_name: map_feedback.name().clone(), stage_max: CAL_STAGE_START, track_stability: false, + restart_helper: ExecutionCountRestartHelper::default(), phantom: PhantomData, } } diff --git a/libafl/src/stages/colorization.rs b/libafl/src/stages/colorization.rs index 6370472e9f..e57420f9b3 100644 --- a/libafl/src/stages/colorization.rs +++ b/libafl/src/stages/colorization.rs @@ -1,24 +1,23 @@ //! The colorization stage from `colorization()` in afl++ -use alloc::{ - collections::binary_heap::BinaryHeap, - string::{String, ToString}, - vec::Vec, -}; +use alloc::{borrow::Cow, collections::binary_heap::BinaryHeap, vec::Vec}; use core::{cmp::Ordering, fmt::Debug, marker::PhantomData, ops::Range}; -use libafl_bolts::{rands::Rand, tuples::MatchName}; +use libafl_bolts::{ + rands::Rand, + tuples::{Handle, Handler}, + Named, +}; use serde::{Deserialize, Serialize}; use crate::{ - corpus::{Corpus, HasCurrentCorpusIdx}, events::EventFirer, executors::{Executor, HasObservers}, inputs::HasBytesVec, mutators::mutations::buffer_copy, observers::{MapObserver, ObserversTuple}, - stages::Stage, - state::{HasCorpus, HasMetadata, HasRand, UsesState}, - Error, + stages::{RetryRestartHelper, Stage}, + state::{HasCorpus, HasCurrentTestcase, HasRand, UsesState}, + Error, HasMetadata, HasNamedMetadata, }; // Bigger range is better @@ -55,30 +54,38 @@ impl Ord for Earlier { /// The mutational stage using power schedules #[derive(Clone, Debug)] -pub struct ColorizationStage { - map_observer_name: String, +pub struct ColorizationStage { + map_observer_ref: Handle, #[allow(clippy::type_complexity)] - phantom: PhantomData<(E, EM, O, Z)>, + phantom: PhantomData<(E, EM, O, E, Z)>, } -impl UsesState for ColorizationStage +impl UsesState for ColorizationStage where E: UsesState, { type State = E::State; } -impl Stage for ColorizationStage +impl Named for ColorizationStage +where + E: UsesState, +{ + fn name(&self) -> &Cow<'static, str> { + self.map_observer_ref.name() + } +} + +impl Stage for ColorizationStage where EM: UsesState + EventFirer, E: HasObservers + Executor, - E::State: HasCorpus + HasMetadata + HasRand, + E::State: HasCorpus + HasMetadata + HasRand + HasNamedMetadata, E::Input: HasBytesVec, O: MapObserver, + C: AsRef + Named, Z: UsesState, { - type Progress = (); // TODO this stage needs resume - #[inline] #[allow(clippy::let_and_return)] fn perform( @@ -89,10 +96,20 @@ where manager: &mut EM, ) -> Result<(), Error> { // Run with the mutated input - Self::colorize(fuzzer, executor, state, manager, &self.map_observer_name)?; + Self::colorize(fuzzer, executor, state, manager, &self.map_observer_ref)?; Ok(()) } + + fn restart_progress_should_run(&mut self, state: &mut Self::State) -> Result { + // TODO this stage needs a proper resume + RetryRestartHelper::restart_progress_should_run(state, self, 3) + } + + fn clear_restart_progress(&mut self, state: &mut Self::State) -> Result<(), Error> { + // TODO this stage needs a proper resume + RetryRestartHelper::clear_restart_progress(state, self) + } } /// Store the taint and the input @@ -134,10 +151,11 @@ impl TaintMetadata { libafl_bolts::impl_serdeany!(TaintMetadata); -impl ColorizationStage +impl ColorizationStage where EM: UsesState + EventFirer, O: MapObserver, + C: AsRef + Named, E: HasObservers + Executor, E::State: HasCorpus + HasMetadata + HasRand, E::Input: HasBytesVec, @@ -150,15 +168,9 @@ where executor: &mut E, state: &mut E::State, manager: &mut EM, - name: &str, + obs_ref: &Handle, ) -> Result { - let Some(corpus_idx) = state.current_corpus_idx()? else { - return Err(Error::illegal_state( - "state is not currently processing a corpus index", - )); - }; - - let mut input = state.corpus().cloned_input_for_id(corpus_idx)?; + let mut input = state.current_input_cloned()?; // The backup of the input let backup = input.clone(); // This is the buffer we'll randomly mutate during type_replace @@ -171,7 +183,7 @@ where // Idea: No need to do this every time let orig_hash = - Self::get_raw_map_hash_run(fuzzer, executor, state, manager, consumed_input, name)?; + Self::get_raw_map_hash_run(fuzzer, executor, state, manager, consumed_input, obs_ref)?; let changed_bytes = changed.bytes_mut(); let input_len = changed_bytes.len(); @@ -216,7 +228,7 @@ where state, manager, consumed_input, - name, + obs_ref, )?; if orig_hash == changed_hash { @@ -287,9 +299,9 @@ where #[must_use] /// Creates a new [`ColorizationStage`] - pub fn new(map_observer_name: &O) -> Self { + pub fn new(map_observer: &C) -> Self { Self { - map_observer_name: map_observer_name.name().to_string(), + map_observer_ref: map_observer.handle(), phantom: PhantomData, } } @@ -301,18 +313,16 @@ where state: &mut E::State, manager: &mut EM, input: E::Input, - name: &str, + obs_ref: &Handle, ) -> Result { executor.observers_mut().pre_exec_all(state, &input)?; let exit_kind = executor.run_target(fuzzer, state, manager, &input)?; - let observer = executor - .observers() - .match_name::(name) - .ok_or_else(|| Error::key_not_found("MapObserver not found".to_string()))?; + let observers = executor.observers(); + let observer = observers[obs_ref].as_ref(); - let hash = observer.hash() as usize; + let hash = observer.hash_simple() as usize; executor .observers_mut() diff --git a/libafl/src/stages/concolic.rs b/libafl/src/stages/concolic.rs index cabd92c83f..54e8f1dd10 100644 --- a/libafl/src/stages/concolic.rs +++ b/libafl/src/stages/concolic.rs @@ -2,104 +2,103 @@ //! and use the results for fuzzer input and mutations. //! -use alloc::string::String; +use alloc::borrow::Cow; +#[cfg(feature = "concolic_mutation")] +use alloc::{string::ToString, vec::Vec}; #[cfg(feature = "concolic_mutation")] -use alloc::{borrow::ToOwned, string::ToString, vec::Vec}; use core::marker::PhantomData; -use super::{Stage, TracingStage}; +use libafl_bolts::{ + tuples::{Handle, MatchNameRef}, + Named, +}; + +#[cfg(all(feature = "concolic_mutation", feature = "introspection"))] +use crate::monitors::PerfFeature; #[cfg(all(feature = "introspection", feature = "concolic_mutation"))] use crate::state::HasClientPerfMonitor; -#[cfg(feature = "concolic_mutation")] -use crate::state::State; use crate::{ - corpus::Corpus, executors::{Executor, HasObservers}, observers::concolic::ConcolicObserver, - state::{HasCorpus, HasExecutions, HasMetadata}, - Error, + stages::{RetryRestartHelper, Stage, TracingStage}, + state::{HasCorpus, HasCurrentTestcase, HasExecutions, UsesState}, + Error, HasMetadata, HasNamedMetadata, +}; +#[cfg(feature = "concolic_mutation")] +use crate::{ + inputs::HasBytesVec, + mark_feature_time, + observers::concolic::{ConcolicMetadata, SymExpr, SymExprRef}, + stages::ExecutionCountRestartHelper, + start_timer, + state::State, + Evaluator, }; /// Wraps a [`TracingStage`] to add concolic observing. #[derive(Clone, Debug)] -pub struct ConcolicTracingStage { +pub struct ConcolicTracingStage<'a, EM, TE, Z> { inner: TracingStage, - observer_name: String, + obs_ref: Handle>, } -impl UsesState for ConcolicTracingStage +impl UsesState for ConcolicTracingStage<'_, EM, TE, Z> where TE: UsesState, { type State = TE::State; } -impl Stage for ConcolicTracingStage +impl Named for ConcolicTracingStage<'_, EM, TE, Z> { + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("ConcolicTracingStage"); + &NAME + } +} + +impl Stage for ConcolicTracingStage<'_, EM, TE, Z> where E: UsesState, EM: UsesState, TE: Executor + HasObservers, - TE::State: HasExecutions + HasCorpus, + TE::State: HasExecutions + HasCorpus + HasNamedMetadata, Z: UsesState, { - type Progress = (); // stage cannot be resumed - #[inline] fn perform( &mut self, fuzzer: &mut Z, - executor: &mut E, + _executor: &mut E, state: &mut TE::State, manager: &mut EM, ) -> Result<(), Error> { - let Some(corpus_idx) = state.current_corpus_idx()? else { - return Err(Error::illegal_state( - "state is not currently processing a corpus index", - )); - }; - - self.inner.perform(fuzzer, executor, state, manager)?; - if let Some(observer) = self - .inner - .executor() - .observers() - .match_name::(&self.observer_name) - { + self.inner.trace(fuzzer, state, manager)?; + if let Some(observer) = self.inner.executor().observers().get(&self.obs_ref) { let metadata = observer.create_metadata_from_current_map(); state - .corpus_mut() - .get(corpus_idx) - .unwrap() - .borrow_mut() + .current_testcase_mut()? .metadata_map_mut() .insert(metadata); } Ok(()) } -} -impl ConcolicTracingStage { - /// Creates a new default tracing stage using the given [`Executor`], observing traces from a [`ConcolicObserver`] with the given name. - pub fn new(inner: TracingStage, observer_name: String) -> Self { - Self { - inner, - observer_name, - } + fn restart_progress_should_run(&mut self, state: &mut Self::State) -> Result { + RetryRestartHelper::restart_progress_should_run(state, self, 3) } -} -use libafl_bolts::tuples::MatchName; + fn clear_restart_progress(&mut self, state: &mut Self::State) -> Result<(), Error> { + RetryRestartHelper::clear_restart_progress(state, self) + } +} -#[cfg(all(feature = "concolic_mutation", feature = "introspection"))] -use crate::monitors::PerfFeature; -use crate::{corpus::HasCurrentCorpusIdx, state::UsesState}; -#[cfg(feature = "concolic_mutation")] -use crate::{ - inputs::HasBytesVec, - mark_feature_time, - observers::concolic::{ConcolicMetadata, SymExpr, SymExprRef}, - start_timer, Evaluator, -}; +impl<'a, EM, TE, Z> ConcolicTracingStage<'a, EM, TE, Z> { + /// Creates a new default tracing stage using the given [`Executor`], observing traces from a + /// [`ConcolicObserver`] with the given name. + pub fn new(inner: TracingStage, obs_ref: Handle>) -> Self { + Self { inner, obs_ref } + } +} #[cfg(feature = "concolic_mutation")] #[allow(clippy::too_many_lines)] @@ -176,6 +175,7 @@ fn generate_mutations(iter: impl Iterator) -> Vec< Some(BV::from_u64(&ctx, value, u32::from(bits)).into()) } SymExpr::Integer128 { high: _, low: _ } => todo!(), + SymExpr::IntegerFromBuffer {} => todo!(), SymExpr::NullPointer => Some(BV::from_u64(&ctx, 0, usize::BITS).into()), SymExpr::True => Some(Bool::from_bool(&ctx, true).into()), SymExpr::False => Some(Bool::from_bool(&ctx, false).into()), @@ -344,9 +344,12 @@ fn generate_mutations(iter: impl Iterator) -> Vec< } /// A mutational stage that uses Z3 to solve concolic constraints attached to the [`crate::corpus::Testcase`] by the [`ConcolicTracingStage`]. +#[cfg(feature = "concolic_mutation")] #[derive(Clone, Debug)] pub struct SimpleConcolicMutationalStage { - _phantom: PhantomData, + /// The helper keeps track of progress for timeouting/restarting targets + restart_helper: ExecutionCountRestartHelper, + phantom: PhantomData, } #[cfg(feature = "concolic_mutation")] @@ -364,10 +367,8 @@ where EM: UsesState, Z: Evaluator, Z::Input: HasBytesVec, - Z::State: State + HasExecutions + HasCorpus, + Z::State: State + HasExecutions + HasCorpus + HasMetadata, { - type Progress = (); // TODO we need a resume for this type - #[inline] fn perform( &mut self, @@ -376,30 +377,25 @@ where state: &mut Z::State, manager: &mut EM, ) -> Result<(), Error> { - let Some(corpus_idx) = state.current_corpus_idx()? else { - return Err(Error::illegal_state( - "state is not currently processing a corpus index", - )); - }; + { + start_timer!(state); + mark_feature_time!(state, PerfFeature::GetInputFromCorpus); + } + let testcase = state.current_testcase()?.clone(); - start_timer!(state); - let testcase = state.corpus().get(corpus_idx)?.clone(); - mark_feature_time!(state, PerfFeature::GetInputFromCorpus); + let mutations = testcase.metadata::().ok().map(|meta| { + start_timer!(state); + let mutations = { generate_mutations(meta.iter_messages()) }; + mark_feature_time!(state, PerfFeature::Mutate); + mutations + }); - let mutations = - if let Some(meta) = testcase.borrow().metadata_map().get::() { - start_timer!(state); - let mutations = generate_mutations(meta.iter_messages()); - mark_feature_time!(state, PerfFeature::Mutate); - Some(mutations) - } else { - None - }; + let post_restart_skip_cnt = + usize::try_from(self.restart_helper.execs_since_progress_start(state)?)?; if let Some(mutations) = mutations { - let input = { testcase.borrow().input().as_ref().unwrap().clone() }; - for mutation in mutations { - let mut input_copy = input.to_owned(); + for mutation in mutations.into_iter().skip(post_restart_skip_cnt) { + let mut input_copy = state.current_input_cloned()?; for (index, new_byte) in mutation { input_copy.bytes_mut()[index] = new_byte; } @@ -409,12 +405,24 @@ where } Ok(()) } + + #[inline] + fn restart_progress_should_run(&mut self, state: &mut Self::State) -> Result { + self.restart_helper.restart_progress_should_run(state) + } + + #[inline] + fn clear_restart_progress(&mut self, state: &mut Self::State) -> Result<(), Error> { + self.restart_helper.clear_restart_progress(state) + } } +#[cfg(feature = "concolic_mutation")] impl Default for SimpleConcolicMutationalStage { fn default() -> Self { Self { - _phantom: PhantomData, + restart_helper: ExecutionCountRestartHelper::default(), + phantom: PhantomData, } } } diff --git a/libafl/src/stages/dump.rs b/libafl/src/stages/dump.rs index 85cccdf89c..be1bd9cfc1 100644 --- a/libafl/src/stages/dump.rs +++ b/libafl/src/stages/dump.rs @@ -11,8 +11,8 @@ use crate::{ corpus::{Corpus, CorpusId}, inputs::UsesInput, stages::Stage, - state::{HasCorpus, HasMetadata, HasRand, HasSolutions, UsesState}, - Error, + state::{HasCorpus, HasRand, HasSolutions, UsesState}, + Error, HasMetadata, }; /// Metadata used to store information about disk dump indexes for names @@ -52,8 +52,6 @@ where Z: UsesState, Z::State: HasCorpus + HasSolutions + HasRand + HasMetadata, { - type Progress = (); // if this fails, we have bigger problems - #[inline] fn perform( &mut self, @@ -115,6 +113,18 @@ where Ok(()) } + + #[inline] + fn restart_progress_should_run(&mut self, _state: &mut Self::State) -> Result { + // Not executing the target, so restart safety is not needed + Ok(true) + } + + #[inline] + fn clear_restart_progress(&mut self, _state: &mut Self::State) -> Result<(), Error> { + // Not executing the target, so restart safety is not needed + Ok(()) + } } impl DumpToDiskStage @@ -132,13 +142,19 @@ where let corpus_dir = corpus_dir.into(); if let Err(e) = fs::create_dir(&corpus_dir) { if !corpus_dir.is_dir() { - return Err(Error::file(e)); + return Err(Error::os_error( + e, + format!("Error creating directory {corpus_dir:?}"), + )); } } let solutions_dir = solutions_dir.into(); if let Err(e) = fs::create_dir(&solutions_dir) { if !corpus_dir.is_dir() { - return Err(Error::file(e)); + return Err(Error::os_error( + e, + format!("Error creating directory {solutions_dir:?}"), + )); } } Ok(Self { diff --git a/libafl/src/stages/generalization.rs b/libafl/src/stages/generalization.rs index 72a3f306c0..8f69e53380 100644 --- a/libafl/src/stages/generalization.rs +++ b/libafl/src/stages/generalization.rs @@ -1,12 +1,12 @@ //! The tracing stage can trace the target and enrich a [`crate::corpus::Testcase`] with metadata, for example for `CmpLog`. -use alloc::{ - string::{String, ToString}, - vec::Vec, -}; +use alloc::{borrow::Cow, vec::Vec}; use core::{fmt::Debug, marker::PhantomData}; -use libafl_bolts::AsSlice; +use libafl_bolts::{ + tuples::{Handle, Handler}, + AsSlice, Named, +}; use crate::{ corpus::{Corpus, HasCurrentCorpusIdx}, @@ -14,11 +14,12 @@ use crate::{ feedbacks::map::MapNoveltiesMetadata, inputs::{BytesInput, GeneralizedInputMetadata, GeneralizedItem, HasBytesVec, UsesInput}, mark_feature_time, - observers::{MapObserver, ObserversTuple}, - stages::Stage, + observers::{CanTrack, MapObserver, ObserversTuple}, + require_novelties_tracking, + stages::{RetryRestartHelper, Stage}, start_timer, - state::{HasCorpus, HasExecutions, HasMetadata, UsesState}, - Error, + state::{HasCorpus, HasExecutions, UsesState}, + Error, HasMetadata, HasNamedMetadata, }; #[cfg(feature = "introspection")] use crate::{monitors::PerfFeature, state::HasClientPerfMonitor}; @@ -41,13 +42,20 @@ fn find_next_char(list: &[Option], mut idx: usize, ch: u8) -> usize { /// A stage that runs a tracer executor #[derive(Clone, Debug)] -pub struct GeneralizationStage { - map_observer_name: String, +pub struct GeneralizationStage { + map_observer_ref: Handle, #[allow(clippy::type_complexity)] phantom: PhantomData<(EM, O, OT, Z)>, } -impl UsesState for GeneralizationStage +impl Named for GeneralizationStage { + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("GeneralizationStage"); + &NAME + } +} + +impl UsesState for GeneralizationStage where EM: UsesState, EM::State: UsesInput, @@ -55,17 +63,17 @@ where type State = EM::State; } -impl Stage for GeneralizationStage +impl Stage for GeneralizationStage where O: MapObserver, + C: CanTrack + AsRef + Named, E: Executor + HasObservers, E::Observers: ObserversTuple, - E::State: UsesInput + HasExecutions + HasMetadata + HasCorpus, + E::State: + UsesInput + HasExecutions + HasMetadata + HasCorpus + HasNamedMetadata, EM: UsesState, Z: UsesState, { - type Progress = (); // TODO this stage needs a resume - #[inline] #[allow(clippy::too_many_lines)] fn perform( @@ -312,29 +320,34 @@ where Ok(()) } + + #[inline] + fn restart_progress_should_run(&mut self, state: &mut Self::State) -> Result { + // TODO: We need to be able to resume better if something crashes or times out + RetryRestartHelper::restart_progress_should_run(state, self, 3) + } + + #[inline] + fn clear_restart_progress(&mut self, state: &mut Self::State) -> Result<(), Error> { + // TODO: We need to be able to resume better if something crashes or times out + RetryRestartHelper::clear_restart_progress(state, self) + } } -impl GeneralizationStage +impl GeneralizationStage where EM: UsesState, O: MapObserver, + C: CanTrack + AsRef + Named, OT: ObserversTuple, EM::State: UsesInput + HasExecutions + HasMetadata + HasCorpus, { /// Create a new [`GeneralizationStage`]. #[must_use] - pub fn new(map_observer: &O) -> Self { - Self { - map_observer_name: map_observer.name().to_string(), - phantom: PhantomData, - } - } - - /// Create a new [`GeneralizationStage`] from name - #[must_use] - pub fn from_name(map_observer_name: &str) -> Self { + pub fn new(map_observer: &C) -> Self { + require_novelties_tracking!("GeneralizationStage", C); Self { - map_observer_name: map_observer_name.to_string(), + map_observer_ref: map_observer.handle(), phantom: PhantomData, } } @@ -368,10 +381,8 @@ where .post_exec_all(state, input, &exit_kind)?; mark_feature_time!(state, PerfFeature::PostExecObservers); - let cnt = executor - .observers() - .match_name::(&self.map_observer_name) - .ok_or_else(|| Error::key_not_found("MapObserver not found".to_string()))? + let cnt = executor.observers()[&self.map_observer_ref] + .as_ref() .how_many_set(novelties); Ok(cnt == novelties.len()) diff --git a/libafl/src/stages/generation.rs b/libafl/src/stages/generation.rs new file mode 100644 index 0000000000..8df777a28a --- /dev/null +++ b/libafl/src/stages/generation.rs @@ -0,0 +1,72 @@ +//! A [`Stage`] that generates a single input via a +//! [`crate::generators::Generator`] and evaluates it using the fuzzer, possibly +//! adding it to the corpus. + +use core::marker::PhantomData; + +use crate::{ + generators::Generator, + inputs::UsesInput, + stages::Stage, + state::{HasCorpus, HasRand, UsesState}, + Error, Evaluator, +}; + +/// A [`Stage`] that generates a single input via a [`Generator`] and evaluates +/// it using the fuzzer, possibly adding it to the corpus. +/// +/// This stage can be used to construct black-box (e.g., grammar-based) fuzzers. +#[derive(Debug)] +pub struct GenStage(G, PhantomData) +where + Z: UsesState, + G: Generator<<::State as UsesInput>::Input, Z::State>; + +impl GenStage +where + Z: UsesState, + G: Generator<<::State as UsesInput>::Input, Z::State>, +{ + /// Create a new [`GenStage`]. + pub fn new(g: G) -> Self { + Self(g, PhantomData) + } +} + +impl UsesState for GenStage +where + Z: UsesState, + G: Generator<<::State as UsesInput>::Input, Z::State>, +{ + type State = Z::State; +} + +impl Stage for GenStage +where + E: UsesState, + EM: UsesState, + Z: Evaluator, + Z::State: HasCorpus + HasRand, + G: Generator<<::State as UsesInput>::Input, Z::State>, +{ + #[inline] + fn perform( + &mut self, + fuzzer: &mut Z, + executor: &mut E, + state: &mut Z::State, + manager: &mut EM, + ) -> Result<(), Error> { + let input = self.0.generate(state)?; + fuzzer.evaluate_input(state, executor, manager, input)?; + Ok(()) + } + + fn restart_progress_should_run(&mut self, _state: &mut Self::State) -> Result { + Ok(true) + } + + fn clear_restart_progress(&mut self, _state: &mut Self::State) -> Result<(), Error> { + Ok(()) + } +} diff --git a/libafl/src/stages/logics.rs b/libafl/src/stages/logics.rs index 2ae9e6b651..9de107bd2a 100644 --- a/libafl/src/stages/logics.rs +++ b/libafl/src/stages/logics.rs @@ -3,36 +3,31 @@ use core::marker::PhantomData; use crate::{ - stages::{HasCurrentStage, HasNestedStageStatus, Stage, StageProgress, StagesTuple}, + stages::{HasCurrentStage, HasNestedStageStatus, Stage, StagesTuple}, state::UsesState, Error, }; /// Progress for nested stages. This merely enters/exits the inner stage's scope. #[derive(Debug)] -pub struct NestedStageProgress; +pub struct NestedStageRestartHelper; -impl StageProgress for NestedStageProgress -where - S: HasNestedStageStatus, -{ - fn initialize_progress(state: &mut S) -> Result<(), Error> { +impl NestedStageRestartHelper { + fn restart_progress_should_run(state: &mut S, _stage: &ST) -> Result + where + S: HasNestedStageStatus, + { state.enter_inner_stage()?; - Ok(()) + Ok(true) } - fn clear_progress(state: &mut S) -> Result<(), Error> { + fn clear_restart_progress(state: &mut S, _stage: &ST) -> Result<(), Error> + where + S: HasNestedStageStatus, + { state.exit_inner_stage()?; Ok(()) } - - fn progress(_state: &S) -> Result<&Self, Error> { - unimplemented!("NestedStageProgress should not be queried") - } - - fn progress_mut(_state: &mut S) -> Result<&mut Self, Error> { - unimplemented!("NestedStageProgress should not be queried") - } } #[derive(Debug)] @@ -70,8 +65,6 @@ where Z: UsesState, E::State: HasNestedStageStatus, { - type Progress = NestedStageProgress; - fn perform( &mut self, fuzzer: &mut Z, @@ -86,6 +79,14 @@ where Ok(()) } + + fn restart_progress_should_run(&mut self, state: &mut Self::State) -> Result { + NestedStageRestartHelper::restart_progress_should_run(state, self) + } + + fn clear_restart_progress(&mut self, state: &mut Self::State) -> Result<(), Error> { + NestedStageRestartHelper::clear_restart_progress(state, self) + } } impl WhileStage @@ -142,8 +143,6 @@ where Z: UsesState, E::State: HasNestedStageStatus, { - type Progress = NestedStageProgress; - fn perform( &mut self, fuzzer: &mut Z, @@ -157,6 +156,14 @@ where } Ok(()) } + + fn restart_progress_should_run(&mut self, state: &mut Self::State) -> Result { + NestedStageRestartHelper::restart_progress_should_run(state, self) + } + + fn clear_restart_progress(&mut self, state: &mut Self::State) -> Result<(), Error> { + NestedStageRestartHelper::clear_restart_progress(state, self) + } } impl IfStage @@ -217,8 +224,6 @@ where Z: UsesState, E::State: HasNestedStageStatus, { - type Progress = NestedStageProgress; - fn perform( &mut self, fuzzer: &mut Z, @@ -252,6 +257,14 @@ where Ok(()) } + + fn restart_progress_should_run(&mut self, state: &mut Self::State) -> Result { + NestedStageRestartHelper::restart_progress_should_run(state, self) + } + + fn clear_restart_progress(&mut self, state: &mut Self::State) -> Result<(), Error> { + NestedStageRestartHelper::clear_restart_progress(state, self) + } } impl IfElseStage @@ -305,8 +318,6 @@ where Z: UsesState, E::State: HasNestedStageStatus, { - type Progress = NestedStageProgress; - fn perform( &mut self, fuzzer: &mut Z, @@ -320,6 +331,14 @@ where Ok(()) } } + + fn restart_progress_should_run(&mut self, state: &mut Self::State) -> Result { + NestedStageRestartHelper::restart_progress_should_run(state, self) + } + + fn clear_restart_progress(&mut self, state: &mut Self::State) -> Result<(), Error> { + NestedStageRestartHelper::clear_restart_progress(state, self) + } } impl OptionalStage @@ -356,178 +375,3 @@ where } } } - -#[cfg(test)] -mod test { - use core::{cell::RefCell, marker::PhantomData}; - - use libafl_bolts::{tuples::tuple_list, Error}; - - use crate::{ - inputs::NopInput, - stages::{ - test::{test_resume, test_resume_stages}, - ClosureStage, IfElseStage, IfStage, Stage, WhileStage, - }, - state::{test::test_std_state, State, UsesState}, - }; - - #[test] - fn check_resumability_while() { - let once = RefCell::new(true); - let (completed, stages) = test_resume_stages(); - let whilestage = WhileStage::new(|_, _, _, _| Ok(once.replace(false)), stages); - let resetstage = ClosureStage::new(|_, _, _, _| { - once.replace(true); - Ok(()) - }); - - let mut state = test_std_state::(); - - test_resume(&completed, &mut state, tuple_list!(whilestage, resetstage)); - } - - #[test] - fn check_resumability_if() { - let once = RefCell::new(true); - let (completed, stages) = test_resume_stages(); - let ifstage = IfStage::new(|_, _, _, _| Ok(once.replace(false)), stages); - let resetstage = ClosureStage::new(|_, _, _, _| { - once.replace(true); - Ok(()) - }); - - let mut state = test_std_state::(); - - test_resume(&completed, &mut state, tuple_list!(ifstage, resetstage)); - } - - #[test] - fn check_resumability_if_deep() { - let (completed, stages) = test_resume_stages(); - let ifstage = IfStage::new( - |_, _, _, _| Ok(true), - tuple_list!(IfStage::new( - |_, _, _, _| Ok(true), - tuple_list!(IfStage::new( - |_, _, _, _| Ok(true), - tuple_list!(IfStage::new( - |_, _, _, _| Ok(true), - tuple_list!(IfStage::new(|_, _, _, _| Ok(true), stages),), - ),), - )) - )), - ); - - let mut state = test_std_state::(); - - test_resume(&completed, &mut state, tuple_list!(ifstage)); - } - - #[derive(Debug)] - pub struct PanicStage { - phantom: PhantomData, - } - - impl PanicStage { - pub fn new() -> Self { - Self { - phantom: PhantomData, - } - } - } - - impl UsesState for PanicStage - where - S: State, - { - type State = S; - } - - impl Stage for PanicStage - where - E: UsesState, - EM: UsesState, - Z: UsesState, - { - type Progress = (); - - fn perform( - &mut self, - _fuzzer: &mut Z, - _executor: &mut E, - _state: &mut Self::State, - _manager: &mut EM, - ) -> Result<(), Error> { - panic!("Test failed; panic stage should never be executed."); - } - } - - #[test] - fn check_resumability_if_else_if() { - let once = RefCell::new(true); - let (completed, stages) = test_resume_stages(); - let ifstage = IfElseStage::new( - |_, _, _, _| Ok(once.replace(false)), - stages, - tuple_list!(PanicStage::new()), - ); - let resetstage = ClosureStage::new(|_, _, _, _| { - once.replace(true); - Ok(()) - }); - - let mut state = test_std_state::(); - - test_resume(&completed, &mut state, tuple_list!(ifstage, resetstage)); - } - - #[test] - fn check_resumability_if_else_else() { - let once = RefCell::new(false); - let (completed, stages) = test_resume_stages(); - let ifstage = IfElseStage::new( - |_, _, _, _| Ok(once.replace(true)), - tuple_list!(PanicStage::new()), - stages, - ); - let resetstage = ClosureStage::new(|_, _, _, _| { - once.replace(false); - Ok(()) - }); - - let mut state = test_std_state::(); - - test_resume(&completed, &mut state, tuple_list!(ifstage, resetstage)); - } - - #[test] - fn check_resumability_if_else_else_deep() { - let (completed, stages) = test_resume_stages(); - let ifstage = IfElseStage::new( - |_, _, _, _| Ok(false), - tuple_list!(PanicStage::new()), - tuple_list!(IfElseStage::new( - |_, _, _, _| Ok(false), - tuple_list!(PanicStage::new()), - tuple_list!(IfElseStage::new( - |_, _, _, _| Ok(false), - tuple_list!(PanicStage::new()), - tuple_list!(IfElseStage::new( - |_, _, _, _| Ok(false), - tuple_list!(PanicStage::new()), - tuple_list!(IfElseStage::new( - |_, _, _, _| Ok(false), - tuple_list!(PanicStage::new()), - stages, - )), - )), - )), - )), - ); - - let mut state = test_std_state::(); - - test_resume(&completed, &mut state, tuple_list!(ifstage)); - } -} diff --git a/libafl/src/stages/mod.rs b/libafl/src/stages/mod.rs index b7783eeb08..d6e711a3c0 100644 --- a/libafl/src/stages/mod.rs +++ b/libafl/src/stages/mod.rs @@ -4,21 +4,28 @@ A well-known [`Stage`], for example, is the mutational stage, running multiple [ Other stages may enrich [`crate::corpus::Testcase`]s with metadata. */ -use core::{convert::From, marker::PhantomData}; +use alloc::{borrow::Cow, boxed::Box, vec::Vec}; +use core::marker::PhantomData; pub use calibrate::CalibrationStage; pub use colorization::*; #[cfg(feature = "std")] pub use concolic::ConcolicTracingStage; -#[cfg(feature = "std")] +#[cfg(all(feature = "std", feature = "concolic_mutation"))] pub use concolic::SimpleConcolicMutationalStage; #[cfg(feature = "std")] pub use dump::*; pub use generalization::GeneralizationStage; -use libafl_bolts::tuples::HasConstLen; +use hashbrown::HashSet; +use libafl_bolts::{ + impl_serdeany, + tuples::{HasConstLen, IntoVec}, + Named, +}; pub use logics::*; pub use mutational::{MutationalStage, StdMutationalStage}; pub use power::{PowerMutationalStage, StdPowerMutationalStage}; +use serde::{Deserialize, Serialize}; pub use stats::AflStatsStage; #[cfg(feature = "unicode")] pub use string::*; @@ -29,17 +36,19 @@ pub use tmin::{ }; pub use tracing::{ShadowTracingStage, TracingStage}; pub use tuneable::*; +use tuple_list::NonEmptyTuple; -use self::push::PushStage; use crate::{ - corpus::HasCurrentCorpusIdx, + corpus::{CorpusId, HasCurrentCorpusIdx}, events::{EventFirer, EventRestarter, HasEventManagerId, ProgressReporter}, executors::{Executor, HasObservers}, inputs::UsesInput, observers::ObserversTuple, schedulers::Scheduler, - state::{HasCorpus, HasExecutions, HasLastReportTime, HasMetadata, HasRand, UsesState}, - Error, EvaluatorObservers, ExecutesInput, ExecutionProcessor, HasScheduler, + stages::push::PushStage, + state::{HasCorpus, HasExecutions, HasLastReportTime, HasRand, State, UsesState}, + Error, EvaluatorObservers, ExecutesInput, ExecutionProcessor, HasMetadata, HasNamedMetadata, + HasScheduler, }; /// Mutational stage is the normal fuzzing stage. @@ -54,6 +63,8 @@ pub mod concolic; #[cfg(feature = "std")] pub mod dump; pub mod generalization; +/// The [`generation::GenStage`] generates a single input and evaluates it. +pub mod generation; pub mod logics; pub mod power; pub mod stats; @@ -72,13 +83,21 @@ where EM: UsesState, Z: UsesState, { - // TODO: default this to () when associated_type_defaults is stable - // TODO: see RFC 2532: https://github.com/rust-lang/rust/issues/29661 - // type Status: ResumableStageStatus = (); - /// The resumption data for this stage. Set to () if resuming is not necessary/possible. - type Progress: StageProgress; + /// This method will be called before every call to [`Stage::perform`]. + /// Initialize the restart tracking for this stage, _if it is not yet initialized_. + /// On restart, this will be called again. + /// As long as [`Stage::clear_restart_progress`], all subsequent calls happen on restart. + /// Returns `true`, if the stage's [`Stage::perform`] method should run, else `false`. + fn restart_progress_should_run(&mut self, state: &mut Self::State) -> Result; + + /// Clear the current status tracking of the associated stage + fn clear_restart_progress(&mut self, state: &mut Self::State) -> Result<(), Error>; - /// Run the stage + /// Run the stage. + /// + /// Before a call to perform, [`Stage::restart_progress_should_run`] will be (must be!) called. + /// After returning (so non-target crash or timeout in a restarting case), [`Stage::clear_restart_progress`] gets called. + /// A call to [`Stage::perform_restartable`] will do these things implicitly. fn perform( &mut self, fuzzer: &mut Z, @@ -86,6 +105,20 @@ where state: &mut Self::State, manager: &mut EM, ) -> Result<(), Error>; + + /// Run the stage, calling [`Stage::restart_progress_should_run`] and [`Stage::clear_restart_progress`] appropriately + fn perform_restartable( + &mut self, + fuzzer: &mut Z, + executor: &mut E, + state: &mut Self::State, + manager: &mut EM, + ) -> Result<(), Error> { + if self.restart_progress_should_run(state)? { + self.perform(fuzzer, executor, state, manager)?; + } + self.clear_restart_progress(state) + } } /// A tuple holding all `Stages` used for fuzzing. @@ -152,9 +185,10 @@ where } Some(idx) if idx == Self::LEN => { // perform the stage, but don't set it - Head::Progress::initialize_progress(state)?; - self.0.perform(fuzzer, executor, state, manager)?; - Head::Progress::clear_progress(state)?; + let stage = &mut self.0; + + stage.perform_restartable(fuzzer, executor, state, manager)?; + state.clear_stage()?; } Some(idx) if idx > Self::LEN => { @@ -163,9 +197,10 @@ where // this is None, but the match can't deduce that _ => { state.set_stage(Self::LEN)?; - Head::Progress::initialize_progress(state)?; - self.0.perform(fuzzer, executor, state, manager)?; - Head::Progress::clear_progress(state)?; + + let stage = &mut self.0; + stage.perform_restartable(fuzzer, executor, state, manager)?; + state.clear_stage()?; } } @@ -175,6 +210,79 @@ where } } +impl + IntoVec>> for (Head, Tail) +where + Head: Stage + 'static, + Tail: StagesTuple + + HasConstLen + + IntoVec>>, + E: UsesState, + EM: UsesState, + Z: UsesState, + Head::State: HasCurrentStage, +{ + fn into_vec_reversed( + self, + ) -> Vec>> { + let (head, tail) = self.uncons(); + let mut ret = tail.0.into_vec_reversed(); + ret.push(Box::new(head)); + ret + } + + fn into_vec(self) -> Vec>> { + let mut ret = self.into_vec_reversed(); + ret.reverse(); + ret + } +} + +impl IntoVec>> + for (Tail,) +where + Tail: UsesState + IntoVec>>, + Z: UsesState, + EM: UsesState, + E: UsesState, +{ + fn into_vec(self) -> Vec>> { + self.0.into_vec() + } +} + +impl IntoVec>> + for Vec>> +where + Z: UsesState, + EM: UsesState, + E: UsesState, +{ + fn into_vec(self) -> Vec>> { + self + } +} + +impl StagesTuple + for Vec>> +where + E: UsesState, + EM: UsesState, + Z: UsesState, + S: UsesInput + HasCurrentStage + State, +{ + fn perform_all( + &mut self, + fuzzer: &mut Z, + executor: &mut E, + state: &mut S, + manager: &mut EM, + ) -> Result<(), Error> { + self.iter_mut() + .try_for_each(|x| x.perform_restartable(fuzzer, executor, state, manager)) + } +} + /// A [`Stage`] that will call a closure #[derive(Debug)] pub struct ClosureStage @@ -194,15 +302,25 @@ where type State = E::State; } +impl Named for ClosureStage +where + CB: FnMut(&mut Z, &mut E, &mut E::State, &mut EM) -> Result<(), Error>, + E: UsesState, +{ + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed(""); + &NAME + } +} + impl Stage for ClosureStage where CB: FnMut(&mut Z, &mut E, &mut E::State, &mut EM) -> Result<(), Error>, E: UsesState, EM: UsesState, Z: UsesState, + E::State: HasNamedMetadata, { - type Progress = (); - fn perform( &mut self, fuzzer: &mut Z, @@ -212,6 +330,17 @@ where ) -> Result<(), Error> { (self.closure)(fuzzer, executor, state, manager) } + + #[inline] + fn restart_progress_should_run(&mut self, state: &mut Self::State) -> Result { + // Make sure we don't get stuck crashing on a single closure + RetryRestartHelper::restart_progress_should_run(state, self, 3) + } + + #[inline] + fn clear_restart_progress(&mut self, state: &mut Self::State) -> Result<(), Error> { + RetryRestartHelper::clear_restart_progress(state, self) + } } /// A stage that takes a closure @@ -285,8 +414,6 @@ where + EvaluatorObservers + HasScheduler, { - type Progress = (); // TODO implement resume - fn perform( &mut self, fuzzer: &mut Z, @@ -304,11 +431,12 @@ where push_stage.set_current_corpus_idx(corpus_idx); - push_stage.init(fuzzer, state, event_mgr, executor.observers_mut())?; + push_stage.init(fuzzer, state, event_mgr, &mut *executor.observers_mut())?; loop { let input = - match push_stage.pre_exec(fuzzer, state, event_mgr, executor.observers_mut()) { + match push_stage.pre_exec(fuzzer, state, event_mgr, &mut *executor.observers_mut()) + { Some(Ok(next_input)) => next_input, Some(Err(err)) => return Err(err), None => break, @@ -320,257 +448,96 @@ where fuzzer, state, event_mgr, - executor.observers_mut(), + &mut *executor.observers_mut(), input, exit_kind, )?; } self.push_stage - .deinit(fuzzer, state, event_mgr, executor.observers_mut()) - } -} - -/// `Stage` Python bindings -#[cfg(feature = "python")] -#[allow(missing_docs)] -pub mod pybind { - use alloc::vec::Vec; - - use pyo3::prelude::*; - - use crate::{ - corpus::HasCurrentCorpusIdx, - events::pybind::PythonEventManager, - executors::pybind::PythonExecutor, - fuzzer::pybind::{PythonStdFuzzer, PythonStdFuzzerWrapper}, - stages::{ - mutational::pybind::PythonStdMutationalStage, HasCurrentStage, Stage, StagesTuple, - }, - state::{ - pybind::{PythonStdState, PythonStdStateWrapper}, - UsesState, - }, - Error, - }; - - #[derive(Clone, Debug)] - pub struct PyObjectStage { - inner: PyObject, - } - - impl PyObjectStage { - #[must_use] - pub fn new(obj: PyObject) -> Self { - PyObjectStage { inner: obj } - } - } - - impl UsesState for PyObjectStage { - type State = PythonStdState; - } - - impl Stage for PyObjectStage { - type Progress = (); // we don't support resumption in python, and maybe can't? - - #[inline] - fn perform( - &mut self, - fuzzer: &mut PythonStdFuzzer, - executor: &mut PythonExecutor, - state: &mut PythonStdState, - manager: &mut PythonEventManager, - ) -> Result<(), Error> { - let Some(corpus_idx) = state.current_corpus_idx()? else { - return Err(Error::illegal_state( - "state is not currently processing a corpus index", - )); - }; - - Python::with_gil(|py| -> PyResult<()> { - self.inner.call_method1( - py, - "perform", - ( - PythonStdFuzzerWrapper::wrap(fuzzer), - executor.clone(), - PythonStdStateWrapper::wrap(state), - manager.clone(), - corpus_idx.0, - ), - )?; - Ok(()) - })?; - Ok(()) - } - } - - #[derive(Clone, Debug)] - pub enum PythonStageWrapper { - StdMutational(Py), - Python(PyObjectStage), - } - - /// Stage Trait binding - #[pyclass(unsendable, name = "Stage")] - #[derive(Clone, Debug)] - pub struct PythonStage { - wrapper: PythonStageWrapper, - } - - macro_rules! unwrap_me_mut { - ($wrapper:expr, $name:ident, $body:block) => { - libafl_bolts::unwrap_me_mut_body!($wrapper, $name, $body, PythonStageWrapper, - { StdMutational }, - { - Python(py_wrapper) => { - let $name = py_wrapper; - $body - } - } - ) - }; - } - - #[pymethods] - impl PythonStage { - #[staticmethod] - #[must_use] - pub fn new_std_mutational( - py_std_havoc_mutations_stage: Py, - ) -> Self { - Self { - wrapper: PythonStageWrapper::StdMutational(py_std_havoc_mutations_stage), - } - } - - #[staticmethod] - #[must_use] - pub fn new_py(obj: PyObject) -> Self { - Self { - wrapper: PythonStageWrapper::Python(PyObjectStage::new(obj)), - } - } - - #[must_use] - pub fn unwrap_py(&self) -> Option { - match &self.wrapper { - PythonStageWrapper::Python(pyo) => Some(pyo.inner.clone()), - PythonStageWrapper::StdMutational(_) => None, - } - } + .deinit(fuzzer, state, event_mgr, &mut *executor.observers_mut()) } - impl UsesState for PythonStage { - type State = PythonStdState; + #[inline] + fn restart_progress_should_run(&mut self, _state: &mut Self::State) -> Result { + // TODO: Proper restart handling - call post_exec at the right time, etc... + Ok(true) } - impl Stage for PythonStage { - // TODO if we implement resumption for StdMutational, we need to apply it here - type Progress = (); - - #[inline] - #[allow(clippy::let_and_return)] - fn perform( - &mut self, - fuzzer: &mut PythonStdFuzzer, - executor: &mut PythonExecutor, - state: &mut PythonStdState, - manager: &mut PythonEventManager, - ) -> Result<(), Error> { - unwrap_me_mut!(self.wrapper, s, { - s.perform(fuzzer, executor, state, manager) - }) - } - } - - #[derive(Clone, Debug)] - #[pyclass(unsendable, name = "StagesTuple")] - pub struct PythonStagesTuple { - list: Vec, - } - - #[pymethods] - impl PythonStagesTuple { - #[new] - fn new(list: Vec) -> Self { - Self { list } - } - - fn len(&self) -> usize { - self.list.len() - } - - fn __getitem__(&self, idx: usize) -> PythonStage { - self.list[idx].clone() - } - } - - impl StagesTuple - for PythonStagesTuple - { - fn perform_all( - &mut self, - fuzzer: &mut PythonStdFuzzer, - executor: &mut PythonExecutor, - state: &mut PythonStdState, - manager: &mut PythonEventManager, - ) -> Result<(), Error> { - for (i, s) in self.list.iter_mut().enumerate() { - if let Some(continued) = state.current_stage()? { - assert!(continued >= i); - if continued > i { - continue; - } - } else { - state.set_stage(i)?; - } - s.perform(fuzzer, executor, state, manager)?; - state.clear_stage()?; - } - Ok(()) - } - } - - /// Register the classes to the python module - pub fn register(_py: Python, m: &PyModule) -> PyResult<()> { - m.add_class::()?; - m.add_class::()?; + #[inline] + fn clear_restart_progress(&mut self, _state: &mut Self::State) -> Result<(), Error> { Ok(()) } } -/// Trait for status tracking of stages which stash data to resume -pub trait StageProgress { - /// Initialize the current status tracking for this stage, if it is not yet initialised - fn initialize_progress(state: &mut S) -> Result<(), Error>; +/// Progress which permits a fixed amount of resumes per round of fuzzing. If this amount is ever +/// exceeded, the input will no longer be executed by this stage. +#[derive(Clone, Deserialize, Serialize, Debug)] +pub struct RetryRestartHelper { + tries_remaining: Option, + skipped: HashSet, +} - /// Clear the current status tracking of the associated stage - fn clear_progress(state: &mut S) -> Result<(), Error>; +impl_serdeany!(RetryRestartHelper); - /// Get the current status tracking of this stage - fn progress(state: &S) -> Result<&Self, Error>; +impl RetryRestartHelper { + /// Initializes (or counts down in) the progress helper, giving it the amount of max retries + /// + /// Returns `true` if the stage should run + pub fn restart_progress_should_run( + state: &mut S, + stage: &ST, + max_retries: usize, + ) -> Result + where + S: HasNamedMetadata + HasCurrentCorpusIdx, + ST: Named, + { + let corpus_idx = state.current_corpus_idx()?.ok_or_else(|| { + Error::illegal_state( + "No current_corpus_idx set in State, but called RetryRestartHelper::should_skip", + ) + })?; + + let initial_tries_remaining = max_retries + 1; + let metadata = state.named_metadata_or_insert_with(stage.name(), || Self { + tries_remaining: Some(initial_tries_remaining), + skipped: HashSet::new(), + }); + let tries_remaining = metadata + .tries_remaining + .unwrap_or(initial_tries_remaining) + .checked_sub(1) + .ok_or_else(|| { + Error::illegal_state( + "Attempted further retries after we had already gotten to none remaining.", + ) + })?; - /// Get the current status tracking of this stage, mutably - fn progress_mut(state: &mut S) -> Result<&mut Self, Error>; -} + metadata.tries_remaining = Some(tries_remaining); -impl StageProgress for () { - fn initialize_progress(_state: &mut S) -> Result<(), Error> { - Ok(()) + Ok(if tries_remaining == 0 { + metadata.skipped.insert(corpus_idx); + false + } else if metadata.skipped.contains(&corpus_idx) { + // skip this testcase, we already retried it often enough... + false + } else { + true + }) } - fn clear_progress(_state: &mut S) -> Result<(), Error> { + /// Clears the progress + pub fn clear_restart_progress(state: &mut S, stage: &ST) -> Result<(), Error> + where + S: HasNamedMetadata, + ST: Named, + { + state + .named_metadata_mut::(stage.name())? + .tries_remaining = None; Ok(()) } - - fn progress(_state: &S) -> Result<&Self, Error> { - unimplemented!("The empty tuple resumable stage status should never be queried") - } - - fn progress_mut(_state: &mut S) -> Result<&mut Self, Error> { - unimplemented!("The empty tuple resumable stage status should never be queried") - } } /// Trait for types which track the current stage @@ -601,21 +568,98 @@ pub trait HasNestedStageStatus: HasCurrentStage { fn exit_inner_stage(&mut self) -> Result<(), Error>; } +impl_serdeany!(ExecutionCountRestartHelperMetadata); + +/// `SerdeAny` metadata used to keep track of executions since start for a given stage. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ExecutionCountRestartHelperMetadata { + /// How many executions we had when we started this stage initially (this round) + started_at_execs: u64, +} + +/// A tool shed of functions to be used for stages that try to run for `n` iterations. +/// +/// # Note +/// This helper assumes resumable mutational stages are not nested. +/// If you want to nest them, you will have to switch all uses of `metadata` in this helper to `named_metadata` instead. +#[derive(Debug, Default, Clone)] +pub struct ExecutionCountRestartHelper { + /// At what exec count this Stage was started (cache) + /// Only used as cache for the value stored in [`MutationalStageMetadata`]. + started_at_execs: Option, +} + +impl ExecutionCountRestartHelper { + /// Create a new [`ExecutionCountRestartHelperMetadata`] + #[must_use] + pub fn new() -> Self { + Self { + started_at_execs: None, + } + } + + /// The execs done since start of this [`Stage`]/helper + pub fn execs_since_progress_start(&mut self, state: &mut S) -> Result + where + S: HasMetadata + HasExecutions, + { + let started_at_execs = if let Some(started_at_execs) = self.started_at_execs { + started_at_execs + } else { + state + .metadata::() + .map(|x| { + self.started_at_execs = Some(x.started_at_execs); + x.started_at_execs + }) + .map_err(|err| { + Error::illegal_state(format!( + "The ExecutionCountRestartHelperMetadata should have been set at this point - {err}" + )) + })? + }; + Ok(state.executions() - started_at_execs) + } + + /// Initialize progress for the stage this wrapper wraps. + pub fn restart_progress_should_run(&mut self, state: &mut S) -> Result + where + S: HasMetadata + HasExecutions, + { + let executions = *state.executions(); + let metadata = state.metadata_or_insert_with(|| ExecutionCountRestartHelperMetadata { + started_at_execs: executions, + }); + self.started_at_execs = Some(metadata.started_at_execs); + Ok(true) + } + + /// Clear progress for the stage this wrapper wraps. + pub fn clear_restart_progress(&mut self, state: &mut S) -> Result<(), Error> + where + S: HasMetadata, + { + self.started_at_execs = None; + let _metadata = state.remove_metadata::(); + debug_assert!(_metadata.is_some(), "Called clear_restart_progress, but restart_progress_should_run was not called before (or did mutational stages get nested?)"); + Ok(()) + } +} + #[cfg(test)] pub mod test { - use alloc::rc::Rc; - use core::{cell::RefCell, marker::PhantomData}; + use alloc::borrow::Cow; + use core::marker::PhantomData; - use libafl_bolts::{impl_serdeany, Error}; + use libafl_bolts::{impl_serdeany, Error, Named}; use serde::{Deserialize, Serialize}; - use tuple_list::{tuple_list, tuple_list_type}; use crate::{ - events::NopEventManager, - executors::test::NopExecutor, - fuzzer::test::NopFuzzer, - stages::{Stage, StageProgress, StagesTuple}, - state::{HasMetadata, State, UsesState}, + corpus::{Corpus, HasCurrentCorpusIdx, Testcase}, + inputs::NopInput, + stages::{RetryRestartHelper, Stage}, + state::{test::test_std_state, HasCorpus, State, UsesState}, + HasMetadata, }; #[derive(Debug)] @@ -623,12 +667,6 @@ pub mod test { phantom: PhantomData, } - #[derive(Debug)] - pub struct ResumeFailedStage { - completed: Rc>, - phantom: PhantomData, - } - #[derive(Serialize, Deserialize, Debug)] pub struct TestProgress { count: usize, @@ -636,34 +674,35 @@ pub mod test { impl_serdeany!(TestProgress); - impl StageProgress for TestProgress - where - S: HasMetadata, - { - fn initialize_progress(state: &mut S) -> Result<(), Error> { + impl TestProgress { + #[allow(clippy::unnecessary_wraps)] + fn restart_progress_should_run(state: &mut S, _stage: &ST) -> Result + where + S: HasMetadata, + { // check if we're resuming - if !state.has_metadata::() { - state.add_metadata(Self { count: 0 }); - } - Ok(()) + let metadata = state.metadata_or_insert_with(|| Self { count: 0 }); + + metadata.count += 1; + assert!( + metadata.count == 1, + "Test failed; we resumed a succeeded stage!" + ); + + Ok(true) } - fn clear_progress(state: &mut S) -> Result<(), Error> { - if state.metadata_map_mut().remove::().is_none() { + fn clear_restart_progress(state: &mut S, _stage: &ST) -> Result<(), Error> + where + S: HasMetadata, + { + if state.remove_metadata::().is_none() { return Err(Error::illegal_state( "attempted to clear status metadata when none was present", )); } Ok(()) } - - fn progress(state: &S) -> Result<&Self, Error> { - state.metadata() - } - - fn progress_mut(state: &mut S) -> Result<&mut Self, Error> { - state.metadata_mut() - } } impl UsesState for ResumeSucceededStage @@ -680,124 +719,91 @@ pub mod test { Z: UsesState, Z::State: HasMetadata, { - type Progress = TestProgress; - fn perform( &mut self, _fuzzer: &mut Z, _executor: &mut E, - state: &mut Self::State, + _state: &mut Self::State, _manager: &mut EM, ) -> Result<(), Error> { - // metadata is attached by the status - let meta = Self::Progress::progress_mut(state)?; - meta.count += 1; - assert!( - meta.count == 1, - "Test failed; we resumed a succeeded stage!" - ); - Ok(()) } - } - - impl UsesState for ResumeFailedStage - where - S: State, - { - type State = S; - } - - impl Stage for ResumeFailedStage - where - E: UsesState, - EM: UsesState, - Z: UsesState, - Z::State: HasMetadata, - { - type Progress = TestProgress; - - fn perform( - &mut self, - _fuzzer: &mut Z, - _executor: &mut E, - state: &mut Self::State, - _manager: &mut EM, - ) -> Result<(), Error> { - // metadata is attached by the status - let meta = Self::Progress::progress_mut(state)?; - meta.count += 1; - - if meta.count == 1 { - return Err(Error::shutting_down()); - } else if meta.count > 2 { - panic!("Resume was somehow corrupted?") - } else { - self.completed.replace(true); - } - Ok(()) + fn restart_progress_should_run(&mut self, state: &mut Self::State) -> Result { + TestProgress::restart_progress_should_run(state, self) } - } - #[must_use] - #[allow(clippy::type_complexity)] - pub fn test_resume_stages() -> ( - Rc>, - tuple_list_type!(ResumeSucceededStage, ResumeFailedStage), - ) { - let completed = Rc::new(RefCell::new(false)); - ( - completed.clone(), - tuple_list!( - ResumeSucceededStage { - phantom: PhantomData - }, - ResumeFailedStage { - completed, - phantom: PhantomData - }, - ), - ) + fn clear_restart_progress(&mut self, state: &mut Self::State) -> Result<(), Error> { + TestProgress::clear_restart_progress(state, self) + } } - pub fn test_resume(completed: &Rc>, state: &mut S, mut stages: ST) - where - ST: StagesTuple, NopEventManager, S, NopFuzzer>, - S: State, - { + #[test] + fn test_tries_progress() -> Result<(), Error> { + // # Safety + // No concurrency per testcase #[cfg(any(not(feature = "serdeany_autoreg"), miri))] unsafe { - TestProgress::register(); + RetryRestartHelper::register(); } - let mut fuzzer = NopFuzzer::new(); - let mut executor = NopExecutor::new(); - let mut manager = NopEventManager::new(); + struct StageWithOneTry; - for _ in 0..2 { - completed.replace(false); - let Err(e) = stages.perform_all(&mut fuzzer, &mut executor, state, &mut manager) else { - panic!("Test failed; stages should fail the first time.") - }; - assert!( - matches!(e, Error::ShuttingDown), - "Unexpected error encountered." - ); - assert!(!*completed.borrow(), "Unexpectedly complete?"); - state - .on_restart() - .expect("Couldn't notify state of restart."); - assert!( - stages - .perform_all(&mut fuzzer, &mut executor, state, &mut manager) - .is_ok(), - "Test failed; stages should pass the second time." - ); - assert!( - *completed.borrow(), - "Test failed; we did not set completed." - ); + impl Named for StageWithOneTry { + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("TestStage"); + &NAME + } } + + let mut state = test_std_state(); + let stage = StageWithOneTry; + + let corpus_idx = state.corpus_mut().add(Testcase::new(NopInput {}))?; + + state.set_corpus_idx(corpus_idx)?; + + for _ in 0..10 { + // used normally, no retries means we never skip + assert!(RetryRestartHelper::restart_progress_should_run( + &mut state, &stage, 1 + )?); + RetryRestartHelper::clear_restart_progress(&mut state, &stage)?; + } + + for _ in 0..10 { + // used normally, only one retry means we never skip + assert!(RetryRestartHelper::restart_progress_should_run( + &mut state, &stage, 2 + )?); + assert!(RetryRestartHelper::restart_progress_should_run( + &mut state, &stage, 2 + )?); + RetryRestartHelper::clear_restart_progress(&mut state, &stage)?; + } + + assert!(RetryRestartHelper::restart_progress_should_run( + &mut state, &stage, 2 + )?); + // task failed, let's resume + // we still have one more try! + assert!(RetryRestartHelper::restart_progress_should_run( + &mut state, &stage, 2 + )?); + + // task failed, let's resume + // out of retries, so now we skip + assert!(!RetryRestartHelper::restart_progress_should_run( + &mut state, &stage, 2 + )?); + RetryRestartHelper::clear_restart_progress(&mut state, &stage)?; + + // we previously exhausted this testcase's retries, so we skip + assert!(!RetryRestartHelper::restart_progress_should_run( + &mut state, &stage, 2 + )?); + RetryRestartHelper::clear_restart_progress(&mut state, &stage)?; + + Ok(()) } } diff --git a/libafl/src/stages/mutational.rs b/libafl/src/stages/mutational.rs index f5c5fda210..19ea5cfa99 100644 --- a/libafl/src/stages/mutational.rs +++ b/libafl/src/stages/mutational.rs @@ -1,20 +1,21 @@ //| The [`MutationalStage`] is the default stage used during fuzzing. //! For the current input, it will perform a range of random mutations, and then run them in the executor. +use alloc::borrow::Cow; use core::marker::PhantomData; -use libafl_bolts::rands::Rand; +use libafl_bolts::{rands::Rand, Named}; use crate::{ - corpus::{Corpus, CorpusId, HasCurrentCorpusIdx, Testcase}, + corpus::{Corpus, CorpusId, Testcase}, fuzzer::Evaluator, inputs::Input, mark_feature_time, mutators::{MultiMutator, MutationResult, Mutator}, - stages::Stage, + stages::{ExecutionCountRestartHelper, RetryRestartHelper, Stage}, start_timer, - state::{HasCorpus, HasRand, UsesState}, - Error, + state::{HasCorpus, HasCurrentTestcase, HasExecutions, HasRand, UsesState}, + Error, HasMetadata, HasNamedMetadata, }; #[cfg(feature = "introspection")] use crate::{monitors::PerfFeature, state::HasClientPerfMonitor}; @@ -26,12 +27,7 @@ use crate::{monitors::PerfFeature, state::HasClientPerfMonitor}; pub trait MutatedTransformPost: Sized { /// Perform any post-execution steps necessary for the transformed input (e.g., updating metadata) #[inline] - fn post_exec( - self, - state: &mut S, - stage_idx: i32, - corpus_idx: Option, - ) -> Result<(), Error> { + fn post_exec(self, state: &mut S, new_corpus_idx: Option) -> Result<(), Error> { Ok(()) } } @@ -51,11 +47,7 @@ where type Post: MutatedTransformPost; /// Transform the provided testcase into this type - fn try_transform_from( - base: &mut Testcase, - state: &S, - corpus_idx: CorpusId, - ) -> Result; + fn try_transform_from(base: &mut Testcase, state: &S) -> Result; /// Transform this instance back into the original input type fn try_transform_into(self, state: &S) -> Result<(I, Self::Post), Error>; @@ -70,11 +62,7 @@ where type Post = (); #[inline] - fn try_transform_from( - base: &mut Testcase, - state: &S, - _corpus_idx: CorpusId, - ) -> Result { + fn try_transform_from(base: &mut Testcase, state: &S) -> Result { state.corpus().load_input_into(base)?; Ok(base.input().as_ref().unwrap().clone()) } @@ -104,7 +92,10 @@ where fn mutator_mut(&mut self) -> &mut M; /// Gets the number of iterations this mutator should run for. - fn iterations(&self, state: &mut Z::State, corpus_idx: CorpusId) -> Result; + fn iterations(&self, state: &mut Z::State) -> Result; + + /// Gets the number of executions this mutator already did since it got first called in this fuzz round. + fn execs_since_progress_start(&mut self, state: &mut Z::State) -> Result; /// Runs this (mutational) stage for the given testcase #[allow(clippy::cast_possible_wrap)] // more than i32 stages on 32 bit system - highly unlikely... @@ -115,27 +106,28 @@ where state: &mut Z::State, manager: &mut EM, ) -> Result<(), Error> { - let Some(corpus_idx) = state.current_corpus_idx()? else { - return Err(Error::illegal_state( - "state is not currently processing a corpus index", - )); - }; + start_timer!(state); - let num = self.iterations(state, corpus_idx)?; + // Here saturating_sub is needed as self.iterations() might be actually smaller than the previous value before reset. + /* + let num = self + .iterations(state)? + .saturating_sub(self.execs_since_progress_start(state)?); + */ + let num = self.iterations(state)?; + let mut testcase = state.current_testcase_mut()?; - start_timer!(state); - let mut testcase = state.corpus().get(corpus_idx)?.borrow_mut(); - let Ok(input) = I::try_transform_from(&mut testcase, state, corpus_idx) else { + let Ok(input) = I::try_transform_from(&mut testcase, state) else { return Ok(()); }; drop(testcase); mark_feature_time!(state, PerfFeature::GetInputFromCorpus); - for i in 0..num { + for _ in 0..num { let mut input = input.clone(); start_timer!(state); - let mutated = self.mutator_mut().mutate(state, &mut input, i as i32)?; + let mutated = self.mutator_mut().mutate(state, &mut input)?; mark_feature_time!(state, PerfFeature::Mutate); if mutated == MutationResult::Skipped { @@ -147,8 +139,8 @@ where let (_, corpus_idx) = fuzzer.evaluate_input(state, executor, manager, untransformed)?; start_timer!(state); - self.mutator_mut().post_exec(state, i as i32, corpus_idx)?; - post.post_exec(state, i as i32, corpus_idx)?; + self.mutator_mut().post_exec(state, corpus_idx)?; + post.post_exec(state, corpus_idx)?; mark_feature_time!(state, PerfFeature::MutatePostExec); } @@ -158,13 +150,17 @@ where /// Default value, how many iterations each stage gets, as an upper bound. /// It may randomly continue earlier. -pub static DEFAULT_MUTATIONAL_MAX_ITERATIONS: u64 = 128; +pub static DEFAULT_MUTATIONAL_MAX_ITERATIONS: usize = 128; /// The default mutational stage #[derive(Clone, Debug)] pub struct StdMutationalStage { + /// The mutator(s) to use mutator: M, - max_iterations: u64, + /// The maximum amount of iterations we should do each round + max_iterations: usize, + /// The progress helper for this mutational stage + restart_helper: ExecutionCountRestartHelper, #[allow(clippy::type_complexity)] phantom: PhantomData<(E, EM, I, Z)>, } @@ -175,7 +171,7 @@ where EM: UsesState, M: Mutator, Z: Evaluator, - Z::State: HasCorpus + HasRand, + Z::State: HasCorpus + HasRand + HasExecutions + HasMetadata, I: MutatedTransform + Clone, { /// The mutator, added to this stage @@ -191,9 +187,13 @@ where } /// Gets the number of iterations as a random number - fn iterations(&self, state: &mut Z::State, _corpus_idx: CorpusId) -> Result { + fn iterations(&self, state: &mut Z::State) -> Result { Ok(1 + state.rand_mut().below(self.max_iterations)) } + + fn execs_since_progress_start(&mut self, state: &mut ::State) -> Result { + self.restart_helper.execs_since_progress_start(state) + } } impl UsesState for StdMutationalStage @@ -213,11 +213,9 @@ where EM: UsesState, M: Mutator, Z: Evaluator, - Z::State: HasCorpus + HasRand, + Z::State: HasCorpus + HasRand + HasMetadata + HasExecutions, I: MutatedTransform + Clone, { - type Progress = (); // TODO should this stage be resumed? - #[inline] #[allow(clippy::let_and_return)] fn perform( @@ -234,6 +232,16 @@ where ret } + + fn restart_progress_should_run(&mut self, _state: &mut Self::State) -> Result { + Ok(true) + // self.restart_helper.restart_progress_should_run(state) + } + + fn clear_restart_progress(&mut self, _state: &mut Self::State) -> Result<(), Error> { + Ok(()) + // self.restart_helper.clear_restart_progress(state) + } } impl StdMutationalStage @@ -250,7 +258,7 @@ where } /// Creates a new mutational stage with the given max iterations - pub fn with_max_iterations(mutator: M, max_iterations: u64) -> Self { + pub fn with_max_iterations(mutator: M, max_iterations: usize) -> Self { Self::transforming_with_max_iterations(mutator, max_iterations) } } @@ -269,16 +277,17 @@ where } /// Creates a new transforming mutational stage with the given max iterations - pub fn transforming_with_max_iterations(mutator: M, max_iterations: u64) -> Self { + pub fn transforming_with_max_iterations(mutator: M, max_iterations: usize) -> Self { Self { mutator, max_iterations, + restart_helper: ExecutionCountRestartHelper::default(), phantom: PhantomData, } } } -/// The default mutational stage +/// A mutational stage that operates on multiple inputs, as returned by [`MultiMutator::multi_mutate`]. #[derive(Clone, Debug)] pub struct MultiMutationalStage { mutator: M, @@ -297,16 +306,40 @@ where type State = Z::State; } -impl Stage for MultiMutationalStage +impl Named for MultiMutationalStage where E: UsesState, EM: UsesState, M: MultiMutator, Z: Evaluator, Z::State: HasCorpus + HasRand, +{ + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("MultiMutational"); + &NAME + } +} + +impl Stage for MultiMutationalStage +where + E: UsesState, + EM: UsesState, + M: MultiMutator, + Z: Evaluator, + Z::State: HasCorpus + HasRand + HasNamedMetadata, I: MutatedTransform + Clone, { - type Progress = (); // TODO implement resume + #[inline] + fn restart_progress_should_run(&mut self, state: &mut Self::State) -> Result { + // TODO: add proper crash/timeout handling + // For now, Make sure we don't get stuck crashing on a single testcase + RetryRestartHelper::restart_progress_should_run(state, self, 3) + } + + #[inline] + fn clear_restart_progress(&mut self, state: &mut Self::State) -> Result<(), Error> { + RetryRestartHelper::clear_restart_progress(state, self) + } #[inline] #[allow(clippy::let_and_return)] @@ -318,26 +351,20 @@ where state: &mut Z::State, manager: &mut EM, ) -> Result<(), Error> { - let Some(corpus_idx) = state.current_corpus_idx()? else { - return Err(Error::illegal_state( - "state is not currently processing a corpus index", - )); - }; - - let mut testcase = state.corpus().get(corpus_idx)?.borrow_mut(); - let Ok(input) = I::try_transform_from(&mut testcase, state, corpus_idx) else { + let mut testcase = state.current_testcase_mut()?; + let Ok(input) = I::try_transform_from(&mut testcase, state) else { return Ok(()); }; drop(testcase); - let generated = self.mutator.multi_mutate(state, &input, 0, None)?; + let generated = self.mutator.multi_mutate(state, &input, None)?; // println!("Generated {}", generated.len()); - for (i, new_input) in generated.into_iter().enumerate() { + for new_input in generated { // Time is measured directly the `evaluate_input` function let (untransformed, post) = new_input.try_transform_into(state)?; let (_, corpus_idx) = fuzzer.evaluate_input(state, executor, manager, untransformed)?; - self.mutator.multi_post_exec(state, i as i32, corpus_idx)?; - post.post_exec(state, i as i32, corpus_idx)?; + self.mutator.multi_post_exec(state, corpus_idx)?; + post.post_exec(state, corpus_idx)?; } // println!("Found {}", found); @@ -353,7 +380,7 @@ where Z: Evaluator, Z::State: HasCorpus + HasRand, { - /// Creates a new default mutational stage + /// Creates a new [`MultiMutationalStage`] pub fn new(mutator: M) -> Self { Self::transforming(mutator) } @@ -375,54 +402,3 @@ where } } } - -#[cfg(feature = "python")] -#[allow(missing_docs)] -#[allow(clippy::unnecessary_fallible_conversions)] -/// `StdMutationalStage` Python bindings -pub mod pybind { - use pyo3::prelude::*; - - use crate::{ - events::pybind::PythonEventManager, - executors::pybind::PythonExecutor, - fuzzer::pybind::PythonStdFuzzer, - inputs::BytesInput, - mutators::pybind::PythonMutator, - stages::{pybind::PythonStage, StdMutationalStage}, - }; - - #[pyclass(unsendable, name = "StdMutationalStage")] - #[derive(Debug)] - /// Python class for StdMutationalStage - pub struct PythonStdMutationalStage { - /// Rust wrapped StdMutationalStage object - pub inner: StdMutationalStage< - PythonExecutor, - PythonEventManager, - BytesInput, - PythonMutator, - PythonStdFuzzer, - >, - } - - #[pymethods] - impl PythonStdMutationalStage { - #[new] - fn new(mutator: PythonMutator) -> Self { - Self { - inner: StdMutationalStage::new(mutator), - } - } - - fn as_stage(slf: Py) -> PythonStage { - PythonStage::new_std_mutational(slf) - } - } - - /// Register the classes to the python module - pub fn register(_py: Python, m: &PyModule) -> PyResult<()> { - m.add_class::()?; - Ok(()) - } -} diff --git a/libafl/src/stages/power.rs b/libafl/src/stages/power.rs index 972a321d5e..961046177f 100644 --- a/libafl/src/stages/power.rs +++ b/libafl/src/stages/power.rs @@ -3,20 +3,22 @@ use core::{fmt::Debug, marker::PhantomData}; use crate::{ - corpus::{Corpus, CorpusId}, executors::{Executor, HasObservers}, fuzzer::Evaluator, mutators::Mutator, schedulers::{testcase_score::CorpusPowerTestcaseScore, TestcaseScore}, - stages::{mutational::MutatedTransform, MutationalStage, Stage}, - state::{HasCorpus, HasMetadata, HasRand, UsesState}, - Error, + stages::{mutational::MutatedTransform, ExecutionCountRestartHelper, MutationalStage, Stage}, + state::{HasCorpus, HasCurrentTestcase, HasExecutions, HasRand, UsesState}, + Error, HasMetadata, }; /// The mutational stage using power schedules #[derive(Clone, Debug)] pub struct PowerMutationalStage { + /// The mutators we use mutator: M, + /// Helper for restarts + restart_helper: ExecutionCountRestartHelper, #[allow(clippy::type_complexity)] phantom: PhantomData<(E, F, EM, I, Z)>, } @@ -34,7 +36,7 @@ where EM: UsesState, F: TestcaseScore, M: Mutator, - E::State: HasCorpus + HasMetadata + HasRand, + E::State: HasCorpus + HasMetadata + HasRand + HasExecutions, Z: Evaluator, I: MutatedTransform + Clone, { @@ -52,13 +54,17 @@ where /// Gets the number of iterations as a random number #[allow(clippy::cast_sign_loss)] - fn iterations(&self, state: &mut E::State, corpus_idx: CorpusId) -> Result { + fn iterations(&self, state: &mut E::State) -> Result { // Update handicap - let mut testcase = state.corpus().get(corpus_idx)?.borrow_mut(); - let score = F::compute(state, &mut *testcase)? as u64; + let mut testcase = state.current_testcase_mut()?; + let score = F::compute(state, &mut testcase)? as usize; Ok(score) } + + fn execs_since_progress_start(&mut self, state: &mut ::State) -> Result { + self.restart_helper.execs_since_progress_start(state) + } } impl Stage for PowerMutationalStage @@ -67,12 +73,10 @@ where EM: UsesState, F: TestcaseScore, M: Mutator, - E::State: HasCorpus + HasMetadata + HasRand, + E::State: HasCorpus + HasMetadata + HasRand + HasExecutions, Z: Evaluator, I: MutatedTransform + Clone, { - type Progress = (); // TODO should we resume this stage? - #[inline] #[allow(clippy::let_and_return)] fn perform( @@ -85,6 +89,16 @@ where let ret = self.perform_mutational(fuzzer, executor, state, manager); ret } + + fn restart_progress_should_run(&mut self, _state: &mut Self::State) -> Result { + Ok(true) + // self.restart_helper.restart_progress_should_run(state) + } + + fn clear_restart_progress(&mut self, _state: &mut Self::State) -> Result<(), Error> { + Ok(()) + // self.restart_helper.clear_restart_progress(state) + } } impl PowerMutationalStage @@ -116,6 +130,7 @@ where Self { mutator, phantom: PhantomData, + restart_helper: ExecutionCountRestartHelper::default(), } } } diff --git a/libafl/src/stages/push/mod.rs b/libafl/src/stages/push/mod.rs index 61884c729a..239363e011 100644 --- a/libafl/src/stages/push/mod.rs +++ b/libafl/src/stages/push/mod.rs @@ -22,8 +22,8 @@ use crate::{ inputs::UsesInput, observers::ObserversTuple, schedulers::Scheduler, - state::{HasCorpus, HasExecutions, HasLastReportTime, HasMetadata, HasRand}, - Error, EvaluatorObservers, ExecutionProcessor, HasScheduler, + state::{HasCorpus, HasExecutions, HasLastReportTime, HasRand}, + Error, EvaluatorObservers, ExecutionProcessor, HasMetadata, HasScheduler, }; /// Send a monitor update all 15 (or more) seconds diff --git a/libafl/src/stages/push/mutational.rs b/libafl/src/stages/push/mutational.rs index f82ff27f9e..976f854b0e 100644 --- a/libafl/src/stages/push/mutational.rs +++ b/libafl/src/stages/push/mutational.rs @@ -20,14 +20,14 @@ use crate::{ observers::ObserversTuple, schedulers::Scheduler, start_timer, - state::{HasCorpus, HasExecutions, HasLastReportTime, HasMetadata, HasRand}, - Error, EvaluatorObservers, ExecutionProcessor, HasScheduler, + state::{HasCorpus, HasExecutions, HasLastReportTime, HasRand}, + Error, EvaluatorObservers, ExecutionProcessor, HasMetadata, HasScheduler, }; #[cfg(feature = "introspection")] use crate::{monitors::PerfFeature, state::HasClientPerfMonitor}; /// The default maximum number of mutations to perform per input. -pub static DEFAULT_MUTATIONAL_MAX_ITERATIONS: u64 = 128; +pub static DEFAULT_MUTATIONAL_MAX_ITERATIONS: usize = 128; /// A Mutational push stage is the stage in a fuzzing run that mutates inputs. /// Mutational push stages will usually have a range of mutations that are /// being applied to the input one by one, between executions. @@ -53,8 +53,6 @@ where testcases_to_do: usize, testcases_done: usize, - stage_idx: i32, - mutator: M, psh: PushStageHelper, @@ -74,7 +72,7 @@ where /// Gets the number of iterations as a random number #[allow(clippy::unused_self, clippy::unnecessary_wraps)] // TODO: we should put this function into a trait later fn iterations(&self, state: &mut CS::State, _corpus_idx: CorpusId) -> Result { - Ok(1 + state.rand_mut().below(DEFAULT_MUTATIONAL_MAX_ITERATIONS) as usize) + Ok(1 + state.rand_mut().below(DEFAULT_MUTATIONAL_MAX_ITERATIONS)) } /// Sets the current corpus index @@ -150,9 +148,7 @@ where mark_feature_time!(state, PerfFeature::GetInputFromCorpus); start_timer!(state); - self.mutator - .mutate(state, &mut input, self.stage_idx) - .unwrap(); + self.mutator.mutate(state, &mut input).unwrap(); mark_feature_time!(state, PerfFeature::Mutate); self.push_stage_helper_mut() @@ -173,11 +169,10 @@ where ) -> Result<(), Error> { // todo: is_interesting, etc. - fuzzer.process_execution(state, event_mgr, last_input, observers, &exit_kind, true)?; + fuzzer.execute_and_process(state, event_mgr, last_input, observers, &exit_kind, true)?; start_timer!(state); - self.mutator - .post_exec(state, self.stage_idx, self.current_corpus_idx)?; + self.mutator.post_exec(state, self.current_corpus_idx)?; mark_feature_time!(state, PerfFeature::MutatePostExec); self.testcases_done += 1; @@ -234,7 +229,6 @@ where mutator: M, shared_state: Rc>>>, exit_kind: Rc>>, - stage_idx: i32, ) -> Self { Self { mutator, @@ -242,7 +236,6 @@ where current_corpus_idx: None, // todo testcases_to_do: 0, testcases_done: 0, - stage_idx, } } } diff --git a/libafl/src/stages/stats.rs b/libafl/src/stages/stats.rs index 4cce498969..59ba9655cb 100644 --- a/libafl/src/stages/stats.rs +++ b/libafl/src/stages/stats.rs @@ -1,7 +1,7 @@ //! Stage to compute/report AFL stats #[cfg(feature = "std")] -use alloc::string::ToString; +use alloc::{borrow::Cow, string::ToString}; use core::{marker::PhantomData, time::Duration}; use libafl_bolts::current_time; @@ -13,8 +13,8 @@ use crate::{ events::EventFirer, schedulers::minimizer::IsFavoredMetadata, stages::Stage, - state::{HasCorpus, HasImported, HasMetadata, UsesState}, - Error, + state::{HasCorpus, HasImported, UsesState}, + Error, HasMetadata, }; #[cfg(feature = "std")] use crate::{ @@ -62,8 +62,6 @@ where Z: UsesState, E::State: HasImported + HasCorpus + HasMetadata, { - type Progress = (); // this stage does not require resume - fn perform( &mut self, _fuzzer: &mut Z, @@ -111,9 +109,9 @@ where _manager.fire( state, Event::UpdateUserStats { - name: "AflStats".to_string(), + name: Cow::from("AflStats"), value: UserStats::new( - UserStatsValue::String(json.to_string()), + UserStatsValue::String(Cow::from(json.to_string())), AggregatorOps::None, ), phantom: PhantomData, @@ -133,6 +131,18 @@ where Ok(()) } + + #[inline] + fn restart_progress_should_run(&mut self, _state: &mut Self::State) -> Result { + // Not running the target so we wont't crash/timeout and, hence, don't need to restore anything + Ok(true) + } + + #[inline] + fn clear_restart_progress(&mut self, _state: &mut Self::State) -> Result<(), Error> { + // Not running the target so we wont't crash/timeout and, hence, don't need to restore anything + Ok(()) + } } impl AflStatsStage diff --git a/libafl/src/stages/string.rs b/libafl/src/stages/string.rs index 5e1333faa8..e2e75f30fe 100644 --- a/libafl/src/stages/string.rs +++ b/libafl/src/stages/string.rs @@ -11,7 +11,8 @@ use crate::{ corpus::HasTestcase, inputs::{BytesInput, HasBytesVec}, stages::Stage, - state::{HasCorpus, HasMetadata, State, UsesState}, + state::{HasCorpus, HasCurrentTestcase, State, UsesState}, + HasMetadata, }; /// Metadata which stores the list of pre-computed string-like ranges in the input @@ -104,8 +105,6 @@ where EM: UsesState, Z: UsesState, { - type Progress = (); // this stage does not need to be resumed - fn perform( &mut self, _fuzzer: &mut Z, @@ -113,13 +112,7 @@ where state: &mut Self::State, _manager: &mut EM, ) -> Result<(), Error> { - let Some(corpus_idx) = state.current_corpus_idx()? else { - return Err(Error::illegal_state( - "state is not currently processing a corpus index", - )); - }; - - let mut tc = state.testcase_mut(corpus_idx)?; + let mut tc = state.current_testcase_mut()?; if tc.has_metadata::() { return Ok(()); // skip recompute } @@ -132,4 +125,16 @@ where Ok(()) } + + #[inline] + fn restart_progress_should_run(&mut self, _state: &mut Self::State) -> Result { + // Stage does not run the target. No reset helper needed. + Ok(true) + } + + #[inline] + fn clear_restart_progress(&mut self, _state: &mut Self::State) -> Result<(), Error> { + // Stage does not run the target. No reset helper needed. + Ok(()) + } } diff --git a/libafl/src/stages/sync.rs b/libafl/src/stages/sync.rs index 081c54a433..520365ba01 100644 --- a/libafl/src/stages/sync.rs +++ b/libafl/src/stages/sync.rs @@ -1,5 +1,6 @@ //! The [`SyncFromDiskStage`] is a stage that imports inputs from disk for e.g. sync with AFL +use alloc::borrow::Cow; use core::marker::PhantomData; use std::{ fs, @@ -7,7 +8,7 @@ use std::{ time::SystemTime, }; -use libafl_bolts::{current_time, shmem::ShMemProvider}; +use libafl_bolts::{current_time, shmem::ShMemProvider, Named}; use serde::{Deserialize, Serialize}; #[cfg(feature = "introspection")] @@ -18,9 +19,9 @@ use crate::{ executors::{Executor, ExitKind, HasObservers}, fuzzer::{Evaluator, EvaluatorObservers, ExecutionProcessor}, inputs::{Input, InputConverter, UsesInput}, - stages::Stage, - state::{HasCorpus, HasExecutions, HasMetadata, HasRand, State, UsesState}, - Error, + stages::{RetryRestartHelper, Stage}, + state::{HasCorpus, HasExecutions, HasRand, State, UsesState}, + Error, HasMetadata, HasNamedMetadata, }; /// Metadata used to store information about disk sync time @@ -59,16 +60,24 @@ where type State = E::State; } +impl Named for SyncFromDiskStage +where + E: UsesState, +{ + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("SyncFromDiskStage"); + &NAME + } +} + impl Stage for SyncFromDiskStage where CB: FnMut(&mut Z, &mut Z::State, &Path) -> Result<::Input, Error>, E: UsesState, EM: UsesState, Z: Evaluator, - Z::State: HasCorpus + HasRand + HasMetadata, + Z::State: HasCorpus + HasRand + HasMetadata + HasNamedMetadata, { - type Progress = (); // TODO load from directory should be resumed - #[inline] fn perform( &mut self, @@ -103,6 +112,18 @@ where Ok(()) } + + #[inline] + fn restart_progress_should_run(&mut self, state: &mut Self::State) -> Result { + // TODO: Needs proper crash handling for when an imported testcase crashes + // For now, Make sure we don't get stuck crashing on this testcase + RetryRestartHelper::restart_progress_should_run(state, self, 3) + } + + #[inline] + fn clear_restart_progress(&mut self, state: &mut Self::State) -> Result<(), Error> { + RetryRestartHelper::clear_restart_progress(state, self) + } } impl SyncFromDiskStage @@ -220,7 +241,7 @@ impl SyncFromBrokerMetadata { /// A stage that loads testcases from disk to sync with other fuzzers such as AFL++ #[derive(Debug)] -pub struct SyncFromBrokerStage +pub struct SyncFromBrokerStage where SP: ShMemProvider + 'static, S: UsesInput, @@ -228,10 +249,10 @@ where ICB: InputConverter, DI: Input, { - client: LlmpEventConverter, + client: LlmpEventConverter, } -impl UsesState for SyncFromBrokerStage +impl UsesState for SyncFromBrokerStage where SP: ShMemProvider + 'static, S: State, @@ -242,7 +263,7 @@ where type State = S; } -impl Stage for SyncFromBrokerStage +impl Stage for SyncFromBrokerStage where EM: UsesState + EventFirer, S: State + HasExecutions + HasCorpus + HasRand + HasMetadata + HasTestcase, @@ -254,8 +275,6 @@ where ICB: InputConverter, DI: Input, { - type Progress = (); // TODO this should be resumed in the case that a testcase causes a crash - #[inline] fn perform( &mut self, @@ -312,9 +331,21 @@ where state.introspection_monitor_mut().finish_stage(); Ok(()) } + + #[inline] + fn restart_progress_should_run(&mut self, _state: &mut Self::State) -> Result { + // No restart handling needed - does not execute the target. + Ok(true) + } + + #[inline] + fn clear_restart_progress(&mut self, _state: &mut Self::State) -> Result<(), Error> { + // Not needed - does not execute the target. + Ok(()) + } } -impl SyncFromBrokerStage +impl SyncFromBrokerStage where SP: ShMemProvider + 'static, S: UsesInput, @@ -324,7 +355,7 @@ where { /// Creates a new [`SyncFromBrokerStage`] #[must_use] - pub fn new(client: LlmpEventConverter) -> Self { + pub fn new(client: LlmpEventConverter) -> Self { Self { client } } } diff --git a/libafl/src/stages/tmin.rs b/libafl/src/stages/tmin.rs index eb603ec30b..26f982893e 100644 --- a/libafl/src/stages/tmin.rs +++ b/libafl/src/stages/tmin.rs @@ -1,25 +1,33 @@ //! The [`TMinMutationalStage`] is a stage which will attempt to minimize corpus entries. -use alloc::string::{String, ToString}; -use core::{fmt::Debug, hash::Hash, marker::PhantomData}; +use alloc::borrow::Cow; +use core::{borrow::BorrowMut, fmt::Debug, hash::Hash, marker::PhantomData}; use ahash::RandomState; -use libafl_bolts::{HasLen, Named}; +use libafl_bolts::{ + tuples::{Handle, Handler, MatchNameRef}, + HasLen, Named, +}; use crate::{ - corpus::{Corpus, CorpusId, HasCurrentCorpusIdx, Testcase}, + corpus::{Corpus, HasCurrentCorpusIdx, Testcase}, events::EventFirer, executors::{Executor, ExitKind, HasObservers}, - feedbacks::{Feedback, FeedbackFactory, HasObserverName}, + feedbacks::{Feedback, FeedbackFactory, HasObserverReference}, inputs::UsesInput, mark_feature_time, mutators::{MutationResult, Mutator}, observers::{MapObserver, ObserversTuple}, schedulers::{RemovableScheduler, Scheduler}, - stages::Stage, + stages::{ + mutational::{MutatedTransform, MutatedTransformPost}, + ExecutionCountRestartHelper, Stage, + }, start_timer, - state::{HasCorpus, HasExecutions, HasMaxSize, HasSolutions, State, UsesState}, - Error, ExecutesInput, ExecutionProcessor, HasFeedback, HasScheduler, + state::{ + HasCorpus, HasCurrentTestcase, HasExecutions, HasMaxSize, HasSolutions, State, UsesState, + }, + Error, ExecutesInput, ExecutionProcessor, HasFeedback, HasMetadata, HasScheduler, }; #[cfg(feature = "introspection")] use crate::{monitors::PerfFeature, state::HasClientPerfMonitor}; @@ -27,7 +35,7 @@ use crate::{monitors::PerfFeature, state::HasClientPerfMonitor}; /// Mutational stage which minimizes corpus entries. /// /// You must provide at least one mutator that actually reduces size. -pub trait TMinMutationalStage: +pub trait TMinMutationalStage: Stage + FeedbackFactory where Self::State: HasCorpus + HasSolutions + HasExecutions + HasMaxSize, @@ -37,12 +45,14 @@ where EM: EventFirer, F1: Feedback, F2: Feedback, - M: Mutator, + M: Mutator, OT: ObserversTuple, Z: ExecutionProcessor + ExecutesInput + HasFeedback + HasScheduler, + IP: MutatedTransformPost + Clone, + I: MutatedTransform + Clone, { /// The mutator registered for this stage fn mutator(&self) -> &M; @@ -51,7 +61,7 @@ where fn mutator_mut(&mut self) -> &mut M; /// Gets the number of iterations this mutator should run for. - fn iterations(&self, state: &mut CS::State, corpus_idx: CorpusId) -> Result; + fn iterations(&self, state: &mut CS::State) -> Result; /// Runs this (mutational) stage for new objectives #[allow(clippy::cast_possible_wrap)] // more than i32 stages on 32 bit system - highly unlikely... @@ -70,17 +80,21 @@ where let orig_max_size = state.max_size(); // basically copy-pasted from mutational.rs - let num = self.iterations(state, base_corpus_idx)?; + let num = self.iterations(state)? + - usize::try_from(self.execs_since_progress_start(state)?).unwrap(); start_timer!(state); - let mut base = state.corpus().cloned_input_for_id(base_corpus_idx)?; + let transformed = I::try_transform_from(state.current_testcase_mut()?.borrow_mut(), state)?; + let mut base = state.current_input_cloned()?; + // potential post operation if base is replaced by a shorter input + let mut base_post = None; let base_hash = RandomState::with_seeds(0, 0, 0, 0).hash_one(&base); mark_feature_time!(state, PerfFeature::GetInputFromCorpus); fuzzer.execute_input(state, executor, manager, &base)?; let observers = executor.observers(); - let mut feedback = self.create_feedback(observers); + let mut feedback = self.create_feedback(&*observers); let mut i = 0; loop { @@ -89,20 +103,21 @@ where } let mut next_i = i + 1; - let mut input = base.clone(); + let mut input_transformed = transformed.clone(); - let before_len = input.len(); + let before_len = base.len(); state.set_max_size(before_len); start_timer!(state); - let mutated = self.mutator_mut().mutate(state, &mut input, i as i32)?; + let mutated = self.mutator_mut().mutate(state, &mut input_transformed)?; mark_feature_time!(state, PerfFeature::Mutate); if mutated == MutationResult::Skipped { continue; } + let (input, post) = input_transformed.try_transform_into(state)?; let corpus_idx = if input.len() < before_len { // run the input let exit_kind = fuzzer.execute_input(state, executor, manager, &input)?; @@ -114,11 +129,11 @@ where // TODO replace if process_execution adds a return value for solution index let solution_count = state.solutions().count(); let corpus_count = state.corpus().count(); - let (_, corpus_idx) = fuzzer.process_execution( + let (_, corpus_idx) = fuzzer.execute_and_process( state, manager, input.clone(), - observers, + &*observers, &exit_kind, false, )?; @@ -127,9 +142,10 @@ where && state.solutions().count() == solution_count { // we do not care about interesting inputs! - if feedback.is_interesting(state, manager, &input, observers, &exit_kind)? { + if feedback.is_interesting(state, manager, &input, &*observers, &exit_kind)? { // we found a reduced corpus entry! use the smaller base base = input; + base_post = Some(post.clone()); // do more runs! maybe we can minify further next_i = 0; @@ -144,7 +160,8 @@ where }; start_timer!(state); - self.mutator_mut().post_exec(state, i as i32, corpus_idx)?; + self.mutator_mut().post_exec(state, corpus_idx)?; + post.post_exec(state, corpus_idx)?; mark_feature_time!(state, PerfFeature::MutatePostExec); i = next_i; @@ -158,63 +175,86 @@ where // marked as interesting above; similarly, it should not trigger objectives fuzzer .feedback_mut() - .is_interesting(state, manager, &base, observers, &exit_kind)?; + .is_interesting(state, manager, &base, &*observers, &exit_kind)?; let mut testcase = Testcase::with_executions(base, *state.executions()); fuzzer .feedback_mut() - .append_metadata(state, observers, &mut testcase)?; + .append_metadata(state, manager, &*observers, &mut testcase)?; let prev = state.corpus_mut().replace(base_corpus_idx, testcase)?; fuzzer .scheduler_mut() .on_replace(state, base_corpus_idx, &prev)?; + // perform the post operation for the new testcase, e.g. to update metadata. + // base_post should be updated along with the base (and is no longer None) + base_post + .ok_or_else(|| Error::empty_optional("Failed to get the MutatedTransformPost"))? + .post_exec(state, Some(base_corpus_idx))?; } state.set_max_size(orig_max_size); Ok(()) } + + /// Gets the number of executions this mutator already did since it got first called in this fuzz round. + fn execs_since_progress_start(&mut self, state: &mut Z::State) -> Result; } /// The default corpus entry minimising mutational stage #[derive(Clone, Debug)] -pub struct StdTMinMutationalStage { +pub struct StdTMinMutationalStage { + /// The mutator(s) this stage uses mutator: M, + /// The factory factory: FF, + /// The runs (=iterations) we are supposed to do runs: usize, + /// The progress helper for this stage, keeping track of resumes after timeouts/crashes + restart_helper: ExecutionCountRestartHelper, #[allow(clippy::type_complexity)] - phantom: PhantomData<(CS, E, EM, F1, F2, OT, Z)>, + phantom: PhantomData<(CS, E, EM, F1, F2, I, IP, OT, Z)>, } -impl UsesState - for StdTMinMutationalStage +impl UsesState + for StdTMinMutationalStage where CS: Scheduler, - M: Mutator, + M: Mutator, Z: ExecutionProcessor, CS::State: HasCorpus, + IP: MutatedTransformPost + Clone, + I: MutatedTransform + Clone, { type State = CS::State; } -impl Stage - for StdTMinMutationalStage +impl Stage + for StdTMinMutationalStage where CS: Scheduler + RemovableScheduler, - CS::State: HasCorpus + HasSolutions + HasExecutions + HasMaxSize + HasCorpus, + CS::State: HasCorpus + HasSolutions + HasExecutions + HasMaxSize + HasCorpus + HasMetadata, ::Input: HasLen + Hash, E: Executor + HasObservers, EM: EventFirer, F1: Feedback, F2: Feedback, FF: FeedbackFactory, - M: Mutator, + M: Mutator, OT: ObserversTuple, Z: ExecutionProcessor + ExecutesInput + HasFeedback + HasScheduler, + IP: MutatedTransformPost + Clone, + I: MutatedTransform + Clone, { - type Progress = (); // TODO this stage desperately needs a resume + fn restart_progress_should_run(&mut self, state: &mut Self::State) -> Result { + self.restart_helper.restart_progress_should_run(state) + } + + fn clear_restart_progress(&mut self, state: &mut Self::State) -> Result<(), Error> { + self.restart_helper.clear_restart_progress(state) + } fn perform( &mut self, @@ -232,8 +272,8 @@ where } } -impl FeedbackFactory - for StdTMinMutationalStage +impl FeedbackFactory + for StdTMinMutationalStage where F2: Feedback, FF: FeedbackFactory, @@ -244,8 +284,8 @@ where } } -impl TMinMutationalStage - for StdTMinMutationalStage +impl TMinMutationalStage + for StdTMinMutationalStage where CS: Scheduler + RemovableScheduler, E: HasObservers + Executor, @@ -254,13 +294,15 @@ where F2: Feedback, FF: FeedbackFactory, ::Input: HasLen + Hash, - M: Mutator, + M: Mutator, OT: ObserversTuple, - CS::State: HasCorpus + HasSolutions + HasExecutions + HasMaxSize, + CS::State: HasCorpus + HasSolutions + HasExecutions + HasMaxSize + HasMetadata, Z: ExecutionProcessor + ExecutesInput + HasFeedback + HasScheduler, + IP: MutatedTransformPost + Clone, + I: MutatedTransform + Clone, { /// The mutator, added to this stage #[inline] @@ -275,24 +317,32 @@ where } /// Gets the number of iterations from a fixed number of runs - fn iterations(&self, _state: &mut CS::State, _corpus_idx: CorpusId) -> Result { + fn iterations(&self, _state: &mut CS::State) -> Result { Ok(self.runs) } + + fn execs_since_progress_start(&mut self, state: &mut ::State) -> Result { + self.restart_helper.execs_since_progress_start(state) + } } -impl StdTMinMutationalStage +impl + StdTMinMutationalStage where CS: Scheduler, - M: Mutator, + M: Mutator, Z: ExecutionProcessor, CS::State: HasCorpus, + IP: MutatedTransformPost + Clone, + I: MutatedTransform + Clone, { - /// Creates a new minimising mutational stage that will minimize provided corpus entries + /// Creates a new minimizing mutational stage that will minimize provided corpus entries pub fn new(mutator: M, factory: FF, runs: usize) -> Self { Self { mutator, factory, runs, + restart_helper: ExecutionCountRestartHelper::default(), phantom: PhantomData, } } @@ -301,41 +351,31 @@ where /// A feedback which checks if the hash of the currently observed map is equal to the original hash /// provided #[derive(Clone, Debug)] -pub struct MapEqualityFeedback { - name: String, - obs_name: String, +pub struct MapEqualityFeedback { + name: Cow<'static, str>, + map_ref: Handle, orig_hash: u64, phantom: PhantomData<(M, S)>, } -impl MapEqualityFeedback { - /// Create a new map equality feedback -- can be used with feedback logic - #[must_use] - pub fn new(name: &str, obs_name: &str, orig_hash: u64) -> Self { - MapEqualityFeedback { - name: name.to_string(), - obs_name: obs_name.to_string(), - orig_hash, - phantom: PhantomData, - } - } -} - -impl Named for MapEqualityFeedback { - fn name(&self) -> &str { +impl Named for MapEqualityFeedback { + fn name(&self) -> &Cow<'static, str> { &self.name } } -impl HasObserverName for MapEqualityFeedback { - fn observer_name(&self) -> &str { - &self.obs_name +impl HasObserverReference for MapEqualityFeedback { + type Observer = C; + + fn observer_ref(&self) -> &Handle { + &self.map_ref } } -impl Feedback for MapEqualityFeedback +impl Feedback for MapEqualityFeedback where M: MapObserver, + C: AsRef, S: State, { fn is_interesting( @@ -351,52 +391,57 @@ where OT: ObserversTuple, { let obs = observers - .match_name::(self.observer_name()) + .get(self.observer_ref()) .expect("Should have been provided valid observer name."); - Ok(obs.hash() == self.orig_hash) + Ok(obs.as_ref().hash_simple() == self.orig_hash) } } /// A feedback factory for ensuring that the maps for minimized inputs are the same #[derive(Debug, Clone)] -pub struct MapEqualityFactory { - obs_name: String, - phantom: PhantomData<(M, S)>, +pub struct MapEqualityFactory { + map_ref: Handle, + phantom: PhantomData<(C, M, S)>, } -impl MapEqualityFactory +impl MapEqualityFactory where M: MapObserver, + C: AsRef + Handler, { /// Creates a new map equality feedback for the given observer - pub fn with_observer(obs: &M) -> Self { + pub fn new(obs: &C) -> Self { Self { - obs_name: obs.name().to_string(), + map_ref: obs.handle(), phantom: PhantomData, } } } -impl HasObserverName for MapEqualityFactory { - fn observer_name(&self) -> &str { - &self.obs_name +impl HasObserverReference for MapEqualityFactory { + type Observer = C; + + fn observer_ref(&self) -> &Handle { + &self.map_ref } } -impl FeedbackFactory, S, OT> for MapEqualityFactory +impl FeedbackFactory, S, OT> + for MapEqualityFactory where M: MapObserver, + C: AsRef + Handler, OT: ObserversTuple, S: State + Debug, { - fn create_feedback(&self, observers: &OT) -> MapEqualityFeedback { + fn create_feedback(&self, observers: &OT) -> MapEqualityFeedback { let obs = observers - .match_name::(self.observer_name()) + .get(self.observer_ref()) .expect("Should have been provided valid observer name."); MapEqualityFeedback { - name: "MapEq".to_string(), - obs_name: self.obs_name.clone(), - orig_hash: obs.hash(), + name: Cow::from("MapEq"), + map_ref: obs.handle(), + orig_hash: obs.as_ref().hash_simple(), phantom: PhantomData, } } diff --git a/libafl/src/stages/tracing.rs b/libafl/src/stages/tracing.rs index f141447b5c..0bc40360a1 100644 --- a/libafl/src/stages/tracing.rs +++ b/libafl/src/stages/tracing.rs @@ -1,16 +1,18 @@ //! The tracing stage can trace the target and enrich a testcase with metadata, for example for `CmpLog`. +use alloc::borrow::Cow; use core::{fmt::Debug, marker::PhantomData}; +use libafl_bolts::Named; + use crate::{ - corpus::{Corpus, HasCurrentCorpusIdx}, executors::{Executor, HasObservers, ShadowExecutor}, mark_feature_time, observers::ObserversTuple, - stages::Stage, + stages::{RetryRestartHelper, Stage}, start_timer, - state::{HasCorpus, HasExecutions, State, UsesState}, - Error, + state::{HasCorpus, HasCurrentTestcase, HasExecutions, State, UsesState}, + Error, HasNamedMetadata, }; #[cfg(feature = "introspection")] use crate::{monitors::PerfFeature, state::HasClientPerfMonitor}; @@ -19,6 +21,7 @@ use crate::{monitors::PerfFeature, state::HasClientPerfMonitor}; #[derive(Clone, Debug)] pub struct TracingStage { tracer_executor: TE, + max_retries: usize, #[allow(clippy::type_complexity)] phantom: PhantomData<(EM, TE, Z)>, } @@ -30,32 +33,24 @@ where type State = TE::State; } -impl Stage for TracingStage +impl TracingStage where - E: UsesState, TE: Executor + HasObservers, - TE::State: HasExecutions + HasCorpus, + TE::State: HasExecutions + HasCorpus + HasNamedMetadata, EM: UsesState, Z: UsesState, { - type Progress = (); // this stage cannot be resumed - - #[inline] - fn perform( + /// Perform tracing on the given `CorpusId`. Useful for if wrapping [`TracingStage`] with your + /// own stage and you need to manage [`super::NestedStageRestartHelper`] differently; see + /// [`super::ConcolicTracingStage`]'s implementation as an example of usage. + pub fn trace( &mut self, fuzzer: &mut Z, - _executor: &mut E, state: &mut TE::State, manager: &mut EM, ) -> Result<(), Error> { - let Some(corpus_idx) = state.current_corpus_idx()? else { - return Err(Error::illegal_state( - "state is not currently processing a corpus index", - )); - }; - start_timer!(state); - let input = state.corpus().cloned_input_for_id(corpus_idx)?; + let input = state.current_input_cloned()?; mark_feature_time!(state, PerfFeature::GetInputFromCorpus); @@ -83,15 +78,59 @@ where } } +impl Stage for TracingStage +where + E: UsesState, + TE: Executor + HasObservers, + TE::State: HasExecutions + HasCorpus + HasNamedMetadata, + EM: UsesState, + Z: UsesState, +{ + #[inline] + fn perform( + &mut self, + fuzzer: &mut Z, + _executor: &mut E, + state: &mut TE::State, + manager: &mut EM, + ) -> Result<(), Error> { + self.trace(fuzzer, state, manager) + } + + fn restart_progress_should_run(&mut self, state: &mut Self::State) -> Result { + RetryRestartHelper::restart_progress_should_run(state, self, self.max_retries) + } + + fn clear_restart_progress(&mut self, state: &mut Self::State) -> Result<(), Error> { + RetryRestartHelper::clear_restart_progress(state, self) + } +} + +impl Named for TracingStage { + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("TracingStage"); + &NAME + } +} + impl TracingStage { /// Creates a new default stage pub fn new(tracer_executor: TE) -> Self { Self { tracer_executor, + max_retries: 10, phantom: PhantomData, } } + /// Specify how many times that this stage will try again to trace the input before giving up + /// and not processing the input again. 0 retries means that the trace will be tried only once. + #[must_use] + pub fn with_retries(mut self, retries: usize) -> Self { + self.max_retries = retries; + self + } + /// Gets the underlying tracer executor pub fn executor(&self) -> &TE { &self.tracer_executor @@ -102,9 +141,11 @@ impl TracingStage { &mut self.tracer_executor } } + /// A stage that runs the shadow executor using also the shadow observers #[derive(Clone, Debug)] pub struct ShadowTracingStage { + max_retries: usize, #[allow(clippy::type_complexity)] phantom: PhantomData<(E, EM, SOT, Z)>, } @@ -116,16 +157,24 @@ where type State = E::State; } +impl Named for ShadowTracingStage +where + E: UsesState, +{ + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("ShadowTracingStage"); + &NAME + } +} + impl Stage, EM, Z> for ShadowTracingStage where E: Executor + HasObservers, EM: UsesState, SOT: ObserversTuple, Z: UsesState, - E::State: State + HasExecutions + HasCorpus + Debug, + E::State: State + HasExecutions + HasCorpus + HasNamedMetadata + Debug, { - type Progress = (); // this stage cannot be resumed - #[inline] fn perform( &mut self, @@ -134,14 +183,8 @@ where state: &mut E::State, manager: &mut EM, ) -> Result<(), Error> { - let Some(corpus_idx) = state.current_corpus_idx()? else { - return Err(Error::illegal_state( - "state is not currently processing a corpus index", - )); - }; - start_timer!(state); - let input = state.corpus().cloned_input_for_id(corpus_idx)?; + let input = state.current_input_cloned()?; mark_feature_time!(state, PerfFeature::GetInputFromCorpus); @@ -169,6 +212,14 @@ where Ok(()) } + + fn restart_progress_should_run(&mut self, state: &mut Self::State) -> Result { + RetryRestartHelper::restart_progress_should_run(state, self, self.max_retries) + } + + fn clear_restart_progress(&mut self, state: &mut Self::State) -> Result<(), Error> { + RetryRestartHelper::clear_restart_progress(state, self) + } } impl ShadowTracingStage @@ -182,7 +233,16 @@ where /// Creates a new default stage pub fn new(_executor: &mut ShadowExecutor) -> Self { Self { + max_retries: 10, phantom: PhantomData, } } + + /// Specify how many times that this stage will try again to trace the input before giving up + /// and not processing the input again. 0 retries means that the trace will be tried only once. + #[must_use] + pub fn with_retries(mut self, retries: usize) -> Self { + self.max_retries = retries; + self + } } diff --git a/libafl/src/stages/tuneable.rs b/libafl/src/stages/tuneable.rs index 66d1a2333f..7ed57a880e 100644 --- a/libafl/src/stages/tuneable.rs +++ b/libafl/src/stages/tuneable.rs @@ -7,16 +7,15 @@ use libafl_bolts::{current_time, impl_serdeany, rands::Rand}; use serde::{Deserialize, Serialize}; use crate::{ - corpus::{Corpus, CorpusId, HasCurrentCorpusIdx}, mark_feature_time, mutators::{MutationResult, Mutator}, stages::{ mutational::{MutatedTransform, MutatedTransformPost, DEFAULT_MUTATIONAL_MAX_ITERATIONS}, - MutationalStage, Stage, + ExecutionCountRestartHelper, MutationalStage, Stage, }, start_timer, - state::{HasCorpus, HasMetadata, HasNamedMetadata, HasRand, UsesState}, - Error, Evaluator, + state::{HasCorpus, HasCurrentTestcase, HasExecutions, HasRand, UsesState}, + Error, Evaluator, HasMetadata, HasNamedMetadata, }; #[cfg(feature = "introspection")] use crate::{monitors::PerfFeature, state::HasClientPerfMonitor}; @@ -150,8 +149,12 @@ where /// A [`crate::stages::MutationalStage`] where the mutator iteration can be tuned at runtime #[derive(Clone, Debug)] pub struct TuneableMutationalStage { + /// The mutator we use mutator: M, + /// The name of this stage name: String, + /// The progress helper we use to keep track of progress across restarts + restart_helper: ExecutionCountRestartHelper, phantom: PhantomData<(E, EM, I, Z)>, } @@ -161,12 +164,11 @@ where EM: UsesState, M: Mutator, Z: Evaluator, - Z::State: HasCorpus + HasRand + HasNamedMetadata + HasMetadata, + Z::State: HasCorpus + HasRand + HasNamedMetadata + HasMetadata + HasExecutions, I: MutatedTransform + Clone, { /// Runs this (mutational) stage for the given `testcase` /// Exactly the same functionality as [`MutationalStage::perform_mutational`], but with added timeout support. - #[allow(clippy::cast_possible_wrap)] // more than i32 stages on 32 bit system - highly unlikely... fn perform_mutational( &mut self, fuzzer: &mut Z, @@ -174,18 +176,12 @@ where state: &mut Z::State, manager: &mut EM, ) -> Result<(), Error> { - let Some(corpus_idx) = state.current_corpus_idx()? else { - return Err(Error::illegal_state( - "state is not currently processing a corpus index", - )); - }; - let fuzz_time = self.seed_fuzz_time(state)?; let iters = self.fixed_iters(state)?; start_timer!(state); - let mut testcase = state.corpus().get(corpus_idx)?.borrow_mut(); - let Ok(input) = I::try_transform_from(&mut testcase, state, corpus_idx) else { + let mut testcase = state.current_testcase_mut()?; + let Ok(input) = I::try_transform_from(&mut testcase, state) else { return Ok(()); }; drop(testcase); @@ -195,36 +191,38 @@ where (Some(fuzz_time), Some(iters)) => { // perform n iterations or fuzz for provided time, whichever comes first let start_time = current_time(); - for i in 1..=iters { + for _ in 1..=iters { if current_time() - start_time >= fuzz_time { break; } - self.perform_mutation(fuzzer, executor, state, manager, &input, i)?; + self.perform_mutation(fuzzer, executor, state, manager, &input)?; } } (Some(fuzz_time), None) => { // fuzz for provided time let start_time = current_time(); - for i in 1.. { + for _ in 1.. { if current_time() - start_time >= fuzz_time { break; } - self.perform_mutation(fuzzer, executor, state, manager, &input, i)?; + self.perform_mutation(fuzzer, executor, state, manager, &input)?; } } (None, Some(iters)) => { // perform n iterations - for i in 1..=iters { - self.perform_mutation(fuzzer, executor, state, manager, &input, i)?; + for _ in 1..=iters { + self.perform_mutation(fuzzer, executor, state, manager, &input)?; } } (None, None) => { // fall back to random - let iters = self.iterations(state, corpus_idx)?; - for i in 1..=iters { - self.perform_mutation(fuzzer, executor, state, manager, &input, i)?; + let iters = self + .iterations(state)? + .saturating_sub(self.execs_since_progress_start(state)? as usize); + for _ in 1..=iters { + self.perform_mutation(fuzzer, executor, state, manager, &input)?; } } } @@ -244,13 +242,16 @@ where } /// Gets the number of iterations as a random number - #[allow(clippy::cast_possible_truncation)] - fn iterations(&self, state: &mut Z::State, _corpus_idx: CorpusId) -> Result { + fn iterations(&self, state: &mut Z::State) -> Result { Ok( // fall back to random 1 + state.rand_mut().below(DEFAULT_MUTATIONAL_MAX_ITERATIONS), ) } + + fn execs_since_progress_start(&mut self, state: &mut ::State) -> Result { + self.restart_helper.execs_since_progress_start(state) + } } impl UsesState for TuneableMutationalStage @@ -259,7 +260,7 @@ where EM: UsesState, M: Mutator, Z: Evaluator, - Z::State: HasCorpus + HasRand, + Z::State: HasCorpus + HasRand + HasExecutions, I: MutatedTransform + Clone, { type State = Z::State; @@ -271,11 +272,9 @@ where EM: UsesState, M: Mutator, Z: Evaluator, - Z::State: HasCorpus + HasRand + HasNamedMetadata + HasMetadata, + Z::State: HasCorpus + HasRand + HasNamedMetadata + HasMetadata + HasExecutions, I: MutatedTransform + Clone, { - type Progress = (); // TODO should this stage be resumed? - #[inline] #[allow(clippy::let_and_return)] fn perform( @@ -292,6 +291,14 @@ where ret } + + fn restart_progress_should_run(&mut self, state: &mut Self::State) -> Result { + self.restart_helper.restart_progress_should_run(state) + } + + fn clear_restart_progress(&mut self, state: &mut Self::State) -> Result<(), Error> { + self.restart_helper.clear_restart_progress(state) + } } impl TuneableMutationalStage @@ -300,7 +307,7 @@ where EM: UsesState, M: Mutator, Z: Evaluator, - Z::State: HasCorpus + HasRand + HasNamedMetadata + HasMetadata, + Z::State: HasCorpus + HasRand + HasNamedMetadata + HasMetadata + HasExecutions, I: MutatedTransform + Clone, { /// Creates a new default tuneable mutational stage @@ -434,14 +441,11 @@ where state: &mut Z::State, manager: &mut EM, input: &I, - stage_idx: u64, ) -> Result<(), Error> { let mut input = input.clone(); start_timer!(state); - let mutated = self - .mutator_mut() - .mutate(state, &mut input, stage_idx as i32)?; + let mutated = self.mutator_mut().mutate(state, &mut input)?; mark_feature_time!(state, PerfFeature::Mutate); if mutated == MutationResult::Skipped { @@ -453,9 +457,8 @@ where let (_, corpus_idx) = fuzzer.evaluate_input(state, executor, manager, untransformed)?; start_timer!(state); - self.mutator_mut() - .post_exec(state, stage_idx as i32, corpus_idx)?; - post.post_exec(state, stage_idx as i32, corpus_idx)?; + self.mutator_mut().post_exec(state, corpus_idx)?; + post.post_exec(state, corpus_idx)?; mark_feature_time!(state, PerfFeature::MutatePostExec); Ok(()) @@ -470,15 +473,14 @@ where Z: Evaluator, Z::State: HasCorpus + HasRand + HasNamedMetadata, { - /// Creates a new tranforming mutational stage + /// Creates a new transforming mutational stage #[must_use] pub fn transforming(state: &mut Z::State, mutator: M, name: &str) -> Self { - if !state.has_named_metadata::(name) { - state.add_named_metadata(TuneableMutationalStageMetadata::default(), name); - } + let _ = state.named_metadata_or_insert_with(name, TuneableMutationalStageMetadata::default); Self { mutator, name: name.to_string(), + restart_helper: ExecutionCountRestartHelper::default(), phantom: PhantomData, } } diff --git a/libafl/src/state/mod.rs b/libafl/src/state/mod.rs index 9bd2af6992..60bf021416 100644 --- a/libafl/src/state/mod.rs +++ b/libafl/src/state/mod.rs @@ -2,6 +2,7 @@ use alloc::vec::Vec; use core::{ + borrow::BorrowMut, cell::{Ref, RefMut}, fmt::Debug, marker::PhantomData, @@ -13,9 +14,11 @@ use std::{ path::{Path, PathBuf}, }; +#[cfg(feature = "std")] +use libafl_bolts::core_affinity::{CoreId, Cores}; use libafl_bolts::{ - rands::Rand, - serdeany::{NamedSerdeAnyMap, SerdeAny, SerdeAnyMap}, + rands::{Rand, StdRand}, + serdeany::{NamedSerdeAnyMap, SerdeAnyMap}, }; use serde::{de::DeserializeOwned, Deserialize, Serialize}; @@ -31,7 +34,7 @@ use crate::{ generators::Generator, inputs::{Input, UsesInput}, stages::{HasCurrentStage, HasNestedStageStatus}, - Error, + Error, HasMetadata, HasNamedMetadata, }; /// The maximum size of a testcase @@ -152,111 +155,13 @@ pub trait HasScalabilityMonitor { fn scalability_monitor_mut(&mut self) -> &mut ScalabilityMonitor; } -/// Trait for elements offering metadata -pub trait HasMetadata { - /// A map, storing all metadata - fn metadata_map(&self) -> &SerdeAnyMap; - /// A map, storing all metadata (mutable) - fn metadata_map_mut(&mut self) -> &mut SerdeAnyMap; - - /// Add a metadata to the metadata map - #[inline] - fn add_metadata(&mut self, meta: M) - where - M: SerdeAny, - { - self.metadata_map_mut().insert(meta); - } - - /// Check for a metadata - #[inline] - fn has_metadata(&self) -> bool - where - M: SerdeAny, - { - self.metadata_map().get::().is_some() - } - - /// To get metadata - #[inline] - fn metadata(&self) -> Result<&M, Error> - where - M: SerdeAny, - { - self.metadata_map().get::().ok_or_else(|| { - Error::key_not_found(format!("{} not found", core::any::type_name::())) - }) - } - - /// To get mutable metadata - #[inline] - fn metadata_mut(&mut self) -> Result<&mut M, Error> - where - M: SerdeAny, - { - self.metadata_map_mut().get_mut::().ok_or_else(|| { - Error::key_not_found(format!("{} not found", core::any::type_name::())) - }) - } -} - -/// Trait for elements offering named metadata -pub trait HasNamedMetadata { - /// A map, storing all metadata - fn named_metadata_map(&self) -> &NamedSerdeAnyMap; - /// A map, storing all metadata (mutable) - fn named_metadata_map_mut(&mut self) -> &mut NamedSerdeAnyMap; - - /// Add a metadata to the metadata map - #[inline] - fn add_named_metadata(&mut self, meta: M, name: &str) - where - M: SerdeAny, - { - self.named_metadata_map_mut().insert(meta, name); - } - - /// Check for a metadata - #[inline] - fn has_named_metadata(&self, name: &str) -> bool - where - M: SerdeAny, - { - self.named_metadata_map().contains::(name) - } - - /// To get named metadata - #[inline] - fn named_metadata(&self, name: &str) -> Result<&M, Error> - where - M: SerdeAny, - { - self.named_metadata_map().get::(name).ok_or_else(|| { - Error::key_not_found(format!("{} not found", core::any::type_name::())) - }) - } - - /// To get mutable named metadata - #[inline] - fn named_metadata_mut(&mut self, name: &str) -> Result<&mut M, Error> - where - M: SerdeAny, - { - self.named_metadata_map_mut() - .get_mut::(name) - .ok_or_else(|| { - Error::key_not_found(format!("{} not found", core::any::type_name::())) - }) - } -} - /// Trait for the execution counter pub trait HasExecutions { /// The executions counter - fn executions(&self) -> &usize; + fn executions(&self) -> &u64; /// The executions counter (mutable) - fn executions_mut(&mut self) -> &mut usize; + fn executions_mut(&mut self) -> &mut u64; } /// Trait for some stats of AFL @@ -288,6 +193,24 @@ pub trait HasLastReportTime { fn last_report_time_mut(&mut self) -> &mut Option; } +/// Struct that holds the options for input loading +#[cfg(feature = "std")] +pub struct LoadConfig<'a, I, S, Z> { + /// Load Input even if it was deemed "uninteresting" by the fuzzer + forced: bool, + /// Function to load input from a Path + loader: &'a mut dyn FnMut(&mut Z, &mut S, &Path) -> Result, + /// Error if Input leads to a Solution. + exit_on_solution: bool, +} + +#[cfg(feature = "std")] +impl<'a, I, S, Z> Debug for LoadConfig<'a, I, S, Z> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "LoadConfig {{}}") + } +} + /// The state a fuzz run. #[derive(Serialize, Deserialize, Clone, Debug)] #[serde(bound = " @@ -299,7 +222,7 @@ pub struct StdState { /// RNG instance rand: R, /// How many times the executor ran the harness/target - executions: usize, + executions: u64, /// At what time the fuzzing started start_time: Duration, /// the number of new paths that imported from other fuzzers @@ -312,7 +235,7 @@ pub struct StdState { metadata: SerdeAnyMap, /// Metadata stored with names named_metadata: NamedSerdeAnyMap, - /// MaxSize testcase size for mutators that appreciate it + /// `MaxSize` testcase size for mutators that appreciate it max_size: usize, /// Performance statistics for this fuzzer #[cfg(feature = "introspection")] @@ -325,6 +248,10 @@ pub struct StdState { #[cfg(feature = "std")] /// Remaining initial inputs to load, if any dont_reenter: Option>, + #[cfg(feature = "std")] + /// If inputs have been processed for multicore loading + /// relevant only for `load_initial_inputs_multicore` + multicore_inputs_processed: Option, /// The last time we reported progress (if available/used). /// This information is used by fuzzer `maybe_report_progress`. last_report_time: Option, @@ -400,7 +327,10 @@ where R: Rand, { /// To get the testcase - fn testcase(&self, id: CorpusId) -> Result::Input>>, Error> { + fn testcase( + &self, + id: CorpusId, + ) -> Result::Input>>, Error> { Ok(self.corpus().get(id)?.borrow()) } @@ -408,7 +338,7 @@ where fn testcase_mut( &self, id: CorpusId, - ) -> Result::Input>>, Error> { + ) -> Result::Input>>, Error> { Ok(self.corpus().get(id)?.borrow_mut()) } } @@ -464,13 +394,13 @@ impl HasNamedMetadata for StdState { impl HasExecutions for StdState { /// The executions counter #[inline] - fn executions(&self) -> &usize { + fn executions(&self) -> &u64 { &self.executions } /// The executions counter (mutable) #[inline] - fn executions_mut(&mut self) -> &mut usize { + fn executions_mut(&mut self) -> &mut u64 { &mut self.executions } } @@ -543,6 +473,64 @@ impl HasCurrentCorpusIdx for StdState { } } +/// Has information about the current [`Testcase`] we are fuzzing +pub trait HasCurrentTestcase +where + I: Input, +{ + /// Gets the current [`Testcase`] we are fuzzing + /// + /// Will return [`Error::key_not_found`] if no `corpus_idx` is currently set. + fn current_testcase(&self) -> Result>, Error>; + //fn current_testcase(&self) -> Result<&Testcase, Error>; + + /// Gets the current [`Testcase`] we are fuzzing (mut) + /// + /// Will return [`Error::key_not_found`] if no `corpus_idx` is currently set. + fn current_testcase_mut(&self) -> Result>, Error>; + //fn current_testcase_mut(&self) -> Result<&mut Testcase, Error>; + + /// Gets a cloned representation of the current [`Testcase`]. + /// + /// Will return [`Error::key_not_found`] if no `corpus_idx` is currently set. + /// + /// # Note + /// This allocates memory and copies the contents! + /// For performance reasons, if you just need to access the testcase, use [`Self::current_testcase`] instead. + fn current_input_cloned(&self) -> Result; +} + +impl HasCurrentTestcase for T +where + I: Input, + T: HasCorpus + HasCurrentCorpusIdx + UsesInput, +{ + fn current_testcase(&self) -> Result>, Error> { + let Some(corpus_id) = self.current_corpus_idx()? else { + return Err(Error::key_not_found( + "We are not currently processing a testcase", + )); + }; + + Ok(self.corpus().get(corpus_id)?.borrow()) + } + + fn current_testcase_mut(&self) -> Result>, Error> { + let Some(corpus_id) = self.current_corpus_idx()? else { + return Err(Error::illegal_state( + "We are not currently processing a testcase", + )); + }; + + Ok(self.corpus().get(corpus_id)?.borrow_mut()) + } + + fn current_input_cloned(&self) -> Result { + let mut testcase = self.current_testcase_mut()?; + Ok(testcase.borrow_mut().load_input(self.corpus())?.clone()) + } +} + impl HasCurrentStage for StdState { fn set_stage(&mut self, idx: usize) -> Result<(), Error> { // ensure we are in the right frame @@ -642,22 +630,14 @@ where } } - /// Loads initial inputs from the passed-in `in_dirs`. - /// If `forced` is true, will add all testcases, no matter what. - fn load_initial_inputs_custom( - &mut self, - fuzzer: &mut Z, - executor: &mut E, - manager: &mut EM, - in_dirs: &[PathBuf], - forced: bool, - loader: &mut dyn FnMut(&mut Z, &mut Self, &Path) -> Result, - ) -> Result<(), Error> - where - E: UsesState, - EM: EventFirer, - Z: Evaluator, - { + /// Resets the state of initial files. + fn reset_initial_files_state(&mut self) { + self.remaining_initial_files = None; + self.dont_reenter = None; + } + + /// Sets canonical paths for provided inputs + fn canonicalize_input_dirs(&mut self, in_dirs: &[PathBuf]) -> Result<(), Error> { if let Some(remaining) = self.remaining_initial_files.as_ref() { // everything was loaded if remaining.is_empty() { @@ -673,8 +653,7 @@ where self.dont_reenter = Some(files.clone()); self.remaining_initial_files = Some(files); } - - self.continue_loading_initial_inputs_custom(fuzzer, executor, manager, forced, loader) + Ok(()) } /// Loads initial inputs from the passed-in `in_dirs`. @@ -686,8 +665,7 @@ where executor: &mut E, manager: &mut EM, file_list: &[PathBuf], - forced: bool, - loader: &mut dyn FnMut(&mut Z, &mut Self, &Path) -> Result, + load_config: LoadConfig, ) -> Result<(), Error> where E: UsesState, @@ -703,19 +681,45 @@ where self.remaining_initial_files = Some(file_list.to_vec()); } - self.continue_loading_initial_inputs_custom(fuzzer, executor, manager, forced, loader) + self.continue_loading_initial_inputs_custom(fuzzer, executor, manager, load_config) } + fn load_file( + &mut self, + path: &PathBuf, + manager: &mut EM, + fuzzer: &mut Z, + executor: &mut E, + config: &mut LoadConfig, + ) -> Result + where + E: UsesState, + EM: EventFirer, + Z: Evaluator, + { + log::info!("Loading file {:?} ...", &path); + let input = (config.loader)(fuzzer, self, path)?; + if config.forced { + let _: CorpusId = fuzzer.add_input(self, executor, manager, input)?; + Ok(ExecuteInputResult::Corpus) + } else { + let (res, _) = fuzzer.evaluate_input(self, executor, manager, input.clone())?; + if res == ExecuteInputResult::None { + fuzzer.add_disabled_input(self, input)?; + log::warn!("input {:?} was not interesting, adding as disabled.", &path); + } + Ok(res) + } + } /// Loads initial inputs from the passed-in `in_dirs`. - /// If `forced` is true, will add all testcases, no matter what. - /// This method takes a list of files. + /// This method takes a list of files and a `LoadConfig` + /// which specifies the special handling of initial inputs fn continue_loading_initial_inputs_custom( &mut self, fuzzer: &mut Z, executor: &mut E, manager: &mut EM, - forced: bool, - loader: &mut dyn FnMut(&mut Z, &mut Self, &Path) -> Result, + mut config: LoadConfig, ) -> Result<(), Error> where E: UsesState, @@ -725,15 +729,12 @@ where loop { match self.next_file() { Ok(path) => { - log::info!("Loading file {:?} ...", &path); - let input = loader(fuzzer, self, &path)?; - if forced { - let _: CorpusId = fuzzer.add_input(self, executor, manager, input)?; - } else { - let (res, _) = fuzzer.evaluate_input(self, executor, manager, input)?; - if res == ExecuteInputResult::None { - log::warn!("File {:?} was not interesting, skipped.", &path); - } + let res = self.load_file(&path, manager, fuzzer, executor, &mut config)?; + if config.exit_on_solution && matches!(res, ExecuteInputResult::Solution) { + return Err(Error::invalid_corpus(format!( + "Input {} resulted in a solution.", + path.display() + ))); } } Err(Error::IteratorEnd(_, _)) => break, @@ -773,8 +774,11 @@ where executor, manager, file_list, - false, - &mut |_, _, path| I::from_file(path), + LoadConfig { + loader: &mut |_, _, path| I::from_file(path), + forced: false, + exit_on_solution: false, + }, ) } @@ -793,16 +797,18 @@ where EM: EventFirer, Z: Evaluator, { - self.load_initial_inputs_custom( + self.canonicalize_input_dirs(in_dirs)?; + self.continue_loading_initial_inputs_custom( fuzzer, executor, manager, - in_dirs, - true, - &mut |_, _, path| I::from_file(path), + LoadConfig { + loader: &mut |_, _, path| I::from_file(path), + forced: true, + exit_on_solution: false, + }, ) } - /// Loads initial inputs from the passed-in `in_dirs`. /// If `forced` is true, will add all testcases, no matter what. /// This method takes a list of files, instead of folders. @@ -823,8 +829,11 @@ where executor, manager, file_list, - true, - &mut |_, _, path| I::from_file(path), + LoadConfig { + loader: &mut |_, _, path| I::from_file(path), + forced: true, + exit_on_solution: false, + }, ) } @@ -841,15 +850,147 @@ where EM: EventFirer, Z: Evaluator, { - self.load_initial_inputs_custom( + self.canonicalize_input_dirs(in_dirs)?; + self.continue_loading_initial_inputs_custom( + fuzzer, + executor, + manager, + LoadConfig { + loader: &mut |_, _, path| I::from_file(path), + forced: false, + exit_on_solution: false, + }, + ) + } + + /// Loads initial inputs from the passed-in `in_dirs`. + /// Will return a `CorpusError` if a solution is found + pub fn load_initial_inputs_disallow_solution( + &mut self, + fuzzer: &mut Z, + executor: &mut E, + manager: &mut EM, + in_dirs: &[PathBuf], + ) -> Result<(), Error> + where + E: UsesState, + EM: EventFirer, + Z: Evaluator, + { + self.canonicalize_input_dirs(in_dirs)?; + self.continue_loading_initial_inputs_custom( fuzzer, executor, manager, - in_dirs, - false, - &mut |_, _, path| I::from_file(path), + LoadConfig { + loader: &mut |_, _, path| I::from_file(path), + forced: false, + exit_on_solution: true, + }, ) } + + fn calculate_corpus_size(&mut self) -> Result { + let mut count: usize = 0; + loop { + match self.next_file() { + Ok(_) => { + count = count.saturating_add(1); + } + Err(Error::IteratorEnd(_, _)) => break, + Err(e) => return Err(e), + } + } + Ok(count) + } + /// Loads initial inputs by dividing the from the passed-in `in_dirs` + /// in a multicore fashion. Divides the corpus in chunks spread across cores. + pub fn load_initial_inputs_multicore( + &mut self, + fuzzer: &mut Z, + executor: &mut E, + manager: &mut EM, + in_dirs: &[PathBuf], + core_id: &CoreId, + cores: &Cores, + ) -> Result<(), Error> + where + E: UsesState, + EM: EventFirer, + Z: Evaluator, + { + if self.multicore_inputs_processed.unwrap_or(false) { + self.continue_loading_initial_inputs_custom( + fuzzer, + executor, + manager, + LoadConfig { + loader: &mut |_, _, path| I::from_file(path), + forced: false, + exit_on_solution: false, + }, + )?; + } else { + self.canonicalize_input_dirs(in_dirs)?; + let corpus_size = self.calculate_corpus_size()?; + log::info!( + "{} total_corpus_size, {} cores", + corpus_size, + cores.ids.len() + ); + self.reset_initial_files_state(); + self.canonicalize_input_dirs(in_dirs)?; + if cores.ids.len() > corpus_size { + log::info!( + "low intial corpus count ({}), no parallelism required.", + corpus_size + ); + } else { + let core_index = cores + .ids + .iter() + .enumerate() + .find(|(_, c)| *c == core_id) + .unwrap_or_else(|| panic!("core id {} not in cores list", core_id.0)) + .0; + let chunk_size = corpus_size.saturating_div(cores.ids.len()); + let mut skip = core_index.saturating_mul(chunk_size); + let mut inputs_todo = chunk_size; + let mut collected_inputs = Vec::new(); + log::info!( + "core = {}, core_index = {}, chunk_size = {}, skip = {}", + core_id.0, + core_index, + chunk_size, + skip + ); + loop { + match self.next_file() { + Ok(path) => { + if skip != 0 { + skip = skip.saturating_sub(1); + continue; + } + if inputs_todo == 0 { + break; + } + collected_inputs.push(path); + inputs_todo = inputs_todo.saturating_sub(1); + } + Err(Error::IteratorEnd(_, _)) => break, + Err(e) => { + return Err(e); + } + } + } + self.remaining_initial_files = Some(collected_inputs); + } + self.multicore_inputs_processed = Some(true); + return self + .load_initial_inputs_multicore(fuzzer, executor, manager, in_dirs, core_id, cores); + } + Ok(()) + } } impl StdState @@ -969,6 +1110,8 @@ where stage_depth: 0, stage_idx_stack: Vec::new(), phantom: PhantomData, + #[cfg(feature = "std")] + multicore_inputs_processed: None, }; feedback.init_state(&mut state)?; objective.init_state(&mut state)?; @@ -998,149 +1141,135 @@ impl HasScalabilityMonitor for StdState { } } -#[cfg(test)] -pub mod test { - use core::{marker::PhantomData, time::Duration}; - - use libafl_bolts::{rands::StdRand, serdeany::SerdeAnyMap, Error}; - use serde::{Deserialize, Serialize}; - - use crate::{ - corpus::{CorpusId, HasCurrentCorpusIdx, InMemoryCorpus}, - inputs::{Input, NopInput, UsesInput}, - stages::test::{test_resume, test_resume_stages}, - state::{ - HasCurrentStage, HasExecutions, HasLastReportTime, HasMetadata, HasRand, State, - StdState, - }, - }; - #[cfg(feature = "introspection")] - use crate::{monitors::ClientPerfMonitor, state::HasClientPerfMonitor}; - #[cfg(feature = "scalability_introspection")] - use crate::{monitors::ScalabilityMonitor, state::HasScalabilityMonitor}; - - /// A very simple state without any bells or whistles, for testing. - #[derive(Debug, Serialize, Deserialize, Default)] - pub struct NopState { - metadata: SerdeAnyMap, - execution: usize, - rand: StdRand, - phantom: PhantomData, - } - - impl NopState { - /// Create a new State that does nothing (for tests) - #[must_use] - pub fn new() -> Self { - NopState { - metadata: SerdeAnyMap::new(), - execution: 0, - rand: StdRand::default(), - phantom: PhantomData, - } +/// A very simple state without any bells or whistles, for testing. +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct NopState { + metadata: SerdeAnyMap, + execution: u64, + rand: StdRand, + phantom: PhantomData, +} + +impl NopState { + /// Create a new State that does nothing (for tests) + #[must_use] + pub fn new() -> Self { + NopState { + metadata: SerdeAnyMap::new(), + execution: 0, + rand: StdRand::default(), + phantom: PhantomData, } } +} - impl UsesInput for NopState - where - I: Input, - { - type Input = I; - } +impl UsesInput for NopState +where + I: Input, +{ + type Input = I; +} - impl HasExecutions for NopState { - fn executions(&self) -> &usize { - &self.execution - } +impl HasExecutions for NopState { + fn executions(&self) -> &u64 { + &self.execution + } - fn executions_mut(&mut self) -> &mut usize { - &mut self.execution - } + fn executions_mut(&mut self) -> &mut u64 { + &mut self.execution } +} - impl HasLastReportTime for NopState { - fn last_report_time(&self) -> &Option { - unimplemented!(); - } +impl HasLastReportTime for NopState { + fn last_report_time(&self) -> &Option { + unimplemented!(); + } - fn last_report_time_mut(&mut self) -> &mut Option { - unimplemented!(); - } + fn last_report_time_mut(&mut self) -> &mut Option { + unimplemented!(); } +} - impl HasMetadata for NopState { - fn metadata_map(&self) -> &SerdeAnyMap { - &self.metadata - } +impl HasMetadata for NopState { + fn metadata_map(&self) -> &SerdeAnyMap { + &self.metadata + } - fn metadata_map_mut(&mut self) -> &mut SerdeAnyMap { - &mut self.metadata - } + fn metadata_map_mut(&mut self) -> &mut SerdeAnyMap { + &mut self.metadata } +} - impl HasRand for NopState { - type Rand = StdRand; +impl HasRand for NopState { + type Rand = StdRand; - fn rand(&self) -> &Self::Rand { - &self.rand - } + fn rand(&self) -> &Self::Rand { + &self.rand + } - fn rand_mut(&mut self) -> &mut Self::Rand { - &mut self.rand - } + fn rand_mut(&mut self) -> &mut Self::Rand { + &mut self.rand } +} - impl State for NopState where I: Input {} +impl State for NopState where I: Input {} - impl HasCurrentCorpusIdx for NopState { - fn set_corpus_idx(&mut self, _idx: CorpusId) -> Result<(), Error> { - Ok(()) - } +impl HasCurrentCorpusIdx for NopState { + fn set_corpus_idx(&mut self, _idx: CorpusId) -> Result<(), Error> { + Ok(()) + } - fn clear_corpus_idx(&mut self) -> Result<(), Error> { - Ok(()) - } + fn clear_corpus_idx(&mut self) -> Result<(), Error> { + Ok(()) + } - fn current_corpus_idx(&self) -> Result, Error> { - Ok(None) - } + fn current_corpus_idx(&self) -> Result, Error> { + Ok(None) } +} - impl HasCurrentStage for NopState { - fn set_stage(&mut self, _idx: usize) -> Result<(), Error> { - Ok(()) - } +impl HasCurrentStage for NopState { + fn set_stage(&mut self, _idx: usize) -> Result<(), Error> { + Ok(()) + } - fn clear_stage(&mut self) -> Result<(), Error> { - Ok(()) - } + fn clear_stage(&mut self) -> Result<(), Error> { + Ok(()) + } - fn current_stage(&self) -> Result, Error> { - Ok(None) - } + fn current_stage(&self) -> Result, Error> { + Ok(None) } +} - #[cfg(feature = "introspection")] - impl HasClientPerfMonitor for NopState { - fn introspection_monitor(&self) -> &ClientPerfMonitor { - unimplemented!(); - } +#[cfg(feature = "introspection")] +impl HasClientPerfMonitor for NopState { + fn introspection_monitor(&self) -> &ClientPerfMonitor { + unimplemented!(); + } - fn introspection_monitor_mut(&mut self) -> &mut ClientPerfMonitor { - unimplemented!(); - } + fn introspection_monitor_mut(&mut self) -> &mut ClientPerfMonitor { + unimplemented!(); } +} - #[cfg(feature = "scalability_introspection")] - impl HasScalabilityMonitor for NopState { - fn scalability_monitor(&self) -> &ScalabilityMonitor { - unimplemented!(); - } +#[cfg(feature = "scalability_introspection")] +impl HasScalabilityMonitor for NopState { + fn scalability_monitor(&self) -> &ScalabilityMonitor { + unimplemented!(); + } - fn scalability_monitor_mut(&mut self) -> &mut ScalabilityMonitor { - unimplemented!(); - } + fn scalability_monitor_mut(&mut self) -> &mut ScalabilityMonitor { + unimplemented!(); } +} + +#[cfg(test)] +pub mod test { + use libafl_bolts::rands::StdRand; + + use super::StdState; + use crate::{corpus::InMemoryCorpus, inputs::Input}; #[must_use] pub fn test_std_state() -> StdState, StdRand, InMemoryCorpus> @@ -1154,155 +1283,4 @@ pub mod test { ) .expect("couldn't instantiate the test state") } - - #[test] - fn resume_simple() { - let mut state = test_std_state::(); - let (completed, stages) = test_resume_stages(); - - test_resume(&completed, &mut state, stages); - } -} - -#[cfg(feature = "python")] -#[allow(missing_docs)] -/// `State` Python bindings -pub mod pybind { - use alloc::{boxed::Box, vec::Vec}; - use std::path::PathBuf; - - use libafl_bolts::{ownedref::OwnedMutPtr, rands::pybind::PythonRand}; - use pyo3::{prelude::*, types::PyDict}; - - use crate::{ - corpus::pybind::PythonCorpus, - events::pybind::PythonEventManager, - executors::pybind::PythonExecutor, - feedbacks::pybind::PythonFeedback, - fuzzer::pybind::PythonStdFuzzerWrapper, - generators::pybind::PythonGenerator, - inputs::BytesInput, - pybind::PythonMetadata, - state::{ - HasCorpus, HasExecutions, HasMaxSize, HasMetadata, HasRand, HasSolutions, StdState, - }, - }; - - /// `StdState` with fixed generics - pub type PythonStdState = StdState; - - #[pyclass(unsendable, name = "StdState")] - #[derive(Debug)] - /// Python class for StdState - pub struct PythonStdStateWrapper { - /// Rust wrapped StdState object - pub inner: OwnedMutPtr, - } - - impl PythonStdStateWrapper { - pub fn wrap(r: &mut PythonStdState) -> Self { - Self { - inner: OwnedMutPtr::Ptr(r), - } - } - - #[must_use] - pub fn unwrap(&self) -> &PythonStdState { - self.inner.as_ref() - } - - pub fn unwrap_mut(&mut self) -> &mut PythonStdState { - self.inner.as_mut() - } - } - - #[pymethods] - impl PythonStdStateWrapper { - #[new] - fn new( - py_rand: PythonRand, - corpus: PythonCorpus, - solutions: PythonCorpus, - feedback: &mut PythonFeedback, - objective: &mut PythonFeedback, - ) -> Self { - Self { - inner: OwnedMutPtr::Owned(Box::new( - StdState::new(py_rand, corpus, solutions, feedback, objective) - .expect("Failed to create a new StdState"), - )), - } - } - - fn metadata(&mut self) -> PyObject { - let meta = self.inner.as_mut().metadata_map_mut(); - if !meta.contains::() { - Python::with_gil(|py| { - let dict: Py = PyDict::new(py).into(); - meta.insert(PythonMetadata::new(dict.to_object(py))); - }); - } - meta.get::().unwrap().map.clone() - } - - fn rand(&self) -> PythonRand { - self.inner.as_ref().rand().clone() - } - - fn corpus(&self) -> PythonCorpus { - self.inner.as_ref().corpus().clone() - } - - fn solutions(&self) -> PythonCorpus { - self.inner.as_ref().solutions().clone() - } - - fn executions(&self) -> usize { - *self.inner.as_ref().executions() - } - - fn max_size(&self) -> usize { - self.inner.as_ref().max_size() - } - - fn generate_initial_inputs( - &mut self, - py_fuzzer: &mut PythonStdFuzzerWrapper, - py_executor: &mut PythonExecutor, - py_generator: &mut PythonGenerator, - py_mgr: &mut PythonEventManager, - num: usize, - ) { - self.inner - .as_mut() - .generate_initial_inputs( - py_fuzzer.unwrap_mut(), - py_executor, - py_generator, - py_mgr, - num, - ) - .expect("Failed to generate the initial corpus"); - } - - #[allow(clippy::needless_pass_by_value)] - fn load_initial_inputs( - &mut self, - py_fuzzer: &mut PythonStdFuzzerWrapper, - py_executor: &mut PythonExecutor, - py_mgr: &mut PythonEventManager, - in_dirs: Vec, - ) { - self.inner - .as_mut() - .load_initial_inputs(py_fuzzer.unwrap_mut(), py_executor, py_mgr, &in_dirs) - .expect("Failed to load the initial corpus"); - } - } - - /// Register the classes to the python module - pub fn register(_py: Python, m: &PyModule) -> PyResult<()> { - m.add_class::()?; - Ok(()) - } } diff --git a/libafl_bolts/Cargo.toml b/libafl_bolts/Cargo.toml index c716fe30c5..feb0938d18 100644 --- a/libafl_bolts/Cargo.toml +++ b/libafl_bolts/Cargo.toml @@ -63,6 +63,13 @@ xxh3 = ["xxhash-rust"] #! ### SerdeAny features +## With this feature, the AnyMap uses [`type_name`](https://doc.rust-lang.org/std/any/fn.type_name.html) +## instead of [`TypeId::of`](https://doc.rust-lang.org/std/any/struct.TypeId.html#method.of) for deserialization. +## With this feature, stored state may remain deserializable across multiple compilations of LibAFL. +## This is **unsafe** and may lead to type confusions. Only use when you know what you are doing/ you have tests in place. +## The rust doc specifically states that "multiple types may map to the same type name"! +unsafe_stable_anymap = [] + ## Automatically register all `#[derive(SerdeAny)]` types at startup. serdeany_autoreg = ["ctor"] @@ -76,7 +83,7 @@ llmp_bind_public = ["alloc"] llmp_compression = ["alloc", "gzip"] ## Enables debug output for LLMP (also needs a `logger` installed) -llmp_debug = ["alloc"] +llmp_debug = ["alloc", "std"] ## Reduces the initial map size for llmp llmp_small_maps = ["alloc"] @@ -85,7 +92,8 @@ llmp_small_maps = ["alloc"] rustversion = "1.0" [dependencies] -libafl_derive = { version = "0.11.2", optional = true, path = "../libafl_derive" } +libafl_derive = { version = "0.12.0", optional = true, path = "../libafl_derive" } +static_assertions = "1.1.0" rustversion = "1.0" tuple_list = { version = "0.1.3" } diff --git a/libafl_bolts/README.md b/libafl_bolts/README.md index eb73f112a0..d29ef59208 100644 --- a/libafl_bolts/README.md +++ b/libafl_bolts/README.md @@ -28,7 +28,7 @@ For bugs, feel free to open issues or contact us directly. Thank you for your su Even though we will gladly assist you in finishing up your PR, try to - keep all the crates compiling with *stable* rust (hide the eventual non-stable code under [`cfg`s](https://github.com/AFLplusplus/LibAFL/blob/main/libafl/build.rs#L26)) -- run `cargo fmt` on your code before pushing +- run `cargo nightly fmt` on your code before pushing - check the output of `cargo clippy --all` or `./clippy.sh` - run `cargo build --no-default-features` to check for `no_std` compatibility (and possibly add `#[cfg(feature = "std")]`) to hide parts of your code. diff --git a/libafl_bolts/examples/llmp_test/main.rs b/libafl_bolts/examples/llmp_test/main.rs index f247c5f31b..0735dac0bb 100644 --- a/libafl_bolts/examples/llmp_test/main.rs +++ b/libafl_bolts/examples/llmp_test/main.rs @@ -3,7 +3,7 @@ This shows how llmp can be used directly, without libafl abstractions */ extern crate alloc; -#[cfg(all(feature = "std", not(target_os = "haiku")))] +#[cfg(not(target_os = "haiku"))] use core::time::Duration; #[cfg(all(feature = "std", not(target_os = "haiku")))] use std::{num::NonZeroUsize, thread, time}; @@ -150,12 +150,11 @@ fn main() -> Result<(), Box> { log::set_logger(&LOGGER).unwrap(); log::set_max_level(log::LevelFilter::Trace); - println!("Launching in mode {mode} on port {port}"); match mode.as_str() { "broker" => { - let mut broker = llmp::LlmpBroker::new(StdShMemProvider::new()?, None)?; + let mut broker = llmp::LlmpBroker::new(StdShMemProvider::new()?)?; broker.launch_tcp_listener_on(port)?; // Exit when we got at least _n_ nodes, and all of them quit. broker.set_exit_cleanly_after(NonZeroUsize::new(1_usize).unwrap()); @@ -166,7 +165,7 @@ fn main() -> Result<(), Box> { ); } "b2b" => { - let mut broker = llmp::LlmpBroker::new(StdShMemProvider::new()?, None)?; + let mut broker = llmp::LlmpBroker::new(StdShMemProvider::new()?)?; broker.launch_tcp_listener_on(b2b_port)?; // connect back to the main broker. broker.connect_b2b(("127.0.0.1", port))?; @@ -203,6 +202,14 @@ fn main() -> Result<(), Box> { } log::info!("Exiting Client exits"); client.sender_mut().send_exiting()?; + + // there is another way to tell that this client wants to exit. + // one is to call client.sender_mut().send_exiting()?; + // you can disconnet the client in this way as long as this client in an unrecoverable state (like in a crash handler) + // another way to do this is through the detach_from_broker() call + // you can call detach_from_broker(port); to notify the broker that this broker wants to exit + // This one is usually for the event restarter to cut off the connection when the client has crashed. + // In that case we don't have access to the llmp client of the client anymore, but we can use detach_from_broker instead } _ => { println!("No valid mode supplied"); diff --git a/libafl_bolts/src/anymap.rs b/libafl_bolts/src/anymap.rs index 7f2bf704aa..2c2bf16673 100644 --- a/libafl_bolts/src/anymap.rs +++ b/libafl_bolts/src/anymap.rs @@ -4,7 +4,7 @@ use alloc::boxed::Box; use core::{ any::{Any, TypeId}, mem::size_of, - ptr::addr_of, + ptr::{addr_of, read_unaligned}, }; /// Convert to an Any trait object @@ -39,7 +39,7 @@ macro_rules! impl_asany { }; } -/// Get a `type_id` from its previously unpacked `u64`. +/// Get a `type_id` from its previously unpacked `u128`. /// Opposite of [`unpack_type_id(id)`]. /// /// # Note @@ -47,26 +47,13 @@ macro_rules! impl_asany { /// The size changed in later rust versions, see #[inline] #[must_use] -#[allow(clippy::cast_ptr_alignment)] pub const fn pack_type_id(id: u128) -> TypeId { - match size_of::() { - 8 => { - let id_64 = id as u64; - // false positive: this branch only executes on 64 bit `TypeId`s - #[allow(clippy::cast_ptr_alignment)] - unsafe { - *(addr_of!(id_64) as *const TypeId) - } - } - 16 => unsafe { *(addr_of!(id) as *const TypeId) }, - _ => { - // TypeId size of this size is not yet supported" - panic!("Unsupported size for TypeId"); - } - } + // TypeId size of other sizes is not yet supported" + static_assertions::const_assert!(size_of::() == 16); + unsafe { *(addr_of!(id) as *const TypeId) } } -/// Unpack a `type_id` to an `u64` +/// Unpack a `type_id` to an `u128` /// Opposite of [`pack_type_id(id)`]. /// /// # Note @@ -75,15 +62,11 @@ pub const fn pack_type_id(id: u128) -> TypeId { #[inline] #[must_use] pub const fn unpack_type_id(id: TypeId) -> u128 { - #[allow(clippy::cast_ptr_alignment)] // we never actually cast to u128 if the type is u64. - match size_of::() { - 8 => unsafe { *(addr_of!(id) as *const u64) as u128 }, - 16 => unsafe { *(addr_of!(id) as *const u128) }, - _ => { - // TypeId size of this size is not yet supported" - panic!("Unsupported size for TypeId"); - } - } + // see any.rs, it's alway u128 hence 16 bytes. + // TypeId size of other sizes is not yet supported" + static_assertions::const_assert!(size_of::() == 16); + let ret: u128 = unsafe { read_unaligned::(addr_of!(id) as *const u128) }; + ret } #[cfg(test)] diff --git a/libafl_bolts/src/cli.rs b/libafl_bolts/src/cli.rs index 981654ee0a..920c5a4c35 100644 --- a/libafl_bolts/src/cli.rs +++ b/libafl_bolts/src/cli.rs @@ -34,14 +34,14 @@ //! use std::env; //! //! // make sure to add `features = ["qemu_cli"]` to the `libafl` crate in `Cargo.toml` -//! use libafl_qemu::Emulator; +//! use libafl_qemu::Qemu; //! //! fn fuzz_with_qemu(mut options: FuzzerOptions) { //! env::remove_var("LD_LIBRARY_PATH"); //! //! let env: Vec<(String, String)> = env::vars().collect(); //! -//! let emu = Emulator::new(&mut options.qemu_args.to_vec(), &mut env).unwrap(); +//! let qemu = Qemu::init(&mut options.qemu_args.to_vec(), &mut env).unwrap(); //! // do other stuff... //! } //! @@ -365,9 +365,6 @@ pub fn parse_args() -> FuzzerOptions { any(feature = "cli", feature = "qemu_cli", feature = "frida_cli") ))] mod tests { - #[cfg(feature = "frida_cli")] - use alloc::string::String; - use super::*; /// pass a standard option and `--` followed by some options that `FuzzerOptions` doesn't know diff --git a/libafl_bolts/src/core_affinity.rs b/libafl_bolts/src/core_affinity.rs index 76ecf17f22..0614876665 100644 --- a/libafl_bolts/src/core_affinity.rs +++ b/libafl_bolts/src/core_affinity.rs @@ -215,27 +215,46 @@ pub fn parse_core_bind_arg(args: &str) -> Result, Error> { // Linux Section -#[cfg(any(target_os = "android", target_os = "linux", target_os = "dragonfly"))] +#[cfg(any( + target_os = "android", + target_os = "linux", + target_os = "dragonfly", + target_os = "freebsd" +))] #[inline] fn get_core_ids_helper() -> Result, Error> { linux::get_core_ids() } -#[cfg(any(target_os = "android", target_os = "linux", target_os = "dragonfly"))] +#[cfg(any( + target_os = "android", + target_os = "linux", + target_os = "dragonfly", + target_os = "freebsd" +))] #[inline] fn set_for_current_helper(core_id: CoreId) -> Result<(), Error> { linux::set_for_current(core_id) } -#[cfg(any(target_os = "android", target_os = "linux", target_os = "dragonfly"))] +#[cfg(any( + target_os = "android", + target_os = "linux", + target_os = "dragonfly", + target_os = "freebsd" +))] mod linux { use alloc::{string::ToString, vec::Vec}; use std::mem; + #[cfg(not(target_os = "freebsd"))] + use libc::cpu_set_t; + #[cfg(target_os = "freebsd")] + use libc::cpuset_t as cpu_set_t; #[cfg(target_os = "dragonfly")] - use libc::{cpu_set_t, sched_getaffinity, sched_setaffinity, CPU_ISSET, CPU_SET}; + use libc::{sched_getaffinity, sched_setaffinity, CPU_ISSET, CPU_SET}; #[cfg(not(target_os = "dragonfly"))] - use libc::{cpu_set_t, sched_getaffinity, sched_setaffinity, CPU_ISSET, CPU_SET, CPU_SETSIZE}; + use libc::{sched_getaffinity, sched_setaffinity, CPU_ISSET, CPU_SET, CPU_SETSIZE}; #[cfg(target_os = "dragonfly")] const CPU_SETSIZE: libc::c_int = 256; @@ -667,134 +686,6 @@ mod apple { } } -// FreeBSD Section - -#[cfg(target_os = "freebsd")] -#[inline] -fn get_core_ids_helper() -> Result, Error> { - freebsd::get_core_ids() -} - -#[cfg(target_os = "freebsd")] -#[inline] -fn set_for_current_helper(core_id: CoreId) -> Result<(), Error> { - freebsd::set_for_current(core_id) -} - -#[cfg(target_os = "freebsd")] -mod freebsd { - use alloc::vec::Vec; - use std::{mem, thread::available_parallelism}; - - use libc::{cpuset_setaffinity, cpuset_t, CPU_LEVEL_WHICH, CPU_SET, CPU_WHICH_PID}; - - use super::CoreId; - use crate::Error; - - #[allow(trivial_numeric_casts)] - pub fn get_core_ids() -> Result, Error> { - Ok((0..(usize::from(available_parallelism()?))) - .map(CoreId) - .collect::>()) - } - - pub fn set_for_current(core_id: CoreId) -> Result<(), Error> { - // Turn `core_id` into a `libc::cpuset_t` with only - let mut set = new_cpuset(); - - unsafe { CPU_SET(core_id.0, &mut set) }; - - // Set the current thread's core affinity. - let result = unsafe { - cpuset_setaffinity( - CPU_LEVEL_WHICH, - CPU_WHICH_PID, - -1, // Defaults to current thread - mem::size_of::(), - &set, - ) - }; - - if result < 0 { - Err(Error::unknown("Failed to set_for_current")) - } else { - Ok(()) - } - } - - #[cfg(test)] - fn get_affinity_mask() -> Result { - let mut set = new_cpuset(); - - // Try to get current core affinity mask. - let result = unsafe { - libc::cpuset_getaffinity( - CPU_LEVEL_WHICH, - CPU_WHICH_PID, - -1, // Defaults to current thread - mem::size_of::(), - &mut set, - ) - }; - - if result == 0 { - Ok(set) - } else { - Err(Error::unknown( - "Failed to retrieve affinity using cpuset_getaffinity", - )) - } - } - - fn new_cpuset() -> cpuset_t { - unsafe { mem::zeroed::() } - } - - // FIXME: unstable for now on freebsd - #[cfg(test)] - mod tests { - use libc::CPU_ISSET; - - use super::*; - - #[test] - #[ignore] - fn test_freebsd_get_affinity_mask() { - get_affinity_mask().unwrap(); - } - - #[test] - #[ignore] - fn test_freebsd_set_for_current() { - let ids = get_core_ids().unwrap(); - - assert!(!ids.is_empty()); - - ids[0].set_affinity().unwrap(); - - // Ensure that the system pinned the current thread - // to the specified core. - let mut core_mask = new_cpuset(); - unsafe { CPU_SET(ids[0].0, &mut core_mask) }; - - let new_mask = get_affinity_mask().unwrap(); - - let mut is_equal = true; - - for i in 0..256 { - let is_set1 = unsafe { CPU_ISSET(i, &core_mask) }; - let is_set2 = unsafe { CPU_ISSET(i, &new_mask) }; - - if is_set1 != is_set2 { - is_equal = false; - } - } - - assert!(is_equal); - } - } -} - // NetBSD Section #[cfg(target_os = "netbsd")] diff --git a/libafl_bolts/src/cpu.rs b/libafl_bolts/src/cpu.rs index c8be696d02..318011c549 100644 --- a/libafl_bolts/src/cpu.rs +++ b/libafl_bolts/src/cpu.rs @@ -1,12 +1,13 @@ //! Architecture agnostic processor features -#[cfg(target_arch = "aarch64")] +#[cfg(any(target_arch = "aarch64", target_arch = "arm"))] use core::arch::asm; #[cfg(not(any( target_arch = "x86_64", target_arch = "x86", target_arch = "aarch64", + target_arch = "arm", target_arch = "riscv64", target_arsch = "riscv32" )))] @@ -50,6 +51,17 @@ pub fn read_time_counter() -> u64 { v } +/// Read a timestamp for measurements +#[cfg(target_arch = "arm")] +#[must_use] +pub fn read_time_counter() -> u64 { + let mut v: u32; + unsafe { + asm!("mrc p15, 0, {v}, c9, c13, 0", v = out(reg) v); + } + u64::from(v) +} + /// Read a timestamp for measurements /// /// Fetches the full 64 bits of the cycle counter in one instruction. @@ -99,6 +111,7 @@ pub fn read_time_counter() -> u64 { target_arch = "x86_64", target_arch = "x86", target_arch = "aarch64", + target_arch = "arm", target_arch = "riscv64", target_arch = "riscv32" )))] diff --git a/libafl_bolts/src/lib.rs b/libafl_bolts/src/lib.rs index 41852a3b57..a42b709ec5 100644 --- a/libafl_bolts/src/lib.rs +++ b/libafl_bolts/src/lib.rs @@ -29,7 +29,8 @@ clippy::missing_docs_in_private_items, clippy::module_name_repetitions, clippy::ptr_cast_constness, - clippy::negative_feature_names + clippy::negative_feature_names, + clippy::too_many_lines )] #![cfg_attr(not(test), warn( missing_debug_implementations, @@ -163,7 +164,7 @@ pub mod bolts_prelude { #[cfg(all(unix, feature = "std"))] use alloc::boxed::Box; #[cfg(feature = "alloc")] -use alloc::vec::Vec; +use alloc::{borrow::Cow, vec::Vec}; #[cfg(all(not(feature = "xxh3"), feature = "alloc"))] use core::hash::BuildHasher; #[cfg(any(feature = "xxh3", feature = "alloc"))] @@ -211,8 +212,8 @@ pub mod launcher {} use core::{ array::TryFromSliceError, fmt::{self, Display}, - iter::Iterator, num::{ParseIntError, TryFromIntError}, + ops::{Deref, DerefMut}, time, }; #[cfg(feature = "std")] @@ -227,10 +228,14 @@ use { core::str::Utf8Error, }; +/// Localhost addr, this is used, for example, for LLMP Client, which connects to this address +pub const IP_LOCALHOST: &str = "127.0.0.1"; + /// We need fixed names for many parts of this lib. +#[cfg(feature = "alloc")] pub trait Named { /// Provide the name of this element. - fn name(&self) -> &str; + fn name(&self) -> &Cow<'static, str>; } #[cfg(feature = "errors_backtrace")] @@ -296,9 +301,6 @@ pub enum Error { /// Compression error #[cfg(feature = "gzip")] Compression(ErrorBacktrace), - /// File related error - #[cfg(feature = "std")] - File(io::Error, ErrorBacktrace), /// Optional val was supposed to be set, but isn't. EmptyOptional(String, ErrorBacktrace), /// Key not in Map @@ -317,8 +319,13 @@ pub enum Error { Unsupported(String, ErrorBacktrace), /// Shutting down, not really an error. ShuttingDown, + /// OS error, wrapping a [`std::io::Error`] + #[cfg(feature = "std")] + OsError(io::Error, String, ErrorBacktrace), /// Something else happened Unknown(String, ErrorBacktrace), + /// Error with the corpora + InvalidCorpus(String, ErrorBacktrace), } impl Error { @@ -336,12 +343,6 @@ impl Error { pub fn compression() -> Self { Error::Compression(ErrorBacktrace::new()) } - #[cfg(feature = "std")] - /// File related error - #[must_use] - pub fn file(arg: io::Error) -> Self { - Error::File(arg, ErrorBacktrace::new()) - } /// Optional val was supposed to be set, but isn't. #[must_use] pub fn empty_optional(arg: S) -> Self @@ -411,6 +412,28 @@ impl Error { { Error::Unsupported(arg.into(), ErrorBacktrace::new()) } + /// OS error with additional message + #[cfg(feature = "std")] + #[must_use] + pub fn os_error(err: io::Error, msg: S) -> Self + where + S: Into, + { + Error::OsError(err, msg.into(), ErrorBacktrace::new()) + } + /// OS error from [`std::io::Error::last_os_error`] with additional message + #[cfg(feature = "std")] + #[must_use] + pub fn last_os_error(msg: S) -> Self + where + S: Into, + { + Error::OsError( + io::Error::last_os_error(), + msg.into(), + ErrorBacktrace::new(), + ) + } /// Something else happened #[must_use] pub fn unknown(arg: S) -> Self @@ -419,6 +442,14 @@ impl Error { { Error::Unknown(arg.into(), ErrorBacktrace::new()) } + /// Error with corpora + #[must_use] + pub fn invalid_corpus(arg: S) -> Self + where + S: Into, + { + Error::InvalidCorpus(arg.into(), ErrorBacktrace::new()) + } } impl Display for Error { @@ -433,17 +464,12 @@ impl Display for Error { write!(f, "Error in decompression")?; display_error_backtrace(f, b) } - #[cfg(feature = "std")] - Self::File(err, b) => { - write!(f, "File IO failed: {:?}", &err)?; - display_error_backtrace(f, b) - } Self::EmptyOptional(s, b) => { write!(f, "Optional value `{0}` was not set", &s)?; display_error_backtrace(f, b) } Self::KeyNotFound(s, b) => { - write!(f, "Key `{0}` not in Corpus", &s)?; + write!(f, "Key: `{0}` - not found", &s)?; display_error_backtrace(f, b) } Self::Empty(s, b) => { @@ -475,10 +501,19 @@ impl Display for Error { display_error_backtrace(f, b) } Self::ShuttingDown => write!(f, "Shutting down!"), + #[cfg(feature = "std")] + Self::OsError(err, s, b) => { + write!(f, "OS error: {0}: {1}", &s, err)?; + display_error_backtrace(f, b) + } Self::Unknown(s, b) => { write!(f, "Unknown error: {0}", &s)?; display_error_backtrace(f, b) } + Self::InvalidCorpus(s, b) => { + write!(f, "Invalid corpus: {0}", &s)?; + display_error_backtrace(f, b) + } } } } @@ -528,7 +563,7 @@ impl From for Error { #[cfg(feature = "std")] impl From for Error { fn from(err: io::Error) -> Self { - Self::file(err) + Self::os_error(err, "io::Error ocurred") } } @@ -640,68 +675,47 @@ pub trait IntoOwned { } /// Can be converted to a slice -pub trait AsSlice { - /// Type of the entries in this slice - type Entry; - /// Convert to a slice - fn as_slice(&self) -> &[Self::Entry]; -} +pub trait AsSlice<'a> { + /// Type of the entries of this slice + type Entry: 'a; + /// Type of the reference to this slice + type SliceRef: Deref; -/// Can be converted to a mutable slice -pub trait AsMutSlice { - /// Type of the entries in this mut slice - type Entry; /// Convert to a slice - fn as_mut_slice(&mut self) -> &mut [Self::Entry]; + fn as_slice(&'a self) -> Self::SliceRef; } -#[cfg(feature = "alloc")] -impl AsSlice for Vec { +impl<'a, T, R> AsSlice<'a> for R +where + T: 'a, + R: Deref, +{ type Entry = T; + type SliceRef = &'a [T]; - fn as_slice(&self) -> &[Self::Entry] { - self + fn as_slice(&'a self) -> Self::SliceRef { + &*self } } -#[cfg(feature = "alloc")] -impl AsMutSlice for Vec { - type Entry = T; - - fn as_mut_slice(&mut self) -> &mut [Self::Entry] { - self - } -} - -impl AsSlice for &[T] { - type Entry = T; - - fn as_slice(&self) -> &[Self::Entry] { - self - } -} - -impl AsSlice for [T] { - type Entry = T; - - fn as_slice(&self) -> &[Self::Entry] { - self - } -} - -impl AsMutSlice for &mut [T] { - type Entry = T; +/// Can be converted to a mutable slice +pub trait AsSliceMut<'a>: AsSlice<'a> { + /// Type of the mutable reference to this slice + type SliceRefMut: DerefMut; - fn as_mut_slice(&mut self) -> &mut [Self::Entry] { - self - } + /// Convert to a slice + fn as_slice_mut(&'a mut self) -> Self::SliceRefMut; } -impl AsMutSlice for [T] { - type Entry = T; +impl<'a, T, R> AsSliceMut<'a> for R +where + T: 'a, + R: DerefMut, +{ + type SliceRefMut = &'a mut [T]; - fn as_mut_slice(&mut self) -> &mut [Self::Entry] { - self + fn as_slice_mut(&'a mut self) -> Self::SliceRefMut { + &mut *self } } @@ -709,22 +723,51 @@ impl AsMutSlice for [T] { pub trait AsIter<'it> { /// The item type type Item: 'it; + /// The ref type + type Ref: Deref; /// The iterator type - type IntoIter: Iterator; + type IntoIter: Iterator; /// Create an iterator from &self fn as_iter(&'it self) -> Self::IntoIter; } +impl<'it, S, T> AsIter<'it> for S +where + S: AsSlice<'it, Entry = T, SliceRef = &'it [T]>, + T: 'it, +{ + type Item = S::Entry; + type Ref = &'it Self::Item; + type IntoIter = core::slice::Iter<'it, Self::Item>; + + fn as_iter(&'it self) -> Self::IntoIter { + self.as_slice().iter() + } +} + /// Create an `Iterator` from a mutable reference -pub trait AsIterMut<'it> { - /// The item type - type Item: 'it; +pub trait AsIterMut<'it>: AsIter<'it> { + /// The ref type + type RefMut: DerefMut; /// The iterator type - type IntoIter: Iterator; + type IntoIterMut: Iterator; /// Create an iterator from &mut self - fn as_iter_mut(&'it mut self) -> Self::IntoIter; + fn as_iter_mut(&'it mut self) -> Self::IntoIterMut; +} + +impl<'it, S, T> AsIterMut<'it> for S +where + S: AsSliceMut<'it, Entry = T, SliceRef = &'it [T], SliceRefMut = &'it mut [T]>, + T: 'it, +{ + type RefMut = &'it mut Self::Item; + type IntoIterMut = core::slice::IterMut<'it, Self::Item>; + + fn as_iter_mut(&'it mut self) -> Self::IntoIterMut { + self.as_slice_mut().iter_mut() + } } /// Has a length field @@ -738,6 +781,14 @@ pub trait HasLen { } } +#[cfg(feature = "alloc")] +impl HasLen for Vec { + #[inline] + fn len(&self) -> usize { + Vec::::len(self) + } +} + /// Has a ref count pub trait HasRefCnt { /// The ref count @@ -849,8 +900,9 @@ impl log::Log for SimpleStdoutLogger { fn log(&self, record: &Record) { println!( - "[{:?}] {}: {}", + "[{:?}, {:?}] {}: {}", current_time(), + std::process::id(), record.level(), record.args() ); @@ -895,8 +947,9 @@ impl log::Log for SimpleStderrLogger { fn log(&self, record: &Record) { eprintln!( - "[{:?}] {}: {}", + "[{:?}, {:?}] {}: {}", current_time(), + std::process::id(), record.level(), record.args() ); @@ -959,8 +1012,9 @@ impl log::Log for SimpleFdLogger { let mut f = unsafe { File::from_raw_fd(self.fd) }; writeln!( f, - "[{:?}] {}: {}", + "[{:?}, {:#?}] {}: {}", current_time(), + std::process::id(), record.level(), record.args() ) @@ -977,6 +1031,7 @@ impl log::Log for SimpleFdLogger { /// # Safety /// The function is arguably safe, but it might have undesirable side effects since it closes `stdout` and `stderr`. #[cfg(all(unix, feature = "std"))] +#[allow(unused_qualifications)] pub unsafe fn dup_and_mute_outputs() -> Result<(RawFd, RawFd), Error> { let old_stdout = stdout().as_raw_fd(); let old_stderr = stderr().as_raw_fd(); @@ -1003,7 +1058,7 @@ pub unsafe fn set_error_print_panic_hook(new_stderr: RawFd) { let mut f = unsafe { File::from_raw_fd(new_stderr) }; writeln!(f, "{panic_info}",) .unwrap_or_else(|err| println!("Failed to log to fd {new_stderr}: {err}")); - std::mem::forget(f); + mem::forget(f); })); } diff --git a/libafl_bolts/src/llmp.rs b/libafl_bolts/src/llmp.rs index bf0a4fd40f..1ff9ecc5ee 100644 --- a/libafl_bolts/src/llmp.rs +++ b/libafl_bolts/src/llmp.rs @@ -91,23 +91,19 @@ use backtrace::Backtrace; use nix::sys::socket::{self, sockopt::ReusePort}; use serde::{Deserialize, Serialize}; -#[cfg(feature = "std")] -use crate::current_time; #[cfg(all(unix, not(miri)))] use crate::os::unix_signals::setup_signal_handler; #[cfg(unix)] use crate::os::unix_signals::{siginfo_t, ucontext_t, Handler, Signal}; #[cfg(all(windows, feature = "std"))] use crate::os::windows_exceptions::{setup_ctrl_handler, CtrlHandler}; +#[cfg(feature = "std")] +use crate::{current_time, IP_LOCALHOST}; use crate::{ shmem::{ShMem, ShMemDescription, ShMemId, ShMemProvider}, ClientId, Error, }; -/// The default timeout in seconds after which a client will be considered stale, and removed. -#[cfg(feature = "std")] -const DEFAULT_CLIENT_TIMEOUT_SECS: u64 = 60 * 5; - /// The max number of pages a [`client`] may have mapped that were not yet read by the [`broker`] /// Usually, this value should not exceed `1`, else the broker cannot keep up with the amount of incoming messages. /// Instead of increasing this value, you may consider sending new messages at a lower rate, else your Sender will eventually `OOM`. @@ -131,6 +127,8 @@ const LLMP_TAG_UNINITIALIZED: Tag = Tag(0xA143AF11); const LLMP_TAG_END_OF_PAGE: Tag = Tag(0xAF1E0F1); /// A new client for this broker got added. const LLMP_TAG_NEW_SHM_CLIENT: Tag = Tag(0xC11E471); +/// A client wants to disconnect from this broker +const LLMP_TAG_CLIENT_EXIT: Tag = Tag(0xC11E472); /// The sender on this map is exiting (if broker exits, clients should exit gracefully); const LLMP_TAG_EXITING: Tag = Tag(0x13C5171); /// Client gave up as the receiver/broker was too slow @@ -155,9 +153,6 @@ const _LLMP_BIND_ADDR: &str = "0.0.0.0"; #[cfg(not(feature = "llmp_bind_public"))] const _LLMP_BIND_ADDR: &str = "127.0.0.1"; -/// LLMP Client connects to this address -const _LLMP_CONNECT_ADDR: &str = "127.0.0.1"; - /// An env var of this value indicates that the set value was a NULL PTR const _NULL_ENV_STR: &str = "_NULL"; @@ -265,6 +260,12 @@ pub enum TcpRequest { /// The hostname of our broker, trying to connect. hostname: String, }, + /// Notify the broker the the othe side is dying so remove this client + /// `client_id` is the pid of the very initial client + ClientQuit { + /// Tell the broker that remove the client with this `client_id`. `client_id` is equal to the one of event restarter + client_id: ClientId, + }, } impl TryFrom<&Vec> for TcpRequest { @@ -324,7 +325,7 @@ pub enum TcpResponse { }, /// Notify the client on the other side that it has been accepted. LocalClientAccepted { - /// The ClientId this client should send messages as. + /// The `ClientId` this client should send messages as. /// Mainly used for client-side deduplication of incoming messages client_id: ClientId, }, @@ -393,14 +394,14 @@ impl Listener { #[inline] #[allow(clippy::cast_ptr_alignment)] unsafe fn shmem2page_mut(afl_shmem: &mut SHM) -> *mut LlmpPage { - afl_shmem.as_mut_slice().as_mut_ptr() as *mut LlmpPage + afl_shmem.as_mut_ptr() as *mut LlmpPage } /// Get sharedmem from a page #[inline] #[allow(clippy::cast_ptr_alignment)] unsafe fn shmem2page(afl_shmem: &SHM) -> *const LlmpPage { - afl_shmem.as_slice().as_ptr() as *const LlmpPage + afl_shmem.as_ptr() as *const LlmpPage } /// Return, if a msg is contained in the current page @@ -456,7 +457,7 @@ fn tcp_bind(port: u16) -> Result { /// Send one message as `u32` len and `[u8;len]` bytes #[cfg(feature = "std")] -fn send_tcp_msg(stream: &mut TcpStream, msg: &T) -> Result<(), Error> +pub fn send_tcp_msg(stream: &mut TcpStream, msg: &T) -> Result<(), Error> where T: Serialize, { @@ -483,7 +484,7 @@ where /// Receive one message of `u32` len and `[u8; len]` bytes #[cfg(feature = "std")] -fn recv_tcp_msg(stream: &mut TcpStream) -> Result, Error> { +pub fn recv_tcp_msg(stream: &mut TcpStream) -> Result, Error> { // Always receive one be u32 of size, then the command. #[cfg(feature = "llmp_debug")] @@ -563,7 +564,7 @@ unsafe fn llmp_next_msg_ptr_checked( alloc_size: usize, ) -> Result<*mut LlmpMsg, Error> { let page = map.page_mut(); - let map_size = map.shmem.as_slice().len(); + let map_size = map.shmem.len(); let msg_begin_min = (page as *const u8).add(size_of::()); // We still need space for this msg (alloc_size). let msg_begin_max = (page as *const u8).add(map_size - alloc_size); @@ -661,7 +662,7 @@ impl LlmpMsg { /// Returns `true`, if the pointer is, indeed, in the page of this shared map. #[inline] pub fn in_shmem(&self, map: &mut LlmpSharedMap) -> bool { - let map_size = map.shmem.as_slice().len(); + let map_size = map.shmem.len(); let buf_ptr = self.buf.as_ptr(); let len = self.buf_len_padded as usize + size_of::(); unsafe { @@ -696,26 +697,24 @@ where { #[cfg(feature = "std")] /// Creates either a broker, if the tcp port is not bound, or a client, connected to this port. - pub fn on_port( - shmem_provider: SP, - port: u16, - client_timeout: Option, - ) -> Result { + /// This will make a new connection to the broker if it ends up a client + /// In that case this function will return its new [`ClientId`], too. + pub fn on_port(shmem_provider: SP, port: u16) -> Result { match tcp_bind(port) { Ok(listener) => { // We got the port. We are the broker! :) log::info!("We're the broker"); - let mut broker = LlmpBroker::new(shmem_provider, client_timeout)?; + let mut broker = LlmpBroker::new(shmem_provider)?; let _listener_thread = broker.launch_listener(Listener::Tcp(listener))?; Ok(LlmpConnection::IsBroker { broker }) } - Err(Error::File(e, _)) if e.kind() == ErrorKind::AddrInUse => { + Err(Error::OsError(e, ..)) if e.kind() == ErrorKind::AddrInUse => { // We are the client :) log::info!("We're the client (internal port already bound by broker, {e:#?})"); - Ok(LlmpConnection::IsClient { - client: LlmpClient::create_attach_to_tcp(shmem_provider, port)?, - }) + let client = LlmpClient::create_attach_to_tcp(shmem_provider, port)?; + let conn = LlmpConnection::IsClient { client }; + Ok(conn) } Err(e) => { log::error!("{e:?}"); @@ -726,22 +725,20 @@ where /// Creates a new broker on the given port #[cfg(feature = "std")] - pub fn broker_on_port( - shmem_provider: SP, - port: u16, - client_timeout: Option, - ) -> Result { + pub fn broker_on_port(shmem_provider: SP, port: u16) -> Result { Ok(LlmpConnection::IsBroker { - broker: LlmpBroker::create_attach_to_tcp(shmem_provider, port, client_timeout)?, + broker: LlmpBroker::create_attach_to_tcp(shmem_provider, port)?, }) } /// Creates a new client on the given port + /// This will make a new connection to the broker if it ends up a client + /// In that case this function will return its new [`ClientId`], too. #[cfg(feature = "std")] pub fn client_on_port(shmem_provider: SP, port: u16) -> Result { - Ok(LlmpConnection::IsClient { - client: LlmpClient::create_attach_to_tcp(shmem_provider, port)?, - }) + let client = LlmpClient::create_attach_to_tcp(shmem_provider, port)?; + let conn = LlmpConnection::IsClient { client }; + Ok(conn) } /// Describe this in a reproducable fashion, if it's a client @@ -792,7 +789,7 @@ pub struct LlmpPage { /// (The os may have tidied up the memory when the receiver starts to map) pub receivers_joined_count: AtomicU16, /// Set to != 1 by the receiver, once it left again after joining. - /// It's not safe for the sender to re-map this page before this is equal to receivers_joined_count + /// It's not safe for the sender to re-map this page before this is equal to `receivers_joined_count` pub receivers_left_count: AtomicU16, #[cfg(target_pointer_width = "64")] /// The current message ID @@ -839,6 +836,16 @@ struct LlmpPayloadSharedMapInfo { pub shm_str: [u8; 20], } +/// Message payload when a client got removed +/// This is an internal message! +/// [`LLMP_TAG_END_OF_PAGE_V1`] +#[derive(Copy, Clone, Debug)] +#[repr(C, align(8))] +struct LlmpClientExitInfo { + /// The restarter process id of the client + pub client_id: u32, +} + /// Sending end on a (unidirectional) sharedmap channel #[derive(Debug)] pub struct LlmpSender @@ -882,6 +889,12 @@ where id: ClientId, keep_pages_forever: bool, ) -> Result { + #[cfg(feature = "llmp_debug")] + log::info!( + "PID: {:#?} Initializing LlmpSender {:#?}", + std::process::id(), + id + ); Ok(Self { id, last_msg_sent: ptr::null_mut(), @@ -948,6 +961,12 @@ where msg_sent_offset, )?; ret.id = Self::client_id_from_env(env_name)?.unwrap_or_default(); + #[cfg(feature = "llmp_debug")] + log::info!( + "PID: {:#?} Initializing LlmpSender from on_existing_from_env {:#?}", + std::process::id(), + &ret.id + ); Ok(ret) } @@ -1015,8 +1034,15 @@ where None => ptr::null_mut(), }; + let client_id = unsafe { (*out_shmem.page()).sender_id }; + #[cfg(feature = "llmp_debug")] + log::info!( + "PID: {:#?} Initializing LlmpSender from on_existing_shmem {:#?}", + std::process::id(), + &client_id + ); Ok(Self { - id: unsafe { (*out_shmem.page()).sender_id }, + id: client_id, last_msg_sent, out_shmems: vec![out_shmem], // drop pages to the broker if it already read them @@ -1144,6 +1170,12 @@ where (*page).size_total ); + // For future allocs, keep track of the maximum (aligned) alloc size we used + (*page).max_alloc_size = max( + (*page).max_alloc_size, + size_of::() + buf_len_padded, + ); + // We need enough space for the current page size_used + payload + padding if (*page).size_used + size_of::() + buf_len_padded + EOP_MSG_SIZE > (*page).size_total @@ -1175,12 +1207,6 @@ where (*_llmp_next_msg_ptr(ret)).tag = LLMP_TAG_UNSET; (*ret).tag = LLMP_TAG_UNINITIALIZED; - // For future allocs, keep track of the maximum (aligned) alloc size we used - (*page).max_alloc_size = max( - (*page).max_alloc_size, - size_of::() + buf_len_padded, - ); - self.has_unsent_message = true; Some(ret) @@ -1749,6 +1775,21 @@ where } } + /// Receive the buffer, also reading the LLMP internal message flags + #[allow(clippy::type_complexity)] + #[inline] + pub fn recv_buf_blocking_with_flags(&mut self) -> Result<(ClientId, Tag, Flags, &[u8]), Error> { + unsafe { + let msg = self.recv_blocking()?; + Ok(( + (*msg).sender, + (*msg).tag, + (*msg).flags, + (*msg).try_as_slice(&mut self.current_recv_shmem)?, + )) + } + } + /// Returns the next sender, tag, buf, looping until it becomes available #[inline] pub fn recv_buf_blocking(&mut self) -> Result<(ClientId, Tag, &[u8]), Error> { @@ -1796,7 +1837,7 @@ where SHM: ShMem, { /// Shmem containg the actual (unsafe) page, - /// shared between one LlmpSender and one LlmpReceiver + /// shared between one `LlmpSender` and one `LlmpReceiver` shmem: SHM, } @@ -1929,7 +1970,7 @@ where let offset = offset as usize; let page = unsafe { self.page_mut() }; - let page_size = self.shmem.as_slice().len() - size_of::(); + let page_size = self.shmem.len() - size_of::(); if offset > page_size { Err(Error::illegal_argument(format!( "Msg offset out of bounds (size: {page_size}, requested offset: {offset})" @@ -1952,23 +1993,20 @@ where /// This allows us to intercept messages right in the broker. /// This keeps the out map clean. /// The backing values of `llmp_clients` [`ClientId`]s will always be sorted (but not gapless) - /// Make sure to always increase `num_clients_total` when pushing a new [`LlmpReceiver`] to `llmp_clients`! + /// Make sure to always increase `num_clients_seen` when pushing a new [`LlmpReceiver`] to `llmp_clients`! llmp_clients: Vec>, /// The own listeners we spawned via `launch_listener` or `crate_attach_to_tcp`. /// Listeners will be ignored for `exit_cleanly_after` and they are never considered to have timed out. listeners: Vec, /// The total amount of clients we had, historically, including those that disconnected, and our listeners. - num_clients_total: usize, + num_clients_seen: usize, /// The amount of total clients that should have connected and (and disconnected) /// after which the broker loop should quit gracefully. pub exit_cleanly_after: Option, - /// Clients that should be removed soon, (offset into llmp_clients) - clients_to_remove: Vec, - /// The ShMemProvider to use + /// Clients that should be removed soon + clients_to_remove: Vec, + /// The `ShMemProvider` to use shmem_provider: SP, - #[cfg(feature = "std")] - /// The timeout after which a client will be considered stale, and removed. - client_timeout: Duration, } /// A signal handler for the [`LlmpBroker`]. @@ -2017,15 +2055,12 @@ where SP: ShMemProvider + 'static, { /// Create and initialize a new [`LlmpBroker`] - pub fn new( - shmem_provider: SP, - #[cfg(feature = "std")] client_timeout: Option, - ) -> Result { + pub fn new(shmem_provider: SP) -> Result { // Broker never cleans up the pages so that new // clients may join at any time #[cfg(feature = "std")] { - Self::with_keep_pages(shmem_provider, true, client_timeout) + Self::with_keep_pages(shmem_provider, true) } #[cfg(not(feature = "std"))] @@ -2038,7 +2073,6 @@ where pub fn with_keep_pages( mut shmem_provider: SP, keep_pages_forever: bool, - #[cfg(feature = "std")] client_timeout: Option, ) -> Result { Ok(LlmpBroker { llmp_out: LlmpSender { @@ -2054,29 +2088,23 @@ where unused_shmem_cache: vec![], }, llmp_clients: vec![], - clients_to_remove: vec![], + clients_to_remove: Vec::new(), shmem_provider, listeners: vec![], exit_cleanly_after: None, - num_clients_total: 0, - #[cfg(feature = "std")] - client_timeout: if let Some(to) = client_timeout { - to - } else { - Duration::from_secs(DEFAULT_CLIENT_TIMEOUT_SECS) - }, + num_clients_seen: 0, }) } /// Gets the [`ClientId`] the next client attaching to this broker will get. /// In its current implememtation, the inner value of the next [`ClientId`] - /// is equal to `self.num_clients_total`. + /// is equal to `self.num_clients_seen`. /// Calling `peek_next_client_id` mutliple times (without adding a client) will yield the same value. #[must_use] #[inline] pub fn peek_next_client_id(&self) -> ClientId { ClientId( - self.num_clients_total + self.num_clients_seen .try_into() .expect("More than u32::MAX clients!"), ) @@ -2084,12 +2112,8 @@ where /// Create a new [`LlmpBroker`] attaching to a TCP port #[cfg(feature = "std")] - pub fn create_attach_to_tcp( - shmem_provider: SP, - port: u16, - client_timeout: Option, - ) -> Result { - Self::with_keep_pages_attach_to_tcp(shmem_provider, port, true, client_timeout) + pub fn create_attach_to_tcp(shmem_provider: SP, port: u16) -> Result { + Self::with_keep_pages_attach_to_tcp(shmem_provider, port, true) } /// Create a new [`LlmpBroker`] attaching to a TCP port and telling if it has to keep pages forever @@ -2098,15 +2122,10 @@ where shmem_provider: SP, port: u16, keep_pages_forever: bool, - client_timeout: Option, ) -> Result { match tcp_bind(port) { Ok(listener) => { - let mut broker = LlmpBroker::with_keep_pages( - shmem_provider, - keep_pages_forever, - client_timeout, - )?; + let mut broker = LlmpBroker::with_keep_pages(shmem_provider, keep_pages_forever)?; let _listener_thread = broker.launch_listener(Listener::Tcp(listener))?; Ok(broker) } @@ -2125,14 +2144,14 @@ where /// Add a client to this broker. /// Will set an appropriate [`ClientId`] before pushing the client to the internal vec. - /// Will increase `num_clients_total`. + /// Will increase `num_clients_seen`. /// The backing values of `llmp_clients` [`ClientId`]s will always be sorted (but not gapless) /// returns the [`ClientId`] of the new client. pub fn add_client(&mut self, mut client_receiver: LlmpReceiver) -> ClientId { let id = self.peek_next_client_id(); client_receiver.id = id; self.llmp_clients.push(client_receiver); - self.num_clients_total += 1; + self.num_clients_seen += 1; id } @@ -2256,37 +2275,37 @@ where where F: FnMut(ClientId, Tag, Flags, &[u8]) -> Result, { - #[cfg(feature = "std")] - let current_time = current_time(); let mut new_messages = false; for i in 0..self.llmp_clients.len() { let client_id = self.llmp_clients[i].id; match unsafe { self.handle_new_msgs(client_id, on_new_msg) } { Ok(has_messages) => { - // See if we need to remove this client, in case no new messages got brokered, and it's not a listener - #[cfg(feature = "std")] - if !has_messages && !self.listeners.iter().any(|&x| x == client_id) { - let last_msg_time = self.llmp_clients[i].last_msg_time; - if last_msg_time < current_time - && current_time - last_msg_time > self.client_timeout - { - self.clients_to_remove.push(i); - #[cfg(feature = "llmp_debug")] - println!("Client #{i} timed out. Removing."); - } - } new_messages = has_messages; } - Err(Error::ShuttingDown) => self.clients_to_remove.push(i), + Err(Error::ShuttingDown) => { + self.clients_to_remove.push(client_id); + } Err(err) => return Err(err), } } - // After brokering, remove all clients we don't want to keep. - for i in self.clients_to_remove.iter().rev() { - log::debug!("Client #{i} disconnected."); - self.llmp_clients.remove(*i); + let possible_remove = self.clients_to_remove.len(); + if possible_remove > 0 { + self.clients_to_remove.sort_unstable(); + self.clients_to_remove.dedup(); + log::trace!("Removing {:#?}", self.clients_to_remove); + // rev() to make it works + // commit the change to llmp_clients + for idx in (0..self.llmp_clients.len()).rev() { + let client_id = self.llmp_clients[idx].id; + if self.clients_to_remove.contains(&client_id) { + log::info!("Client {:#?} wants to exit. Removing.", client_id); + self.llmp_clients.remove(idx); + } + } + // log::trace!("{:#?}", self.llmp_clients); } + self.clients_to_remove.clear(); Ok(new_messages) } @@ -2378,12 +2397,12 @@ where // log::trace!( // "Clients connected: {} && > {} - {} >= {}", // self.has_clients(), - // self.num_clients_total, + // self.num_clients_seen, // self.listeners.len(), // exit_after_count // ); if !self.has_clients() - && (self.num_clients_total - self.listeners.len()) >= exit_after_count.into() + && (self.num_clients_seen - self.listeners.len()) >= exit_after_count.into() { // No more clients connected, and the amount of clients we were waiting for was previously connected. // exit cleanly. @@ -2423,7 +2442,7 @@ where if let Some(exit_after_count) = self.exit_cleanly_after { if !self.has_clients() - && (self.num_clients_total - self.listeners.len()) > exit_after_count.into() + && (self.num_clients_seen - self.listeners.len()) > exit_after_count.into() { // No more clients connected, and the amount of clients we were waiting for was previously connected. // exit cleanly. @@ -2487,6 +2506,21 @@ where } } + /// Tell the broker to disconnect this client from it. + #[cfg(feature = "std")] + fn announce_client_exit(sender: &mut LlmpSender, client_id: u32) -> Result<(), Error> { + unsafe { + let msg = sender + .alloc_next(size_of::()) + .expect("Could not allocate a new message in shared map."); + (*msg).tag = LLMP_TAG_CLIENT_EXIT; + #[allow(clippy::cast_ptr_alignment)] + let exitinfo = (*msg).buf.as_mut_ptr() as *mut LlmpClientExitInfo; + (*exitinfo).client_id = client_id; + sender.send(msg, true) + } + } + /// For broker to broker connections: /// Launches a proxy thread. /// It will read outgoing messages from the given broker map (and handle EOP by mapping a new page). @@ -2612,7 +2646,7 @@ where .expect("B2B: Error forwarding message. Exiting."); } Err(e) => { - if let Error::File(e, _) = e { + if let Error::OsError(e, ..) = e { if e.kind() == ErrorKind::UnexpectedEof { log::info!( "Broker {peer_address} seems to have disconnected, exiting" @@ -2648,6 +2682,13 @@ where broker_shmem_description: &ShMemDescription, ) { match request { + TcpRequest::ClientQuit { client_id } => { + // todo search the ancestor_id and remove it. + match Self::announce_client_exit(sender, client_id.0) { + Ok(()) => (), + Err(e) => log::info!("Error announcing client exit: {e:?}"), + } + } TcpRequest::LocalClientHello { shmem_description } => { match Self::announce_new_client(sender, shmem_description) { Ok(()) => (), @@ -2768,6 +2809,8 @@ where continue; } }; + + // log::info!("{:#?}", buf); let req = match buf.try_into() { Ok(req) => req, Err(e) => { @@ -2812,6 +2855,7 @@ where // TODO: We could memcpy a range of pending messages, instead of one by one. loop { + // log::trace!("{:#?}", self.llmp_clients); let msg = { let pos = if (client_id.0 as usize) < self.llmp_clients.len() && self.llmp_clients[client_id.0 as usize].id == client_id @@ -2846,6 +2890,25 @@ where LLMP_SLOW_RECEIVER_PANIC => { return Err(Error::unknown(format!("The broker was too slow to handle messages of client {client_id:?} in time, so it quit. Either the client sent messages too fast, or we (the broker) got stuck!"))); } + LLMP_TAG_CLIENT_EXIT => { + let msg_buf_len_padded = (*msg).buf_len_padded; + if (*msg).buf_len < size_of::() as u64 { + log::info!("Ignoring broken CLIENT_EXIT msg due to incorrect size. Expected {} but got {}", + msg_buf_len_padded, + size_of::() + ); + #[cfg(not(feature = "std"))] + return Err(Error::unknown(format!("Broken CLIENT_EXIT msg with incorrect size received. Expected {} but got {}", + msg_buf_len_padded, + size_of::() + ))); + } + let exitinfo = (*msg).buf.as_mut_ptr() as *mut LlmpClientExitInfo; + let client_id = ClientId((*exitinfo).client_id); + log::info!("Client exit message received!, we are removing clients whose client_group_id is {:#?}", client_id); + + self.clients_to_remove.push(client_id); + } LLMP_TAG_NEW_SHM_CLIENT => { /* This client informs us about yet another new client add it to the list! Also, no need to forward this msg. */ @@ -2862,7 +2925,6 @@ where ))); } let pageinfo = (*msg).buf.as_mut_ptr() as *mut LlmpPayloadSharedMapInfo; - match self.shmem_provider.shmem_from_id_and_size( ShMemId::from_array(&(*pageinfo).shm_str), (*pageinfo).map_size, @@ -2871,7 +2933,7 @@ where let mut new_page = LlmpSharedMap::existing(new_shmem); new_page.mark_safe_to_unmap(); - self.add_client(LlmpReceiver { + let _new_client = self.add_client(LlmpReceiver { id: ClientId(0), // will be auto-filled current_recv_shmem: new_page, last_msg_recvd: ptr::null_mut(), @@ -3129,26 +3191,6 @@ where self.sender.send_buf_with_flags(tag, flags, buf) } - /// Informs the broker about a new client in town, with the given map id - pub fn send_client_added_msg( - &mut self, - shm_str: &[u8; 20], - shm_id: usize, - ) -> Result<(), Error> { - // We write this by hand to get around checks in send_buf - unsafe { - let msg = self - .alloc_next(size_of::()) - .expect("Could not allocate a new message in shared map."); - (*msg).tag = LLMP_TAG_NEW_SHM_CLIENT; - #[allow(clippy::cast_ptr_alignment)] - let pageinfo = (*msg).buf.as_mut_ptr() as *mut LlmpPayloadSharedMapInfo; - (*pageinfo).shm_str = *shm_str; - (*pageinfo).map_size = shm_id; - self.send(msg) - } - } - /// A client receives a broadcast message. /// Returns null if no message is availiable /// # Safety @@ -3195,6 +3237,12 @@ where self.receiver.recv_buf_with_flags() } + /// Receive a `buf` from the broker, including the `flags` used during transmission. + #[allow(clippy::type_complexity)] + pub fn recv_buf_blocking_with_flags(&mut self) -> Result<(ClientId, Tag, Flags, &[u8]), Error> { + self.receiver.recv_buf_blocking_with_flags() + } + #[cfg(feature = "std")] /// Creates a new [`LlmpClient`], reading the map id and len from env pub fn create_using_env(mut shmem_provider: SP, env_var: &str) -> Result { @@ -3204,16 +3252,17 @@ where } #[cfg(feature = "std")] - /// Create a [`LlmpClient`], getting the ID from a given port + /// Create a [`LlmpClient`], getting the ID from a given port, then also tell the restarter's ID so we ask to be removed later + /// This is called when, for the first time, the restarter attaches to this process. pub fn create_attach_to_tcp(mut shmem_provider: SP, port: u16) -> Result { - let mut stream = match TcpStream::connect((_LLMP_CONNECT_ADDR, port)) { + let mut stream = match TcpStream::connect((IP_LOCALHOST, port)) { Ok(stream) => stream, Err(e) => { match e.kind() { - std::io::ErrorKind::ConnectionRefused => { + ErrorKind::ConnectionRefused => { //connection refused. loop till the broker is up loop { - match TcpStream::connect((_LLMP_CONNECT_ADDR, port)) { + match TcpStream::connect((IP_LOCALHOST, port)) { Ok(stream) => break stream, Err(_) => { log::info!("Connection Refused.. Retrying"); @@ -3292,15 +3341,13 @@ mod tests { pub fn test_llmp_connection() { #[allow(unused_variables)] let shmem_provider = StdShMemProvider::new().unwrap(); - let mut broker = match LlmpConnection::on_port(shmem_provider.clone(), 1337, None).unwrap() - { + let mut broker = match LlmpConnection::on_port(shmem_provider.clone(), 1337).unwrap() { IsClient { client: _ } => panic!("Could not bind to port as broker"), IsBroker { broker } => broker, }; // Add the first client (2nd, actually, because of the tcp listener client) - let mut client = match LlmpConnection::on_port(shmem_provider.clone(), 1337, None).unwrap() - { + let mut client = match LlmpConnection::on_port(shmem_provider.clone(), 1337).unwrap() { IsBroker { broker: _ } => panic!("Second connect should be a client!"), IsClient { client } => client, }; diff --git a/libafl_bolts/src/os/mod.rs b/libafl_bolts/src/os/mod.rs index 2af85e2651..4eaabf92a9 100644 --- a/libafl_bolts/src/os/mod.rs +++ b/libafl_bolts/src/os/mod.rs @@ -53,7 +53,7 @@ impl ChildHandle { unsafe { libc::waitpid(self.pid, &mut status, 0); } - status + libc::WEXITSTATUS(status) } } @@ -107,7 +107,7 @@ pub fn startable_self() -> Result { #[cfg(all(unix, feature = "std"))] pub fn dup(fd: RawFd) -> Result { match unsafe { libc::dup(fd) } { - -1 => Err(Error::file(std::io::Error::last_os_error())), + -1 => Err(Error::last_os_error(format!("Error calling dup({fd})"))), new_fd => Ok(new_fd), } } @@ -119,7 +119,9 @@ pub fn dup(fd: RawFd) -> Result { #[cfg(all(unix, feature = "std"))] pub fn dup2(fd: RawFd, device: RawFd) -> Result<(), Error> { match unsafe { libc::dup2(fd, device) } { - -1 => Err(Error::file(std::io::Error::last_os_error())), + -1 => Err(Error::last_os_error(format!( + "Error calling dup2({fd}, {device})" + ))), _ => Ok(()), } } diff --git a/libafl_bolts/src/os/unix_shmem_server.rs b/libafl_bolts/src/os/unix_shmem_server.rs index a402902a97..38780a27fc 100644 --- a/libafl_bolts/src/os/unix_shmem_server.rs +++ b/libafl_bolts/src/os/unix_shmem_server.rs @@ -10,7 +10,11 @@ use alloc::{ string::{String, ToString}, vec::Vec, }; -use core::{mem::ManuallyDrop, ptr::addr_of}; +use core::{ + mem::ManuallyDrop, + ops::{Deref, DerefMut}, + ptr::addr_of, +}; #[cfg(target_vendor = "apple")] use std::fs; use std::{ @@ -40,7 +44,7 @@ use uds::{UnixListenerExt, UnixSocketAddr, UnixStreamExt}; use crate::{ shmem::{ShMem, ShMemDescription, ShMemId, ShMemProvider}, - AsMutSlice, AsSlice, Error, + Error, }; /// The default server name for our abstract shmem server @@ -79,36 +83,33 @@ where server_fd: i32, } -impl ShMem for ServedShMem +impl Deref for ServedShMem where SH: ShMem, { - fn id(&self) -> ShMemId { - let client_id = self.inner.id(); - ShMemId::from_string(&format!("{}:{client_id}", self.server_fd)) - } + type Target = [u8]; - fn len(&self) -> usize { - self.inner.len() + fn deref(&self) -> &Self::Target { + &self.inner } } -impl AsSlice for ServedShMem +impl DerefMut for ServedShMem where SH: ShMem, { - type Entry = u8; - fn as_slice(&self) -> &[u8] { - self.inner.as_slice() + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner } } -impl AsMutSlice for ServedShMem + +impl ShMem for ServedShMem where SH: ShMem, { - type Entry = u8; - fn as_mut_slice(&mut self) -> &mut [u8] { - self.inner.as_mut_slice() + fn id(&self) -> ShMemId { + let client_id = self.inner.id(); + ShMemId::from_string(&format!("{}:{client_id}", self.server_fd)) } } @@ -179,7 +180,7 @@ where /// Connect to the server and return a new [`ServedShMemProvider`] /// Will try to spawn a [`ShMemService`]. This will only work for the first try. fn new() -> Result { - // Needed for MacOS and Android to get sharedmaps working. + // Needed for `MacOS` and Android to get sharedmaps working. let service = ShMemService::::start(); let mut res = Self { @@ -282,7 +283,7 @@ pub enum ServedShMemRequest { PreFork(), /// The client's child re-registers with us after it forked. PostForkChildHello(i32), - /// The ShMem Service should exit. This is sually sent internally on `drop`, but feel free to do whatever with it? + /// The `ShMem` Service should exit. This is sually sent internally on `drop`, but feel free to do whatever with it? Exit, } diff --git a/libafl_bolts/src/os/unix_signals.rs b/libafl_bolts/src/os/unix_signals.rs index a105303937..b01e0638f1 100644 --- a/libafl_bolts/src/os/unix_signals.rs +++ b/libafl_bolts/src/os/unix_signals.rs @@ -174,11 +174,11 @@ pub struct arm_neon_state64 { #[repr(C)] #[allow(clippy::pub_underscore_fields)] pub struct mcontext64 { - /// _STRUCT_ARM_EXCEPTION_STATE64 + /// `_STRUCT_ARM_EXCEPTION_STATE64` pub __es: arm_exception_state64, - /// _STRUCT_ARM_THREAD_STATE64 + /// `_STRUCT_ARM_THREAD_STATE64` pub __ss: arm_thread_state64, - /// _STRUCT_ARM_NEON_STATE64 + /// `_STRUCT_ARM_NEON_STATE64` pub __ns: arm_neon_state64, } @@ -189,7 +189,7 @@ pub struct mcontext64 { /// __darwin_size_t ss_size; /* signal stack length */ /// int ss_flags; /* SA_DISABLE and/or SA_ONSTACK */ /// }; -/// ```` +/// ``` #[cfg(all(target_vendor = "apple", target_arch = "aarch64"))] #[derive(Debug)] #[allow(non_camel_case_types)] @@ -199,7 +199,7 @@ pub struct sigaltstack { pub ss_sp: *mut c_void, /// signal stack length pub ss_size: libc::size_t, - /// SA_DISABLE and/or SA_ONSTACK + /// `SA_DISABLE` and/or `SA_ONSTACK` pub ss_flags: c_int, } @@ -263,7 +263,7 @@ use crate::Error; extern "C" { /// The `libc` `getcontext` - /// For some reason, it's not available on MacOS. + /// For some reason, it's not available on `MacOS`. /// fn getcontext(ucp: *mut ucontext_t) -> c_int; } diff --git a/libafl_bolts/src/os/windows_exceptions.rs b/libafl_bolts/src/os/windows_exceptions.rs index 336aa89232..5eb5b82126 100644 --- a/libafl_bolts/src/os/windows_exceptions.rs +++ b/libafl_bolts/src/os/windows_exceptions.rs @@ -5,13 +5,13 @@ use alloc::vec::Vec; use core::{ cell::UnsafeCell, fmt::{self, Display, Formatter}, - ptr::{self, addr_of_mut, write_volatile}, + ptr::{self, addr_of, addr_of_mut, write_volatile}, sync::atomic::{compiler_fence, Ordering}, }; use std::os::raw::{c_long, c_void}; use log::info; -use num_enum::TryFromPrimitive; +use num_enum::FromPrimitive; pub use windows::Win32::{ Foundation::{BOOL, NTSTATUS}, System::{ @@ -94,65 +94,130 @@ pub const STATUS_SXS_EARLY_DEACTIVATION: i32 = 0xC015000F; pub const STATUS_SXS_INVALID_DEACTIVATION: i32 = 0xC0150010; pub const STATUS_NOT_IMPLEMENTED: i32 = 0xC0000002; -#[derive(Debug, TryFromPrimitive, Clone, Copy)] +// from https://github.com/x64dbg/x64dbg/blob/4d631707b89d97e199844c08f5b65d8ea5d5d3f3/bin/exceptiondb.txt +pub const STATUS_WX86_UNSIMULATE: i32 = 0x4000001C; +pub const STATUS_WX86_CONTINUE: i32 = 0x4000001D; +pub const STATUS_WX86_SINGLE_STEP: i32 = 0x4000001E; +pub const STATUS_WX86_BREAKPOINT: i32 = 0x4000001F; +pub const STATUS_WX86_EXCEPTION_CONTINUE: i32 = 0x40000020; +pub const STATUS_WX86_EXCEPTION_LASTCHANCE: i32 = 0x40000021; +pub const STATUS_WX86_EXCEPTION_CHAIN: i32 = 0x40000022; +pub const STATUS_WX86_CREATEWX86TIB: i32 = 0x40000028; +pub const DBG_TERMINATE_THREAD: i32 = 0x40010003; +pub const DBG_TERMINATE_PROCESS: i32 = 0x40010004; +pub const DBG_CONTROL_C: i32 = 0x40010005; +pub const DBG_PRINTEXCEPTION_C: i32 = 0x40010006; +pub const DBG_RIPEXCEPTION: i32 = 0x40010007; +pub const DBG_CONTROL_BREAK: i32 = 0x40010008; +pub const DBG_COMMAND_EXCEPTION: i32 = 0x40010009; +pub const DBG_PRINTEXCEPTION_WIDE_C: i32 = 0x4001000A; +pub const EXCEPTION_RO_ORIGINATEERROR: i32 = 0x40080201; +pub const EXCEPTION_RO_TRANSFORMERROR: i32 = 0x40080202; +pub const MS_VC_EXCEPTION: i32 = 0x406D1388; +pub const DBG_EXCEPTION_NOT_HANDLED: i32 = 0x80010001; +pub const STATUS_INVALID_PARAMETER: i32 = 0xC000000D; +pub const STATUS_ILLEGAL_FLOAT_CONTEXT: i32 = 0xC000014A; +pub const EXCEPTION_POSSIBLE_DEADLOCK: i32 = 0xC0000194; +pub const STATUS_INVALID_EXCEPTION_HANDLER: i32 = 0xC00001A5; +pub const STATUS_DATATYPE_MISALIGNMENT_ERROR: i32 = 0xC00002C5; +pub const STATUS_USER_CALLBACK: i32 = 0xC000041D; +pub const CLR_EXCEPTION: i32 = 0xE0434352; +pub const CPP_EH_EXCEPTION: i32 = 0xE06D7363; +pub const VCPP_EXCEPTION_ERROR_INVALID_PARAMETER: i32 = 0xC06D0057; +pub const VCPP_EXCEPTION_ERROR_MOD_NOT_FOUND: i32 = 0xC06D007E; +pub const VCPP_EXCEPTION_ERROR_PROC_NOT_FOUND: i32 = 0xC06D007F; + +#[derive(Debug, FromPrimitive, Clone, Copy)] #[repr(i32)] pub enum ExceptionCode { - // From https://docs.microsoft.com/en-us/windows/win32/debug/getexceptioncode - AccessViolation = STATUS_ACCESS_VIOLATION, - ArrayBoundsExceeded = STATUS_ARRAY_BOUNDS_EXCEEDED, - Breakpoint = STATUS_BREAKPOINT, - DatatypeMisalignment = STATUS_DATATYPE_MISALIGNMENT, - FltDenormalOperand = STATUS_FLOAT_DENORMAL_OPERAND, - FltDivideByZero = STATUS_FLOAT_DIVIDE_BY_ZERO, - FltInexactResult = STATUS_FLOAT_INEXACT_RESULT, - FltInvalidOperation = STATUS_FLOAT_INVALID_OPERATION, - FltOverflow = STATUS_FLOAT_OVERFLOW, - FltStackCheck = STATUS_FLOAT_STACK_CHECK, - FltUnderflow = STATUS_FLOAT_UNDERFLOW, + // From https://github.com/wine-mirror/wine/blob/master/include/winnt.h#L611 + WaitZero = STATUS_WAIT_0, + AbandonedWaitZero = STATUS_ABANDONED_WAIT_0, + UserApc = STATUS_USER_APC, + Timeout = STATUS_TIMEOUT, + Pending = STATUS_PENDING, + SegmentNotification = STATUS_SEGMENT_NOTIFICATION, + FatalAppExit = STATUS_FATAL_APP_EXIT, GuardPageViolation = STATUS_GUARD_PAGE_VIOLATION, - IllegalInstruction = STATUS_ILLEGAL_INSTRUCTION, + DatatypeMisalignment = STATUS_DATATYPE_MISALIGNMENT, + Breakpoint = STATUS_BREAKPOINT, + SingleStep = STATUS_SINGLE_STEP, + Longjump = STATUS_LONGJUMP, + UnwindConsolidate = STATUS_UNWIND_CONSOLIDATE, + AccessViolation = STATUS_ACCESS_VIOLATION, InPageError = STATUS_IN_PAGE_ERROR, - IntegerDivideByZero = STATUS_INTEGER_DIVIDE_BY_ZERO, - IntegerOverflow = STATUS_INTEGER_OVERFLOW, - InvalidDisposition = STATUS_INVALID_DISPOSITION, InvalidHandle = STATUS_INVALID_HANDLE, + NoMemory = STATUS_NO_MEMORY, + IllegalInstruction = STATUS_ILLEGAL_INSTRUCTION, NoncontinuableException = STATUS_NONCONTINUABLE_EXCEPTION, + InvalidDisposition = STATUS_INVALID_DISPOSITION, + ArrayBoundsExceeded = STATUS_ARRAY_BOUNDS_EXCEEDED, + FloatDenormalOperand = STATUS_FLOAT_DENORMAL_OPERAND, + FloatDivideByZero = STATUS_FLOAT_DIVIDE_BY_ZERO, + FloatInexactResult = STATUS_FLOAT_INEXACT_RESULT, + FloatInvalidOperation = STATUS_FLOAT_INVALID_OPERATION, + FloatOverflow = STATUS_FLOAT_OVERFLOW, + FloatStackCheck = STATUS_FLOAT_STACK_CHECK, + FloatUnderflow = STATUS_FLOAT_UNDERFLOW, + IntegerDivideByZero = STATUS_INTEGER_DIVIDE_BY_ZERO, + IntegerOverflow = STATUS_INTEGER_OVERFLOW, PrivilegedInstruction = STATUS_PRIVILEGED_INSTRUCTION, - SingleStep = STATUS_SINGLE_STEP, StackOverflow = STATUS_STACK_OVERFLOW, - UnwindConsolidate = STATUS_UNWIND_CONSOLIDATE, - // Addition exceptions - Wait0 = STATUS_WAIT_0, - AbandonedWait0 = STATUS_ABANDONED_WAIT_0, - UserAPC = STATUS_USER_APC, - Timeout = STATUS_TIMEOUT, - Pending = STATUS_PENDING, - SegmentNotification = STATUS_SEGMENT_NOTIFICATION, - FatalAppExit = STATUS_FATAL_APP_EXIT, - Longjump = STATUS_LONGJUMP, - DLLNotFound = STATUS_DLL_NOT_FOUND, + DllNotFound = STATUS_DLL_NOT_FOUND, OrdinalNotFound = STATUS_ORDINAL_NOT_FOUND, - EntryPointNotFound = STATUS_ENTRYPOINT_NOT_FOUND, + EntrypointNotFound = STATUS_ENTRYPOINT_NOT_FOUND, ControlCExit = STATUS_CONTROL_C_EXIT, DllInitFailed = STATUS_DLL_INIT_FAILED, - FltMultipleFaults = STATUS_FLOAT_MULTIPLE_FAULTS, - FltMultipleTraps = STATUS_FLOAT_MULTIPLE_TRAPS, + FloatMultipleFaults = STATUS_FLOAT_MULTIPLE_FAULTS, + FloatMultipleTraps = STATUS_FLOAT_MULTIPLE_TRAPS, RegNatConsumption = STATUS_REG_NAT_CONSUMPTION, HeapCorruption = STATUS_HEAP_CORRUPTION, StackBufferOverrun = STATUS_STACK_BUFFER_OVERRUN, - InvalidCRuntimeParameter = STATUS_INVALID_CRUNTIME_PARAMETER, + InvalidCruntimeParameter = STATUS_INVALID_CRUNTIME_PARAMETER, AssertionFailure = STATUS_ASSERTION_FAILURE, - SXSEarlyDeactivation = STATUS_SXS_EARLY_DEACTIVATION, - SXSInvalidDeactivation = STATUS_SXS_INVALID_DEACTIVATION, + SxsEarlyDeactivation = STATUS_SXS_EARLY_DEACTIVATION, + SxsInvalidDeactivation = STATUS_SXS_INVALID_DEACTIVATION, NotImplemented = STATUS_NOT_IMPLEMENTED, - #[num_enum(default)] - Other, + // from https://github.com/x64dbg/x64dbg/blob/4d631707b89d97e199844c08f5b65d8ea5d5d3f3/bin/exceptiondb.txt + Wx86Unsimulate = STATUS_WX86_UNSIMULATE, + Wx86Continue = STATUS_WX86_CONTINUE, + Wx86SingleStep = STATUS_WX86_SINGLE_STEP, + Wx86Breakpoint = STATUS_WX86_BREAKPOINT, + Wx86ExceptionContinue = STATUS_WX86_EXCEPTION_CONTINUE, + Wx86ExceptionLastchance = STATUS_WX86_EXCEPTION_LASTCHANCE, + Wx86ExceptionChain = STATUS_WX86_EXCEPTION_CHAIN, + Wx86Createwx86Tib = STATUS_WX86_CREATEWX86TIB, + DbgTerminateThread = DBG_TERMINATE_THREAD, + DbgTerminateProcess = DBG_TERMINATE_PROCESS, + DbgControlC = DBG_CONTROL_C, + DbgPrintexceptionC = DBG_PRINTEXCEPTION_C, + DbgRipexception = DBG_RIPEXCEPTION, + DbgControlBreak = DBG_CONTROL_BREAK, + DbgCommandException = DBG_COMMAND_EXCEPTION, + DbgPrintexceptionWideC = DBG_PRINTEXCEPTION_WIDE_C, + ExceptionRoOriginateError = EXCEPTION_RO_ORIGINATEERROR, + ExceptionRoTransformError = EXCEPTION_RO_TRANSFORMERROR, + MsVcException = MS_VC_EXCEPTION, + DbgExceptionNotHandled = DBG_EXCEPTION_NOT_HANDLED, + InvalidParameter = STATUS_INVALID_PARAMETER, + IllegalFloatContext = STATUS_ILLEGAL_FLOAT_CONTEXT, + ExceptionPossibleDeadlock = EXCEPTION_POSSIBLE_DEADLOCK, + InvalidExceptionHandler = STATUS_INVALID_EXCEPTION_HANDLER, + DatatypeMisalignmentError = STATUS_DATATYPE_MISALIGNMENT_ERROR, + UserCallback = STATUS_USER_CALLBACK, + ClrException = CLR_EXCEPTION, + CppEhException = CPP_EH_EXCEPTION, + VcppExceptionErrorInvalidParameter = VCPP_EXCEPTION_ERROR_INVALID_PARAMETER, + VcppExceptionErrorModNotFound = VCPP_EXCEPTION_ERROR_MOD_NOT_FOUND, + VcppExceptionErrorProcNotFound = VCPP_EXCEPTION_ERROR_PROC_NOT_FOUND, + #[default] + Others, } pub static CRASH_EXCEPTIONS: &[ExceptionCode] = &[ ExceptionCode::AccessViolation, ExceptionCode::ArrayBoundsExceeded, - ExceptionCode::FltDivideByZero, + ExceptionCode::FloatDivideByZero, ExceptionCode::GuardPageViolation, ExceptionCode::IllegalInstruction, ExceptionCode::InPageError, @@ -177,111 +242,181 @@ impl Eq for ExceptionCode {} unsafe impl Sync for ExceptionCode {} impl Display for ExceptionCode { - fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), fmt::Error> { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { match self { - ExceptionCode::AccessViolation => write!(f, "STATUS_ACCESS_VIOLATION")?, - ExceptionCode::ArrayBoundsExceeded => write!(f, "STATUS_ARRAY_BOUNDS_EXCEEDED")?, - ExceptionCode::Breakpoint => write!(f, "STATUS_BREAKPOINT")?, - ExceptionCode::DatatypeMisalignment => write!(f, "STATUS_DATATYPE_MISALIGNMENT")?, - ExceptionCode::FltDenormalOperand => write!(f, "STATUS_FLOAT_DENORMAL_OPERAND")?, - ExceptionCode::FltDivideByZero => write!(f, "STATUS_FLOAT_DIVIDE_BY_ZERO")?, - ExceptionCode::FltInexactResult => write!(f, "STATUS_FLOAT_INEXACT_RESULT")?, - ExceptionCode::FltInvalidOperation => write!(f, "STATUS_FLOAT_INVALID_OPERATION")?, - ExceptionCode::FltOverflow => write!(f, "STATUS_FLOAT_OVERFLOW")?, - ExceptionCode::FltStackCheck => write!(f, "STATUS_FLOAT_STACK_CHECK")?, - ExceptionCode::FltUnderflow => write!(f, "STATUS_FLOAT_UNDERFLOW")?, - ExceptionCode::GuardPageViolation => write!(f, "STATUS_GUARD_PAGE_VIOLATION")?, - ExceptionCode::IllegalInstruction => write!(f, "STATUS_ILLEGAL_INSTRUCTION")?, - ExceptionCode::InPageError => write!(f, "STATUS_IN_PAGE_ERROR")?, - ExceptionCode::IntegerDivideByZero => write!(f, "STATUS_INTEGER_DIVIDE_BY_ZERO")?, - ExceptionCode::IntegerOverflow => write!(f, "STATUS_INTEGER_OVERFLOW")?, - ExceptionCode::InvalidDisposition => write!(f, "STATUS_INVALID_DISPOSITION")?, - ExceptionCode::InvalidHandle => write!(f, "STATUS_INVALID_HANDLE")?, - ExceptionCode::NoncontinuableException => write!(f, "STATUS_NONCONTINUABLE_EXCEPTION")?, - ExceptionCode::PrivilegedInstruction => write!(f, "STATUS_PRIVILEGED_INSTRUCTION")?, - ExceptionCode::SingleStep => write!(f, "STATUS_SINGLE_STEP")?, - ExceptionCode::StackOverflow => write!(f, "STATUS_STACK_OVERFLOW")?, - ExceptionCode::UnwindConsolidate => write!(f, "STATUS_UNWIND_CONSOLIDATE")?, - ExceptionCode::Wait0 => write!(f, "STATUS_WAIT_0")?, - ExceptionCode::AbandonedWait0 => write!(f, "STATUS_ABANDONED_WAIT_0")?, - ExceptionCode::UserAPC => write!(f, "STATUS_USER_APC")?, - ExceptionCode::Timeout => write!(f, "STATUS_TIMEOUT")?, - ExceptionCode::Pending => write!(f, "STATUS_PENDING")?, - ExceptionCode::SegmentNotification => write!(f, "STATUS_SEGMENT_NOTIFICATION")?, - ExceptionCode::FatalAppExit => write!(f, "STATUS_FATAL_APP_EXIT")?, - ExceptionCode::Longjump => write!(f, "STATUS_LONGJUMP")?, - ExceptionCode::DLLNotFound => write!(f, "STATUS_DLL_NOT_FOUND")?, - ExceptionCode::OrdinalNotFound => write!(f, "STATUS_ORDINAL_NOT_FOUND")?, - ExceptionCode::EntryPointNotFound => write!(f, "STATUS_ENTRYPOINT_NOT_FOUND")?, - ExceptionCode::ControlCExit => write!(f, "STATUS_CONTROL_C_EXIT")?, - ExceptionCode::DllInitFailed => write!(f, "STATUS_DLL_INIT_FAILED")?, - ExceptionCode::FltMultipleFaults => write!(f, "STATUS_FLOAT_MULTIPLE_FAULTS")?, - ExceptionCode::FltMultipleTraps => write!(f, "STATUS_FLOAT_MULTIPLE_TRAPS")?, - ExceptionCode::RegNatConsumption => write!(f, "STATUS_REG_NAT_CONSUMPTION")?, - ExceptionCode::HeapCorruption => write!(f, "STATUS_HEAP_CORRUPTION")?, - ExceptionCode::StackBufferOverrun => write!(f, "STATUS_STACK_BUFFER_OVERRUN")?, - ExceptionCode::InvalidCRuntimeParameter => { - write!(f, "STATUS_INVALID_CRUNTIME_PARAMETER")?; + ExceptionCode::WaitZero => write!(f, "STATUS_WAIT_0"), + ExceptionCode::AbandonedWaitZero => write!(f, "STATUS_ABANDONED_WAIT_0"), + ExceptionCode::UserApc => write!(f, "STATUS_USER_APC"), + ExceptionCode::Timeout => write!(f, "STATUS_TIMEOUT"), + ExceptionCode::Pending => write!(f, "STATUS_PENDING"), + ExceptionCode::SegmentNotification => write!(f, "STATUS_SEGMENT_NOTIFICATION"), + ExceptionCode::FatalAppExit => write!(f, "STATUS_FATAL_APP_EXIT"), + ExceptionCode::GuardPageViolation => write!(f, "STATUS_GUARD_PAGE_VIOLATION"), + ExceptionCode::DatatypeMisalignment => write!(f, "STATUS_DATATYPE_MISALIGNMENT"), + ExceptionCode::Breakpoint => write!(f, "STATUS_BREAKPOINT"), + ExceptionCode::SingleStep => write!(f, "STATUS_SINGLE_STEP"), + ExceptionCode::Longjump => write!(f, "STATUS_LONGJUMP"), + ExceptionCode::UnwindConsolidate => write!(f, "STATUS_UNWIND_CONSOLIDATE"), + ExceptionCode::AccessViolation => write!(f, "STATUS_ACCESS_VIOLATION"), + ExceptionCode::InPageError => write!(f, "STATUS_IN_PAGE_ERROR"), + ExceptionCode::InvalidHandle => write!(f, "STATUS_INVALID_HANDLE"), + ExceptionCode::NoMemory => write!(f, "STATUS_NO_MEMORY"), + ExceptionCode::IllegalInstruction => write!(f, "STATUS_ILLEGAL_INSTRUCTION"), + ExceptionCode::NoncontinuableException => write!(f, "STATUS_NONCONTINUABLE_EXCEPTION"), + ExceptionCode::InvalidDisposition => write!(f, "STATUS_INVALID_DISPOSITION"), + ExceptionCode::ArrayBoundsExceeded => write!(f, "STATUS_ARRAY_BOUNDS_EXCEEDED"), + ExceptionCode::FloatDenormalOperand => write!(f, "STATUS_FLOAT_DENORMAL_OPERAND"), + ExceptionCode::FloatDivideByZero => write!(f, "STATUS_FLOAT_DIVIDE_BY_ZERO"), + ExceptionCode::FloatInexactResult => write!(f, "STATUS_FLOAT_INEXACT_RESULT"), + ExceptionCode::FloatInvalidOperation => write!(f, "STATUS_FLOAT_INVALID_OPERATION"), + ExceptionCode::FloatOverflow => write!(f, "STATUS_FLOAT_OVERFLOW"), + ExceptionCode::FloatStackCheck => write!(f, "STATUS_FLOAT_STACK_CHECK"), + ExceptionCode::FloatUnderflow => write!(f, "STATUS_FLOAT_UNDERFLOW"), + ExceptionCode::IntegerDivideByZero => write!(f, "STATUS_INTEGER_DIVIDE_BY_ZERO"), + ExceptionCode::IntegerOverflow => write!(f, "STATUS_INTEGER_OVERFLOW"), + ExceptionCode::PrivilegedInstruction => write!(f, "STATUS_PRIVILEGED_INSTRUCTION"), + ExceptionCode::StackOverflow => write!(f, "STATUS_STACK_OVERFLOW"), + ExceptionCode::DllNotFound => write!(f, "STATUS_DLL_NOT_FOUND"), + ExceptionCode::OrdinalNotFound => write!(f, "STATUS_ORDINAL_NOT_FOUND"), + ExceptionCode::EntrypointNotFound => write!(f, "STATUS_ENTRYPOINT_NOT_FOUND"), + ExceptionCode::ControlCExit => write!(f, "STATUS_CONTROL_C_EXIT"), + ExceptionCode::DllInitFailed => write!(f, "STATUS_DLL_INIT_FAILED"), + ExceptionCode::FloatMultipleFaults => write!(f, "STATUS_FLOAT_MULTIPLE_FAULTS"), + ExceptionCode::FloatMultipleTraps => write!(f, "STATUS_FLOAT_MULTIPLE_TRAPS"), + ExceptionCode::RegNatConsumption => write!(f, "STATUS_REG_NAT_CONSUMPTION"), + ExceptionCode::HeapCorruption => write!(f, "STATUS_HEAP_CORRUPTION"), + ExceptionCode::StackBufferOverrun => write!(f, "STATUS_STACK_BUFFER_OVERRUN"), + ExceptionCode::InvalidCruntimeParameter => { + write!(f, "STATUS_INVALID_CRUNTIME_PARAMETER") + } + ExceptionCode::AssertionFailure => write!(f, "STATUS_ASSERTION_FAILURE"), + ExceptionCode::SxsEarlyDeactivation => write!(f, "STATUS_SXS_EARLY_DEACTIVATION"), + ExceptionCode::SxsInvalidDeactivation => write!(f, "STATUS_SXS_INVALID_DEACTIVATION"), + ExceptionCode::NotImplemented => write!(f, "STATUS_NOT_IMPLEMENTED"), + ExceptionCode::Wx86Unsimulate => write!(f, "STATUS_WX86_UNSIMULATE"), + ExceptionCode::Wx86Continue => write!(f, "STATUS_WX86_CONTINUE"), + ExceptionCode::Wx86SingleStep => write!(f, "STATUS_WX86_SINGLE_STEP"), + ExceptionCode::Wx86Breakpoint => write!(f, "STATUS_WX86_BREAKPOINT"), + ExceptionCode::Wx86ExceptionContinue => write!(f, "STATUS_WX86_EXCEPTION_CONTINUE"), + ExceptionCode::Wx86ExceptionLastchance => write!(f, "STATUS_WX86_EXCEPTION_LASTCHANCE"), + ExceptionCode::Wx86ExceptionChain => write!(f, "STATUS_WX86_EXCEPTION_CHAIN"), + ExceptionCode::Wx86Createwx86Tib => write!(f, "STATUS_WX86_CREATEWX86TIB"), + ExceptionCode::DbgTerminateThread => write!(f, "DBG_TERMINATE_THREAD"), + ExceptionCode::DbgTerminateProcess => write!(f, "DBG_TERMINATE_PROCESS"), + ExceptionCode::DbgControlC => write!(f, "DBG_CONTROL_C"), + ExceptionCode::DbgPrintexceptionC => write!(f, "DBG_PRINTEXCEPTION_C"), + ExceptionCode::DbgRipexception => write!(f, "DBG_RIPEXCEPTION"), + ExceptionCode::DbgControlBreak => write!(f, "DBG_CONTROL_BREAK"), + ExceptionCode::DbgCommandException => write!(f, "DBG_COMMAND_EXCEPTION"), + ExceptionCode::DbgPrintexceptionWideC => write!(f, "DBG_PRINTEXCEPTION_WIDE_C"), + ExceptionCode::ExceptionRoOriginateError => write!(f, "EXCEPTION_RO_ORIGINATEERROR"), + ExceptionCode::ExceptionRoTransformError => write!(f, "EXCEPTION_RO_TRANSFORMERROR"), + ExceptionCode::MsVcException => write!(f, "MS_VC_EXCEPTION"), + ExceptionCode::DbgExceptionNotHandled => write!(f, "DBG_EXCEPTION_NOT_HANDLED"), + ExceptionCode::InvalidParameter => write!(f, "STATUS_INVALID_PARAMETER"), + ExceptionCode::IllegalFloatContext => write!(f, "STATUS_ILLEGAL_FLOAT_CONTEXT"), + ExceptionCode::ExceptionPossibleDeadlock => write!(f, "EXCEPTION_POSSIBLE_DEADLOCK"), + ExceptionCode::InvalidExceptionHandler => write!(f, "STATUS_INVALID_EXCEPTION_HANDLER"), + ExceptionCode::DatatypeMisalignmentError => { + write!(f, "STATUS_DATATYPE_MISALIGNMENT_ERROR") } - ExceptionCode::AssertionFailure => write!(f, "STATUS_ASSERTION_FAILURE")?, - ExceptionCode::SXSEarlyDeactivation => write!(f, "STATUS_SXS_EARLY_DEACTIVATION")?, - ExceptionCode::SXSInvalidDeactivation => write!(f, "STATUS_SXS_INVALID_DEACTIVATION")?, - ExceptionCode::NotImplemented => write!(f, "STATUS_NOT_IMPLEMENTED")?, - ExceptionCode::Other => write!(f, "Other/User defined exception")?, - }; - - Ok(()) + ExceptionCode::UserCallback => write!(f, "STATUS_USER_CALLBACK"), + ExceptionCode::ClrException => write!(f, "CLR_EXCEPTION"), + ExceptionCode::CppEhException => write!(f, "CPP_EH_EXCEPTION"), + ExceptionCode::VcppExceptionErrorInvalidParameter => { + write!(f, "VCPP_EXCEPTION_ERROR_INVALID_PARAMETER") + } + ExceptionCode::VcppExceptionErrorModNotFound => { + write!(f, "VCPP_EXCEPTION_ERROR_MOD_NOT_FOUND") + } + ExceptionCode::VcppExceptionErrorProcNotFound => { + write!(f, "VCPP_EXCEPTION_ERROR_PROC_NOT_FOUND") + } + ExceptionCode::Others => write!(f, "Unknown exception code"), + } } } -pub static EXCEPTION_CODES_MAPPING: [ExceptionCode; 47] = [ - ExceptionCode::AccessViolation, - ExceptionCode::ArrayBoundsExceeded, - ExceptionCode::Breakpoint, - ExceptionCode::DatatypeMisalignment, - ExceptionCode::FltDenormalOperand, - ExceptionCode::FltDivideByZero, - ExceptionCode::FltInexactResult, - ExceptionCode::FltInvalidOperation, - ExceptionCode::FltOverflow, - ExceptionCode::FltStackCheck, - ExceptionCode::FltUnderflow, +pub static EXCEPTION_CODES_MAPPING: [ExceptionCode; 79] = [ + ExceptionCode::WaitZero, + ExceptionCode::AbandonedWaitZero, + ExceptionCode::UserApc, + ExceptionCode::Timeout, + ExceptionCode::Pending, + ExceptionCode::SegmentNotification, + ExceptionCode::FatalAppExit, ExceptionCode::GuardPageViolation, - ExceptionCode::IllegalInstruction, + ExceptionCode::DatatypeMisalignment, + ExceptionCode::Breakpoint, + ExceptionCode::SingleStep, + ExceptionCode::Longjump, + ExceptionCode::UnwindConsolidate, + ExceptionCode::AccessViolation, ExceptionCode::InPageError, - ExceptionCode::IntegerDivideByZero, - ExceptionCode::IntegerOverflow, - ExceptionCode::InvalidDisposition, ExceptionCode::InvalidHandle, + ExceptionCode::NoMemory, + ExceptionCode::IllegalInstruction, ExceptionCode::NoncontinuableException, + ExceptionCode::InvalidDisposition, + ExceptionCode::ArrayBoundsExceeded, + ExceptionCode::FloatDenormalOperand, + ExceptionCode::FloatDivideByZero, + ExceptionCode::FloatInexactResult, + ExceptionCode::FloatInvalidOperation, + ExceptionCode::FloatOverflow, + ExceptionCode::FloatStackCheck, + ExceptionCode::FloatUnderflow, + ExceptionCode::IntegerDivideByZero, + ExceptionCode::IntegerOverflow, ExceptionCode::PrivilegedInstruction, - ExceptionCode::SingleStep, ExceptionCode::StackOverflow, - ExceptionCode::UnwindConsolidate, - ExceptionCode::Wait0, - ExceptionCode::AbandonedWait0, - ExceptionCode::UserAPC, - ExceptionCode::Timeout, - ExceptionCode::Pending, - ExceptionCode::SegmentNotification, - ExceptionCode::FatalAppExit, - ExceptionCode::Longjump, - ExceptionCode::DLLNotFound, + ExceptionCode::DllNotFound, ExceptionCode::OrdinalNotFound, - ExceptionCode::EntryPointNotFound, + ExceptionCode::EntrypointNotFound, ExceptionCode::ControlCExit, ExceptionCode::DllInitFailed, - ExceptionCode::FltMultipleFaults, - ExceptionCode::FltMultipleTraps, + ExceptionCode::FloatMultipleFaults, + ExceptionCode::FloatMultipleTraps, ExceptionCode::RegNatConsumption, ExceptionCode::HeapCorruption, ExceptionCode::StackBufferOverrun, - ExceptionCode::InvalidCRuntimeParameter, + ExceptionCode::InvalidCruntimeParameter, ExceptionCode::AssertionFailure, - ExceptionCode::SXSEarlyDeactivation, - ExceptionCode::SXSInvalidDeactivation, + ExceptionCode::SxsEarlyDeactivation, + ExceptionCode::SxsInvalidDeactivation, ExceptionCode::NotImplemented, - ExceptionCode::Other, + ExceptionCode::Wx86Unsimulate, + ExceptionCode::Wx86Continue, + ExceptionCode::Wx86SingleStep, + ExceptionCode::Wx86Breakpoint, + ExceptionCode::Wx86ExceptionContinue, + ExceptionCode::Wx86ExceptionLastchance, + ExceptionCode::Wx86ExceptionChain, + ExceptionCode::Wx86Createwx86Tib, + ExceptionCode::DbgTerminateThread, + ExceptionCode::DbgTerminateProcess, + ExceptionCode::DbgControlC, + ExceptionCode::DbgPrintexceptionC, + ExceptionCode::DbgRipexception, + ExceptionCode::DbgControlBreak, + ExceptionCode::DbgCommandException, + ExceptionCode::DbgPrintexceptionWideC, + ExceptionCode::ExceptionRoOriginateError, + ExceptionCode::ExceptionRoTransformError, + ExceptionCode::MsVcException, + ExceptionCode::DbgExceptionNotHandled, + ExceptionCode::InvalidParameter, + ExceptionCode::IllegalFloatContext, + ExceptionCode::ExceptionPossibleDeadlock, + ExceptionCode::InvalidExceptionHandler, + ExceptionCode::DatatypeMisalignmentError, + ExceptionCode::UserCallback, + ExceptionCode::ClrException, + ExceptionCode::CppEhException, + ExceptionCode::VcppExceptionErrorInvalidParameter, + ExceptionCode::VcppExceptionErrorModNotFound, + ExceptionCode::VcppExceptionErrorProcNotFound, + ExceptionCode::Others, ]; #[cfg(feature = "alloc")] @@ -300,7 +435,7 @@ struct HandlerHolder { handler: UnsafeCell<*mut dyn Handler>, } -pub const EXCEPTION_HANDLERS_SIZE: usize = 64; +pub const EXCEPTION_HANDLERS_SIZE: usize = 96; unsafe impl Send for HandlerHolder {} @@ -310,6 +445,8 @@ static mut EXCEPTION_HANDLERS: [Option; EXCEPTION_HANDLERS_SIZE] None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, + None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, + None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, ]; unsafe fn internal_handle_exception( @@ -361,17 +498,23 @@ pub unsafe extern "system" fn handle_exception( .as_mut() .unwrap() .ExceptionCode; - let exception_code = match ExceptionCode::try_from(code.0) { - Ok(x) => x, - Err(_) => ExceptionCode::Other, - }; + let exception_code = From::from(code.0); log::info!("Received exception; code: {}", exception_code); internal_handle_exception(exception_code, exception_pointers) } +/// Return `SIGIGN` this is 1 (when represented as u64) +/// Check `https://github.com/ziglang/zig/blob/956f53beb09c07925970453d4c178c6feb53ba70/lib/libc/include/any-windows-any/signal.h#L51` +/// # Safety +/// It is just casting into another type, nothing unsafe. +#[must_use] +pub const unsafe fn sig_ign() -> NativeSignalHandlerType { + core::mem::transmute(1u64) +} + type NativeSignalHandlerType = unsafe extern "C" fn(i32); extern "C" { - fn signal(signum: i32, func: NativeSignalHandlerType) -> *const c_void; + pub fn signal(signum: i32, func: NativeSignalHandlerType) -> *const c_void; } unsafe extern "C" fn handle_signal(_signum: i32) { @@ -463,7 +606,8 @@ pub(crate) unsafe fn setup_ctrl_handler( } unsafe extern "system" fn ctrl_handler(ctrl_type: u32) -> BOOL { - match &CTRL_HANDLER { + let handler = ptr::read_volatile(addr_of!(CTRL_HANDLER)); + match handler { Some(handler_holder) => { info!("{:?}: Handling ctrl {}", std::process::id(), ctrl_type); let handler = &mut *handler_holder.handler.get(); diff --git a/libafl_bolts/src/ownedref.rs b/libafl_bolts/src/ownedref.rs index f778f1b31e..9cf4e19568 100644 --- a/libafl_bolts/src/ownedref.rs +++ b/libafl_bolts/src/ownedref.rs @@ -6,11 +6,16 @@ use alloc::{ slice::{Iter, IterMut}, vec::Vec, }; -use core::{clone::Clone, fmt::Debug, slice}; +use core::{ + clone::Clone, + fmt::Debug, + ops::{Deref, DerefMut}, + slice, +}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; -use crate::{shmem::ShMem, AsMutSlice, AsSlice, IntoOwned, Truncate}; +use crate::{shmem::ShMem, AsSlice, AsSliceMut, IntoOwned, Truncate}; /// Private part of the unsafe marker, making sure this cannot be initialized directly. #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] @@ -455,11 +460,10 @@ impl<'a, T> From> for OwnedSlice<'a, T> { } } -impl<'a, T: Sized> AsSlice for OwnedSlice<'a, T> { - type Entry = T; - /// Get the [`OwnedSlice`] as slice. - #[must_use] - fn as_slice(&self) -> &[T] { +impl<'a, T: Sized> Deref for OwnedSlice<'a, T> { + type Target = [T]; + + fn deref(&self) -> &Self::Target { match &self.inner { OwnedSliceInner::Ref(r) => r, OwnedSliceInner::RefRaw(rr, len, _) => unsafe { slice::from_raw_parts(*rr, *len) }, @@ -562,7 +566,7 @@ impl<'a, 'it, T> IntoIterator for &'it mut OwnedMutSlice<'a, T> { type IntoIter = IterMut<'it, T>; fn into_iter(self) -> Self::IntoIter { - self.as_mut_slice().iter_mut() + self.as_slice_mut().iter_mut() } } @@ -639,11 +643,10 @@ impl<'a, T: 'a + Sized> OwnedMutSlice<'a, T> { } } -impl<'a, T: Sized> AsSlice for OwnedMutSlice<'a, T> { - type Entry = T; - /// Get the value as slice - #[must_use] - fn as_slice(&self) -> &[T] { +impl<'a, T: Sized> Deref for OwnedMutSlice<'a, T> { + type Target = [T]; + + fn deref(&self) -> &Self::Target { match &self.inner { OwnedMutSliceInner::RefRaw(rr, len, _) => unsafe { slice::from_raw_parts(*rr, *len) }, OwnedMutSliceInner::Ref(r) => r, @@ -651,17 +654,15 @@ impl<'a, T: Sized> AsSlice for OwnedMutSlice<'a, T> { } } } -impl<'a, T: Sized> AsMutSlice for OwnedMutSlice<'a, T> { - type Entry = T; - /// Get the value as mut slice - #[must_use] - fn as_mut_slice(&mut self) -> &mut [T] { + +impl<'a, T: Sized> DerefMut for OwnedMutSlice<'a, T> { + fn deref_mut(&mut self) -> &mut [T] { match &mut self.inner { OwnedMutSliceInner::RefRaw(rr, len, _) => unsafe { slice::from_raw_parts_mut(*rr, *len) }, OwnedMutSliceInner::Ref(r) => r, - OwnedMutSliceInner::Owned(v) => v.as_mut_slice(), + OwnedMutSliceInner::Owned(v) => v.as_slice_mut(), } } } @@ -761,6 +762,17 @@ pub enum OwnedPtr { Owned(Box), } +impl OwnedPtr { + /// Creates a new [`OwnedPtr`] from a raw pointer + /// + /// # Safety + /// The raw pointer will later be dereferenced. + /// It must outlive this `OwnedPtr` type and remain valid. + pub unsafe fn from_raw(ptr: *const T) -> Self { + Self::Ptr(ptr) + } +} + impl Serialize for OwnedPtr { fn serialize(&self, se: S) -> Result where @@ -822,6 +834,17 @@ pub enum OwnedMutPtr { Owned(Box), } +impl OwnedMutPtr { + /// Creates a new [`OwnedMutPtr`] from a raw pointer + /// + /// # Safety + /// The raw pointer will later be dereferenced. + /// It must outlive this `OwnedPtr` type and remain valid. + pub unsafe fn from_raw_mut(ptr: *mut T) -> Self { + Self::Ptr(ptr) + } +} + impl Serialize for OwnedMutPtr { fn serialize(&self, se: S) -> Result where diff --git a/libafl_bolts/src/rands.rs b/libafl_bolts/src/rands.rs index 7925435e28..ef434b9407 100644 --- a/libafl_bolts/src/rands.rs +++ b/libafl_bolts/src/rands.rs @@ -1,20 +1,95 @@ //! The random number generators of `LibAFL` -use core::{debug_assert, fmt::Debug}; -#[cfg(feature = "rand_trait")] -use rand_core::{self, impls::fill_bytes_via_next, RngCore}; +use core::{ + debug_assert, + fmt::Debug, + sync::atomic::{AtomicUsize, Ordering}, +}; + use serde::{de::DeserializeOwned, Deserialize, Serialize}; +/// Return a pseudo-random seed. For `no_std` environments, a single deterministic sequence is used. +#[must_use] +#[allow(unreachable_code)] +pub fn random_seed() -> u64 { + #[cfg(feature = "std")] + return random_seed_from_random_state(); + #[cfg(all(not(feature = "std"), target_has_atomic = "ptr"))] + return random_seed_deterministic(); + // no_std and no atomics; https://xkcd.com/221/ + 4 +} + +static SEED_COUNTER: AtomicUsize = AtomicUsize::new(0); + +#[allow(dead_code)] +#[cfg(target_has_atomic = "ptr")] +fn random_seed_deterministic() -> u64 { + let mut seed = SEED_COUNTER.fetch_add(1, Ordering::Relaxed) as u64; + splitmix64(&mut seed) +} + +#[allow(dead_code)] #[cfg(feature = "std")] -use crate::current_nanos; -#[cfg(any(feature = "xxh3", feature = "alloc"))] -use crate::hash_std; +fn random_seed_from_random_state() -> u64 { + use std::{ + collections::hash_map::RandomState, + hash::{BuildHasher, Hasher}, + }; + RandomState::new().build_hasher().finish() +} + +// https://prng.di.unimi.it/splitmix64.c +fn splitmix64(x: &mut u64) -> u64 { + *x = x.wrapping_add(0x9e3779b97f4a7c15); + let mut z = *x; + z = (z ^ (z >> 30)).wrapping_mul(0xbf58476d1ce4e5b9); + z = (z ^ (z >> 27)).wrapping_mul(0x94d049bb133111eb); + z ^ (z >> 31) +} /// The standard rand implementation for `LibAFL`. /// It is usually the right choice, with very good speed and a reasonable randomness. /// Not cryptographically secure (which is not what you want during fuzzing ;) ) pub type StdRand = RomuDuoJrRand; +/// Choose an item at random from the given iterator, sampling uniformly. +/// +/// Note: the runtime cost is bound by the iterator's [`nth`][`Iterator::nth`] implementation +/// * For `Vec`, slice, array, this is O(1) +/// * For `HashMap`, `HashSet`, this is O(n) +pub fn choose(from: I, rand: u64) -> I::Item +where + I: IntoIterator, + I::IntoIter: ExactSizeIterator, +{ + // create iterator + let mut iter = from.into_iter(); + + // make sure there is something to choose from + debug_assert!(iter.len() > 0, "choosing from an empty iterator"); + + // pick a random, valid index + let index = fast_bound(rand, iter.len()); + + // return the item chosen + iter.nth(index).unwrap() +} + +/// Faster and almost unbiased alternative to `rand % n`. +/// +/// For N-bit bound, probability of getting a biased value is 1/2^(64-N). +/// At least 2^2*(64-N) samples are required to detect this amount of bias. +/// +/// See: [An optimal algorithm for bounded random integers](https://github.com/apple/swift/pull/39143). +#[inline] +#[must_use] +pub fn fast_bound(rand: u64, n: usize) -> usize { + debug_assert_ne!(n, 0); + let mul = u128::from(rand).wrapping_mul(u128::from(n as u64)); + (mul >> 64) as usize +} + /// Ways to get random around here. /// Please note that these are not cryptographically secure. /// Or, even if some might be by accident, at least they are not seeded in a cryptographically secure fashion. @@ -25,107 +100,77 @@ pub trait Rand: Debug + Serialize + DeserializeOwned { /// Gets the next 64 bit value fn next(&mut self) -> u64; - /// Gets a value below the given 64 bit val (exclusive) - fn below(&mut self, upper_bound_excl: u64) -> u64 { - if upper_bound_excl <= 1 { - return 0; - } + /// Gets a value between 0.0 (inclusive) and 1.0 (exclusive) + #[inline] + #[allow(clippy::cast_precision_loss)] + fn next_float(&mut self) -> f64 { + // both 2^53 and 2^-53 can be represented in f64 exactly + const MAX: u64 = 1u64 << 53; + const MAX_DIV: f64 = 1.0 / (MAX as f64); + let u = self.next() & MAX.wrapping_sub(1); + u as f64 * MAX_DIV + } - /* - Modulo is biased - we don't want our fuzzing to be biased so let's do it - right. See - https://stackoverflow.com/questions/10984974/why-do-people-say-there-is-modulo-bias-when-using-a-random-number-generator - */ - let mut unbiased_rnd: u64; - loop { - unbiased_rnd = self.next(); - if unbiased_rnd < (u64::MAX - (u64::MAX % upper_bound_excl)) { - break; - } - } + /// Returns true with specified probability + #[inline] + fn coinflip(&mut self, success_prob: f64) -> bool { + debug_assert!((0.0..=1.0).contains(&success_prob)); + self.next_float() < success_prob + } - unbiased_rnd % upper_bound_excl + /// Gets a value below the given bound (exclusive) + #[inline] + fn below(&mut self, upper_bound_excl: usize) -> usize { + fast_bound(self.next(), upper_bound_excl) } /// Gets a value between the given lower bound (inclusive) and upper bound (inclusive) - fn between(&mut self, lower_bound_incl: u64, upper_bound_incl: u64) -> u64 { + #[inline] + fn between(&mut self, lower_bound_incl: usize, upper_bound_incl: usize) -> usize { debug_assert!(lower_bound_incl <= upper_bound_incl); lower_bound_incl + self.below(upper_bound_incl - lower_bound_incl + 1) } - /// Choose an item at random from the given iterator, sampling uniformly. - /// - /// Note: the runtime cost is bound by the iterator's [`nth`][`Iterator::nth`] implementation - /// * For `Vec`, slice, array, this is O(1) - /// * For `HashMap`, `HashSet`, this is O(n) - fn choose(&mut self, from: I) -> T + /// Convenient variant of [`choose`]. + fn choose(&mut self, from: I) -> I::Item where - I: IntoIterator, - E: ExactSizeIterator + Iterator, + I: IntoIterator, + I::IntoIter: ExactSizeIterator, { - // create iterator - let mut iter = from.into_iter(); - - // make sure there is something to choose from - debug_assert!(iter.len() > 0, "choosing from an empty iterator"); - - // pick a random, valid index - let index = self.below(iter.len() as u64) as usize; - - // return the item chosen - iter.nth(index).unwrap() + choose(from, self.next()) } } -// helper macro for deriving Default -macro_rules! default_rand { +macro_rules! impl_default_new { ($rand: ty) => { - /// A default RNG will usually produce a nondeterministic stream of random numbers. - /// As we do not have any way to get random seeds for `no_std`, they have to be reproducible there. - /// Use [`$rand::with_seed`] to generate a reproducible RNG. impl Default for $rand { - #[cfg(feature = "std")] + /// Creates a generator seeded with [`random_seed`]. fn default() -> Self { - Self::new() + Self::with_seed(random_seed()) } - #[cfg(not(feature = "std"))] - fn default() -> Self { - Self::with_seed(0xAF1) + } + + impl $rand { + /// Creates a generator seeded with [`random_seed`]. + #[must_use] + pub fn new() -> Self { + Self::with_seed(random_seed()) } } }; } -// Derive Default by calling `new(DEFAULT_SEED)` on each of the following Rand types. -#[cfg(any(feature = "xxh3", feature = "alloc"))] -default_rand!(Xoshiro256StarRand); -default_rand!(XorShift64Rand); -default_rand!(Lehmer64Rand); -default_rand!(RomuTrioRand); -default_rand!(RomuDuoJrRand); +impl_default_new!(Xoshiro256PlusPlusRand); +impl_default_new!(XorShift64Rand); +impl_default_new!(Lehmer64Rand); +impl_default_new!(RomuTrioRand); +impl_default_new!(RomuDuoJrRand); +impl_default_new!(Sfc64Rand); -/// Initialize Rand types from a source of randomness. -/// -/// Default implementations are provided with the "std" feature enabled, using system time in -/// nanoseconds as the initial seed. -pub trait RandomSeed: Rand + Default { - /// Creates a new [`RandomSeed`]. - fn new() -> Self; -} - -// helper macro to impl RandomSeed -macro_rules! impl_random { +macro_rules! impl_rng_core { ($rand: ty) => { - #[cfg(feature = "std")] - impl RandomSeed for $rand { - /// Creates a rand instance, pre-seeded with the current time in nanoseconds. - fn new() -> Self { - Self::with_seed(current_nanos()) - } - } - #[cfg(feature = "rand_trait")] - impl RngCore for $rand { + impl rand_core::RngCore for $rand { fn next_u32(&mut self) -> u32 { self.next() as u32 } @@ -135,7 +180,7 @@ macro_rules! impl_random { } fn fill_bytes(&mut self, dest: &mut [u8]) { - fill_bytes_via_next(self, dest) + rand_core::impls::fill_bytes_via_next(self, dest) } fn try_fill_bytes(&mut self, dest: &mut [u8]) -> Result<(), rand_core::Error> { @@ -145,112 +190,108 @@ macro_rules! impl_random { }; } -#[cfg(any(feature = "xxh3", feature = "alloc"))] -impl_random!(Xoshiro256StarRand); -impl_random!(XorShift64Rand); -impl_random!(Lehmer64Rand); -impl_random!(RomuTrioRand); -impl_random!(RomuDuoJrRand); +impl_rng_core!(Xoshiro256PlusPlusRand); +impl_rng_core!(XorShift64Rand); +impl_rng_core!(Lehmer64Rand); +impl_rng_core!(RomuTrioRand); +impl_rng_core!(RomuDuoJrRand); +impl_rng_core!(Sfc64Rand); -/// XXH3 Based, hopefully speedy, rnd implementation +/// xoshiro256++ PRNG: #[derive(Copy, Clone, Debug, Serialize, Deserialize)] -pub struct Xoshiro256StarRand { - rand_seed: [u64; 4], +pub struct Xoshiro256PlusPlusRand { + s: [u64; 4], } -// TODO: re-enable ahash works without alloc -#[cfg(any(feature = "xxh3", feature = "alloc"))] -impl Rand for Xoshiro256StarRand { - #[allow(clippy::unreadable_literal)] - fn set_seed(&mut self, seed: u64) { - self.rand_seed[0] = hash_std(&seed.to_be_bytes()); - self.rand_seed[1] = self.rand_seed[0] ^ 0x1234567890abcdef; - self.rand_seed[2] = self.rand_seed[0] & 0x0123456789abcdef; - self.rand_seed[3] = self.rand_seed[0] | 0x01abcde43f567908; +impl Rand for Xoshiro256PlusPlusRand { + fn set_seed(&mut self, mut seed: u64) { + self.s[0] = splitmix64(&mut seed); + self.s[1] = splitmix64(&mut seed); + self.s[2] = splitmix64(&mut seed); + self.s[3] = splitmix64(&mut seed); } #[inline] fn next(&mut self) -> u64 { - let ret: u64 = self.rand_seed[0] - .wrapping_add(self.rand_seed[3]) + let ret: u64 = self.s[0] + .wrapping_add(self.s[3]) .rotate_left(23) - .wrapping_add(self.rand_seed[0]); - let t: u64 = self.rand_seed[1] << 17; + .wrapping_add(self.s[0]); + let t: u64 = self.s[1] << 17; - self.rand_seed[2] ^= self.rand_seed[0]; - self.rand_seed[3] ^= self.rand_seed[1]; - self.rand_seed[1] ^= self.rand_seed[2]; - self.rand_seed[0] ^= self.rand_seed[3]; + self.s[2] ^= self.s[0]; + self.s[3] ^= self.s[1]; + self.s[1] ^= self.s[2]; + self.s[0] ^= self.s[3]; - self.rand_seed[2] ^= t; + self.s[2] ^= t; - self.rand_seed[3] = self.rand_seed[3].rotate_left(45); + self.s[3] = self.s[3].rotate_left(45); ret } } -#[cfg(any(feature = "xxh3", feature = "alloc"))] -impl Xoshiro256StarRand { - /// Creates a new Xoshiro rand with the given seed +impl Xoshiro256PlusPlusRand { + /// Creates a new xoshiro256++ rand with the given seed #[must_use] pub fn with_seed(seed: u64) -> Self { - let mut rand = Self { rand_seed: [0; 4] }; - rand.set_seed(seed); // TODO: Proper random seed? + let mut rand = Self { s: [0; 4] }; + rand.set_seed(seed); rand } } -/// XXH3 Based, hopefully speedy, rnd implementation +/// Xorshift64 PRNG #[derive(Copy, Clone, Debug, Serialize, Deserialize)] pub struct XorShift64Rand { - rand_seed: u64, + s: u64, } impl Rand for XorShift64Rand { - #[allow(clippy::unreadable_literal)] - fn set_seed(&mut self, seed: u64) { - self.rand_seed = seed ^ 0x1234567890abcdef; + fn set_seed(&mut self, mut seed: u64) { + self.s = splitmix64(&mut seed) | 1; } #[inline] fn next(&mut self) -> u64 { - let mut x = self.rand_seed; + let mut x = self.s; x ^= x << 13; x ^= x >> 7; x ^= x << 17; - self.rand_seed = x; + self.s = x; x } } impl XorShift64Rand { - /// Creates a new Xoshiro rand with the given seed + /// Creates a new xorshift64 rand with the given seed #[must_use] pub fn with_seed(seed: u64) -> Self { - let mut ret: Self = Self { rand_seed: 0 }; - ret.set_seed(seed); // TODO: Proper random seed? + let mut ret: Self = Self { s: 0 }; + ret.set_seed(seed); ret } } -/// XXH3 Based, hopefully speedy, rnd implementation +/// Lehmer64 PRNG #[derive(Copy, Clone, Debug, Serialize, Deserialize)] pub struct Lehmer64Rand { - rand_seed: u128, + s: u128, } impl Rand for Lehmer64Rand { - #[allow(clippy::unreadable_literal)] - fn set_seed(&mut self, seed: u64) { - self.rand_seed = u128::from(seed) ^ 0x1234567890abcdef; + fn set_seed(&mut self, mut seed: u64) { + let hi = splitmix64(&mut seed); + let lo = splitmix64(&mut seed) | 1; + self.s = u128::from(hi) << 64 | u128::from(lo); } #[inline] #[allow(clippy::unreadable_literal)] fn next(&mut self) -> u64 { - self.rand_seed *= 0xda942042e4dd58b5; - (self.rand_seed >> 64) as u64 + self.s *= 0xda942042e4dd58b5; + (self.s >> 64) as u64 } } @@ -258,7 +299,7 @@ impl Lehmer64Rand { /// Creates a new Lehmer rand with the given seed #[must_use] pub fn with_seed(seed: u64) -> Self { - let mut ret: Self = Self { rand_seed: 0 }; + let mut ret: Self = Self { s: 0 }; ret.set_seed(seed); ret } @@ -288,10 +329,10 @@ impl RomuTrioRand { } impl Rand for RomuTrioRand { - fn set_seed(&mut self, seed: u64) { - self.x_state = seed ^ 0x12345; - self.y_state = seed ^ 0x6789A; - self.z_state = seed ^ 0xBCDEF; + fn set_seed(&mut self, mut seed: u64) { + self.x_state = splitmix64(&mut seed); + self.y_state = splitmix64(&mut seed); + self.z_state = splitmix64(&mut seed); } #[inline] @@ -328,9 +369,9 @@ impl RomuDuoJrRand { } impl Rand for RomuDuoJrRand { - fn set_seed(&mut self, seed: u64) { - self.x_state = seed ^ 0x12345; - self.y_state = seed ^ 0x6789A; + fn set_seed(&mut self, mut seed: u64) { + self.x_state = splitmix64(&mut seed); + self.y_state = splitmix64(&mut seed); } #[inline] @@ -343,6 +384,54 @@ impl Rand for RomuDuoJrRand { } } +/// [SFC64][1] algorithm by Chris Doty-Humphrey. +/// +/// [1]: https://numpy.org/doc/stable/reference/random/bit_generators/sfc64.html +#[derive(Copy, Clone, Debug, Serialize, Deserialize)] +pub struct Sfc64Rand { + a: u64, + b: u64, + c: u64, + w: u64, +} + +impl Sfc64Rand { + /// Creates a new [`Sfc64Rand`] with the given seed. + #[must_use] + pub fn with_seed(seed: u64) -> Self { + let mut s = Sfc64Rand { + a: 0, + b: 0, + c: 0, + w: 0, + }; + s.set_seed(seed); + s + } +} + +impl Rand for Sfc64Rand { + fn set_seed(&mut self, seed: u64) { + self.a = seed; + self.b = seed; + self.c = seed; + self.w = 1; + for _ in 0..12 { + self.next(); + } + } + + #[inline] + fn next(&mut self) -> u64 { + let out = self.a.wrapping_add(self.b).wrapping_add(self.w); + self.w = self.w.wrapping_add(1); + self.a = self.b ^ (self.b >> 11); + self.b = self.c.wrapping_add(self.c << 3); + self.c = self.c.rotate_left(24).wrapping_add(out); + out + } +} + /// fake rand, for testing purposes #[derive(Copy, Clone, Debug, Default, Serialize, Deserialize)] #[allow(clippy::upper_case_acronyms)] @@ -351,8 +440,8 @@ pub struct XkcdRand { } impl Rand for XkcdRand { - fn set_seed(&mut self, val: u64) { - self.val = val; + fn set_seed(&mut self, mut seed: u64) { + self.val = splitmix64(&mut seed); } fn next(&mut self) -> u64 { @@ -363,26 +452,26 @@ impl Rand for XkcdRand { /// A test rng that will return the same value (chose by fair dice roll) for testing. impl XkcdRand { /// Creates a new [`XkcdRand`] with the rand of 4, [chosen by fair dice roll, guaranteed to be random](https://xkcd.com/221/). - /// Will always return this seed. #[must_use] pub fn new() -> Self { - Self::with_seed(4) + Self { val: 4 } } - /// Creates a new [`XkcdRand`] with the given seed. Will always return this seed. + /// Creates a new [`XkcdRand`] with the given seed. #[must_use] pub fn with_seed(seed: u64) -> Self { - Self { val: seed } + let mut rand = XkcdRand { val: 0 }; + rand.set_seed(seed); + rand } } #[cfg(test)] mod tests { - //use xxhash_rust::xxh3::xxh3_64_with_seed; - - #[cfg(any(feature = "xxh3", feature = "alloc"))] - use crate::rands::Xoshiro256StarRand; - use crate::rands::{Rand, RomuDuoJrRand, RomuTrioRand, StdRand, XorShift64Rand}; + use crate::rands::{ + Rand, RomuDuoJrRand, RomuTrioRand, Sfc64Rand, StdRand, XorShift64Rand, + Xoshiro256PlusPlusRand, + }; fn test_single_rand(rand: &mut R) { assert_ne!(rand.next(), rand.next()); @@ -399,51 +488,116 @@ mod tests { test_single_rand(&mut RomuTrioRand::with_seed(0)); test_single_rand(&mut RomuDuoJrRand::with_seed(0)); test_single_rand(&mut XorShift64Rand::with_seed(0)); - #[cfg(any(feature = "xxh3", feature = "alloc"))] - test_single_rand(&mut Xoshiro256StarRand::with_seed(0)); + test_single_rand(&mut Xoshiro256PlusPlusRand::with_seed(0)); + test_single_rand(&mut Sfc64Rand::with_seed(0)); } - #[cfg(feature = "std")] #[test] - fn test_random_seed() { - use crate::rands::RandomSeed; - - let mut rand_fixed = StdRand::with_seed(0); - let mut rand = StdRand::new(); - - // The seed should be reasonably random so these never fail - assert_ne!(rand.next(), rand_fixed.next()); - test_single_rand(&mut rand); + fn test_romutrio_golden() { + // https://github.com/ziglang/zig/blob/130fb5cb0fb9039e79450c9db58d6590c5bee3b3/lib/std/Random/RomuTrio.zig#L75-L95 + let golden: [u64; 10] = [ + 16294208416658607535, + 13964609475759908645, + 4703697494102998476, + 3425221541186733346, + 2285772463536419399, + 9454187757529463048, + 13695907680080547496, + 8328236714879408626, + 12323357569716880909, + 12375466223337721820, + ]; + + let mut s = RomuTrioRand::with_seed(0); + for v in golden { + let u = s.next(); + assert_eq!(v, u); + } } #[test] - #[cfg(feature = "rand_trait")] - fn test_rgn_core_support() { - use rand_core::RngCore; - - use crate::rands::StdRand; - pub struct Mutator { - rng: R, + fn test_romuduojr_golden() { + // https://github.com/eqv/rand_romu/blob/c0379dc3c21ffac8440197e2f8fe95c226c44bfe/src/lib.rs#L65-L79 + let golden: [u64; 9] = [ + 0x3c91b13ee3913664, + 0xdc1980b78df3115, + 0x1c163b704996d2ad, + 0xa000c594bb28313b, + 0xfb6c42e69a523526, + 0x1fcebd6988ab21d8, + 0x5e0a8abf025f8f02, + 0x29554b00ffab0263, + 0xff5b6bb1551cf66, + ]; + + let mut s = RomuDuoJrRand { + x_state: 0x3c91b13ee3913664u64, + y_state: 0x863f0e37c2637d1fu64, + }; + for v in golden { + let u = s.next(); + assert_eq!(v, u); } + } - let mut mutator = Mutator { - rng: StdRand::with_seed(0), - }; + #[test] + fn test_xoshiro256pp_golden() { + // https://github.com/ziglang/zig/blob/130fb5cb0fb9039e79450c9db58d6590c5bee3b3/lib/std/Random/Xoshiro256.zig#L96-L103 + let golden: [u64; 6] = [ + 0x53175d61490b23df, + 0x61da6f3dc380d507, + 0x5c0fdf91ec9a7bfc, + 0x02eebf8c3bbe5e1a, + 0x7eca04ebaf4a5eea, + 0x0543c37757f08d9a, + ]; + + let mut s = Xoshiro256PlusPlusRand::with_seed(0); + for v in golden { + let u = s.next(); + assert_eq!(v, u); + } + } - log::info!("random value: {}", mutator.rng.next_u32()); + #[test] + fn test_sfc64_golden() { + // https://github.com/ziglang/zig/blob/130fb5cb0fb9039e79450c9db58d6590c5bee3b3/lib/std/Random/Sfc64.zig#L73-L99 + let golden: [u64; 16] = [ + 0x3acfa029e3cc6041, + 0xf5b6515bf2ee419c, + 0x1259635894a29b61, + 0xb6ae75395f8ebd6, + 0x225622285ce302e2, + 0x520d28611395cb21, + 0xdb909c818901599d, + 0x8ffd195365216f57, + 0xe8c4ad5e258ac04a, + 0x8f8ef2c89fdb63ca, + 0xf9865b01d98d8e2f, + 0x46555871a65d08ba, + 0x66868677c6298fcd, + 0x2ce15a7e6329f57d, + 0xb2f1833ca91ca79, + 0x4b0890ac9bf453ca, + ]; + + let mut s = Sfc64Rand::with_seed(0); + for v in golden { + let u = s.next(); + assert_eq!(v, u); + } } } #[cfg(feature = "python")] -#[allow(clippy::unnecessary_fallible_conversions)] +#[allow(clippy::unnecessary_fallible_conversions, unused_qualifications)] #[allow(missing_docs)] /// `Rand` Python bindings pub mod pybind { use pyo3::prelude::*; use serde::{Deserialize, Serialize}; - use super::Rand; - use crate::{current_nanos, rands::StdRand}; + use super::{random_seed, Rand, StdRand}; #[pyclass(unsendable, name = "StdRand")] #[allow(clippy::unsafe_derive_deserialize)] @@ -457,9 +611,9 @@ pub mod pybind { #[pymethods] impl PythonStdRand { #[staticmethod] - fn with_current_nanos() -> Self { + fn with_random_seed() -> Self { Self { - inner: StdRand::with_seed(current_nanos()), + inner: StdRand::with_seed(random_seed()), } } diff --git a/libafl_bolts/src/serdeany.rs b/libafl_bolts/src/serdeany.rs index 6eb429318f..356aeb8088 100644 --- a/libafl_bolts/src/serdeany.rs +++ b/libafl_bolts/src/serdeany.rs @@ -1,11 +1,54 @@ //! Poor-rust-man's downcasts for stuff we send over the wire (or shared maps) use alloc::boxed::Box; +#[cfg(feature = "unsafe_stable_anymap")] +use alloc::string::{String, ToString}; +#[cfg(feature = "unsafe_stable_anymap")] +use core::any::type_name; +#[cfg(not(feature = "unsafe_stable_anymap"))] +use core::any::TypeId; use core::{any::Any, fmt::Debug}; use serde::{de::DeserializeSeed, Deserialize, Deserializer, Serialize, Serializer}; pub use serdeany_registry::*; +#[cfg(not(feature = "unsafe_stable_anymap"))] +use crate::anymap::unpack_type_id; + +/// The type of a stored type in this anymap (`u128`) +#[cfg(not(feature = "unsafe_stable_anymap"))] +pub type TypeRepr = u128; + +/// The type of a stored type in this anymap (`String`) +#[cfg(feature = "unsafe_stable_anymap")] +pub type TypeRepr = String; + +#[cfg(not(feature = "unsafe_stable_anymap"))] +fn type_repr() -> TypeRepr +where + T: 'static, +{ + unpack_type_id(TypeId::of::()) +} + +#[cfg(not(feature = "unsafe_stable_anymap"))] +fn type_repr_owned() -> TypeRepr +where + T: 'static, +{ + unpack_type_id(TypeId::of::()) +} + +#[cfg(feature = "unsafe_stable_anymap")] +fn type_repr_owned() -> TypeRepr { + type_name::().to_string() +} + +#[cfg(feature = "unsafe_stable_anymap")] +fn type_repr() -> &'static str { + type_name::() +} + /// A (de)serializable Any trait pub trait SerdeAny: Any + erased_serde::Serialize + Debug { /// returns this as Any trait @@ -66,23 +109,29 @@ where /// Each element needs to be registered so that it can be deserialized. pub mod serdeany_registry { - use alloc::boxed::Box; - use core::{any::TypeId, fmt}; + use alloc::{ + boxed::Box, + string::{String, ToString}, + }; + use core::{any::TypeId, fmt, hash::BuildHasherDefault}; use hashbrown::{ - hash_map::{Keys, Values, ValuesMut}, + hash_map::{Values, ValuesMut}, HashMap, }; - use postcard; use serde::{Deserialize, Serialize}; use crate::{ - anymap::{pack_type_id, unpack_type_id}, - hash_std, - serdeany::{DeserializeCallback, DeserializeCallbackSeed}, + serdeany::{ + type_repr, type_repr_owned, DeserializeCallback, DeserializeCallbackSeed, SerdeAny, + TypeRepr, + }, Error, }; + /// A [`HashMap`] that maps from [`TypeRepr`] to a deserializer and its [`TypeId`]. + type DeserializeCallbackMap = HashMap, TypeId)>; + /// Visitor object used internally for the [`crate::serdeany::SerdeAny`] registry. #[derive(Debug)] pub struct BoxDynVisitor {} @@ -98,14 +147,15 @@ pub mod serdeany_registry { where V: serde::de::SeqAccess<'de>, { - let id: u128 = visitor.next_element()?.unwrap(); + let id: TypeRepr = visitor.next_element()?.unwrap(); let cb = unsafe { - *REGISTRY + REGISTRY .deserializers .as_ref() .expect("Empty types registry") .get(&id) .expect("Cannot deserialize an unregistered type") + .0 }; let seed = DeserializeCallbackSeed:: { cb }; let obj: Self::Value = visitor.next_element_seed(seed)?.unwrap(); @@ -115,7 +165,7 @@ pub mod serdeany_registry { #[allow(unused_qualifications)] struct Registry { - deserializers: Option>>, + deserializers: Option, finalized: bool, } @@ -128,9 +178,17 @@ pub mod serdeany_registry { assert!(!self.finalized, "Registry is already finalized!"); let deserializers = self.deserializers.get_or_insert_with(HashMap::default); - deserializers.insert(unpack_type_id(TypeId::of::()), |de| { - Ok(Box::new(erased_serde::deserialize::(de)?)) - }); + let _entry = deserializers + .entry(type_repr_owned::()) + .or_insert_with(|| { + ( + |de| Ok(Box::new(erased_serde::deserialize::(de)?)), + TypeId::of::(), + ) + }); + + #[cfg(feature = "unsafe_stable_anymap")] + assert_eq!(_entry.1, TypeId::of::(), "Fatal safety error: TypeId of type {} is not equals to the deserializer's TypeId for this type! Two registered types have the same type_name!", type_repr::()); } pub fn finalize(&mut self) { @@ -181,7 +239,7 @@ pub mod serdeany_registry { #[allow(clippy::unsafe_derive_deserialize)] #[derive(Debug, Serialize, Deserialize)] pub struct SerdeAnyMap { - map: HashMap>, + map: HashMap>, } // Cloning by serializing and deserializing. It ain't fast, but it's honest work. @@ -218,8 +276,12 @@ pub mod serdeany_registry { where T: crate::serdeany::SerdeAny, { + let type_repr = type_repr::(); + #[cfg(not(feature = "unsafe_stable_anymap"))] + let type_repr = &type_repr; + self.map - .get(&unpack_type_id(TypeId::of::())) + .get(type_repr) .map(|x| x.as_ref().as_any().downcast_ref::().unwrap()) } @@ -230,8 +292,12 @@ pub mod serdeany_registry { where T: crate::serdeany::SerdeAny, { + let type_repr = type_repr::(); + #[cfg(not(feature = "unsafe_stable_anymap"))] + let type_repr = &type_repr; + self.map - .get_mut(&unpack_type_id(TypeId::of::())) + .get_mut(type_repr) .map(|x| x.as_mut().as_any_mut().downcast_mut::().unwrap()) } @@ -242,8 +308,12 @@ pub mod serdeany_registry { where T: crate::serdeany::SerdeAny, { + let type_repr = type_repr::(); + #[cfg(not(feature = "unsafe_stable_anymap"))] + let type_repr = &type_repr; + self.map - .remove(&unpack_type_id(TypeId::of::())) + .remove(type_repr) .map(|x| x.as_any_boxed().downcast::().unwrap()) } @@ -258,25 +328,65 @@ pub mod serdeany_registry { /// Insert a boxed element into the map. #[inline] - pub fn insert_boxed(&mut self, t: Box) + pub fn insert_boxed(&mut self, value: Box) + where + T: crate::serdeany::SerdeAny, + { + self.raw_entry_mut::() + .insert(type_repr_owned::(), value); + } + + /// Get an entry to an element in this map. + #[inline] + #[allow(unused_qualifications)] + pub fn raw_entry_mut( + &mut self, + ) -> hashbrown::hash_map::RawEntryMut< + '_, + TypeRepr, + Box, + BuildHasherDefault, + > where T: crate::serdeany::SerdeAny, { - let id = unpack_type_id(TypeId::of::()); + let type_repr = type_repr::(); + #[cfg(not(feature = "unsafe_stable_anymap"))] + let type_repr = &type_repr; + assert!( unsafe { REGISTRY .deserializers .as_ref() .expect("Empty types registry") - .get(&id) + .get(type_repr) .is_some() }, - "Type {} was inserted without registration! Call {}::register or use serde_autoreg.", + "Type {} was inserted without registration! Call RegistryBuilder::register::<{}>() or use serde_autoreg.", core::any::type_name::(), core::any::type_name::() ); - self.map.insert(id, t); + self.map.raw_entry_mut().from_key(type_repr) + } + + /// Gets a value by type, or inserts it using the given construction function `default` + pub fn get_or_insert_with(&mut self, default: impl FnOnce() -> T) -> &mut T + where + T: SerdeAny, + { + self.get_or_insert_with_boxed::(|| Box::new(default())) + } + + /// Gets a value by type, or inserts it using the given construction function `default` (returning a boxed value) + pub fn get_or_insert_with_boxed(&mut self, default: impl FnOnce() -> Box) -> &mut T + where + T: SerdeAny + 'static, + { + let ret = self + .raw_entry_mut::() + .or_insert_with(|| (type_repr_owned::(), default())); + ret.1.as_any_mut().downcast_mut::().unwrap() } /// Returns the count of elements in this map. @@ -299,7 +409,11 @@ pub mod serdeany_registry { where T: crate::serdeany::SerdeAny, { - self.map.contains_key(&unpack_type_id(TypeId::of::())) + let type_repr = type_repr::(); + #[cfg(not(feature = "unsafe_stable_anymap"))] + let type_repr = &type_repr; + + self.map.contains_key(type_repr) } /// Create a new [`SerdeAnyMap`]. @@ -322,7 +436,7 @@ pub mod serdeany_registry { #[allow(unused_qualifications)] #[derive(Debug, Serialize, Deserialize)] pub struct NamedSerdeAnyMap { - map: HashMap>>, + map: HashMap>>, } // Cloning by serializing and deserializing. It ain't fast, but it's honest work. @@ -343,58 +457,54 @@ pub mod serdeany_registry { where T: crate::serdeany::SerdeAny, { - match self.map.get(&unpack_type_id(TypeId::of::())) { + let type_repr = type_repr::(); + #[cfg(not(feature = "unsafe_stable_anymap"))] + let type_repr = &type_repr; + + match self.map.get(type_repr) { None => None, - Some(h) => h - .get(&hash_std(name.as_bytes())) - .map(|x| x.as_any().downcast_ref::().unwrap()), + Some(h) => h.get(name).map(|x| x.as_any().downcast_ref::().unwrap()), } } - /// Get an element of a given type contained in this map by [`TypeId`]. + /// Remove an element by type and name #[must_use] - #[allow(unused_qualifications)] #[inline] - pub fn by_typeid( - &self, - name: &str, - typeid: &TypeId, - ) -> Option<&dyn crate::serdeany::SerdeAny> { - match self.map.get(&unpack_type_id(*typeid)) { + pub fn remove(&mut self, name: &str) -> Option> + where + T: crate::serdeany::SerdeAny, + { + let type_repr = type_repr::(); + #[cfg(not(feature = "unsafe_stable_anymap"))] + let type_repr = &type_repr; + + match self.map.get_mut(type_repr) { None => None, - Some(h) => h.get(&hash_std(name.as_bytes())).map(AsRef::as_ref), + Some(h) => h + .remove(name) + .map(|x| x.as_any_boxed().downcast::().unwrap()), } } - /// Get an element of a given type contained in this map by [`TypeId`], as mut. + /// Get an element of a given type contained in this map by type `T`, as mut. #[must_use] #[inline] pub fn get_mut(&mut self, name: &str) -> Option<&mut T> where T: crate::serdeany::SerdeAny, { - match self.map.get_mut(&unpack_type_id(TypeId::of::())) { + let type_repr = type_repr::(); + #[cfg(not(feature = "unsafe_stable_anymap"))] + let type_repr = &type_repr; + + match self.map.get_mut(type_repr) { None => None, Some(h) => h - .get_mut(&hash_std(name.as_bytes())) + .get_mut(name) .map(|x| x.as_any_mut().downcast_mut::().unwrap()), } } - /// Get an element of a given type contained in this map by [`TypeId`], as mut. - #[must_use] - #[inline] - pub fn by_typeid_mut( - &mut self, - name: &str, - typeid: &TypeId, - ) -> Option<&mut dyn crate::serdeany::SerdeAny> { - match self.map.get_mut(&unpack_type_id(*typeid)) { - None => None, - Some(h) => h.get_mut(&hash_std(name.as_bytes())).map(AsMut::as_mut), - } - } - /// Get all elements of a type contained in this map. #[must_use] #[allow(unused_qualifications)] @@ -404,38 +514,21 @@ pub mod serdeany_registry { &self, ) -> Option< core::iter::Map< - Values<'_, u64, Box>, + Values<'_, String, Box>, fn(&Box) -> &T, >, > where T: crate::serdeany::SerdeAny, { - #[allow(clippy::manual_map)] - match self.map.get(&unpack_type_id(TypeId::of::())) { - None => None, - Some(h) => Some(h.values().map(|x| x.as_any().downcast_ref::().unwrap())), - } - } + let type_repr = type_repr::(); + #[cfg(not(feature = "unsafe_stable_anymap"))] + let type_repr = &type_repr; - /// Get all elements of a given type contained in this map by [`TypeId`]. - #[must_use] - #[allow(unused_qualifications)] - #[inline] - #[allow(clippy::type_complexity)] - pub fn all_by_typeid( - &self, - typeid: &TypeId, - ) -> Option< - core::iter::Map< - Values<'_, u64, Box>, - fn(&Box) -> &dyn crate::serdeany::SerdeAny, - >, - > { #[allow(clippy::manual_map)] - match self.map.get(&unpack_type_id(*typeid)) { + match self.map.get(type_repr) { None => None, - Some(h) => Some(h.values().map(|x| x.as_ref())), + Some(h) => Some(h.values().map(|x| x.as_any().downcast_ref::().unwrap())), } } @@ -447,15 +540,19 @@ pub mod serdeany_registry { &mut self, ) -> Option< core::iter::Map< - ValuesMut<'_, u64, Box>, + ValuesMut<'_, String, Box>, fn(&mut Box) -> &mut T, >, > where T: crate::serdeany::SerdeAny, { + let type_repr = type_repr::(); + #[cfg(not(feature = "unsafe_stable_anymap"))] + let type_repr = &type_repr; + #[allow(clippy::manual_map)] - match self.map.get_mut(&unpack_type_id(TypeId::of::())) { + match self.map.get_mut(type_repr) { None => None, Some(h) => Some( h.values_mut() @@ -464,51 +561,18 @@ pub mod serdeany_registry { } } - /// Get all [`TypeId`]`s` contained in this map, as mut. - #[inline] - #[allow(unused_qualifications)] - #[allow(clippy::type_complexity)] - pub fn all_by_typeid_mut( - &mut self, - typeid: &TypeId, - ) -> Option< - core::iter::Map< - ValuesMut<'_, u64, Box>, - fn(&mut Box) -> &mut dyn crate::serdeany::SerdeAny, - >, - > { - #[allow(clippy::manual_map)] - match self.map.get_mut(&unpack_type_id(*typeid)) { - None => None, - Some(h) => Some(h.values_mut().map(|x| x.as_mut())), - } - } - - /// Get all [`TypeId`]`s` contained in this map. - #[inline] - #[allow(unused_qualifications)] - #[allow(clippy::type_complexity)] - pub fn all_typeids( - &self, - ) -> core::iter::Map< - Keys<'_, u128, HashMap>>, - fn(&u128) -> TypeId, - > { - self.map.keys().map(|x| pack_type_id(*x)) - } - /// Run `func` for each element in this map. #[inline] #[allow(unused_qualifications)] pub fn for_each< - F: FnMut(&TypeId, &Box) -> Result<(), Error>, + F: FnMut(&TypeRepr, &Box) -> Result<(), Error>, >( &self, func: &mut F, ) -> Result<(), Error> { for (id, h) in &self.map { for x in h.values() { - func(&pack_type_id(*id), x)?; + func(id, x)?; } } Ok(()) @@ -517,14 +581,14 @@ pub mod serdeany_registry { /// Run `func` for each element in this map, getting a mutable borrow. #[inline] pub fn for_each_mut< - F: FnMut(&TypeId, &mut Box) -> Result<(), Error>, + F: FnMut(&TypeRepr, &mut Box) -> Result<(), Error>, >( &mut self, func: &mut F, ) -> Result<(), Error> { for (id, h) in &mut self.map { for x in h.values_mut() { - func(&pack_type_id(*id), x)?; + func(id, x)?; } } Ok(()) @@ -533,31 +597,107 @@ pub mod serdeany_registry { /// Insert an element into this map. #[inline] #[allow(unused_qualifications)] - pub fn insert(&mut self, val: T, name: &str) + pub fn insert(&mut self, name: &str, val: T) where T: crate::serdeany::SerdeAny, { - let id = unpack_type_id(TypeId::of::()); + self.entry::(name.into()).insert(Box::new(val)); + } + + /// Get a reference to the type map. + #[inline] + #[allow(unused_qualifications)] + fn outer_map_mut( + &mut self, + ) -> &mut hashbrown::hash_map::HashMap> + where + T: crate::serdeany::SerdeAny, + { + let type_repr = type_repr::(); + #[cfg(not(feature = "unsafe_stable_anymap"))] + let type_repr = &type_repr; + assert!( unsafe { REGISTRY .deserializers .as_ref() .expect("Empty types registry") - .get(&id) + .get(type_repr) .is_some() }, - "Type {} was inserted without registration! Call {}::register or use serde_autoreg.", + "Type {} was inserted without registration! Call RegistryBuilder::register::<{}>() or use serde_autoreg.", core::any::type_name::(), core::any::type_name::() ); - if !self.map.contains_key(&id) { - self.map.insert(id, HashMap::default()); - } self.map - .get_mut(&id) - .unwrap() - .insert(hash_std(name.as_bytes()), Box::new(val)); + .raw_entry_mut() + .from_key(type_repr) + .or_insert_with(|| (type_repr_owned::(), HashMap::default())) + .1 + } + + /// Get an entry to an element into this map. + /// Prefer [`Self::raw_entry_mut`] as it won't need an owned key. + #[inline] + #[allow(unused_qualifications)] + fn entry( + &mut self, + name: String, + ) -> hashbrown::hash_map::Entry< + '_, + String, + Box, + BuildHasherDefault, + > + where + T: crate::serdeany::SerdeAny, + { + self.outer_map_mut::().entry(name) + } + + /// Get a raw entry to an element into this map. + #[inline] + #[allow(unused_qualifications)] + fn raw_entry_mut( + &mut self, + name: &str, + ) -> hashbrown::hash_map::RawEntryMut< + '_, + String, + Box, + BuildHasherDefault, + > + where + T: crate::serdeany::SerdeAny, + { + self.outer_map_mut::().raw_entry_mut().from_key(name) + } + + /// Gets a value by name, or inserts it using the given construction function `default` + pub fn get_or_insert_with(&mut self, name: &str, default: impl FnOnce() -> T) -> &mut T + where + T: SerdeAny, + { + let ret = self + .raw_entry_mut::(name) + .or_insert_with(|| (name.to_string(), Box::new(default()))); + ret.1.as_any_mut().downcast_mut::().unwrap() + } + + /// Gets a value by name, or inserts it using the given construction function `default` (returning a boxed value) + pub fn get_or_insert_with_boxed( + &mut self, + name: &str, + default: impl FnOnce() -> Box, + ) -> &mut T + where + T: SerdeAny + 'static, + { + let ret = self + .raw_entry_mut::(name) + .or_insert_with(|| (name.to_string(), default())); + ret.1.as_any_mut().downcast_mut::().unwrap() } /// Returns the `len` of this map. @@ -580,7 +720,11 @@ pub mod serdeany_registry { where T: crate::serdeany::SerdeAny, { - self.map.contains_key(&unpack_type_id(TypeId::of::())) + let type_repr = type_repr::(); + #[cfg(not(feature = "unsafe_stable_anymap"))] + let type_repr = &type_repr; + + self.map.contains_key(type_repr) } /// Returns if the element by a given `name` is contained in this map. @@ -590,9 +734,13 @@ pub mod serdeany_registry { where T: crate::serdeany::SerdeAny, { - match self.map.get(&unpack_type_id(TypeId::of::())) { + let type_repr = type_repr::(); + #[cfg(not(feature = "unsafe_stable_anymap"))] + let type_repr = &type_repr; + + match self.map.get(type_repr) { None => false, - Some(h) => h.contains_key(&hash_std(name.as_bytes())), + Some(h) => h.contains_key(name), } } @@ -737,6 +885,7 @@ macro_rules! impl_serdeany { /// /// # Safety /// This may never be called concurrently as it dereferences the `RegistryBuilder` without acquiring a lock. + #[allow(unused)] pub unsafe fn register() { $crate::serdeany::RegistryBuilder::register::<$struct_name>(); } @@ -745,3 +894,39 @@ macro_rules! impl_serdeany { $crate::create_register!($struct_name); }; } + +#[cfg(test)] +mod tests { + use serde::{Deserialize, Serialize}; + + use crate::serdeany::RegistryBuilder; + + #[derive(Debug, Serialize, Deserialize)] + struct MyType(u32); + impl_serdeany!(MyType); + + mod inner { + use serde::{Deserialize, Serialize}; + + #[derive(Debug, Serialize, Deserialize)] + pub(super) struct MyType(f32); + impl_serdeany!(MyType); + } + + #[test] + fn test_deserialize_serialize() { + unsafe { + RegistryBuilder::register::(); + RegistryBuilder::register::(); + } + + let val = MyType(1); + let serialized = postcard::to_allocvec(&val).unwrap(); + + assert_eq!( + postcard::from_bytes::(&serialized).unwrap().0, + val.0 + ); + assert!(postcard::from_bytes::(&serialized).is_err()); + } +} diff --git a/libafl_bolts/src/shmem.rs b/libafl_bolts/src/shmem.rs index c85546ab14..a14c3e29f7 100644 --- a/libafl_bolts/src/shmem.rs +++ b/libafl_bolts/src/shmem.rs @@ -7,7 +7,11 @@ use alloc::{rc::Rc, string::ToString}; use core::fmt::Display; #[cfg(feature = "alloc")] use core::{cell::RefCell, fmt, mem::ManuallyDrop}; -use core::{fmt::Debug, mem}; +use core::{ + fmt::Debug, + mem, + ops::{Deref, DerefMut}, +}; #[cfg(feature = "std")] use std::env; #[cfg(all(unix, feature = "std", not(target_os = "haiku")))] @@ -31,7 +35,7 @@ pub use win32_shmem::{Win32ShMem, Win32ShMemProvider}; use crate::os::pipes::Pipe; #[cfg(all(feature = "std", unix, not(target_os = "haiku")))] pub use crate::os::unix_shmem_server::{ServedShMemProvider, ShMemService}; -use crate::{AsMutSlice, AsSlice, Error}; +use crate::Error; /// The standard sharedmem provider #[cfg(all(windows, feature = "std"))] @@ -168,9 +172,10 @@ impl ShMemId { alloc::str::from_utf8(&self.id[..self.null_pos()]).unwrap() } } -impl AsSlice for ShMemId { - type Entry = u8; - fn as_slice(&self) -> &[u8] { + +impl Deref for ShMemId { + type Target = [u8]; + fn deref(&self) -> &[u8] { &self.id } } @@ -192,23 +197,15 @@ impl Display for ShMemId { /// A [`ShMem`] is an interface to shared maps. /// They are the backbone of [`crate::llmp`] for inter-process communication. /// All you need for scaling on a new target is to implement this interface, as well as the respective [`ShMemProvider`]. -pub trait ShMem: Sized + Debug + Clone + AsSlice + AsMutSlice { +pub trait ShMem: Sized + Debug + Clone + DerefMut { /// Get the id of this shared memory mapping fn id(&self) -> ShMemId; - /// Get the size of this mapping - fn len(&self) -> usize; - - /// Check if the mapping is empty - fn is_empty(&self) -> bool { - self.len() == 0 - } - /// Convert to a ptr of a given type, checking the size. /// If the map is too small, returns `None` fn as_ptr_of(&self) -> Option<*const T> { if self.len() >= mem::size_of::() { - Some(self.as_slice().as_ptr() as *const T) + Some(self.as_ptr() as *const T) } else { None } @@ -218,7 +215,7 @@ pub trait ShMem: Sized + Debug + Clone + AsSlice + AsMutSlice(&mut self) -> Option<*mut T> { if self.len() >= mem::size_of::() { - Some(self.as_mut_slice().as_mut_ptr() as *mut T) + Some(self.as_mut_ptr() as *mut T) } else { None } @@ -321,7 +318,7 @@ pub trait ShMemProvider: Clone + Default + Debug { } } -/// A Reference Counted shared map, +/// A Handle Counted shared map, /// that can use internal mutability. /// Useful if the `ShMemProvider` needs to keep local state. #[cfg(feature = "alloc")] @@ -339,31 +336,27 @@ where fn id(&self) -> ShMemId { self.internal.id() } - - fn len(&self) -> usize { - self.internal.len() - } } #[cfg(feature = "alloc")] -impl AsSlice for RcShMem +impl Deref for RcShMem where T: ShMemProvider + Debug, { - type Entry = u8; - fn as_slice(&self) -> &[u8] { - self.internal.as_slice() + type Target = [u8]; + + fn deref(&self) -> &[u8] { + &self.internal } } #[cfg(feature = "alloc")] -impl AsMutSlice for RcShMem +impl DerefMut for RcShMem where T: ShMemProvider + Debug, { - type Entry = u8; - fn as_mut_slice(&mut self) -> &mut [u8] { - self.internal.as_mut_slice() + fn deref_mut(&mut self) -> &mut [u8] { + &mut self.internal } } @@ -374,7 +367,7 @@ impl Drop for RcShMem { } } -/// A Reference Counted `ShMemProvider`, +/// A Handle Counted `ShMemProvider`, /// that can use internal mutability. /// Useful if the `ShMemProvider` needs to keep local state. #[derive(Debug, Clone)] @@ -562,6 +555,13 @@ where /// Is needed on top. #[cfg(all(unix, feature = "std", not(target_os = "haiku")))] pub mod unix_shmem { + /// Mmap [`ShMem`] for Unix + #[cfg(not(target_os = "android"))] + pub use default::MmapShMem; + /// Mmap [`ShMemProvider`] for Unix + #[cfg(not(target_os = "android"))] + pub use default::MmapShMemProvider; + #[cfg(doc)] use crate::shmem::{ShMem, ShMemProvider}; @@ -578,29 +578,24 @@ pub mod unix_shmem { #[cfg(not(target_os = "android"))] pub type UnixShMem = default::CommonUnixShMem; - /// Mmap [`ShMem`] for Unix - #[cfg(not(target_os = "android"))] - pub use default::MmapShMem; - /// Mmap [`ShMemProvider`] for Unix - #[cfg(not(target_os = "android"))] - pub use default::MmapShMemProvider; - #[cfg(all(unix, feature = "std", not(target_os = "android")))] mod default { - use alloc::string::ToString; - use core::{ptr, slice}; + use core::{ + ops::{Deref, DerefMut}, + ptr, slice, + }; use std::{io::Write, process}; use libc::{ - c_int, c_uchar, close, ftruncate, mmap, munmap, perror, shm_open, shm_unlink, shmat, - shmctl, shmdt, shmget, + c_int, c_uchar, close, ftruncate, mmap, munmap, shm_open, shm_unlink, shmat, shmctl, + shmdt, shmget, }; use crate::{ - rands::{Rand, RandomSeed, StdRand}, + rands::{Rand, StdRand}, shmem::{ShMem, ShMemId, ShMemProvider}, - AsMutSlice, AsSlice, Error, + Error, }; // This is macOS's limit @@ -648,17 +643,15 @@ pub mod unix_shmem { 0o600, ); if shm_fd == -1 { - perror(b"shm_open\0".as_ptr() as *const _); - return Err(Error::unknown(format!( + return Err(Error::last_os_error(format!( "Failed to shm_open map with id {filename_path:?}", ))); } /* configure the size of the shared memory segment */ if ftruncate(shm_fd, map_size.try_into()?) != 0 { - perror(b"ftruncate\0".as_ptr() as *const _); shm_unlink(filename_path.as_ptr() as *const _); - return Err(Error::unknown(format!( + return Err(Error::last_os_error(format!( "setup_shm(): ftruncate() failed for map with id {filename_path:?}", ))); } @@ -673,10 +666,9 @@ pub mod unix_shmem { 0, ); if map == libc::MAP_FAILED || map.is_null() { - perror(b"mmap\0".as_ptr() as *const _); close(shm_fd); shm_unlink(filename_path.as_ptr() as *const _); - return Err(Error::unknown(format!( + return Err(Error::last_os_error(format!( "mmap() failed for map with id {filename_path:?}", ))); } @@ -705,9 +697,8 @@ pub mod unix_shmem { 0, ); if map == libc::MAP_FAILED || map.is_null() { - perror(b"mmap\0".as_ptr() as *const _); close(shm_fd); - return Err(Error::unknown(format!( + return Err(Error::last_os_error(format!( "mmap() failed for map with fd {shm_fd:?}" ))); } @@ -729,7 +720,7 @@ pub mod unix_shmem { } } - /// A [`ShMemProvider`] which uses `shmget`/`shmat`/`shmctl` to provide shared memory mappings. + /// A [`ShMemProvider`] which uses [`shm_open`] and [`mmap`] to provide shared memory mappings. #[cfg(unix)] #[derive(Clone, Debug)] pub struct MmapShMemProvider { @@ -773,22 +764,18 @@ pub mod unix_shmem { fn id(&self) -> ShMemId { self.id } - - fn len(&self) -> usize { - self.map_size - } } - impl AsSlice for MmapShMem { - type Entry = u8; - fn as_slice(&self) -> &[u8] { + impl Deref for MmapShMem { + type Target = [u8]; + + fn deref(&self) -> &[u8] { unsafe { slice::from_raw_parts(self.map, self.map_size) } } } - impl AsMutSlice for MmapShMem { - type Entry = u8; - fn as_mut_slice(&mut self) -> &mut [u8] { + impl DerefMut for MmapShMem { + fn deref_mut(&mut self) -> &mut [u8] { unsafe { slice::from_raw_parts_mut(self.map, self.map_size) } } } @@ -852,11 +839,7 @@ pub mod unix_shmem { let map = shmat(os_id, ptr::null(), 0) as *mut c_uchar; if map as c_int == -1 || map.is_null() { - perror(b"shmat\0".as_ptr() as *const _); - shmctl(os_id, libc::IPC_RMID, ptr::null_mut()); - return Err(Error::unknown( - "Failed to map the shared mapping".to_string(), - )); + return Err(Error::last_os_error("Failed to map the shared mapping")); } Ok(Self { @@ -874,8 +857,7 @@ pub mod unix_shmem { let map = shmat(id_int, ptr::null(), 0) as *mut c_uchar; if map.is_null() || map == ptr::null_mut::().wrapping_sub(1) { - perror(b"shmat\0".as_ptr() as *const _); - return Err(Error::unknown(format!( + return Err(Error::last_os_error(format!( "Failed to map the shared mapping with id {id_int}" ))); } @@ -890,22 +872,18 @@ pub mod unix_shmem { fn id(&self) -> ShMemId { self.id } - - fn len(&self) -> usize { - self.map_size - } } - impl AsSlice for CommonUnixShMem { - type Entry = u8; - fn as_slice(&self) -> &[u8] { + impl Deref for CommonUnixShMem { + type Target = [u8]; + + fn deref(&self) -> &[u8] { unsafe { slice::from_raw_parts(self.map, self.map_size) } } } - impl AsMutSlice for CommonUnixShMem { - type Entry = u8; - fn as_mut_slice(&mut self) -> &mut [u8] { + impl DerefMut for CommonUnixShMem { + fn deref_mut(&mut self) -> &mut [u8] { unsafe { slice::from_raw_parts_mut(self.map, self.map_size) } } } @@ -963,7 +941,10 @@ pub mod unix_shmem { #[cfg(all(unix, feature = "std"))] pub mod ashmem { use alloc::string::ToString; - use core::{ptr, slice}; + use core::{ + ops::{Deref, DerefMut}, + ptr, slice, + }; use std::ffi::CString; use libc::{ @@ -973,7 +954,7 @@ pub mod unix_shmem { use crate::{ shmem::{ShMem, ShMemId, ShMemProvider}, - AsMutSlice, AsSlice, Error, + Error, }; /// An ashmem based impl for linux/android @@ -1100,23 +1081,18 @@ pub mod unix_shmem { fn id(&self) -> ShMemId { self.id } - - fn len(&self) -> usize { - self.map_size - } } - impl AsSlice for AshmemShMem { - type Entry = u8; - fn as_slice(&self) -> &[u8] { + impl Deref for AshmemShMem { + type Target = [u8]; + + fn deref(&self) -> &[u8] { unsafe { slice::from_raw_parts(self.map, self.map_size) } } } - impl AsMutSlice for AshmemShMem { - type Entry = u8; - - fn as_mut_slice(&mut self) -> &mut [u8] { + impl DerefMut for AshmemShMem { + fn deref_mut(&mut self) -> &mut [u8] { unsafe { slice::from_raw_parts_mut(self.map, self.map_size) } } } @@ -1186,23 +1162,15 @@ pub mod unix_shmem { /// Then `win32` implementation for shared memory. #[cfg(all(feature = "std", windows))] pub mod win32_shmem { - use alloc::string::String; use core::{ ffi::c_void, fmt::{self, Debug, Formatter}, + ops::{Deref, DerefMut}, slice, }; use uuid::Uuid; - - use crate::{ - shmem::{ShMem, ShMemId, ShMemProvider}, - AsMutSlice, AsSlice, Error, - }; - - const INVALID_HANDLE_VALUE: isize = -1; - use windows::{ core::PCSTR, Win32::{ @@ -1214,6 +1182,13 @@ pub mod win32_shmem { }, }; + use crate::{ + shmem::{ShMem, ShMemId, ShMemProvider}, + Error, + }; + + const INVALID_HANDLE_VALUE: isize = -1; + /// The default [`ShMem`] impl for Windows using `shmctl` & `shmget` #[derive(Clone)] pub struct Win32ShMem { @@ -1300,21 +1275,16 @@ pub mod win32_shmem { fn id(&self) -> ShMemId { self.id } - - fn len(&self) -> usize { - self.map_size - } } - impl AsSlice for Win32ShMem { - type Entry = u8; - fn as_slice(&self) -> &[u8] { + impl Deref for Win32ShMem { + type Target = [u8]; + fn deref(&self) -> &[u8] { unsafe { slice::from_raw_parts(self.map, self.map_size) } } } - impl AsMutSlice for Win32ShMem { - type Entry = u8; - fn as_mut_slice(&mut self) -> &mut [u8] { + impl DerefMut for Win32ShMem { + fn deref_mut(&mut self) -> &mut [u8] { unsafe { slice::from_raw_parts_mut(self.map, self.map_size) } } } @@ -1404,7 +1374,8 @@ impl ShMemCursor { /// Slice from the current location on this map to the end, mutable fn empty_slice_mut(&mut self) -> &mut [u8] { - &mut (self.inner.as_mut_slice()[self.pos..]) + use crate::AsSliceMut; + &mut (self.inner.as_slice_mut()[self.pos..]) } } @@ -1451,6 +1422,7 @@ impl std::io::Seek for ShMemCursor { let effective_new_pos = match pos { std::io::SeekFrom::Start(s) => s, std::io::SeekFrom::End(offset) => { + use crate::AsSlice; let map_len = self.inner.as_slice().len(); let signed_pos = i64::try_from(map_len).unwrap(); let effective = signed_pos.checked_add(offset).unwrap(); @@ -1479,7 +1451,7 @@ mod tests { use crate::{ shmem::{ShMemProvider, StdShMemProvider}, - AsMutSlice, AsSlice, + AsSlice, AsSliceMut, }; #[test] @@ -1488,7 +1460,7 @@ mod tests { fn test_shmem_service() { let mut provider = StdShMemProvider::new().unwrap(); let mut map = provider.new_shmem(1024).unwrap(); - map.as_mut_slice()[0] = 1; + map.as_slice_mut()[0] = 1; assert!(map.as_slice()[0] == 1); } } diff --git a/libafl_bolts/src/tuples.rs b/libafl_bolts/src/tuples.rs index c1c5c1061d..9d93fdddb8 100644 --- a/libafl_bolts/src/tuples.rs +++ b/libafl_bolts/src/tuples.rs @@ -1,17 +1,27 @@ //! Compiletime lists/tuples used throughout the `LibAFL` universe -#[rustversion::not(nightly)] -use core::any::type_name; +#[cfg(feature = "alloc")] +use alloc::{borrow::Cow, vec::Vec}; +#[cfg(feature = "alloc")] +use core::ops::{Deref, DerefMut}; use core::{ - any::TypeId, + any::{type_name, TypeId}, + fmt::{Debug, Formatter}, + marker::PhantomData, + mem::transmute, + ops::{Index, IndexMut}, ptr::{addr_of, addr_of_mut}, }; +#[cfg(feature = "alloc")] +use serde::{Deserialize, Serialize}; pub use tuple_list::{tuple_list, tuple_list_type, TupleList}; #[cfg(any(feature = "xxh3", feature = "alloc"))] use crate::hash_std; -use crate::{HasLen, Named}; +use crate::HasLen; +#[cfg(feature = "alloc")] +use crate::Named; /// Returns if the type `T` is equal to `U` /// From @@ -88,6 +98,33 @@ where } } +/// Create a [`Vec`] from a tuple list or similar +/// (We need this trait since we cannot implement `Into` for foreign types) +#[cfg(feature = "alloc")] +pub trait IntoVec { + /// Convert this into a [`Vec`], reversed. + /// (Having this method around makes some implementations more performant) + fn into_vec_reversed(self) -> Vec + where + Self: Sized, + { + let mut ret = self.into_vec(); + ret.reverse(); + ret + } + + /// Convert this into a [`Vec`]. + fn into_vec(self) -> Vec; +} + +#[cfg(feature = "alloc")] +impl IntoVec for () { + #[inline] + fn into_vec(self) -> Vec { + Vec::new() + } +} + /// Gets the length of the element pub trait HasConstLen { /// The length as constant `usize` @@ -105,16 +142,30 @@ where const LEN: usize = 1 + Tail::LEN; } -impl HasLen for C +impl HasLen for (Head, Tail) where - C: HasConstLen, + Tail: HasLen, { + #[inline] fn len(&self) -> usize { - Self::LEN + self.1.len() + 1 } +} - fn is_empty(&self) -> bool { - Self::LEN != 0 +impl HasLen for (Tail,) +where + Tail: HasLen, +{ + #[inline] + fn len(&self) -> usize { + self.0.len() + } +} + +impl HasLen for () { + #[inline] + fn len(&self) -> usize { + 0 } } @@ -172,9 +223,9 @@ where /// Returns the first element with the given type pub trait MatchFirstType { - /// Returns the first element with the given type as borrow, or [`Option::None`] + /// Returns the first element with the given type as borrow, or [`None`] fn match_first_type(&self) -> Option<&T>; - /// Returns the first element with the given type as mutable borrow, or [`Option::None`] + /// Returns the first element with the given type as mutable borrow, or [`None`] fn match_first_type_mut(&mut self) -> Option<&mut T>; } @@ -211,7 +262,7 @@ where /// Returns the first element with the given type (dereference mut version) pub trait ExtractFirstRefType { - /// Returns the first element with the given type as borrow, or [`Option::None`] + /// Returns the first element with the given type as borrow, or [`None`] fn take<'a, T: 'static>(self) -> (Option<&'a T>, Self); } @@ -229,7 +280,7 @@ where fn take<'a, T: 'static>(mut self) -> (Option<&'a T>, Self) { if TypeId::of::() == TypeId::of::() { let r = self.0.take(); - (unsafe { core::mem::transmute(r) }, self) + (unsafe { transmute::, Option<&T>>(r) }, self) } else { let (r, tail) = self.1.take::(); (r, (self.0, tail)) @@ -245,7 +296,10 @@ where fn take<'a, T: 'static>(mut self) -> (Option<&'a T>, Self) { if TypeId::of::() == TypeId::of::() { let r = self.0.take(); - (unsafe { core::mem::transmute(r) }, self) + ( + unsafe { transmute::, Option<&T>>(r) }, + self, + ) } else { let (r, tail) = self.1.take::(); (r, (self.0, tail)) @@ -255,7 +309,7 @@ where /// Returns the first element with the given type (dereference mut version) pub trait ExtractFirstRefMutType { - /// Returns the first element with the given type as borrow, or [`Option::None`] + /// Returns the first element with the given type as borrow, or [`None`] fn take<'a, T: 'static>(self) -> (Option<&'a mut T>, Self); } @@ -273,7 +327,10 @@ where fn take<'a, T: 'static>(mut self) -> (Option<&'a mut T>, Self) { if TypeId::of::() == TypeId::of::() { let r = self.0.take(); - (unsafe { core::mem::transmute(r) }, self) + ( + unsafe { transmute::, Option<&mut T>>(r) }, + self, + ) } else { let (r, tail) = self.1.take::(); (r, (self.0, tail)) @@ -357,31 +414,36 @@ where } } +#[cfg(feature = "alloc")] /// A named tuple pub trait NamedTuple: HasConstLen { /// Gets the name of this tuple - fn name(&self, index: usize) -> Option<&str>; + fn name(&self, index: usize) -> Option<&Cow<'static, str>>; } +#[cfg(feature = "alloc")] impl NamedTuple for () { - fn name(&self, _index: usize) -> Option<&str> { + fn name(&self, _index: usize) -> Option<&Cow<'static, str>> { None } } +#[cfg(feature = "alloc")] impl Named for () { #[inline] - fn name(&self) -> &str { - "Empty" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("Empty"); + &NAME } } +#[cfg(feature = "alloc")] impl NamedTuple for (Head, Tail) where Head: Named, Tail: NamedTuple, { - fn name(&self, index: usize) -> Option<&str> { + fn name(&self, index: usize) -> Option<&Cow<'static, str>> { if index == 0 { Some(self.0.name()) } else { @@ -395,13 +457,17 @@ where /// # Note /// This operation may not be 100% accurate with Rust stable, see the notes for [`type_eq`] /// (in `nightly`, it uses [specialization](https://stackoverflow.com/a/60138532/7658998)). +#[cfg(feature = "alloc")] pub trait MatchName { /// Match for a name and return the borrowed value + #[deprecated = "Use `.reference` and either `.get` (fallible access) or `[]` (infallible access) instead"] fn match_name(&self, name: &str) -> Option<&T>; /// Match for a name and return the mut borrowed value + #[deprecated = "Use `.reference` and either `.get` (fallible access) or `[]` (infallible access) instead"] fn match_name_mut(&mut self, name: &str) -> Option<&mut T>; } +#[cfg(feature = "alloc")] impl MatchName for () { fn match_name(&self, _name: &str) -> Option<&T> { None @@ -411,6 +477,8 @@ impl MatchName for () { } } +#[cfg(feature = "alloc")] +#[allow(deprecated)] impl MatchName for (Head, Tail) where Head: Named, @@ -433,44 +501,154 @@ where } } -/// Finds an element of a `type` by the given `name`. -pub trait MatchNameAndType { - /// Finds an element of a `type` by the given `name`, and returns a borrow, or [`Option::None`]. - fn match_name_type(&self, name: &str) -> Option<&T>; - /// Finds an element of a `type` by the given `name`, and returns a mut borrow, or [`Option::None`]. - fn match_name_type_mut(&mut self, name: &str) -> Option<&mut T>; +/// Structs that has `Handle ` +/// You should use this when you want to avoid specifying types using `match_name_type_mut` +#[cfg(feature = "alloc")] +pub trait Handler: Named { + /// Return the `Handle ` + fn handle(&self) -> Handle { + Handle { + name: Named::name(self).clone(), + phantom: PhantomData, + } + } } -impl MatchNameAndType for () { - fn match_name_type(&self, _name: &str) -> Option<&T> { - None +#[cfg(feature = "alloc")] +impl Handler for N where N: Named {} + +/// Object with the type T and the name associated with its concrete value +#[derive(Serialize, Deserialize)] +#[cfg(feature = "alloc")] +pub struct Handle { + name: Cow<'static, str>, + #[serde(skip)] + phantom: PhantomData, +} + +#[cfg(feature = "alloc")] +impl Handle { + /// Fetch the name of the referenced instance. + /// + /// We explicitly do *not* implement [`Named`], as this could potentially lead to confusion + /// where we make a [`Handle`] of a [`Handle`] as [`Named`] is blanket implemented. + #[must_use] + pub fn name(&self) -> &Cow<'static, str> { + &self.name } - fn match_name_type_mut(&mut self, _name: &str) -> Option<&mut T> { - None +} + +#[cfg(feature = "alloc")] +impl Clone for Handle { + fn clone(&self) -> Self { + Self { + name: self.name.clone(), + phantom: PhantomData, + } + } +} + +#[cfg(feature = "alloc")] +impl Debug for Handle { + fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { + f.debug_struct("Handle") + .field("name", self.name()) + .field("type", &type_name::()) + .finish() } } -impl MatchNameAndType for (Head, Tail) +/// Search using `Handle ` +#[cfg(feature = "alloc")] +pub trait MatchNameRef { + /// Search using name and `Handle ` + fn get(&self, rf: &Handle) -> Option<&T>; + + /// Search using name and `Handle ` + fn get_mut(&mut self, rf: &Handle) -> Option<&mut T>; +} + +#[cfg(feature = "alloc")] +#[allow(deprecated)] +impl MatchNameRef for M where - Head: 'static + Named, - Tail: MatchNameAndType, + M: MatchName, { - fn match_name_type(&self, name: &str) -> Option<&T> { - // Switch this check to https://stackoverflow.com/a/60138532/7658998 when in stable and remove 'static - if TypeId::of::() == TypeId::of::() && name == self.0.name() { - unsafe { (addr_of!(self.0) as *const T).as_ref() } - } else { - self.1.match_name_type::(name) - } + fn get(&self, rf: &Handle) -> Option<&T> { + self.match_name::(&rf.name) } - fn match_name_type_mut(&mut self, name: &str) -> Option<&mut T> { - // Switch this check to https://stackoverflow.com/a/60138532/7658998 when in stable and remove 'static - if TypeId::of::() == TypeId::of::() && name == self.0.name() { - unsafe { (addr_of_mut!(self.0) as *mut T).as_mut() } - } else { - self.1.match_name_type_mut::(name) - } + fn get_mut(&mut self, rf: &Handle) -> Option<&mut T> { + self.match_name_mut::(&rf.name) + } +} + +/// A wrapper type to enable the indexing of [`MatchName`] implementors with `[]`. +#[cfg(feature = "alloc")] +#[derive(Copy, Clone, Debug)] +#[repr(transparent)] +pub struct RefIndexable(RM, PhantomData); + +#[cfg(feature = "alloc")] +impl From for RefIndexable +where + RM: Deref, + M: MatchName, +{ + fn from(value: RM) -> Self { + RefIndexable(value, PhantomData) + } +} + +#[cfg(feature = "alloc")] +impl Deref for RefIndexable +where + RM: Deref, +{ + type Target = RM::Target; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +#[cfg(feature = "alloc")] +impl DerefMut for RefIndexable +where + RM: DerefMut, +{ + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +#[cfg(feature = "alloc")] +impl Index<&Handle> for RefIndexable +where + RM: Deref, + M: MatchName, +{ + type Output = T; + + fn index(&self, index: &Handle) -> &Self::Output { + let Some(e) = self.get(index) else { + panic!("Could not find entry matching {index:?}") + }; + e + } +} + +#[cfg(feature = "alloc")] +impl IndexMut<&Handle> for RefIndexable +where + RM: DerefMut, + M: MatchName, +{ + fn index_mut(&mut self, index: &Handle) -> &mut Self::Output { + let Some(e) = self.get_mut(index) else { + panic!("Could not find entry matching {index:?}") + }; + e } } diff --git a/libafl_cc/build.rs b/libafl_cc/build.rs index 9d9a84c757..3b63ed86de 100644 --- a/libafl_cc/build.rs +++ b/libafl_cc/build.rs @@ -238,8 +238,9 @@ fn main() { println!("cargo:rerun-if-env-changed=LLVM_CXXFLAGS"); println!("cargo:rerun-if-env-changed=LLVM_LDFLAGS"); println!("cargo:rerun-if-env-changed=LLVM_VERSION"); - println!("cargo:rerun-if-env-changed=LIBAFL_EDGES_MAP_SIZE"); + println!("cargo:rerun-if-env-changed=LIBAFL_EDGES_MAP_SIZE_IN_USE"); println!("cargo:rerun-if-env-changed=LIBAFL_ACCOUNTING_MAP_SIZE"); + println!("cargo:rerun-if-env-changed=LIBAFL_DDG_MAP_SIZE"); println!("cargo:rerun-if-changed=src/common-llvm.h"); println!("cargo:rerun-if-changed=build.rs"); @@ -256,7 +257,7 @@ fn main() { && llvm_version.is_ok()) { println!( - "cargo:warning=Failed to find llvm-config, we will not build LLVM passes. If you need them, set the LLVM_CONFIG environment variable to a recent llvm-config." + "cargo:warning=Failed to find llvm-config, we will not build LLVM passes. If you need them, set the LLVM_CONFIG environment variable to a recent llvm-config, else just ignore this message." ); write!( @@ -310,15 +311,23 @@ pub const LIBAFL_CC_LLVM_VERSION: Option = None; }; let mut cxxflags: Vec = cxxflags.split_whitespace().map(String::from).collect(); - let edges_map_size: usize = option_env!("LIBAFL_EDGES_MAP_SIZE") + let edges_map_size_in_use: usize = option_env!("LIBAFL_EDGES_MAP_SIZE_IN_USE") + .map_or(Ok(65_536), str::parse) + .expect("Could not parse LIBAFL_EDGES_MAP_SIZE_IN_USE"); + let edges_map_size_max: usize = option_env!("LIBAFL_EDGES_MAP_SIZE_MAX") .map_or(Ok(2_621_440), str::parse) - .expect("Could not parse LIBAFL_EDGES_MAP_SIZE"); - cxxflags.push(format!("-DLIBAFL_EDGES_MAP_SIZE={edges_map_size}")); + .expect("Could not parse LIBAFL_EDGES_MAP_SIZE_IN_USE"); + cxxflags.push(format!("-DEDGES_MAP_SIZE_IN_USE={edges_map_size_in_use}")); let acc_map_size: usize = option_env!("LIBAFL_ACCOUNTING_MAP_SIZE") .map_or(Ok(65_536), str::parse) .expect("Could not parse LIBAFL_ACCOUNTING_MAP_SIZE"); - cxxflags.push(format!("-DLIBAFL_ACCOUNTING_MAP_SIZE={acc_map_size}")); + cxxflags.push(format!("-DACCOUNTING_MAP_SIZE={acc_map_size}")); + + let ddg_map_size: usize = option_env!("LIBAFL_DDG_MAP_SIZE") + .map_or(Ok(65_536), str::parse) + .expect("Could not parse LIBAFL_DDG_MAP_SIZE"); + cxxflags.push(format!("-DDDG_MAP_SIZE={ddg_map_size}")); let llvm_version = find_llvm_version(); @@ -337,12 +346,17 @@ pub const LIBAFL_CC_LLVM_VERSION: Option = None; /// The path to the `clang++` executable pub const CLANGXX_PATH: &str = {clangcpp:?}; - /// The size of the edges map - pub const EDGES_MAP_SIZE: usize = {edges_map_size}; + /// The default size of the edges map the fuzzer uses + pub const EDGES_MAP_SIZE_IN_USE: usize = {edges_map_size_in_use}; + /// The real allocated size of the edges map + pub const EDGES_MAP_SIZE_MAX: usize = {edges_map_size_max}; /// The size of the accounting maps pub const ACCOUNTING_MAP_SIZE: usize = {acc_map_size}; + /// The size of the ddg maps + pub const DDG_MAP_SIZE: usize = {acc_map_size}; + /// The llvm version used to build llvm passes pub const LIBAFL_CC_LLVM_VERSION: Option = {llvm_version:?}; ", @@ -402,12 +416,23 @@ pub const LIBAFL_CC_LLVM_VERSION: Option = None; ldflags.push(&sdk_path); }; + build_pass( + bindir_path, + out_dir, + &cxxflags, + &ldflags, + src_dir, + "ddg-instr.cc", + Some(&vec!["ddg-utils.cc"]), + false, + ); + for pass in &[ "cmplog-routines-pass.cc", - "afl-coverage-pass.cc", "autotokens-pass.cc", "coverage-accounting-pass.cc", "cmplog-instructions-pass.cc", + "ctx-pass.cc", ] { build_pass( bindir_path, diff --git a/libafl_cc/src/afl-coverage-pass.cc b/libafl_cc/src/afl-coverage-pass.cc deleted file mode 100644 index 921013bb52..0000000000 --- a/libafl_cc/src/afl-coverage-pass.cc +++ /dev/null @@ -1,872 +0,0 @@ -/* - american fuzzy lop++ - LLVM-mode instrumentation pass - --------------------------------------------------- - - Written by Laszlo Szekeres , - Adrian Herrera , - Michal Zalewski - - LLVM integration design comes from Laszlo Szekeres. C bits copied-and-pasted - from afl-as.c are Michal's fault. - - NGRAM previous location coverage comes from Adrian Herrera. - - Copyright 2015, 2016 Google Inc. All rights reserved. - Copyright 2019-2020 AFLplusplus Project. All rights reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at: - - http://www.apache.org/licenses/LICENSE-2.0 - - This library is plugged into LLVM when invoking clang through afl-clang-fast. - It tells the compiler to add code roughly equivalent to the bits discussed - in ../afl-as.h. - - */ - -#include "common-llvm.h" - -#include -#include -#include -#ifndef _WIN32 - #include - #include -#else - #include -#endif -#include -#include -#include -#include - -#include -#include -#include - -#include "llvm/Support/CommandLine.h" -#include "llvm/IR/IRBuilder.h" -#include "llvm/IR/BasicBlock.h" -#include "llvm/IR/Module.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/MathExtras.h" -#include "llvm/ADT/DenseMap.h" -#include "llvm/Support/FormatVariadic.h" - -// Without this, Can't build with llvm-14 & old PM -#if LLVM_VERSION_MAJOR >= 14 && !defined(USE_NEW_PM) - #include "llvm/Pass.h" -#endif - -#if LLVM_VERSION_MAJOR > 3 || \ - (LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR > 4) - #include "llvm/IR/DebugInfo.h" - #include "llvm/IR/CFG.h" -#else - #include "llvm/DebugInfo.h" - #include "llvm/Support/CFG.h" -#endif - -typedef uint32_t prev_loc_t; - -/* Maximum ngram size */ -#define NGRAM_SIZE_MAX 16U - -/* Maximum K for top-K context sensitivity */ -#define CTX_MAX_K 32U - -#define MAP_SIZE LIBAFL_EDGES_MAP_SIZE - -using namespace llvm; - -static cl::opt Debug("debug_afl_coverage", cl::desc("Debug prints"), - cl::init(false), cl::NotHidden); -static cl::opt InstRatio( - "inst_ratio_afl_coverage", cl::desc("Instrumentation ratio in percentage"), - cl::init(100), cl::NotHidden); -static cl::opt NotZero("not_zero", - cl::desc("Never hit 0 again in the hitcount"), - cl::init(true), cl::NotHidden); -static cl::opt Ngram( - "ngram", cl::desc("Size of the Ngram instrumentation (0 to disable)"), - cl::init(0), cl::NotHidden); -static cl::opt CtxK( - "ctx_k", - cl::desc( - "Size of the context for K-Ctx context sensitivity (0 to disable)"), - cl::init(0), cl::NotHidden); -static cl::opt Ctx("ctx", - cl::desc("Enable full context sensitive coverage"), - cl::init(false), cl::NotHidden); -static cl::opt ThreadSafe("thread_safe_afl_coverage", - cl::desc("Use the thread safe instrumentation"), - cl::init(false), cl::NotHidden); -static cl::opt DumpCFG( - "dump_afl_cfg", cl::desc("Dump CFG containing AFL-style edge index"), - cl::init(false), cl::NotHidden); -static cl::opt DumpCFGPath( - "dump_afl_cfg_path", - cl::desc("Path to dump CFG containing AFL-style edge index"), - cl::init(".cfg"), cl::NotHidden); - -namespace { - -#ifdef USE_NEW_PM -class AFLCoverage : public PassInfoMixin { - public: - AFLCoverage() { -#else -class AFLCoverage : public ModulePass { - public: - static char ID; - AFLCoverage() : ModulePass(ID) { -#endif - - // initInstrumentList(); - } - -#ifdef USE_NEW_PM - PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); -#else - bool runOnModule(Module &M) override; -#endif - - protected: - uint32_t map_size = MAP_SIZE; - uint32_t function_minimum_size = 1; - DenseMap bb_to_cur_loc; - DenseMap entry_bb; -}; - -} // namespace - -#ifdef USE_NEW_PM -extern "C" ::llvm::PassPluginLibraryInfo LLVM_ATTRIBUTE_WEAK -llvmGetPassPluginInfo() { - return {LLVM_PLUGIN_API_VERSION, "AFLCoverage", "v0.1", - /* lambda to insert our pass into the pass pipeline. */ - [](PassBuilder &PB) { - #if 1 - #if LLVM_VERSION_MAJOR <= 13 - using OptimizationLevel = typename PassBuilder::OptimizationLevel; - #endif - PB.registerOptimizerLastEPCallback( - [](ModulePassManager &MPM, OptimizationLevel OL) { - MPM.addPass(AFLCoverage()); - }); - /* TODO LTO registration */ - #else - using PipelineElement = typename PassBuilder::PipelineElement; - PB.registerPipelineParsingCallback([](StringRef Name, - ModulePassManager &MPM, - ArrayRef) { - if (Name == "AFLCoverage") { - MPM.addPass(AFLCoverage()); - return true; - } else { - return false; - } - }); - #endif - }}; -} -#else - -char AFLCoverage::ID = 0; -#endif - -#ifdef USE_NEW_PM -PreservedAnalyses AFLCoverage::run(Module &M, ModuleAnalysisManager &MAM) { -#else -bool AFLCoverage::runOnModule(Module &M) { -#endif - if (Ctx && DumpCFG) { - FATAL( - "Does not support dumping CFG with full context sensitive coverage " - "enabled."); - } - LLVMContext &C = M.getContext(); - - IntegerType *Int8Ty = IntegerType::getInt8Ty(C); - IntegerType *Int32Ty = IntegerType::getInt32Ty(C); -#ifdef HAVE_VECTOR_INTRINSICS - IntegerType *IntLocTy = - IntegerType::getIntNTy(C, sizeof(prev_loc_t) * CHAR_BIT); -#endif - uint32_t rand_seed; - unsigned int cur_loc = 0; - -#ifdef USE_NEW_PM - auto PA = PreservedAnalyses::all(); -#endif - - /* Setup random() so we get Actually Random(TM) */ - rand_seed = time(NULL); - srand(rand_seed); - - /* - char *ptr; - if ((ptr = getenv("AFL_MAP_SIZE")) || (ptr = getenv("AFL_MAPSIZE"))) { - - map_size = atoi(ptr); - if (map_size < 8 || map_size > (1 << 29)) { - FATAL("illegal AFL_MAP_SIZE %u, must be between 2^3 and 2^30", - } - map_size); if (map_size % 8) {map_size = (((map_size >> 3) + 1) << 3);} - - } - - */ - - /* Decide instrumentation ratio */ - - if (!InstRatio || InstRatio > 100) { - FATAL("Bad value of the instrumentation ratio (must be between 1 and 100)"); - } - - unsigned PrevLocSize = 0; - unsigned PrevCallerSize = 0; - - bool instrument_ctx = Ctx || CtxK > 0; - bool instrument_caller = false; - -#ifdef HAVE_VECTOR_INTRINSICS - /* Decide previous location vector size (must be a power of two) */ - VectorType *PrevLocTy = NULL; - - if (Ngram && (Ngram < 2 || Ngram > NGRAM_SIZE_MAX)) { - FATAL( - "Bad value of the Ngram size (must be between 2 and NGRAM_SIZE_MAX " - "(%u))", - NGRAM_SIZE_MAX); - } - - if (Ngram) { - PrevLocSize = Ngram - 1; - } else { - PrevLocSize = 1; - } - - /* Decide K-Ctx vector size (must be a power of two) */ - VectorType *PrevCallerTy = NULL; - - if (CtxK > CTX_MAX_K) { - FATAL( - "Bad value of K for K-context sensitivity (must be between 1 and " - "CTX_MAX_K (%u))", - CTX_MAX_K); - } - - if (CtxK == 1) { - CtxK = 0; - instrument_ctx = true; - instrument_caller = true; // Enable CALLER instead - } - - if (CtxK) { - PrevCallerSize = CtxK; - instrument_ctx = true; - } - -#else - if (Ngram) - #ifndef LLVM_VERSION_PATCH - FATAL( - "Sorry, NGRAM branch coverage is not supported with llvm version " - "%d.%d.%d!", - LLVM_VERSION_MAJOR, LLVM_VERSION_MINOR, 0); - #else - FATAL( - "Sorry, NGRAM branch coverage is not supported with llvm version " - "%d.%d.%d!", - LLVM_VERSION_MAJOR, LLVM_VERSION_MINOR, LLVM_VERSION_PATCH); - #endif - if (CtxK) - #ifndef LLVM_VERSION_PATCH - FATAL( - "Sorry, K-CTX branch coverage is not supported with llvm version " - "%d.%d.%d!", - LLVM_VERSION_MAJOR, LLVM_VERSION_MINOR, 0); - #else - FATAL( - "Sorry, K-CTX branch coverage is not supported with llvm version " - "%d.%d.%d!", - LLVM_VERSION_MAJOR, LLVM_VERSION_MINOR, LLVM_VERSION_PATCH); - #endif - PrevLocSize = 1; -#endif - -#ifdef HAVE_VECTOR_INTRINSICS - int PrevLocVecSize = PowerOf2Ceil(PrevLocSize); - if (Ngram) - PrevLocTy = VectorType::get(IntLocTy, PrevLocVecSize - #if LLVM_VERSION_MAJOR >= 12 - , - false - #endif - ); -#endif - -#ifdef HAVE_VECTOR_INTRINSICS - int PrevCallerVecSize = PowerOf2Ceil(PrevCallerSize); - if (CtxK) - PrevCallerTy = VectorType::get(IntLocTy, PrevCallerVecSize - #if LLVM_VERSION_MAJOR >= 12 - , - false - #endif - ); -#endif - - /* Get globals for the SHM region and the previous location. Note that - __afl_prev_loc is thread-local. */ - - GlobalVariable *AFLMapPtr = - new GlobalVariable(M, PointerType::get(Int8Ty, 0), false, - GlobalValue::ExternalLinkage, 0, "__afl_area_ptr"); - GlobalVariable *AFLPrevLoc; - GlobalVariable *AFLPrevCaller; - GlobalVariable *AFLContext = NULL; - - if (Ctx || instrument_caller) -#if defined(__ANDROID__) || defined(__HAIKU__) - AFLContext = new GlobalVariable( - M, Int32Ty, false, GlobalValue::ExternalLinkage, 0, "__afl_prev_ctx"); -#else - AFLContext = new GlobalVariable( - M, Int32Ty, false, GlobalValue::ExternalLinkage, 0, "__afl_prev_ctx", 0, - GlobalVariable::GeneralDynamicTLSModel, 0, false); -#endif - -#ifdef HAVE_VECTOR_INTRINSICS - if (Ngram) - #if defined(__ANDROID__) || defined(__HAIKU__) - AFLPrevLoc = new GlobalVariable( - M, PrevLocTy, /* isConstant */ false, GlobalValue::ExternalLinkage, - /* Initializer */ nullptr, "__afl_prev_loc"); - #else - AFLPrevLoc = new GlobalVariable( - M, PrevLocTy, /* isConstant */ false, GlobalValue::ExternalLinkage, - /* Initializer */ nullptr, "__afl_prev_loc", - /* InsertBefore */ nullptr, GlobalVariable::GeneralDynamicTLSModel, - /* AddressSpace */ 0, /* IsExternallyInitialized */ false); - #endif - else -#endif -#if defined(__ANDROID__) || defined(__HAIKU__) - AFLPrevLoc = new GlobalVariable( - M, Int32Ty, false, GlobalValue::ExternalLinkage, 0, "__afl_prev_loc"); -#else - AFLPrevLoc = new GlobalVariable( - M, Int32Ty, false, GlobalValue::ExternalLinkage, 0, "__afl_prev_loc", 0, - GlobalVariable::GeneralDynamicTLSModel, 0, false); -#endif - -#ifdef HAVE_VECTOR_INTRINSICS - if (CtxK) - #if defined(__ANDROID__) || defined(__HAIKU__) - AFLPrevCaller = new GlobalVariable( - M, PrevCallerTy, /* isConstant */ false, GlobalValue::ExternalLinkage, - /* Initializer */ nullptr, "__afl_prev_caller"); - #else - AFLPrevCaller = new GlobalVariable( - M, PrevCallerTy, /* isConstant */ false, GlobalValue::ExternalLinkage, - /* Initializer */ nullptr, "__afl_prev_caller", - /* InsertBefore */ nullptr, GlobalVariable::GeneralDynamicTLSModel, - /* AddressSpace */ 0, /* IsExternallyInitialized */ false); - #endif - else -#endif -#if defined(__ANDROID__) || defined(__HAIKU__) - AFLPrevCaller = - new GlobalVariable(M, Int32Ty, false, GlobalValue::ExternalLinkage, 0, - "__afl_prev_caller"); -#else - AFLPrevCaller = new GlobalVariable( - M, Int32Ty, false, GlobalValue::ExternalLinkage, 0, "__afl_prev_caller", - 0, GlobalVariable::GeneralDynamicTLSModel, 0, false); -#endif - -#ifdef HAVE_VECTOR_INTRINSICS - /* Create the vector shuffle mask for updating the previous block history. - Note that the first element of the vector will store cur_loc, so just set - it to undef to allow the optimizer to do its thing. */ - - SmallVector PrevLocShuffle = {UndefValue::get(Int32Ty)}; - - for (unsigned I = 0; I < PrevLocSize - 1; ++I) { - PrevLocShuffle.push_back(ConstantInt::get(Int32Ty, I)); - } - - for (int I = PrevLocSize; I < PrevLocVecSize; ++I) { - PrevLocShuffle.push_back(ConstantInt::get(Int32Ty, PrevLocSize)); - } - - Constant *PrevLocShuffleMask = ConstantVector::get(PrevLocShuffle); - - Constant *PrevCallerShuffleMask = NULL; - SmallVector PrevCallerShuffle = {UndefValue::get(Int32Ty)}; - - if (CtxK) { - for (unsigned I = 0; I < PrevCallerSize - 1; ++I) { - PrevCallerShuffle.push_back(ConstantInt::get(Int32Ty, I)); - } - - for (int I = PrevCallerSize; I < PrevCallerVecSize; ++I) { - PrevCallerShuffle.push_back(ConstantInt::get(Int32Ty, PrevCallerSize)); - } - - PrevCallerShuffleMask = ConstantVector::get(PrevCallerShuffle); - } - -#endif - - // other constants we need - ConstantInt *One = ConstantInt::get(Int8Ty, 1); - - Value *PrevCtx = NULL; // CTX sensitive coverage - LoadInst *PrevCaller = NULL; // K-CTX coverage - - /* Instrument all the things! */ - - int inst_blocks = 0; - // scanForDangerousFunctions(&M); - - for (auto &F : M) { - int has_calls = 0; - if (Debug) - fprintf(stderr, "FUNCTION: %s (%zu)\n", F.getName().str().c_str(), - F.size()); - - if (isIgnoreFunction(&F)) { continue; } - - if (F.size() < function_minimum_size) { continue; } - if (DumpCFG) { entry_bb[F.getName()] = &F.getEntryBlock(); } - - std::list todo; - for (auto &BB : F) { - BasicBlock::iterator IP = BB.getFirstInsertionPt(); - IRBuilder<> IRB(&(*IP)); - - // Context sensitive coverage - if (instrument_ctx && &BB == &F.getEntryBlock()) { -#ifdef HAVE_VECTOR_INTRINSICS - if (CtxK) { - PrevCaller = IRB.CreateLoad( - #if LLVM_VERSION_MAJOR >= 14 - PrevCallerTy, - #endif - AFLPrevCaller); - PrevCaller->setMetadata(M.getMDKindID("nosanitize"), - MDNode::get(C, None)); - PrevCtx = - IRB.CreateZExt(IRB.CreateXorReduce(PrevCaller), IRB.getInt32Ty()); - - } else -#endif - { - - // load the context ID of the previous function and write to to a - // local variable on the stack - LoadInst *PrevCtxLoad = IRB.CreateLoad( -#if LLVM_VERSION_MAJOR >= 14 - IRB.getInt32Ty(), -#endif - AFLContext); - PrevCtxLoad->setMetadata(M.getMDKindID("nosanitize"), - MDNode::get(C, None)); - PrevCtx = PrevCtxLoad; - } - - // does the function have calls? and is any of the calls larger than one - // basic block? - for (auto &BB_2 : F) { - if (has_calls) { break; } - for (auto &IN : BB_2) { - CallInst *callInst = nullptr; - if ((callInst = dyn_cast(&IN))) { - Function *Callee = callInst->getCalledFunction(); - if (!Callee || Callee->size() < function_minimum_size) - continue; - else { - has_calls = 1; - break; - } - } - } - } - - // if yes we store a context ID for this function in the global var - if (has_calls) { - Value *NewCtx = ConstantInt::get(Int32Ty, RandBelow(map_size)); -#ifdef HAVE_VECTOR_INTRINSICS - if (CtxK) { - Value *ShuffledPrevCaller = IRB.CreateShuffleVector( - PrevCaller, UndefValue::get(PrevCallerTy), - PrevCallerShuffleMask); - Value *UpdatedPrevCaller = IRB.CreateInsertElement( - ShuffledPrevCaller, NewCtx, (uint64_t)0); - - StoreInst *Store = - IRB.CreateStore(UpdatedPrevCaller, AFLPrevCaller); - Store->setMetadata(M.getMDKindID("nosanitize"), - MDNode::get(C, None)); - - } else -#endif - { - - if (Ctx) { NewCtx = IRB.CreateXor(PrevCtx, NewCtx); } - StoreInst *StoreCtx = IRB.CreateStore(NewCtx, AFLContext); - StoreCtx->setMetadata(M.getMDKindID("nosanitize"), - MDNode::get(C, None)); - } - } - } - - if (RandBelow(100) >= InstRatio) { continue; } - - /* Make up cur_loc */ - - // cur_loc++; - cur_loc = RandBelow(map_size); - if (DumpCFG) { bb_to_cur_loc[&BB] = cur_loc; } -/* There is a problem with Ubuntu 18.04 and llvm 6.0 (see issue #63). - The inline function successors() is not inlined and also not found at runtime - :-( As I am unable to detect Ubuntu18.04 heree, the next best thing is to - disable this optional optimization for LLVM 6.0.0 and Linux */ -#if !(LLVM_VERSION_MAJOR == 6 && LLVM_VERSION_MINOR == 0) || !defined __linux__ - // only instrument if this basic block is the destination of a previous - // basic block that has multiple successors - // this gets rid of ~5-10% of instrumentations that are unnecessary - // result: a little more speed and less map pollution - int more_than_one = -1; - // fprintf(stderr, "BB %u: ", cur_loc); - for (pred_iterator PI = pred_begin(&BB), E = pred_end(&BB); PI != E; - ++PI) { - BasicBlock *Pred = *PI; - - int count = 0; - if (more_than_one == -1) { more_than_one = 0; } - // fprintf(stderr, " %p=>", Pred); - - for (succ_iterator SI = succ_begin(Pred), E = succ_end(Pred); SI != E; - ++SI) { - BasicBlock *Succ = *SI; - - // if (count > 0) - // fprintf(stderr, "|"); - if (Succ != NULL) { count++; } - // fprintf(stderr, "%p", Succ); - } - - if (count > 1) { more_than_one = 1; } - } - - // fprintf(stderr, " == %d\n", more_than_one); - if (F.size() > 1 && more_than_one != 1) { - // in CTX mode we have to restore the original context for the caller - - // she might be calling other functions which need the correct CTX - if (instrument_ctx && has_calls) { - Instruction *Inst = BB.getTerminator(); - if (isa(Inst) || isa(Inst)) { - IRBuilder<> Post_IRB(Inst); - - StoreInst *RestoreCtx; - #ifdef HAVE_VECTOR_INTRINSICS - if (CtxK) - RestoreCtx = IRB.CreateStore(PrevCaller, AFLPrevCaller); - else - #endif - RestoreCtx = Post_IRB.CreateStore(PrevCtx, AFLContext); - RestoreCtx->setMetadata(M.getMDKindID("nosanitize"), - MDNode::get(C, None)); - } - } - - continue; - } - -#endif - - ConstantInt *CurLoc; - -#ifdef HAVE_VECTOR_INTRINSICS - if (Ngram) - CurLoc = ConstantInt::get(IntLocTy, cur_loc); - else -#endif - CurLoc = ConstantInt::get(Int32Ty, cur_loc); - - /* Load prev_loc */ - - LoadInst *PrevLoc; - - if (Ngram) { - PrevLoc = IRB.CreateLoad( -#if LLVM_VERSION_MAJOR >= 14 - PrevLocTy, -#endif - AFLPrevLoc); - } else { - PrevLoc = IRB.CreateLoad( -#if LLVM_VERSION_MAJOR >= 14 - IRB.getInt32Ty(), -#endif - AFLPrevLoc); - } - PrevLoc->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); - Value *PrevLocTrans; - -#ifdef HAVE_VECTOR_INTRINSICS - /* "For efficiency, we propose to hash the tuple as a key into the - hit_count map as (prev_block_trans << 1) ^ curr_block_trans, where - prev_block_trans = (block_trans_1 ^ ... ^ block_trans_(n-1)" */ - - if (Ngram) - PrevLocTrans = - IRB.CreateZExt(IRB.CreateXorReduce(PrevLoc), IRB.getInt32Ty()); - else -#endif - PrevLocTrans = PrevLoc; - - if (instrument_ctx) - PrevLocTrans = - IRB.CreateZExt(IRB.CreateXor(PrevLocTrans, PrevCtx), Int32Ty); - else - PrevLocTrans = IRB.CreateZExt(PrevLocTrans, IRB.getInt32Ty()); - - /* Load SHM pointer */ - - LoadInst *MapPtr = IRB.CreateLoad( -#if LLVM_VERSION_MAJOR >= 14 - PointerType::get(Int8Ty, 0), -#endif - AFLMapPtr); - MapPtr->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); - - Value *MapPtrIdx; -#ifdef HAVE_VECTOR_INTRINSICS - if (Ngram) - MapPtrIdx = IRB.CreateGEP( - #if LLVM_VERSION_MAJOR >= 14 - Int8Ty, - #endif - MapPtr, - IRB.CreateZExt( - IRB.CreateXor(PrevLocTrans, IRB.CreateZExt(CurLoc, Int32Ty)), - Int32Ty)); - else -#endif - MapPtrIdx = IRB.CreateGEP( -#if LLVM_VERSION_MAJOR >= 14 - Int8Ty, -#endif - MapPtr, IRB.CreateXor(PrevLocTrans, CurLoc)); - - /* Update bitmap */ - - if (ThreadSafe) { /* Atomic */ - /* - #if LLVM_VERSION_MAJOR < 9 - if (neverZero_counters_str != - NULL) { // with llvm 9 we make this the default as the bug - in llvm - // is then fixed - #else - if (NotZero) { - - #endif - // register MapPtrIdx in a todo list - todo.push_back(MapPtrIdx); - - } else { - - */ - IRB.CreateAtomicRMW(llvm::AtomicRMWInst::BinOp::Add, MapPtrIdx, One, -#if LLVM_VERSION_MAJOR >= 13 - llvm::MaybeAlign(1), -#endif - llvm::AtomicOrdering::Monotonic); - /* - - } - - */ - - } else { - LoadInst *Counter = IRB.CreateLoad( -#if LLVM_VERSION_MAJOR >= 14 - IRB.getInt8Ty(), -#endif - MapPtrIdx); - Counter->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); - - Value *Incr = IRB.CreateAdd(Counter, One); - -#if LLVM_VERSION_MAJOR < 9 - if (neverZero_counters_str != - NULL) { // with llvm 9 we make this the default as the bug in llvm - // is then fixed -#else - if (NotZero) { - -#endif - /* hexcoder: Realize a counter that skips zero during overflow. - * Once this counter reaches its maximum value, it next increments to - * 1 - * - * Instead of - * Counter + 1 -> Counter - * we inject now this - * Counter + 1 -> {Counter, OverflowFlag} - * Counter + OverflowFlag -> Counter - */ - - ConstantInt *Zero = ConstantInt::get(Int8Ty, 0); - auto cf = IRB.CreateICmpEQ(Incr, Zero); - auto carry = IRB.CreateZExt(cf, Int8Ty); - Incr = IRB.CreateAdd(Incr, carry); - } - - IRB.CreateStore(Incr, MapPtrIdx) - ->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); - - } /* non atomic case */ - - /* Update prev_loc history vector (by placing cur_loc at the head of the - vector and shuffle the other elements back by one) */ - - StoreInst *Store; - -#ifdef HAVE_VECTOR_INTRINSICS - if (Ngram) { - Value *ShuffledPrevLoc = IRB.CreateShuffleVector( - PrevLoc, UndefValue::get(PrevLocTy), PrevLocShuffleMask); - Value *UpdatedPrevLoc = IRB.CreateInsertElement( - ShuffledPrevLoc, IRB.CreateLShr(CurLoc, (uint64_t)1), (uint64_t)0); - - Store = IRB.CreateStore(UpdatedPrevLoc, AFLPrevLoc); - Store->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); - - } else -#endif - { - - Store = IRB.CreateStore(ConstantInt::get(Int32Ty, cur_loc >> 1), - AFLPrevLoc); - Store->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); - } - - // in CTX mode we have to restore the original context for the caller - - // she might be calling other functions which need the correct CTX. - // Currently this is only needed for the Ubuntu clang-6.0 bug - if (instrument_ctx && has_calls) { - Instruction *Inst = BB.getTerminator(); - if (isa(Inst) || isa(Inst)) { - IRBuilder<> Post_IRB(Inst); - - StoreInst *RestoreCtx; -#ifdef HAVE_VECTOR_INTRINSICS - if (CtxK) - RestoreCtx = IRB.CreateStore(PrevCaller, AFLPrevCaller); - else -#endif - RestoreCtx = Post_IRB.CreateStore(PrevCtx, AFLContext); - RestoreCtx->setMetadata(M.getMDKindID("nosanitize"), - MDNode::get(C, None)); - } - } - - inst_blocks++; - } - } - if (DumpCFG) { - int fd; -#ifndef _WIN32 - if ((fd = open(DumpCFGPath.c_str(), O_WRONLY | O_CREAT | O_APPEND, 0644)) < - 0) -#else - if ((fd = _open(DumpCFGPath.c_str(), O_WRONLY | O_CREAT | O_APPEND, 0644)) < - 0) -#endif - FATAL("Could not open/create CFG dump file."); - std::string cfg = ""; - for (auto record = entry_bb.begin(); record != entry_bb.end(); record++) { - // Dump function BB entry points - cfg += formatv("$${0}+{1}\n", record->getFirst(), - bb_to_cur_loc[record->getSecond()]); - } - for (auto record = bb_to_cur_loc.begin(); record != bb_to_cur_loc.end(); - record++) { - // Dump CFG information - auto current_bb = record->getFirst(); - Function *calling_func = current_bb->getParent(); - if (calling_func) { - auto function_name = calling_func->getName().str(); - cfg += formatv("%%{0}", function_name); - } else - cfg += "%%__"; - auto current_cur_loc = record->getSecond(); - cfg += formatv("+{0}\n", current_cur_loc); - for (auto bb_successor = succ_begin(current_bb); - bb_successor != succ_end(current_bb); bb_successor++) { - cfg += formatv("->{0}\n", bb_to_cur_loc[*bb_successor]).str(); - } - } - if (Debug) { errs() << "CFG: \n" << cfg; } - if (cfg.size() > 0 && write(fd, cfg.c_str(), cfg.length()) <= 0) - FATAL("Failed to dump CFG.\n"); - } - - /* Say something nice. */ - - /*if (!be_quiet) { - - if (!inst_blocks) - WARNF("No instrumentation targets found."); - else { - - char modeline[100]; - snprintf(modeline, sizeof(modeline), "%s%s%s%s%s", - getenv("AFL_HARDEN") ? "hardened" : "non-hardened", - getenv("AFL_USE_ASAN") ? ", ASAN" : "", - getenv("AFL_USE_MSAN") ? ", MSAN" : "", - getenv("AFL_USE_CFISAN") ? ", CFISAN" : "", - getenv("AFL_USE_UBSAN") ? ", UBSAN" : ""); - OKF("Instrumented %d locations (%s mode, ratio %u%%).", inst_blocks, - modeline, InstRatio); - - } - - }*/ - - if (Debug) { - if (!inst_blocks) - fprintf(stderr, "No instrumentation targets found.\n"); - else - fprintf(stderr, "Instrumented %d locations (ratio %u%%).\n", inst_blocks, - (unsigned)InstRatio); - } - -#ifdef USE_NEW_PM - return PA; -#else - return true; -#endif -} - -#ifndef USE_NEW_PM -static void registerAFLPass(const PassManagerBuilder &, - legacy::PassManagerBase &PM) { - PM.add(new AFLCoverage()); -} - -static RegisterStandardPasses RegisterAFLPass( - PassManagerBuilder::EP_OptimizerLast, registerAFLPass); - -static RegisterStandardPasses RegisterAFLPass0( - PassManagerBuilder::EP_EnabledOnOptLevel0, registerAFLPass); -#endif diff --git a/libafl_cc/src/ar.rs b/libafl_cc/src/ar.rs index 42eba53c47..eb528c1962 100644 --- a/libafl_cc/src/ar.rs +++ b/libafl_cc/src/ar.rs @@ -1,7 +1,7 @@ //! Ar Wrapper from `LibAFL` // pass to e.g. cmake with -DCMAKE_AR=/path/to/fuzzer/target/release/libafl_ar -use std::{convert::Into, env, path::PathBuf, str::FromStr, string::String, vec::Vec}; +use std::{env, path::PathBuf, str::FromStr}; use crate::{Error, ToolWrapper, LIB_EXT, LIB_PREFIX}; @@ -162,7 +162,7 @@ impl ToolWrapper for ArWrapper { .base_args .iter() .map(|r| { - let arg_as_path = std::path::PathBuf::from(r); + let arg_as_path = PathBuf::from(r); if r.ends_with('.') { r.to_string() } else { @@ -185,7 +185,7 @@ impl ToolWrapper for ArWrapper { }) .collect::>(); - let Ok(ar_path) = std::env::var("LLVM_AR_PATH") else { + let Ok(ar_path) = env::var("LLVM_AR_PATH") else { panic!("Couldn't find llvm-ar. Specify the `LLVM_AR_PATH` environment variable"); }; diff --git a/libafl_cc/src/cfg.rs b/libafl_cc/src/cfg.rs index a7519d37fe..9c353ffa52 100644 --- a/libafl_cc/src/cfg.rs +++ b/libafl_cc/src/cfg.rs @@ -95,9 +95,9 @@ where /// Inserts an edge into CFG. #[must_use] pub fn new() -> Self { - let map_size = option_env!("LIBAFL_EDGES_MAP_SIZE") - .map_or(Ok(2621440), str::parse) - .expect("Could not parse LIBAFL_EDGES_MAP_SIZE"); + let map_size = option_env!("LIBAFL_EDGES_MAP_SIZE_IN_USE") + .map_or(Ok(65536), str::parse) + .expect("Could not parse LIBAFL_EDGES_MAP_SIZE_IN_USE"); Self { edges: (0..map_size).map(|_| None).collect(), func_to_entry_bb: HashMap::default(), @@ -396,6 +396,6 @@ mod tests { assert_eq!(*distances.get(&((41864 >> 1) ^ 26911)).unwrap(), 1); assert_eq!(*distances.get(&((26911 >> 1) ^ 52706)).unwrap(), 2); assert_eq!(*distances.get(&((26911 >> 1) ^ 41925)).unwrap(), 2); - assert!(distances.get(&((41864 >> 1) ^ 52706)).is_none()); + assert!(!distances.contains_key(&((41864 >> 1) ^ 52706))); } } diff --git a/libafl_cc/src/clang.rs b/libafl_cc/src/clang.rs index 2b0fe36a69..8151f9e6a6 100644 --- a/libafl_cc/src/clang.rs +++ b/libafl_cc/src/clang.rs @@ -1,12 +1,9 @@ //! LLVM compiler Wrapper from `LibAFL` use std::{ - convert::Into, env, path::{Path, PathBuf}, str::FromStr, - string::String, - vec::Vec, }; use crate::{CompilerWrapper, Error, ToolWrapper, LIB_EXT, LIB_PREFIX}; @@ -31,10 +28,8 @@ include!(concat!(env!("OUT_DIR"), "/clang_constants.rs")); #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum LLVMPasses { //CmpLogIns, - /// The CmpLog pass + /// The `CmpLog` pass CmpLogRtn, - /// The AFL coverage pass - AFLCoverage, /// The Autotoken pass AutoTokens, /// The Coverage Accouting (BB metric) pass @@ -42,8 +37,12 @@ pub enum LLVMPasses { /// The dump cfg pass DumpCfg, #[cfg(unix)] - /// The CmpLog Instruction pass + /// The `CmpLog` Instruction pass CmpLogInstructions, + /// Instrument caller for sancov coverage + Ctx, + /// Data dependency instrumentation + DDG, } impl LLVMPasses { @@ -53,8 +52,6 @@ impl LLVMPasses { match self { LLVMPasses::CmpLogRtn => PathBuf::from(env!("OUT_DIR")) .join(format!("cmplog-routines-pass.{}", dll_extension())), - LLVMPasses::AFLCoverage => PathBuf::from(env!("OUT_DIR")) - .join(format!("afl-coverage-pass.{}", dll_extension())), LLVMPasses::AutoTokens => { PathBuf::from(env!("OUT_DIR")).join(format!("autotokens-pass.{}", dll_extension())) } @@ -66,6 +63,12 @@ impl LLVMPasses { #[cfg(unix)] LLVMPasses::CmpLogInstructions => PathBuf::from(env!("OUT_DIR")) .join(format!("cmplog-instructions-pass.{}", dll_extension())), + LLVMPasses::Ctx => { + PathBuf::from(env!("OUT_DIR")).join(format!("ctx-pass.{}", dll_extension())) + } + LLVMPasses::DDG => { + PathBuf::from(env!("OUT_DIR")).join(format!("ddg-instr.{}", dll_extension())) + } } } } @@ -154,7 +157,7 @@ impl ToolWrapper for ClangWrapper { let mut suppress_linking = 0; let mut i = 1; while i < args.len() { - let arg_as_path = std::path::Path::new(args[i].as_ref()); + let arg_as_path = Path::new(args[i].as_ref()); if arg_as_path .extension() @@ -275,7 +278,7 @@ impl ToolWrapper for ClangWrapper { if linking { new_args.push("-lrt".into()); } - // MacOS has odd linker behavior sometimes + // `MacOS` has odd linker behavior sometimes #[cfg(target_vendor = "apple")] if linking || shared { new_args.push("-undefined".into()); @@ -331,7 +334,7 @@ impl ToolWrapper for ClangWrapper { .base_args .iter() .map(|r| { - let arg_as_path = std::path::PathBuf::from(r); + let arg_as_path = PathBuf::from(r); if r.ends_with('.') { r.to_string() } else { @@ -368,7 +371,7 @@ impl ToolWrapper for ClangWrapper { // No output specified, we need to rewrite the single .c file's name into a -o // argument. for arg in &base_args { - let arg_as_path = std::path::PathBuf::from(arg); + let arg_as_path = PathBuf::from(arg); if !arg.ends_with('.') && !arg.starts_with('-') { if let Some(extension) = arg_as_path.extension() { let extension = extension.to_str().unwrap(); @@ -378,7 +381,7 @@ impl ToolWrapper for ClangWrapper { args.push("-o".to_string()); args.push(if self.linking { configuration - .replace_extension(&std::path::PathBuf::from("a.out")) + .replace_extension(&PathBuf::from("a.out")) .into_os_string() .into_string() .unwrap() diff --git a/libafl_cc/src/cmplog-routines-pass.cc b/libafl_cc/src/cmplog-routines-pass.cc index 25fe1d6e76..0a68ab544f 100644 --- a/libafl_cc/src/cmplog-routines-pass.cc +++ b/libafl_cc/src/cmplog-routines-pass.cc @@ -71,7 +71,7 @@ class CmpLogRoutines : public ModulePass { #if USE_NEW_PM PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); #else - bool runOnModule(Module &M) override; + bool runOnModule(Module &M) override; #if LLVM_VERSION_MAJOR < 4 const char *getPassName() const override { diff --git a/libafl_cc/src/coverage-accounting-pass.cc b/libafl_cc/src/coverage-accounting-pass.cc index f6b96e635f..a7416bb307 100644 --- a/libafl_cc/src/coverage-accounting-pass.cc +++ b/libafl_cc/src/coverage-accounting-pass.cc @@ -45,7 +45,7 @@ typedef uint32_t prev_loc_t; -#define MAP_SIZE LIBAFL_ACCOUNTING_MAP_SIZE +#define MAP_SIZE ACCOUNTING_MAP_SIZE #define SECURITY_SENSITIVE_FUNCS(CF) \ static CF securitySensitiveFunctions[] = { \ diff --git a/libafl_cc/src/ctx-pass.cc b/libafl_cc/src/ctx-pass.cc new file mode 100644 index 0000000000..9f70445e2f --- /dev/null +++ b/libafl_cc/src/ctx-pass.cc @@ -0,0 +1,228 @@ +/* + LibAFL - Ctx LLVM pass + -------------------------------------------------- + + Written by Dongjia Zhang + + Copyright 2022-2023 AFLplusplus Project. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at: + + http://www.apache.org/licenses/LICENSE-2.0 + +*/ + +#include +#include +#include "common-llvm.h" +#ifndef _WIN32 + #include + #include +#else + #include +#endif +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "llvm/Config/llvm-config.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/IR/IRBuilder.h" + +#if USE_NEW_PM + #include "llvm/Passes/PassPlugin.h" + #include "llvm/Passes/PassBuilder.h" + #include "llvm/IR/PassManager.h" +#else + #include "llvm/IR/LegacyPassManager.h" + #include "llvm/Transforms/IPO/PassManagerBuilder.h" +#endif + +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/DebugInfo.h" +#include "llvm/IR/CFG.h" +#include "llvm/IR/Verifier.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Transforms/Utils/BasicBlockUtils.h" +#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/ValueTracking.h" +#include "llvm/Pass.h" +#include "llvm/IR/Constants.h" + +#include + +using namespace llvm; + +#define MAP_SIZE EDGES_MAP_SIZE_IN_USE + +namespace { + +#if USE_NEW_PM +class CtxPass : public PassInfoMixin { + public: + CtxPass() { +#else +class CtxPass : public ModulePass { + public: + static char ID; + + CtxPass() : ModulePass(ID) { +#endif + } + +#if USE_NEW_PM + PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); +#else + bool runOnModule(Module &M) override; +#endif + + protected: + uint32_t map_size = MAP_SIZE; + + private: + bool isLLVMIntrinsicFn(StringRef &n) { + // Not interested in these LLVM's functions +#if LLVM_VERSION_MAJOR >= 18 + if (n.starts_with("llvm.")) { +#else + if (n.startswith("llvm.")) { +#endif + return true; + } else { + return false; + } + } +}; + +} // namespace + +#if USE_NEW_PM +extern "C" ::llvm::PassPluginLibraryInfo LLVM_ATTRIBUTE_WEAK +llvmGetPassPluginInfo() { + return {LLVM_PLUGIN_API_VERSION, "CtxPass", "v0.1", + /* lambda to insert our pass into the pass pipeline. */ + [](PassBuilder &PB) { + + #if LLVM_VERSION_MAJOR <= 13 + using OptimizationLevel = typename PassBuilder::OptimizationLevel; + #endif + PB.registerOptimizerLastEPCallback( + [](ModulePassManager &MPM, OptimizationLevel OL) { + MPM.addPass(CtxPass()); + }); + }}; +} +#else +char CtxPass::ID = 0; +#endif + +#if USE_NEW_PM +PreservedAnalyses CtxPass::run(Module &M, ModuleAnalysisManager &MAM) { +#else +bool CtxPass::runOnModule(Module &M) { + +#endif + LLVMContext &C = M.getContext(); + auto moduleName = M.getName(); + IntegerType *Int8Ty = IntegerType::getInt8Ty(C); + IntegerType *Int32Ty = IntegerType::getInt32Ty(C); + + uint32_t rand_seed; + + rand_seed = time(NULL); + srand(rand_seed); + + GlobalVariable *AFLContext = new GlobalVariable( + M, Int32Ty, false, GlobalValue::ExternalLinkage, 0, "__afl_prev_ctx"); + Value *PrevCtx = + NULL; // the ctx value up until now that we save on the stack + + for (auto &F : M) { + int has_calls = 0; + + if (isIgnoreFunction(&F)) { continue; } + if (F.size() < 1) { continue; } + for (auto &BB : F) { + BasicBlock::iterator IP = BB.getFirstInsertionPt(); + IRBuilder<> IRB(&(*IP)); + if (&BB == &F.getEntryBlock()) { + // if this is the first block.. + LoadInst *PrevCtxLoad = IRB.CreateLoad(IRB.getInt32Ty(), AFLContext); + PrevCtxLoad->setMetadata(M.getMDKindID("nosanitize"), + MDNode::get(C, None)); + PrevCtx = PrevCtxLoad; + + // now check for if calls exists + for (auto &BB_2 : F) { + if (has_calls) { break; } + for (auto &IN : BB_2) { + CallInst *callInst = nullptr; + if ((callInst = dyn_cast(&IN))) { + Function *Callee = callInst->getCalledFunction(); + if (!Callee || Callee->size() < 1) { + continue; + } else { + has_calls = 1; + break; + } + } + } + } + + if (has_calls) { + Value *NewCtx = ConstantInt::get(Int32Ty, RandBelow(map_size)); + NewCtx = IRB.CreateXor(PrevCtx, NewCtx); + StoreInst *StoreCtx = IRB.CreateStore(NewCtx, AFLContext); + StoreCtx->setMetadata(M.getMDKindID("nosanitize"), + MDNode::get(C, None)); + } + } + // Restore the ctx at the end of BB + Instruction *Inst = BB.getTerminator(); + if (has_calls) { + if (isa(Inst) || isa(Inst)) { + IRBuilder<> Post_IRB(Inst); + StoreInst *RestoreCtx; + RestoreCtx = Post_IRB.CreateStore(PrevCtx, AFLContext); + RestoreCtx->setMetadata(M.getMDKindID("nosanitize"), + MDNode::get(C, None)); + } + } + } + } + +#if USE_NEW_PM + auto PA = PreservedAnalyses::all(); + return PA; +#else + return true; +#endif +} + +#if USE_NEW_PM + +#else +static void registerCtxPass(const PassManagerBuilder &, + legacy::PassManagerBase &PM) { + PM.add(new CtxPass()); +} + +static RegisterPass X("ctx", "ctx instrumentation pass", false, false); + +static RegisterStandardPasses RegisterCtxPass( + PassManagerBuilder::EP_OptimizerLast, registerCtxPass); + +static RegisterStandardPasses RegisterCtxPass0( + PassManagerBuilder::EP_EnabledOnOptLevel0, registerCtxPass); +#endif diff --git a/libafl_cc/src/ddg-instr.cc b/libafl_cc/src/ddg-instr.cc new file mode 100644 index 0000000000..87539911eb --- /dev/null +++ b/libafl_cc/src/ddg-instr.cc @@ -0,0 +1,786 @@ +#include "llvm/IR/Function.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/PassManager.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DepthFirstIterator.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/Twine.h" +#include "llvm/Analysis/MemoryBuiltins.h" +#include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Analysis/ValueTracking.h" +#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/CFG.h" +#include "llvm/BinaryFormat/MachO.h" +#include "llvm/IR/Argument.h" +#include "llvm/IR/Attributes.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/Comdat.h" +#include "llvm/IR/Constant.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/DIBuilder.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/DebugInfoMetadata.h" +#include "llvm/IR/DebugLoc.h" +#include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/Dominators.h" +#include "llvm/Analysis/PostDominators.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/GlobalAlias.h" +#include "llvm/IR/GlobalValue.h" +#include "llvm/IR/GlobalVariable.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/InlineAsm.h" +#include "llvm/IR/InstVisitor.h" +#include "llvm/IR/InstrTypes.h" +#include "llvm/IR/Instruction.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/MDBuilder.h" +#include "llvm/IR/Metadata.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/Type.h" +#include "llvm/IR/Use.h" +#include "llvm/IR/Value.h" +#include "llvm/IR/Verifier.h" +#include "llvm/IR/DebugInfo.h" +#include "llvm/IR/LegacyPassManager.h" +#include "llvm/MC/MCSectionMachO.h" +#include "llvm/Pass.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/MathExtras.h" +#include "llvm/Support/ScopedPrinter.h" +#include "llvm/Support/raw_ostream.h" +#include +#include "llvm/Transforms/Instrumentation.h" +#include "llvm/Transforms/Utils/ASanStackFrameLayout.h" +#include "llvm/Transforms/Utils/BasicBlockUtils.h" +#include "llvm/Transforms/Utils/Local.h" +#include "llvm/Transforms/Utils/ModuleUtils.h" +#include "llvm/Transforms/Utils/PromoteMemToReg.h" + +// #include "WPA/WPAPass.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ddg-utils.h" +#include "common-llvm.h" + +#define MAX_DEPTH 3 +#define MIN_FCN_SIZE 1 +#define VAR_NAME_LEN 264 + +#define MAP_SIZE DDG_MAP_SIZE +// #define MAP_SIZE 65536 +#define ALL_BIT_SET (MAP_SIZE - 1) +// #define MAP_SIZE 255 + +// #define INTERPROCEDURAL 1 // unset if you want only intraprocedural ret +// values management BUT #define LOAD_INSTR // considers loads as +// stores + +// #define DEBUG 1 // set if you want debug prints enabled + +#define AFL_SR(s) (srandom(s)) +#define AFL_R(x) (random() % (x)) + +#ifdef DEBUG + #define DEBUG(X) \ + do { \ + X; \ + } while (false) +#else + #define DEBUG(X) ((void)0) +#endif + +using namespace llvm; +// using namespace svf; + +class DDGInstrModulePass : public PassInfoMixin { + private: + void InsertDataFlow(Value *Operand, Value *Res) { + std::map>::iterator it = + this->DataFlowTracker.begin(); + while (it != this->DataFlowTracker.end()) { + std::vector Slice = it->second; + std::vector::iterator jt; + for (jt = Slice.begin(); jt != Slice.end(); ++jt) { + if (Operand == *jt) { + this->DataFlowTracker[it->first].push_back(Res); + break; + } + } + it++; + } + } + + void RetrieveDataFlow(Value *V, std::vector *Dependencies) { + std::map>::iterator it = + this->DataFlowTracker.begin(); + while (it != this->DataFlowTracker.end()) { + std::vector Slice = it->second; + std::vector::iterator jt; + for (jt = Slice.begin(); jt != Slice.end(); ++jt) { + if (V == *jt) { + Dependencies->push_back(it->first); + break; + } + } + it++; + } + } + + bool isSourceCodeVariable(Value *Variable) { + std::map>::iterator it = + this->DataFlowTracker.find(Variable); + return it != this->DataFlowTracker.end(); + } + + bool isLLVMVariable(Value *Variable, + std::map *LLVMVariables) { + std::map::iterator it = + LLVMVariables->find(Variable); + return it != LLVMVariables->end(); + } + + void CreateDataFlow(Value *Variable) { + std::map>::iterator it = + this->DataFlowTracker.find(Variable); + if (it == this->DataFlowTracker.end()) { + this->DataFlowTracker[Variable].push_back(Variable); + } + } + + // When we have `Store A, B`, we want to know that exactly B reperensents. In + // the default case, it is a source code variable and so we're done. BUT, in + // many cases B could represent the field of a struct, or a location whithin a + // buffer. So, we need to recover what B represents to be more precise when we + // define the dependency relationship. + void RetrieveAccessedVariable(Value *Variable, std::vector *Flows, + std::map *LLVMVariables, + Value **ActualSrcVariable) { + if (isLLVMVariable(Variable, LLVMVariables)) { + // If it is an LLVM variable (mostly for struct fields), we have it + // tracked down in the LLVMVariables list, so we just need to parse the + // GEP inst + Instruction *DefiningInstruction = (*LLVMVariables)[Variable]; + // For now we only handle the GEP instructions, maybe in future + // it could be useful to implement other instructions + if (auto GEP = dyn_cast(DefiningInstruction)) { + Value *PtrOperand = GEP->getPointerOperand(); + Variable = PtrOperand; + *ActualSrcVariable = PtrOperand; + if (isSourceCodeVariable(PtrOperand)) { + // We finally could connect an LLVM variable to an actual Source code + // Variable! + for (unsigned int i = 1; i < DefiningInstruction->getNumOperands(); + i++) { // Starts from 1, since 0 is thr PtrOperand + Value *Op = DefiningInstruction->getOperand(i); + if (!isa(Op)) { RetrieveDataFlow(Op, Flows); } + } + return; + } else { + // Re-itereate the Variable analysis + RetrieveAccessedVariable(Variable, Flows, LLVMVariables, + ActualSrcVariable); + } + for (unsigned int i = 1; i < DefiningInstruction->getNumOperands(); + i++) { // Starts from 1, since 0 is thr PtrOperand + Value *Op = DefiningInstruction->getOperand(i); + if (!isa(Op)) { RetrieveDataFlow(Op, Flows); } + } + } + } else { + // If it is not a GEP-defined llvm variable, we basically use the DataFlow + // Tracker, to retrieve the dependency of this variable. The idea is that, + // if this llvm variable is not GEP-depending, it should be easier to + // retrieve what it does represent + std::vector TmpFlow; + RetrieveDataFlow(Variable, &TmpFlow); + if (TmpFlow.size() == 1) { + *ActualSrcVariable = TmpFlow[0]; + // We found a Source Code variable (Variable->getName()) + return; + } else if (TmpFlow.size() > 1) { + *ActualSrcVariable = TmpFlow[0]; + DEBUG(errs() << "[Warning] multiple flows for the same GEP access, " + "choosing the first one\n"); + } else { + return; + } + } + } + + public: + static char ID; + FunctionCallee logger; + Type *VoidTy; + std::map> DataFlowTracker; + + PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM) { + LLVMContext &C = M.getContext(); + + auto &FAM = + MAM.getResult(M).getManager(); + auto DTCallback = [&FAM](Function &F) -> DominatorTree * { + return &FAM.getResult(F); + }; + + auto PDTCallback = [&FAM](Function &F) -> PostDominatorTree * { + return &FAM.getResult(F); + }; + + auto LICallback = [&FAM](Function &F) -> LoopInfo * { + return &FAM.getResult(F); + }; + + IntegerType *Int16Ty = IntegerType::getInt16Ty(C); + IntegerType *Int8Ty = IntegerType::getInt8Ty(C); + // IntegerType *Int32Ty = IntegerType::getInt32Ty(C); + ConstantInt *Zero = ConstantInt::get(Int8Ty, 0); + ConstantInt *One = ConstantInt::get(Int8Ty, 1); + unsigned int instrumentedLocations = 0; + + std::map BlocksLocs; + std::map VisitedBlocks; + ConstantInt *Visited = ConstantInt::get(Int16Ty, 0xff); + ConstantInt *NonVisited = ConstantInt::get(Int16Ty, 0); + ConstantInt *CurLoc; + char *name = nullptr; + unsigned BBCounter = 0; + + unsigned bb_count = 0; + unsigned int cur_loc = 0; + uint32_t map_size = MAP_SIZE; + + struct timeval tv; + struct timezone tz; + unsigned int rand_seed; + + /* Setup random() so we get Actually Random(TM) outputs from AFL_R() */ + gettimeofday(&tv, &tz); + rand_seed = tv.tv_sec ^ tv.tv_usec ^ getpid(); + AFL_SR(rand_seed); + + GlobalVariable *DDGMapPtr = M.getGlobalVariable("__ddg_area_ptr"); + if (DDGMapPtr == nullptr) + DDGMapPtr = + new GlobalVariable(M, PointerType::get(Int8Ty, 0), false, + GlobalValue::ExternalLinkage, 0, "__ddg_area_ptr"); + +#ifdef INTERPROCEDURAL + // For each function we store the return Values + std::map> ReturnValues; + + for (auto &F : M) { + if (F.size() < MIN_FCN_SIZE) continue; + + for (auto &BB : F) { + for (auto &I : BB) { + if (auto RI = dyn_cast(&I)) { + Value *RetVal = RI->getReturnValue(); + if (RetVal) { ReturnValues[&F].push_back(RI); } + } + } + } + } +#endif + + for (auto &F : M) { + if (F.size() < MIN_FCN_SIZE) continue; + + std::map> + Stores; // Represents the nodes of our DataDep Graph + std::vector> + StoreEdges; // Contains the edges of the DDG + std::map> + IncomingEdges; // Map s.t. key is a BB and value is a set of BBs + // whose data flow reaches the key + std::map + LLVMVariables; // LLVM IR Variables which are used as Operands for + // the store (for instance, the ones resulting from a + // GEP) + + BasicBlock &EntryBB = F.getEntryBlock(); + Instruction *FirstInst = EntryBB.getFirstNonPHIOrDbg(); + + // First we add the function params to track the dataflow + for (Function::arg_iterator arg_it = F.arg_begin(); arg_it != F.arg_end(); + arg_it++) { + Argument *Arg = arg_it; + if (Value *ArgVariable = dyn_cast(Arg)) { + CreateDataFlow(ArgVariable); + FlowWriteInstruction *MyStore = + new FlowWriteInstruction(&EntryBB, FirstInst, declaration); + Stores[ArgVariable].push_back(MyStore); + } + } + + LoopInfo *LI = LICallback(F); + DominatorTree *DT = DTCallback(F); + PostDominatorTree *PT = PDTCallback(F); + + // We basically want to track data flow between memory instructions + // and call instructions (i.e., the arguments) + + // Here we extract the data dependence info for function F + for (auto &BB : F) { + BBCounter += 1; + for (auto &I : BB) { + // We track all variables "Alloca" derived and we add them to the + // RootNode + if (auto AI = dyn_cast(&I)) { + Value *Variable = static_cast(AI); + CreateDataFlow(Variable); + } + + if (auto LOI = dyn_cast(&I)) { + Value *Variable = LOI->getPointerOperand(); + CreateDataFlow(Variable); +#ifdef LOAD_INSTR + std::vector Flows; + RetrieveDataFlow(Variable, &Flows); + + // If `Variable` does not directly represent a Src code variable, we + // fetch what it represents (e.g., the field of a struct) + if (!isSourceCodeVariable(Variable)) { + Value *ActualSrcVariable = nullptr; + RetrieveAccessedVariable(Variable, &Flows, &LLVMVariables, + &ActualSrcVariable); + if (ActualSrcVariable) Variable = ActualSrcVariable; + } + + for (std::vector::iterator it = Flows.begin(); + it != Flows.end(); ++it) { + Value *Dependency = *it; + + // First we find the edges between the current store and the + // previous ones (i.e., when we wrote into `c` and `b` if the + // current store is `a = c + b`) + std::vector AllStoresPerVariable = + Stores[Dependency]; + unsigned ConsideredStores = 0; + bool *ReachingStores = isReachableByStore( + &AllStoresPerVariable, LOI, &DT, &LI, &ConsideredStores); + + // ReachingStores[0] refers to the last Store instruction that we + // met (i.e., the last in `AllStoresPerVariable` This is why we + // iterate the vector in a reverse way BUT the array in the + // forward + unsigned i = 0; + for (std::vector::reverse_iterator it = + AllStoresPerVariable.rbegin(); + it != AllStoresPerVariable.rend(); it++) { + if (ReachingStores[i] && (i < ConsideredStores)) { + Instruction *Src = (*it)->I; + if (Src == + LOI) // Already managed in the `reachableByStores` method + continue; + if (Src->getParent() != LOI->getParent()) { + StoreEdges.push_back(edge); + IncomingEdges[LOI->getParent()].insert(LOI->getParent()); + DEBUG(errs() << "+++++++++++\nAdding edge\n"); + DEBUG(debug_instruction(Src)); + DEBUG(debug_instruction(LOI)); + DEBUG(errs() << "-----------\n"); + } + } + i++; + } + + delete[] ReachingStores; + } + // Then we insert the new Store in our map that contains all the + // stores, so we build forward deps + FlowWriteInstruction *MyStore = + new FlowWriteInstruction(LOI->getParent(), LOI, declaration); + Stores[Variable].push_back(MyStore); +#endif + } + + if (auto GEP = dyn_cast( + &I)) { // We dedicate an list for GEPs defined llvm vars. + Value *Var = static_cast( + &I); // For other LLVM variables, we use the DataflowTracker + LLVMVariables[Var] = GEP; + } + + // We propagate the dependency info + Value *Result = static_cast(&I); + if (Result and + !isa( + I)) { // We exclude CallInst, as they're managed separately + // (Not excluding them now, would introduce a double + // dependency leading to the same value) + for (unsigned int i = 0; i < I.getNumOperands(); i++) { + Value *Op = I.getOperand(i); + if (!isa(Op)) InsertDataFlow(Op, Result); + } + } +#ifdef INTERPROCEDURAL + else if (Result and isa(I)) { + CallInst *CI = dyn_cast(&I); + Function *CalledFunction = CI->getCalledFunction(); + std::map>::iterator it = + ReturnValues.find(CalledFunction); + if (it != ReturnValues.end()) { + std::vector RetValsInstrs = it->second; + for (std::vector::iterator jt = + RetValsInstrs.begin(); + jt != RetValsInstrs.end(); jt++) { + Instruction *In = *jt; + ReturnInst *Ret = static_cast(In); + Value *RV = Ret->getReturnValue(); + CreateDataFlow(RV); + InsertDataFlow(RV, Result); // We indicate dependency between + // retval and call site + Stores[RV].push_back(new FlowWriteInstruction( + Ret->getParent(), Ret, declaration)); + } + } + } +#endif + // We create the actual DDG depending on mem accesses and Call + // instructions + if (auto ST = dyn_cast(&I)) { + Value *Variable = ST->getPointerOperand(); // Where we're writing + Value *Access = ST->getValueOperand(); // What we're writing, this + // gives us the dependencies + // The current Store is writing `Access` into `Variable` + + std::vector Flows; + RetrieveDataFlow(Access, &Flows); + + // If `Variable` does not directly represent a Src code variable, we + // fetch what it represents (e.g., the field of a struct) + if (!isSourceCodeVariable(Variable)) { + Value *ActualSrcVariable = nullptr; + RetrieveAccessedVariable(Variable, &Flows, &LLVMVariables, + &ActualSrcVariable); + if (ActualSrcVariable) Variable = ActualSrcVariable; + } + + StoreType Type = declaration; // Usually we have `a = c + b` + for (std::vector::iterator it = Flows.begin(); + it != Flows.end(); ++it) { + Value *Dependency = *it; + if (Dependency == Variable) // If we fall into `a += c + b`, we + // manage differently + Type = modification; // Probably we dont need this distinction + // anymore, but keep it for future + // experiments + + // First we find the edges between the current store and the + // previous ones (i.e., when we wrote into `c` and `b` if the + // current store is `a = c + b`) + std::vector AllStoresPerVariable = + Stores[Dependency]; + unsigned ConsideredStores = 0; + bool *ReachingStores = isReachableByStore( + &AllStoresPerVariable, ST, DT, LI, &ConsideredStores); + + // ReachingStores[0] refers to the last Store instruction that we + // met (i.e., the last in `AllStoresPerVariable` This is why we + // iterate the vector in a reverse way BUT the array in the + // forward + unsigned i = 0; + for (std::vector::reverse_iterator it = + AllStoresPerVariable.rbegin(); + it != AllStoresPerVariable.rend(); it++) { + if (ReachingStores[i] && (i < ConsideredStores)) { + Instruction *Src = (*it)->I; + if (Src == + ST) // Already managed in the `reachableByStores` method + continue; + if (isPredecessorBB(Src, + ST)) // Already managed by edge coverage + continue; +#if LLVM_VERSION_MAJOR == 9 + BasicBlock *SrcParent = Src->getParent(); + BasicBlock *STParent = ST->getParent(); + if (PT->dominates(SrcParent, STParent)) +#else + if (PT->dominates(Src, ST)) +#endif + continue; + if (Src->getParent() != ST->getParent()) { + std::tuple edge = + decltype(edge){Src->getParent(), ST->getParent()}; + StoreEdges.push_back(edge); + IncomingEdges[ST->getParent()].insert(Src->getParent()); + DEBUG(errs() << "+++++++++++\nAdding edge\n"); + DEBUG(debug_instruction(Src)); + DEBUG(debug_instruction(ST)); + DEBUG(errs() << "-----------\n"); + } + } + i++; + } + + delete[] ReachingStores; + } + // Then we insert the new Store in our map that contains all the + // stores, so we build forward deps + FlowWriteInstruction *MyStore = + new FlowWriteInstruction(ST->getParent(), ST, Type); + Stores[Variable].push_back(MyStore); + + } + // Three major cases: + // 1) a = foo(x) => a depends on the result of foo() applied + // on x and x depends on its previous values and return value 2) + // memcpy(src, dst, N) => dst depends on src and N && the triple src, + // dst, N depends on their previous value (memcpy or any other API) 3) + // foo(x, out_y, out_z) => out_x, out_y are writen within foo + // depending on x. Thus here the dependency is managed internally to + // the function when passing on it + else if (CallInst *Call = dyn_cast(&I)) { + FlowWriteInstruction *MyStore = nullptr; + Value *Variable = nullptr; + Function *FC = Call->getCalledFunction(); + // DEBUG(errs() << "Looking for dependencies when calling " << + // FC->getName() << "\n"); + int argStart = + 0; // In some cases, we dont want to track dependencies for + // each argument. For instance, for memcpy(src, dst, n), we + // can ignore previous `src` dependencies, since it is being + // written. Rather, for this specific case, we generate a + // FlowWriteInstruction object to save the fact that `src` + // internal value has been modified according to `dst` and + // `n` + + if (FC == nullptr) continue; + if (FC->isIntrinsic()) { + switch (FC->getIntrinsicID()) { + case Intrinsic::memcpy: { + Variable = Call->getArgOperand(0); + std::vector Flows; + RetrieveDataFlow(Variable, &Flows); + if (Flows.size() != 0) Variable = Flows[0]; + MyStore = new FlowWriteInstruction(Call->getParent(), Call, + declaration); + argStart = 1; + break; + } + case Intrinsic::memset: { + // memset does not produce a real dataflow + // errs() << "memset to implement\n"; + break; + } + case Intrinsic::memmove: { + Variable = Call->getArgOperand(0); + std::vector Flows; + RetrieveDataFlow(Variable, &Flows); + if (Flows.size() != 0) Variable = Flows[0]; + MyStore = new FlowWriteInstruction(Call->getParent(), Call, + declaration); + argStart = 1; + break; + } + default: { + // errs() << "Not implemented/interesting intrinsic for data + // flow\n"; + break; + } + } + } + for (unsigned int i = argStart; i < Call->arg_size(); i++) { + Value *ArgOp = Call->getArgOperand(i); + if (!isa(ArgOp)) { + std::vector Flows; + RetrieveDataFlow(ArgOp, &Flows); + + for (std::vector::iterator it = Flows.begin(); + it != Flows.end(); ++it) { + Value *Dependency = *it; + // DEBUG(errs() << "Call depending on: {" << + // Dependency->getName() << "}\n"); + std::vector AllStoresPerVariable = + Stores[Dependency]; + unsigned ConsideredStores = 0; + bool *ReachingStores = isReachableByStore( + &AllStoresPerVariable, Call, DT, LI, &ConsideredStores); + unsigned i = 0; + for (std::vector::reverse_iterator + it = AllStoresPerVariable.rbegin(); + it != AllStoresPerVariable.rend(); it++) { + if (ReachingStores[i] && (i < ConsideredStores)) { + Instruction *Src = (*it)->I; + if (Src == Call) // Already managed in the + // `reachableByStores` method + continue; + if (isPredecessorBB(Src, Call)) continue; +#if LLVM_VERSION_MAJOR == 9 + BasicBlock *SrcParent = Src->getParent(); + BasicBlock *CallParent = Call->getParent(); + if (PT->dominates(SrcParent, CallParent)) +#else + if (PT->dominates(Src, Call)) +#endif + continue; + if (Src->getParent() != Call->getParent()) { + std::tuple edge = + decltype(edge){Src->getParent(), Call->getParent()}; + StoreEdges.push_back(edge); + IncomingEdges[Call->getParent()].insert( + Src->getParent()); + DEBUG(errs() << "+++++++++++\nAdding edge\n"); + DEBUG(debug_instruction(Src)); + DEBUG(debug_instruction(Call)); + DEBUG(errs() << "-----------\n"); + } + } + i++; + } + } + } + } + if (Variable != nullptr && MyStore != nullptr) { + Stores[Variable].push_back(MyStore); + } + } else + continue; + } + } + + // Instrument the locations in the function + BasicBlock::iterator IP = EntryBB.getFirstInsertionPt(); + IRBuilder<> IRB(&(*IP)); + Value *IsCurrentBlockVisited; + + for (auto &BB : F) { + bb_count++; + name = new char[VAR_NAME_LEN]; + memset(name, 0, VAR_NAME_LEN); + snprintf(name, VAR_NAME_LEN, "my_var_%d", BBCounter++); + AllocaInst *AllocaIsCurrentlyBlockVisited = + IRB.CreateAlloca(Int16Ty, nullptr, StringRef(name)); + AllocaIsCurrentlyBlockVisited->setMetadata(M.getMDKindID("nosanitize"), + MDNode::get(C, None)); + IsCurrentBlockVisited = + static_cast(AllocaIsCurrentlyBlockVisited); + StoreInst *InitializeVisited; + if (&EntryBB == &BB) + InitializeVisited = IRB.CreateStore(Visited, IsCurrentBlockVisited); + else + InitializeVisited = + IRB.CreateStore(NonVisited, IsCurrentBlockVisited); + + if (InitializeVisited) + InitializeVisited->setMetadata(M.getMDKindID("nosanitize"), + MDNode::get(C, None)); + + VisitedBlocks[&BB] = IsCurrentBlockVisited; + + // errs() << "MAP SIZE " << std::to_string(map_size) << "\n"; + cur_loc = AFL_R(map_size); + CurLoc = ConstantInt::get(Int16Ty, cur_loc); + BlocksLocs[&BB] = CurLoc; + } + + for (auto &BB : F) { + if (&BB == &EntryBB) continue; + + IP = BB.getFirstInsertionPt(); + IRBuilder<> IRB(&(*IP)); + IsCurrentBlockVisited = VisitedBlocks[&BB]; + + StoreInst *StoreIsVisited = + IRB.CreateStore(Visited, IsCurrentBlockVisited); + StoreIsVisited->setMetadata(M.getMDKindID("nosanitize"), + MDNode::get(C, None)); + + Value *HashedLoc = nullptr; + if (IncomingEdges[&BB].size() <= 1) continue; + for (std::set::iterator it = IncomingEdges[&BB].begin(); + it != IncomingEdges[&BB].end(); ++it) { + Value *isVisited = VisitedBlocks[*it]; + ConstantInt *PotentiallyPreviousLoc = BlocksLocs[*it]; + if (!isVisited or !PotentiallyPreviousLoc) continue; + LoadInst *LoadIsVisited = + IRB.CreateLoad(isVisited->getType(), isVisited); + LoadIsVisited->setMetadata(M.getMDKindID("nosanitize"), + MDNode::get(C, None)); + + Value *PrevLocIfVisited = + IRB.CreateAnd(LoadIsVisited, PotentiallyPreviousLoc); + CurLoc = BlocksLocs[&BB]; + if (HashedLoc == nullptr) + HashedLoc = IRB.CreateXor(CurLoc, PrevLocIfVisited); + else + HashedLoc = IRB.CreateXor(HashedLoc, PrevLocIfVisited); + } + if (HashedLoc == nullptr) continue; + + HashedLoc = IRB.CreateZExt(HashedLoc, IRB.getInt16Ty()); + + LoadInst *MapPtr = + IRB.CreateLoad(PointerType::get(Int8Ty, 0), DDGMapPtr); + MapPtr->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); + + Value *MapPtrIdx = IRB.CreateGEP(Int8Ty, MapPtr, HashedLoc); + LoadInst *Counter = IRB.CreateLoad(Int8Ty, MapPtrIdx); + Counter->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); + + Value *Incr = IRB.CreateAdd(Counter, One); + auto cf = IRB.CreateICmpEQ(Incr, Zero); + auto carry = IRB.CreateZExt(cf, Int8Ty); + Incr = IRB.CreateAdd(Incr, carry); + + StoreInst *StoreMapPtr = IRB.CreateStore(Incr, MapPtrIdx); + StoreMapPtr->setMetadata(M.getMDKindID("nosanitize"), + MDNode::get(C, None)); + + instrumentedLocations++; + } + } + + errs() << "DDG - Instrumented " << instrumentedLocations + << " locations over a total of " << bb_count << " \t\n"; + + auto PA = PreservedAnalyses::all(); + return PA; + } +}; + +extern "C" ::llvm::PassPluginLibraryInfo LLVM_ATTRIBUTE_WEAK +llvmGetPassPluginInfo() { + return {LLVM_PLUGIN_API_VERSION, "DDGInstrPass", "v0.1", + /* lambda to insert our pass into the pass pipeline. */ + [](PassBuilder &PB) { + +#if LLVM_VERSION_MAJOR <= 13 + using OptimizationLevel = typename PassBuilder::OptimizationLevel; +#endif + PB.registerOptimizerLastEPCallback( + [](ModulePassManager &MPM, OptimizationLevel OL) { + MPM.addPass(DDGInstrModulePass()); + }); + }}; +} diff --git a/libafl_cc/src/ddg-utils.cc b/libafl_cc/src/ddg-utils.cc new file mode 100644 index 0000000000..e04fceddbf --- /dev/null +++ b/libafl_cc/src/ddg-utils.cc @@ -0,0 +1,114 @@ +#include "llvm/Analysis/CFG.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/MathExtras.h" +#include "llvm/Support/ScopedPrinter.h" +#include "llvm/Support/raw_ostream.h" +#include +#include "llvm/Transforms/Instrumentation.h" +#include "llvm/Transforms/Utils/ASanStackFrameLayout.h" +#include "llvm/Transforms/Utils/BasicBlockUtils.h" +#include "llvm/Transforms/Utils/Local.h" +#include "llvm/Transforms/Utils/ModuleUtils.h" +#include "llvm/Transforms/Utils/PromoteMemToReg.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ddg-utils.h" + +#define BB_THRESHOLD 16 + +void debug_instruction(Instruction *I) { + DILocation *D = I->getDebugLoc(); + + if (D != NULL) { + errs() << "Line: " << D->getLine() << "\n"; + return; + } + errs() << "[DEBUG] No dbg info recovered\n"; +} + +// void debug_DDG(std::map> graph) { +// std::map>::iterator it = +// graph.begin(); while(it != graph.end()) { +// CustomDDGNode* src = it->first; +// std::vector sinks = it->second; +// +// it++; +// } +// } + +// Checks if Src is in the predecessor BB of To +bool isPredecessorBB(Instruction *Src, Instruction *To) { + BasicBlock *ToParent = To->getParent(); + BasicBlock *SrcParent = Src->getParent(); + for (auto it = pred_begin(ToParent); it != pred_end(ToParent); ++it) { + BasicBlock *predecessor = *it; + if (predecessor == SrcParent) return true; + } + return false; +} + +bool *isReachableByStore(std::vector *From, + Instruction *To, DominatorTree *DT, LoopInfo *LI, + unsigned *ConsideredStores) { + size_t NumberOfStores = From->size(); + unsigned bb_threshold = + NumberOfStores < BB_THRESHOLD ? NumberOfStores : BB_THRESHOLD; + *ConsideredStores = bb_threshold; + FlowWriteInstruction *TopNstores[bb_threshold]; + bool *ReachingStores = new bool[bb_threshold]; + SmallPtrSet ExclusionSet; + unsigned idx = 0; + for (std::vector::reverse_iterator it = + From->rbegin(); + it != From->rend(); it++) { + FlowWriteInstruction *MyStore = *it; + // TopNStores contains the last N stores, which are the ones that we check + // if are reachable. These are put in reverse order, i.e., the position `0` + // (TopNstores[0]) is the last store that we met (which is the last in the + // vector From) + TopNstores[idx] = MyStore; + ExclusionSet.insert(MyStore->BB); + idx++; + if (idx >= bb_threshold) break; + } + + // We need the ExclusionSet to be complete, before startintg with the actual + // check loop + for (int i = 0; i < bb_threshold; i++) { + Instruction *FromInstruction = TopNstores[i]->I; + if (TopNstores[i]->BB == To->getParent()) { + // If the two BBs are the same, we discard this flow. It is not + // interesting since if we reach the BB we cover it + ReachingStores[i] = false; + // continue; // RE-ENABLE THIS WHEN NO DEBUGGING IS NEEDED; + } + ExclusionSet.erase(TopNstores[i]->BB); + if (FromInstruction != To) { + bool r = + isPotentiallyReachable(FromInstruction, To, &ExclusionSet, DT, LI); + // errs() << "isPotentiallyReachable " << r << "\n"; + ReachingStores[i] = r; + } else + ReachingStores[i] = false; // Same instruction not reachable by itself + ExclusionSet.insert(TopNstores[i]->BB); + } + // ReachingStores[0] refers to the last Store instruction that we met + + return ReachingStores; +} \ No newline at end of file diff --git a/libafl_cc/src/ddg-utils.h b/libafl_cc/src/ddg-utils.h new file mode 100644 index 0000000000..b00b1548ab --- /dev/null +++ b/libafl_cc/src/ddg-utils.h @@ -0,0 +1,120 @@ + +#include "llvm/IR/Function.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/PassManager.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DepthFirstIterator.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/Twine.h" +#include "llvm/Analysis/MemoryBuiltins.h" +#include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Analysis/ValueTracking.h" +#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/CFG.h" +#include "llvm/BinaryFormat/MachO.h" +#include "llvm/IR/Argument.h" +#include "llvm/IR/Attributes.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/Comdat.h" +#include "llvm/IR/Constant.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/DIBuilder.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/DebugInfoMetadata.h" +#include "llvm/IR/DebugLoc.h" +#include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/Dominators.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/GlobalAlias.h" +#include "llvm/IR/GlobalValue.h" +#include "llvm/IR/GlobalVariable.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/InlineAsm.h" +#include "llvm/IR/InstVisitor.h" +#include "llvm/IR/InstrTypes.h" +#include "llvm/IR/Instruction.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/MDBuilder.h" +#include "llvm/IR/Metadata.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/Type.h" +#include "llvm/IR/Use.h" +#include "llvm/IR/Value.h" +#include "llvm/IR/Verifier.h" +#include "llvm/IR/DebugInfo.h" +#include "llvm/IR/LegacyPassManager.h" +#include "llvm/MC/MCSectionMachO.h" +#include "llvm/Pass.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/MathExtras.h" +#include "llvm/Support/ScopedPrinter.h" +#include "llvm/Support/raw_ostream.h" +#include +#include "llvm/Transforms/Instrumentation.h" +#include "llvm/Transforms/Utils/ASanStackFrameLayout.h" +#include "llvm/Transforms/Utils/BasicBlockUtils.h" +#include "llvm/Transforms/Utils/Local.h" +#include "llvm/Transforms/Utils/ModuleUtils.h" +#include "llvm/Transforms/Utils/PromoteMemToReg.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace llvm; + +enum StoreType { declaration, modification }; + +struct FlowWriteInstruction { + BasicBlock *BB; + Instruction *I; + // Value* WrittenVar; + // Value* WhatWeAreWriting; + // std::vector* WhatWeAreDepending; + StoreType Type; + + FlowWriteInstruction(BasicBlock *_BB, Instruction *_I, StoreType _T) { + this->BB = _BB; + this->I = _I; + this->Type = _T; + } + + FlowWriteInstruction(struct FlowWriteInstruction *S) { + this->BB = S->BB; + this->I = S->I; + this->Type = S->Type; + } +}; + +// Debug + +void debug_instruction(Instruction *I); +// void debug_DDG(std::map> graph); + +// Other util methods + +bool *isReachableByStore(std::vector *From, + Instruction *To, DominatorTree *DT, LoopInfo *LI, + unsigned *ConsideredStores); +bool isPredecessorBB(Instruction *Src, Instruction *To); diff --git a/libafl_cc/src/dump-cfg-pass.cc b/libafl_cc/src/dump-cfg-pass.cc index 01daa24303..8fda3e0a9c 100644 --- a/libafl_cc/src/dump-cfg-pass.cc +++ b/libafl_cc/src/dump-cfg-pass.cc @@ -101,12 +101,18 @@ class DumpCfgPass : public ModulePass { private: bool isLLVMIntrinsicFn(StringRef &n) { // Not interested in these LLVM's functions +#if LLVM_VERSION_MAJOR >= 18 + if (n.starts_with("llvm.")) { +#else if (n.startswith("llvm.")) { - return true; - } else { - return false; +#endif } + return true; } + else { + return false; + } +} }; } // namespace diff --git a/libafl_cc/src/lib.rs b/libafl_cc/src/lib.rs index a442ab0bcc..b0c013dc83 100644 --- a/libafl_cc/src/lib.rs +++ b/libafl_cc/src/lib.rs @@ -27,14 +27,12 @@ ))] #![cfg_attr(test, deny( missing_debug_implementations, - missing_docs, //trivial_casts, trivial_numeric_casts, unused_extern_crates, unused_import_braces, unused_qualifications, unused_must_use, - missing_docs, //unused_results ))] #![cfg_attr( @@ -57,7 +55,7 @@ ) )] -use std::{convert::Into, path::Path, process::Command, string::String, vec::Vec}; +use std::{path::Path, process::Command}; pub mod ar; pub use ar::ArWrapper; @@ -146,12 +144,12 @@ impl Configuration { let output = output.to_str().unwrap(); let new_filename = if let Some((filename, extension)) = output.split_once('.') { - if let crate::Configuration::Default = self { + if let Configuration::Default = self { format!("{filename}.{extension}") } else { format!("{filename}.{self}.{extension}") } - } else if let crate::Configuration::Default = self { + } else if let Configuration::Default = self { output.to_string() } else { format!("{output}.{self}") diff --git a/libafl_cc/src/libtool.rs b/libafl_cc/src/libtool.rs index 67527e5d30..4117efc0af 100644 --- a/libafl_cc/src/libtool.rs +++ b/libafl_cc/src/libtool.rs @@ -1,7 +1,7 @@ //! Libtool Wrapper from `LibAFL` // call make passing LIBTOOL=/path/to/target/release/libafl_libtool -use std::{convert::Into, env, path::PathBuf, str::FromStr, string::String, vec::Vec}; +use std::{env, path::PathBuf, str::FromStr}; use crate::{Error, ToolWrapper, LIB_EXT, LIB_PREFIX}; @@ -170,7 +170,7 @@ impl ToolWrapper for LibtoolWrapper { .base_args .iter() .map(|r| { - let arg_as_path = std::path::PathBuf::from(r); + let arg_as_path = PathBuf::from(r); if r.ends_with('.') { r.to_string() } else { @@ -191,7 +191,7 @@ impl ToolWrapper for LibtoolWrapper { }) .collect::>(); - let libtool_path = if let Ok(libtool_dir) = std::env::var("LIBTOOL_DIR") { + let libtool_path = if let Ok(libtool_dir) = env::var("LIBTOOL_DIR") { format!("{libtool_dir}/libtool") } else { "./libtool".to_string() diff --git a/libafl_concolic/symcc_libafl/src/lib.rs b/libafl_concolic/symcc_libafl/src/lib.rs index ba7f84070e..1583f8af82 100644 --- a/libafl_concolic/symcc_libafl/src/lib.rs +++ b/libafl_concolic/symcc_libafl/src/lib.rs @@ -5,7 +5,7 @@ /// The URL of the `LibAFL` `SymCC` fork. pub const SYMCC_REPO_URL: &str = "https://github.com/AFLplusplus/symcc.git"; /// The commit of the `LibAFL` `SymCC` fork. -pub const SYMCC_REPO_COMMIT: &str = "6010402596f02da6de1c2dc88794f339d7c4dfe7"; +pub const SYMCC_REPO_COMMIT: &str = "1330e29d28bce706d9f7c0864da3b0a5ae218e03"; #[cfg(feature = "clone")] mod clone { diff --git a/libafl_concolic/symcc_runtime/Cargo.toml b/libafl_concolic/symcc_runtime/Cargo.toml index 3d51669052..ff5e635f97 100644 --- a/libafl_concolic/symcc_runtime/Cargo.toml +++ b/libafl_concolic/symcc_runtime/Cargo.toml @@ -25,12 +25,12 @@ no-cpp-runtime = [] unchecked_unwrap = "4" ctor = "0.2" libc = "0.2" -libafl = { path = "../../libafl", version = "0.11.2", default-features=false, features=["std", "serdeany_autoreg"] } -libafl_bolts = { path = "../../libafl_bolts", version = "0.11.2", default-features=false, features=["std", "serdeany_autoreg"] } +libafl = { path = "../../libafl", version = "0.12.0", default-features=false, features=["std", "serdeany_autoreg"] } +libafl_bolts = { path = "../../libafl_bolts", version = "0.12.0", default-features=false, features=["std", "serdeany_autoreg"] } [build-dependencies] cmake = "0.1" -bindgen = "0.69" +bindgen = "0.69.4" regex = "1" which = "4.4" -symcc_libafl = { path = "../symcc_libafl", version = "0.11.2" } +symcc_libafl = { path = "../symcc_libafl", version = "0.12.0" } diff --git a/libafl_concolic/symcc_runtime/src/filter/coverage.rs b/libafl_concolic/symcc_runtime/src/filter/coverage.rs index ff96f5b0f0..1a56557a93 100644 --- a/libafl_concolic/symcc_runtime/src/filter/coverage.rs +++ b/libafl_concolic/symcc_runtime/src/filter/coverage.rs @@ -195,9 +195,7 @@ where // # Safety // The index is modulo by the length, therefore it is always in bounds let len = self.hitcounts_map.len(); - self.hitcounts_map - .as_mut_slice() - .get_unchecked_mut(hash % len) + self.hitcounts_map.get_unchecked_mut(hash % len) }; *val = val.saturating_add(1); } diff --git a/libafl_concolic/symcc_runtime/src/lib.rs b/libafl_concolic/symcc_runtime/src/lib.rs index deef3214ea..1d59a9fd26 100644 --- a/libafl_concolic/symcc_runtime/src/lib.rs +++ b/libafl_concolic/symcc_runtime/src/lib.rs @@ -27,14 +27,19 @@ //! # SymCC and SymQEMU expect to runtime file to be called `libSymRuntime.so`. Setting the name to `SymRuntime` achieves this. //! name = "SymRuntime" //! ``` -#![allow(clippy::module_name_repetitions, clippy::missing_panics_doc)] -#![allow(clippy::pub_underscore_fields)] +#![allow( + clippy::module_name_repetitions, + clippy::missing_panics_doc, + clippy::pub_underscore_fields +)] + pub mod filter; pub mod tracing; // The following exports are used by the `export_runtime` macro. They are therefore exported, but hidden from docs, as they are not supposed to be used directly by the user. #[doc(hidden)] #[cfg(target_os = "linux")] +#[allow(clippy::mixed_attributes_style)] pub mod cpp_runtime { #![allow(non_upper_case_globals)] #![allow(non_camel_case_types)] @@ -155,6 +160,18 @@ macro_rules! export_rust_runtime_fn { } } }; + // special case for build_integer_from_buffer cuz the next one just doesn't work!!!!!!! + (pub fn build_integer_from_buffer( + buffer: *mut ::std::os::raw::c_void, + num_bits: ::std::os::raw::c_uint,) -> RSymExpr,$c_name:ident; $rt_cb:path) => { + #[allow(clippy::missing_safety_doc)] + #[no_mangle] + pub unsafe extern "C" fn _rsym_build_integer_from_buffer(buffer: *mut ::std::os::raw::c_void, num_bits: ::std::os::raw::c_uint) { + $rt_cb(|rt| { + rt.build_integer_from_buffer(buffer, num_bits); + }) + } + }; // all other methods are handled by this (pub fn $name:ident($( $arg:ident : $(::)?$($type:ident)::+ ),*$(,)?)$( -> $($ret:ident)::+)?, $c_name:ident; $rt_cb:path) => { #[allow(clippy::missing_safety_doc)] diff --git a/libafl_concolic/symcc_runtime/src/tracing.rs b/libafl_concolic/symcc_runtime/src/tracing.rs index 58676d4753..ade5da9d75 100644 --- a/libafl_concolic/symcc_runtime/src/tracing.rs +++ b/libafl_concolic/symcc_runtime/src/tracing.rs @@ -62,6 +62,17 @@ macro_rules! binary_expression_builder { } impl Runtime for TracingRuntime { + #[allow(clippy::missing_safety_doc)] + #[no_mangle] + fn build_integer_from_buffer( + &mut self, + _buffer: *mut core::ffi::c_void, + _num_bits: core::ffi::c_uint, + ) -> Option { + // todo + self.write_message(SymExpr::IntegerFromBuffer {}) + } + expression_builder!(get_input_byte(offset: usize, value: u8) => InputByte); expression_builder!(build_integer(value: u64, bits: u8) => Integer); diff --git a/libafl_concolic/test/dump_constraints/src/main.rs b/libafl_concolic/test/dump_constraints/src/main.rs index 082a527870..34d12b6322 100644 --- a/libafl_concolic/test/dump_constraints/src/main.rs +++ b/libafl_concolic/test/dump_constraints/src/main.rs @@ -8,10 +8,9 @@ use std::{ io::{BufWriter, Write}, path::PathBuf, process::{exit, Command}, - string::ToString, }; -use clap::{self, Parser}; +use clap::Parser; use libafl::observers::concolic::{ serialization_format::{MessageFileReader, MessageFileWriter, DEFAULT_ENV_NAME}, EXPRESSION_PRUNING, HITMAP_ENV_NAME, NO_FLOAT_ENV_NAME, SELECTIVE_SYMBOLICATION_ENV_NAME, diff --git a/libafl_concolic/test/smoke_test.sh b/libafl_concolic/test/smoke_test.sh index 4e23543e9e..e47db28f4a 100755 --- a/libafl_concolic/test/smoke_test.sh +++ b/libafl_concolic/test/smoke_test.sh @@ -16,7 +16,7 @@ if [ ! -d "symcc" ]; then echo "cloning symcc" git clone https://github.com/AFLplusplus/symcc.git symcc cd symcc - git checkout 2a3229da6101596af220f20fef5085e59537abcb + git checkout 1330e29d28bce706d9f7c0864da3b0a5ae218e03 cd .. fi @@ -46,4 +46,4 @@ cat constraints.txt sed 's/, location: .* / /' < constraints.txt > constraints_filtered.txt sed 's/, location: .* / /' < expected_constraints.txt > expected_constraints_filtered.txt -diff constraints_filtered.txt expected_constraints_filtered.txt \ No newline at end of file +diff constraints_filtered.txt expected_constraints_filtered.txt diff --git a/libafl_derive/src/lib.rs b/libafl_derive/src/lib.rs index fa91639283..d10489dde8 100644 --- a/libafl_derive/src/lib.rs +++ b/libafl_derive/src/lib.rs @@ -28,14 +28,12 @@ ))] #![cfg_attr(test, deny( missing_debug_implementations, - missing_docs, //trivial_casts, trivial_numeric_casts, unused_extern_crates, unused_import_braces, unused_qualifications, unused_must_use, - missing_docs, //unused_results ))] #![cfg_attr( diff --git a/libafl_frida/Cargo.toml b/libafl_frida/Cargo.toml index e146ea87c9..338aa8a3e1 100644 --- a/libafl_frida/Cargo.toml +++ b/libafl_frida/Cargo.toml @@ -36,17 +36,17 @@ yaxpeax-x86 = "1.2.2" iced-x86 = { version = "1.20.0", features = ["code_asm"], optional = true } [dependencies] -libafl = { path = "../libafl", default-features = false, version = "0.11.2", features = [ +libafl = { path = "../libafl", default-features = false, version = "0.12.0", features = [ "std", "derive", "frida_cli", ] } -libafl_bolts = { path = "../libafl_bolts", version = "0.11.2", default-features = false, features = [ +libafl_bolts = { path = "../libafl_bolts", version = "0.12.0", default-features = false, features = [ "std", "derive", "frida_cli" ] } -libafl_targets = { path = "../libafl_targets", version = "0.11.2", features = [ +libafl_targets = { path = "../libafl_targets", version = "0.12.0", features = [ "std", "sancov_cmplog", ] } @@ -55,7 +55,7 @@ nix = { version = "0.27", features = ["mman"] } libc = "0.2" hashbrown = "0.14" rangemap = "1.3" -frida-gum-sys = { path = "../../frida-rust/frida-gum-sys/", version = "0.8.3", features = [ +frida-gum-sys = { path = "../../frida-rust/frida-gum-sys/", version = "0.13.6", features = [ "auto-download", "event-sink", "invocation-listener", diff --git a/libafl_frida/src/alloc.rs b/libafl_frida/src/alloc.rs index 0e70932d2e..5020aed0d0 100644 --- a/libafl_frida/src/alloc.rs +++ b/libafl_frida/src/alloc.rs @@ -10,9 +10,9 @@ use std::{collections::BTreeMap, ffi::c_void}; use backtrace::Backtrace; +use frida_gum::{PageProtection, RangeDetails}; use hashbrown::HashMap; use libafl_bolts::cli::FuzzerOptions; -use mmap_rs::Protection; #[cfg(any( windows, target_os = "linux", @@ -22,7 +22,7 @@ use mmap_rs::Protection; target_os = "android" ) ))] -use mmap_rs::{MemoryAreas, MmapFlags, MmapMut, MmapOptions, ReservedMut}; +use mmap_rs::{MmapFlags, MmapMut, MmapOptions, ReservedMut}; use rangemap::RangeSet; use serde::{Deserialize, Serialize}; @@ -247,14 +247,14 @@ impl Allocator { //log::trace!("freeing address: {:?}", ptr); let Some(metadata) = self.allocations.get_mut(&(ptr as usize)) else { if !ptr.is_null() { - AsanErrors::get_mut() + AsanErrors::get_mut_blocking() .report_error(AsanError::UnallocatedFree((ptr as usize, Backtrace::new()))); } return; }; if metadata.freed { - AsanErrors::get_mut().report_error(AsanError::DoubleFree(( + AsanErrors::get_mut_blocking().report_error(AsanError::DoubleFree(( ptr as usize, metadata.clone(), Backtrace::new(), @@ -279,7 +279,7 @@ impl Allocator { ) -> Option<&mut AllocationMetadata> { let mut metadatas: Vec<&mut AllocationMetadata> = self.allocations.values_mut().collect(); metadatas.sort_by(|a, b| a.address.cmp(&b.address)); - let mut offset_to_closest = i64::max_value(); + let mut offset_to_closest = i64::MAX; let mut closest = None; let ptr: i64 = ptr.try_into().unwrap(); for metadata in metadatas { @@ -348,7 +348,9 @@ impl Allocator { let remainder = size % 8; if remainder > 0 { - ((start + size / 8) as *mut u8).write((1 << remainder) - 1); + let mut current_value = ((start + size / 8) as *const u8).read(); + current_value = current_value | (0xff << (8 - remainder)); + ((start + size / 8) as *mut u8).write(current_value); } } } @@ -361,7 +363,11 @@ impl Allocator { let remainder = size % 8; if remainder > 0 { - ((start + size / 8) as *mut u8).write(0x00); + let mask = !(0xff << (8 - remainder)); + let mut current_value = ((start + size / 8) as *const u8).read(); + + current_value = current_value & mask; + ((start + size / 8) as *mut u8).write(current_value); } } } @@ -432,7 +438,7 @@ impl Allocator { #[inline] #[must_use] pub fn check_shadow(&mut self, address: *const c_void, size: usize) -> bool { - if size == 0 || !self.is_managed(address as *mut c_void){ + if size == 0 || !self.is_managed(address as *mut c_void) { return true; } let address = address as usize; @@ -454,7 +460,7 @@ impl Allocator { // if we are not aligned to 8 bytes, we need to check the high bits of the shadow if offset != 0 { let val = (unsafe { (shadow_addr as *const u16).read() }) >> offset; - let mask = (1 << (size % 9)) -1; + let mask = (1 << (size % 9)) - 1; if val & mask != mask { return false; } @@ -471,7 +477,7 @@ impl Allocator { .all(|&x| x == 0xffffffffffffffffffffffffffffffffu128) { if size % 8 != 0 { - let val = unsafe { ((shadow_addr + shadow_size) as *mut u8).read()}; + let val = unsafe { ((shadow_addr + shadow_size) as *mut u8).read() }; let mask = (1 << (size % 8)) - 1; if val & mask != mask { return false; @@ -479,11 +485,9 @@ impl Allocator { } return true; } - - } if size % 8 != 0 { - let val = unsafe { ((shadow_addr + shadow_size) as *mut u8).read()}; + let val = unsafe { ((shadow_addr + shadow_size) as *mut u8).read() }; let mask = (1 << (size % 8)) - 1; if val & mask != mask { return false; @@ -502,14 +506,14 @@ impl Allocator { #[inline] pub fn is_managed(&self, ptr: *mut c_void) -> bool { //self.allocations.contains_key(&(ptr as usize)) - self.base_mapping_addr <= ptr as usize && (ptr as usize) < self.current_mapping_addr + self.shadow_offset <= ptr as usize && (ptr as usize) < self.current_mapping_addr } /// Checks if any of the allocations has not been freed pub fn check_for_leaks(&self) { for metadata in self.allocations.values() { if !metadata.freed { - AsanErrors::get_mut() + AsanErrors::get_mut_blocking() .report_error(AsanError::Leak((metadata.address, metadata.clone()))); } } @@ -517,19 +521,31 @@ impl Allocator { /// Unpoison all the memory that is currently mapped with read/write permissions. pub fn unpoison_all_existing_memory(&mut self) { - for area in MemoryAreas::open(None).unwrap() { - let area = area.unwrap(); - if area - .protection() - .intersects(Protection::READ | Protection::WRITE) - && !self.is_managed(area.start() as *mut c_void) - { - if self.using_pre_allocated_shadow_mapping && area.start() == 1 << self.shadow_bit { - continue; + log::trace!( + "Shadow Mapping: {:#x}-{:#x}", + self.shadow_offset, + self.current_mapping_addr + ); + RangeDetails::enumerate_with_prot( + PageProtection::Read, + &mut |range: &RangeDetails| -> bool { + let start = range.memory_range().base_address().0 as usize; + let end = start + range.memory_range().size(); + log::trace!( + "Mapping: {:#x}-{:#x}, is_managed: {}", + start, + end, + self.is_managed(start as *mut c_void) + ); + + if !self.is_managed(start as *mut c_void) { + log::trace!("Unpoisoning: {:#x}-{:#x}", start, end); + self.map_shadow_for_region(start, end, true); } - self.map_shadow_for_region(area.start(), area.end(), true); - } - } + + return true; + }, + ); } /// Initialize the allocator, making sure a valid shadow bit is selected. @@ -546,28 +562,35 @@ impl Allocator { let mut userspace_max: usize = 0; // Enumerate memory ranges that are already occupied. - for area in MemoryAreas::open(None).unwrap() { - let start = area.as_ref().unwrap().start(); - let end = area.unwrap().end(); - occupied_ranges.push((start, end)); - let base: usize = 2; - // On x64, if end > 2**48, then that's in vsyscall or something. - #[cfg(all(unix, target_arch = "x86_64"))] - if end <= base.pow(48) && end > userspace_max { - userspace_max = end; - } - #[cfg(all(not(unix), target_arch = "x86_64"))] - if (end >> 3) <= base.pow(44) && (end >> 3) > userspace_max { - userspace_max = end >> 3; - } + RangeDetails::enumerate_with_prot( + PageProtection::NoAccess, + &mut |range: &RangeDetails| -> bool { + let start = range.memory_range().base_address().0 as usize; + let end = start + range.memory_range().size(); + log::trace!("Start: {:#x}, end: {:#x}", start, end); + occupied_ranges.push((start, end)); + let base: usize = 2; + // On x64, if end > 2**48, then that's in vsyscall or something. + #[cfg(all(unix, target_arch = "x86_64"))] + if end <= base.pow(48) && end > userspace_max { + userspace_max = end; + } - // On aarch64, if end > 2**52, then range is not in userspace - #[cfg(target_arch = "aarch64")] - if end <= base.pow(52) && end > userspace_max { - userspace_max = end; - } - } + #[cfg(all(not(unix), target_arch = "x86_64"))] + if (end >> 3) <= base.pow(44) && (end >> 3) > userspace_max { + userspace_max = end >> 3; + } + + // On aarch64, if end > 2**52, then range is not in userspace + #[cfg(target_arch = "aarch64")] + if end <= base.pow(52) && end > userspace_max { + userspace_max = end; + } + + return true; + }, + ); let mut maxbit = 0; for power in 1..64 { @@ -699,18 +722,18 @@ fn check_shadow() { assert!(allocator.check_shadow(allocation, 8) == true); assert!(allocator.check_shadow(allocation, 9) == false); assert!(allocator.check_shadow(allocation, 10) == false); - assert!(allocator.check_shadow(unsafe {allocation.offset(1) }, 7) == true); - assert!(allocator.check_shadow(unsafe {allocation.offset(2) }, 6) == true); - assert!(allocator.check_shadow(unsafe {allocation.offset(3) }, 5) == true); - assert!(allocator.check_shadow(unsafe {allocation.offset(4) }, 4) == true); - assert!(allocator.check_shadow(unsafe {allocation.offset(5) }, 3) == true); - assert!(allocator.check_shadow(unsafe {allocation.offset(6) }, 2) == true); - assert!(allocator.check_shadow(unsafe {allocation.offset(7) }, 1) == true); - assert!(allocator.check_shadow(unsafe {allocation.offset(8) }, 0) == true); - assert!(allocator.check_shadow(unsafe {allocation.offset(9) }, 1) == false); - assert!(allocator.check_shadow(unsafe {allocation.offset(9) }, 8) == false); - assert!(allocator.check_shadow(unsafe {allocation.offset(1) }, 9) == false); - assert!(allocator.check_shadow(unsafe {allocation.offset(1) }, 8) == false); - assert!(allocator.check_shadow(unsafe {allocation.offset(2) }, 8) == false); - assert!(allocator.check_shadow(unsafe {allocation.offset(3) }, 8) == false); + assert!(allocator.check_shadow(unsafe { allocation.offset(1) }, 7) == true); + assert!(allocator.check_shadow(unsafe { allocation.offset(2) }, 6) == true); + assert!(allocator.check_shadow(unsafe { allocation.offset(3) }, 5) == true); + assert!(allocator.check_shadow(unsafe { allocation.offset(4) }, 4) == true); + assert!(allocator.check_shadow(unsafe { allocation.offset(5) }, 3) == true); + assert!(allocator.check_shadow(unsafe { allocation.offset(6) }, 2) == true); + assert!(allocator.check_shadow(unsafe { allocation.offset(7) }, 1) == true); + assert!(allocator.check_shadow(unsafe { allocation.offset(8) }, 0) == true); + assert!(allocator.check_shadow(unsafe { allocation.offset(9) }, 1) == false); + assert!(allocator.check_shadow(unsafe { allocation.offset(9) }, 8) == false); + assert!(allocator.check_shadow(unsafe { allocation.offset(1) }, 9) == false); + assert!(allocator.check_shadow(unsafe { allocation.offset(1) }, 8) == false); + assert!(allocator.check_shadow(unsafe { allocation.offset(2) }, 8) == false); + assert!(allocator.check_shadow(unsafe { allocation.offset(3) }, 8) == false); } diff --git a/libafl_frida/src/asan/asan_rt.rs b/libafl_frida/src/asan/asan_rt.rs index ad0ba0085d..08b69c4baf 100644 --- a/libafl_frida/src/asan/asan_rt.rs +++ b/libafl_frida/src/asan/asan_rt.rs @@ -12,11 +12,14 @@ use core::{ }; use std::{ ffi::c_void, - // num::NonZeroUsize, - ptr::{addr_of, write_volatile}, + num::NonZeroUsize, + ptr::write_volatile, rc::Rc, + sync::MutexGuard, }; +use nix::sys::mman::{ProtFlags, mmap, MapFlags}; + use backtrace::Backtrace; use dynasmrt::{dynasm, DynasmApi, DynasmLabelApi}; #[cfg(target_arch = "x86_64")] @@ -94,7 +97,7 @@ pub const ASAN_SAVE_REGISTER_NAMES: [&str; ASAN_SAVE_REGISTER_COUNT] = [ #[cfg(target_arch = "aarch64")] pub const ASAN_SAVE_REGISTER_COUNT: usize = 32; -#[cfg(target = "aarch64")] +#[cfg(target_arch = "aarch64")] const ASAN_EH_FRAME_DWORD_COUNT: usize = 14; #[cfg(target_arch = "aarch64")] const ASAN_EH_FRAME_FDE_OFFSET: u32 = 20; @@ -161,9 +164,9 @@ impl FridaRuntime for AsanRuntime { ) { self.allocator.init(); - unsafe { - ASAN_ERRORS = Some(AsanErrors::new(self.continue_on_error)); - } + AsanErrors::get_mut_blocking().set_continue_on_error(self.continue_on_error); + + self.generate_shadow_check_function(); self.generate_instrumentation_blobs(); @@ -329,10 +332,11 @@ impl AsanRuntime { self.allocator.check_for_leaks(); } - /// Returns the `AsanErrors` from the recent run + /// Returns the `AsanErrors` from the recent run. + /// Will block if some other thread holds on to the `ASAN_ERRORS` Mutex. #[allow(clippy::unused_self)] - pub fn errors(&mut self) -> &Option { - unsafe { &*addr_of!(ASAN_ERRORS) } + pub fn errors(&mut self) -> MutexGuard<'static, AsanErrors> { + ASAN_ERRORS.lock().unwrap() } /// Make sure the specified memory is unpoisoned @@ -550,7 +554,41 @@ impl AsanRuntime { } pub fn register_hooks(hook_rt: &mut HookRuntime) { + #[allow(unused)] macro_rules! hook_func { + ($lib:expr, $name:ident, ($($param:ident : $param_type:ty),*)) => { + paste::paste! { + // extern "system" { + // fn $name($($param: $param_type),*) -> $return_type; + // } + let address = Module::find_export_by_name($lib, stringify!($name)).expect("Failed to find function").0 as usize; + log::trace!("hooking {} at {:x}", stringify!($name), address); + hook_rt.register_hook(address, move |_address, mut _context, _asan_rt| { + let mut index = 0; + + let asan_rt = _asan_rt.unwrap(); + + #[cfg(target_arch = "x86_64")] + asan_rt.set_pc(_context.rip() as usize); + + #[cfg(target_arch = "aarch64")] + asan_rt.set_pc(_context.pc() as usize); + + + log::trace!("hooked {} from {:x}", stringify!($name), asan_rt.pc()); + #[allow(trivial_numeric_casts)] + #[allow(unused_assignments)] + asan_rt.[]($(_context.arg({ + let $param = index; + index += 1; + $param + }) as _),*); + + asan_rt.unset_pc(); + + }); + }; + }; ($lib:expr, $name:ident, ($($param:ident : $param_type:ty),*), $return_type:ty) => { paste::paste! { // extern "system" { @@ -562,9 +600,15 @@ impl AsanRuntime { let mut index = 0; let asan_rt = _asan_rt.unwrap(); + + #[cfg(target_arch = "x86_64")] asan_rt.set_pc(_context.rip() as usize); - log::trace!("hooked {} from {:x}", stringify!($name), _context.rip()); + #[cfg(target_arch = "aarch64")] + asan_rt.set_pc(_context.pc() as usize); + + + log::trace!("hooked {} from {:x}", stringify!($name), asan_rt.pc()); #[allow(trivial_numeric_casts)] #[allow(unused_assignments)] _context.set_return_value(asan_rt.[]($(_context.arg({ @@ -576,8 +620,11 @@ impl AsanRuntime { asan_rt.unset_pc(); }); } - } + }; + } + + #[cfg(target_os = "windows")] macro_rules! hook_func_with_alt { ($lib:expr, $alt_name:ident, $name:ident, ($($param:ident : $param_type:ty),*), $return_type:ty) => { paste::paste! { @@ -590,9 +637,13 @@ impl AsanRuntime { let mut index = 0; let asan_rt = _asan_rt.unwrap(); + #[cfg(target_arch = "x86_64")] asan_rt.set_pc(_context.rip() as usize); - log::trace!("hooked {} from {:x}", stringify!($alt_name), _context.rip()); + #[cfg(target_arch = "aarch64")] + asan_rt.set_pc(_context.pc() as usize); + + log::trace!("hooked {} from {:x}", stringify!($alt_name), asan_rt.pc()); #[allow(trivial_numeric_casts)] #[allow(unused_assignments)] _context.set_return_value(asan_rt.[]($(_context.arg({ @@ -616,10 +667,17 @@ impl AsanRuntime { let address = Module::find_export_by_name($lib, stringify!($name)).expect("Failed to find function").0 as usize; log::trace!("hooking {} at {:x}", stringify!($name), address); hook_rt.register_hook(address, move |_address, mut _context, _asan_rt| { - log::trace!("hooked {} from {:x}", stringify!($name), _context.rip()); + let asan_rt = _asan_rt.unwrap(); let mut index = 0; + + #[cfg(target_arch = "x86_64")] asan_rt.set_pc(_context.rip() as usize); + + #[cfg(target_arch = "aarch64")] + asan_rt.set_pc(_context.pc() as usize); + + log::trace!("hooked {} from {:x}", stringify!($name), asan_rt.pc()); #[allow(trivial_numeric_casts)] #[allow(unused_assignments)] let result = if asan_rt.[]($(_context.arg({ @@ -653,6 +711,7 @@ impl AsanRuntime { } } + #[cfg(target_os = "windows")] macro_rules! hook_func_with_check_with_alt { ($lib:expr, $alt_name:ident, $name:ident, ($($param:ident : $param_type:ty),*), $return_type:ty) => { paste::paste! { @@ -917,15 +976,21 @@ impl AsanRuntime { } } - #[cfg(not(windows))] - for libname in [ + #[cfg(target_os = "linux")] + let cpp_libs = [ "libc++.so", "libc++.so.1", "libc++abi.so.1", "libc++_shared.so", "libstdc++.so", "libstdc++.so.6", - ] { + ]; + + #[cfg(target_vendor = "apple")] + let cpp_libs = ["libc++.1.dylib", "libc++abi.dylib", "libsystem_c.dylib"]; + + #[cfg(not(windows))] + for libname in cpp_libs { log::info!("Hooking c++ functions in {}", libname); for export in Module::enumerate_exports(libname) { match &export.name[..] { @@ -1256,22 +1321,19 @@ impl AsanRuntime { hook_func!( None, memset_pattern4, - (s: *mut c_void, c: *const c_void, n: usize), - usize + (s: *mut c_void, c: *const c_void, n: usize) ); #[cfg(target_vendor = "apple")] hook_func!( None, memset_pattern8, - (s: *mut c_void, c: *const c_void, n: usize), - usize + (s: *mut c_void, c: *const c_void, n: usize) ); #[cfg(target_vendor = "apple")] hook_func!( None, memset_pattern16, - (s: *mut c_void, c: *const c_void, n: usize), - usize + (s: *mut c_void, c: *const c_void, n: usize) ); } @@ -1421,11 +1483,11 @@ impl AsanRuntime { backtrace, )) }; - AsanErrors::get_mut().report_error(error); + AsanErrors::get_mut_blocking().report_error(error); // This is not even a mem instruction?? } else { - AsanErrors::get_mut().report_error(AsanError::Unknown(( + AsanErrors::get_mut_blocking().report_error(AsanError::Unknown(( self.regs, actual_pc, (None, None, 0, fault_address), @@ -1559,7 +1621,7 @@ impl AsanRuntime { backtrace, )) }; - AsanErrors::get_mut().report_error(error); + AsanErrors::get_mut_blocking().report_error(error); } #[cfg(target_arch = "x86_64")] @@ -1634,6 +1696,414 @@ impl AsanRuntime { } // https://godbolt.org/z/EvWPzqjeK + + + // https://godbolt.org/z/oajhcP5sv + /* + #include + #include + uint8_t shadow_bit = 44; + + uint64_t generate_shadow_check_function(uint64_t start, uint64_t size){ + // calculate the shadow address + uint64_t addr = 0; + addr = addr + (start >> 3); + uint64_t mask = (1ULL << (shadow_bit + 1)) - 1; + addr = addr & mask; + addr = addr + (1ULL << shadow_bit); + + if(size == 0){ + // goto return_success + return 1; + } + else{ + // check if the ptr is not aligned to 8 bytes + uint8_t remainder = start & 0b111; + if(remainder != 0){ + // we need to test the high bits from the first shadow byte + uint8_t shift; + if(size < 8){ + shift = size; + } + else{ + shift = 8 - remainder; + } + // goto check_bits + uint8_t mask = (1 << shift) - 1; + + // bitwise reverse for amd64 :< + // https://gist.github.com/yantonov/4359090 + // we need 16bit number here, (not 8bit) + uint16_t val = *(uint16_t *)addr; + val = (val & 0xff00) >> 8 | (val & 0x00ff) << 8; + val = (val & 0xf0f0) >> 4 | (val & 0x0f0f) << 4; + val = (val & 0xcccc) >> 2 | (val & 0x3333) << 2; + val = (val & 0xaaaa) >> 1 | (val & 0x5555) << 1; + val = (val >> 8) | (val << 8); // swap the byte + val = (val >> remainder); + if((val & mask) != mask){ + // goto return failure + return 0; + } + + size = size - shift; + addr += 1; + } + + // no_start_offset + uint64_t num_shadow_bytes = size >> 3; + uint64_t mask = -1; + + while(true){ + if(num_shadow_bytes < 8){ + // goto less_than_8_shadow_bytes_remaining + break; + } + else{ + uint64_t val = *(uint64_t *)addr; + addr += 8; + if(val != mask){ + // goto return failure + return 0; + } + num_shadow_bytes -= 8; + size -= 64; + } + } + + while(true){ + if(num_shadow_bytes < 1){ + // goto check_trailing_bits + break; + } + else{ + uint8_t val = *(uint8_t *)addr; + addr += 1; + if(val != 0xff){ + // goto return failure + return 0; + } + num_shadow_bytes -= 1; + size -= 8; + } + } + + if(size == 0){ + // goto return success + return 1; + } + + uint8_t mask2 = ((1 << (size & 0b111)) - 1); + uint8_t val = *(uint8_t *)addr; + val = (val & 0xf0) >> 4 | (val & 0x0f) << 4; + val = (val & 0xff) >> 2 | (val & 0x33) << 2; + val = (val & 0xaa) >> 1 | (val & 0x55) << 1; + + if((val & mask2) != mask2){ + // goto return failure + return 0; + } + return 1; + } + } + */ + #[cfg(target_arch = "x86_64")] + #[allow(clippy::unused_self, clippy::identity_op)] + #[allow(clippy::too_many_lines)] + fn generate_shadow_check_function(&mut self) { + use std::fs::File; + + let shadow_bit = self.allocator.shadow_bit(); + let mut ops = dynasmrt::VecAssembler::::new(0); + + // Rdi start, Rsi size + dynasm!(ops + ; .arch x64 + ; mov cl, BYTE shadow_bit as i8 + ; mov r10, -2 + ; shl r10, cl + ; mov eax, 1 + ; mov edx, 1 + ; shl rdx, cl + ; test rsi, rsi + ; je >LBB0_15 + ; mov rcx, rdi + ; shr rcx, 3 + ; not r10 + ; and r10, rcx + ; add r10, rdx + ; and edi, 7 + ; je >LBB0_4 + ; mov cl, 8 + ; sub cl, dil + ; cmp rsi, 8 + ; movzx ecx, cl + ; mov r8d, esi + ; cmovae r8d, ecx + ; mov r9d, -1 + ; mov ecx, r8d + ; shl r9d, cl + ; movzx ecx, WORD [r10] + ; rol cx, 8 + ; mov edx, ecx + ; shr edx, 4 + ; and edx, 3855 + ; shl ecx, 4 + ; and ecx, -3856 + ; or ecx, edx + ; mov edx, ecx + ; shr edx, 2 + ; and edx, 13107 + ; and ecx, -3277 + ; lea ecx, [rdx + 4*rcx] + ; mov edx, ecx + ; shr edx, 1 + ; and edx, 21845 + ; and ecx, -10923 + ; lea ecx, [rdx + 2*rcx] + ; rol cx, 8 + ; movzx edx, cx + ; mov ecx, edi + ; shr edx, cl + ; not r9d + ; movzx ecx, r9b + ; and edx, ecx + ; cmp edx, ecx + ; jne >LBB0_11 + ; movzx ecx, r8b + ; sub rsi, rcx + ; add r10, 1 + ;LBB0_4: + ; mov r8, rsi + ; shr r8, 3 + ; mov r9, r8 + ; and r9, -8 + ; mov edi, r8d + ; and edi, 7 + ; add r9, r10 + ; and esi, 63 + ; mov rdx, r8 + ; mov rcx, r10 + ;LBB0_5: + ; cmp rdx, 7 + ; jbe >LBB0_8 + ; add rdx, -8 + ; cmp QWORD [rcx], -1 + ; lea rcx, [rcx + 8] + ; je LBB0_11 + ;LBB0_8: + ; lea rcx, [8*rdi] + ; sub rsi, rcx + ;LBB0_9: + ; test rdi, rdi + ; je >LBB0_13 + ; add rdi, -1 + ; cmp BYTE [r9], -1 + ; lea r9, [r9 + 1] + ; je LBB0_15 + ; and sil, 7 + ; mov dl, -1 + ; mov ecx, esi + ; shl dl, cl + ; not dl + ; mov cl, BYTE [r8 + r10] + ; rol cl, 4 + ; mov eax, ecx + ; shr al, 2 + ; shl cl, 2 + ; and cl, -52 + ; or cl, al + ; mov eax, ecx + ; shr al, 1 + ; and al, 85 + ; add cl, cl + ; and cl, -86 + ; or cl, al + ; and cl, dl + ; xor eax, eax + ; cmp cl, dl + ; sete al + ;LBB0_15: + ; ret + ); + let blob = ops.finalize().unwrap(); + unsafe { + let mapping = mmap::( + None, + NonZeroUsize::new_unchecked(0x1000), + ProtFlags::all(), + MapFlags::MAP_ANON | MapFlags::MAP_PRIVATE, + None, + 0, + ) + .unwrap(); + blob.as_ptr() + .copy_to_nonoverlapping(mapping as *mut u8, blob.len()); + self.shadow_check_func = Some(std::mem::transmute::< + *mut u8, + extern "C" fn(*const c_void, usize) -> bool, + >(mapping as *mut u8)); + } + } + + #[cfg(target_arch = "aarch64")] + // identity_op appears to be a false positive in ubfx + #[allow(clippy::unused_self, clippy::identity_op, clippy::too_many_lines)] + fn generate_shadow_check_function(&mut self) { + use std::fs::File; + + let shadow_bit = self.allocator.shadow_bit(); + let mut ops = dynasmrt::VecAssembler::::new(0); + dynasm!(ops + ; .arch aarch64 + + // calculate the shadow address + ; mov x5, #0 + // ; add x5, xzr, x5, lsl #shadow_bit + ; add x5, x5, x0, lsr #3 + ; ubfx x5, x5, #0, #(shadow_bit + 1) + ; mov x6, #1 + ; add x5, x5, x6, lsl #shadow_bit + + ; cmp x1, #0 + ; b.eq >return_success + // check if the ptr is not aligned to 8 bytes + ; ands x6, x0, #7 + ; b.eq >no_start_offset + + // we need to test the high bits from the first shadow byte + ; ldrh w7, [x5, #0] + ; rev16 w7, w7 + ; rbit w7, w7 + ; lsr x7, x7, #16 + ; lsr x7, x7, x6 + + ; cmp x1, #8 + ; b.lt >dont_fill_to_8 + ; mov x2, #8 + ; sub x6, x2, x6 + ; b >check_bits + ; dont_fill_to_8: + ; mov x6, x1 + ; check_bits: + ; mov x2, #1 + ; lsl x2, x2, x6 + ; sub x4, x2, #1 + + // if shadow_bits & size_to_test != size_to_test: fail + ; and x7, x7, x4 + ; cmp x7, x4 + ; b.ne >return_failure + + // size -= size_to_test + ; sub x1, x1, x6 + // shadow_addr += 1 (we consumed the initial byte in the above test) + ; add x5, x5, 1 + + ; no_start_offset: + // num_shadow_bytes = size / 8 + ; lsr x4, x1, #3 + ; eor x3, x3, x3 + ; sub x3, x3, #1 + + // if num_shadow_bytes < 8; then goto check_bytes; else check_8_shadow_bytes + ; check_8_shadow_bytes: + ; cmp x4, #0x8 + ; b.lt >less_than_8_shadow_bytes_remaining + ; ldr x7, [x5], #8 + ; cmp x7, x3 + ; b.ne >return_failure + ; sub x4, x4, #8 + ; sub x1, x1, #64 + ; b check_trailing_bits + ; ldrb w7, [x5], #1 + ; cmp w7, #0xff + ; b.ne >return_failure + ; sub x4, x4, #1 + ; sub x1, x1, #8 + ; b return_success + + ; and x4, x1, #7 + ; mov x2, #1 + ; lsl x2, x2, x4 + ; sub x4, x2, #1 + + ; ldrh w7, [x5, #0] + ; rev16 w7, w7 + ; rbit w7, w7 + ; lsr x7, x7, #16 + ; and x7, x7, x4 + ; cmp x7, x4 + ; b.ne >return_failure + + ; return_success: + ; mov x0, #1 + ; b >prologue + + ; return_failure: + ; mov x0, #0 + + + ; prologue: + ; ret + ); + + let blob = ops.finalize().unwrap(); + + // apple aarch64 requires MAP_JIT to allocates WX pages + #[cfg(target_vendor = "apple")] + let map_flags = MapFlags::MAP_ANON | MapFlags::MAP_PRIVATE | MapFlags::MAP_JIT; + #[cfg(not(target_vendor = "apple"))] + let map_flags = MapFlags::MAP_ANON | MapFlags::MAP_PRIVATE; + + unsafe { + let mapping = mmap::( + None, + NonZeroUsize::try_from(0x1000).unwrap(), + ProtFlags::all(), + map_flags, + None, + 0, + ) + .unwrap(); + + // on apple aarch64, WX pages can't be both writable and executable at the same time. + // pthread_jit_write_protect_np flips them from executable (1) to writable (0) + #[cfg(all(target_vendor = "apple", target_arch = "aarch64"))] + { + libc::pthread_jit_write_protect_np(0); + } + + blob.as_ptr() + .copy_to_nonoverlapping(mapping as *mut u8, blob.len()); + + #[cfg(all(target_vendor = "apple", target_arch = "aarch64"))] + { + libc::pthread_jit_write_protect_np(1); + } + self.shadow_check_func = Some(std::mem::transmute::< + *mut u8, + extern "C" fn(*const c_void, usize) -> bool, + >(mapping as *mut u8)); + } + } + + // https://godbolt.org/z/ah8vG8sWo /* #include #include @@ -1648,7 +2118,7 @@ impl AsanRuntime { uint64_t addr = 0; addr = addr + (start >> 3); - uint64_t mask = (1ULL << (shadow_bit + 1)) - 1; + uint64_t mask = (1ULL << (shadow_bit + 1)) - 1; addr = addr & mask; addr = addr + (1ULL << shadow_bit); @@ -1663,91 +2133,103 @@ impl AsanRuntime { handle_trap(true_rip); } return 0; - + } + */ + + /* + + FRIDA ASAN IMPLEMENTATION DETAILS + + The format of Frida's ASAN is signficantly different from LLVM ASAN. + + In Frida ASAN, we attempt to find the lowest possible bit such that there is no mapping with that bit. That is to say, for some bit x, there is no mapping greater than + 1 << x. This is our shadow base and is similar to Ultra compact shadow in LLVM ASAN. Unlike ASAN where 0 represents a poisoned byte and 1 represents an unpoisoned byte, in Frida-ASAN + + The reasoning for this is that new pages are zeroed, so, by default, every qword is poisoned and we must explicitly unpoison any byte. + + Much like LLVM ASAN, shadow bytes are qword based. This is to say that each shadow byte maps to one qword. The shadow calculation is as follows: + (1ULL << shadow_bit) | (address >> 3) + + The format of a shadow bit is a bitmask. Each bit represents if a byte in the qword is valid starting from the first bit. So, something like 0b11100000 indicates that only the first 3 bytes in the associated qword are valid. + */ #[cfg(target_arch = "x86_64")] #[allow(clippy::unused_self)] - fn generate_shadow_check_blob(&mut self, bit: u32) -> Box<[u8]> { + fn generate_shadow_check_blob(&mut self, size: u32) -> Box<[u8]> { let shadow_bit = self.allocator.shadow_bit(); // Rcx, Rax, Rdi, Rdx, Rsi, R8 are used, so we save them in emit_shadow_check + //at this point RDI contains the + let mask_shift = 32 - size; macro_rules! shadow_check{ ($ops:ident, $bit:expr) => {dynasm!($ops ; .arch x64 - // ; int3 - ; mov cl, BYTE shadow_bit as i8 - ; mov edx, 3 - ; shl rdx, cl - ; mov eax, 4 - ; shl rax, cl - ; cmp rdx, rdi - ; ja >done - ; cmp rax, rdi - ; jbe >done - ; mov eax, 1 - ; shl rax, cl - ; mov rdx, rdi - ; shr rdx, 3 - ; mov r8d, 2 - ; shl r8, cl - ; dec r8 - ; and r8, rdx - ; movzx eax, WORD [r8 + rax] - ; and dil, 7 - ; mov ecx, edi - ; shr eax, cl - ; mov cl, BYTE bit as i8 - ; mov edx, -1 - ; shl edx, cl - ; not edx - ; not eax - ; test dl, al - ; xor eax, eax +// ; int3 + ; mov rdx, 1 + ; shl rdx, shadow_bit as i8 //rcx now contains the mask + ; mov rcx, rdi //copy address into rdx + ; and rcx, 7 //rsi now contains the offset for unaligned accesses + ; shr rdi, 3 //rdi now contains the shadow byte offset + ; add rdi, rdx //add rdx and rdi to get the address of the shadow byte. rdi now contains the shadow address + ; mov edx, [rdi] //load 4 shadow bytes. We load 4 just in case of an unaligned access + ; bswap edx //bswap to get it into an acceptable form + ; shl edx, cl //this shifts by the unaligned access offset. why does x86 require cl... + ; mov edi, -1 //fill edi with all 1s + ; shl edi, mask_shift as i8 //edi now contains mask. this shl functionally creates a bitmask with the top `size` bits as 1s + ; and edx, edi //and it to see if the top bits are enabled in edx + ; cmp edx, edi //if the mask and the and'd value are the same, we're good ; je >done - ; lea rsi, [>done] // leap 10 bytes forward - ; nop // jmp takes 10 bytes at most so we want to allocate 10 bytes buffer (?) - ; nop - ; nop - ; nop - ; nop - ; nop - ; nop - ; nop - ; nop - ; nop + ; lea rsi, [>done] // leap 10 bytes forward + ; nop // jmp takes 10 bytes at most so we want to allocate 10 bytes buffer (?) + ; nop + ; nop + ; nop + ; nop + ; nop + ; nop + ; nop + ; nop + ; nop ;done: );}; } let mut ops = dynasmrt::VecAssembler::::new(0); shadow_check!(ops, bit); let ops_vec = ops.finalize().unwrap(); - ops_vec[..ops_vec.len() - 10].to_vec().into_boxed_slice() //???? + ops_vec[..ops_vec.len() - 10].to_vec().into_boxed_slice() //subtract 10 because } #[cfg(target_arch = "aarch64")] #[allow(clippy::unused_self)] - fn generate_shadow_check_blob(&mut self, bit: u32) -> Box<[u8]> { + fn generate_shadow_check_blob(&mut self, width: u32) -> Box<[u8]> { + /*x0 contains the shadow address + x0 and x1 are saved by the asan_check + The maximum size this supports is up to 25 bytes. This is because we load 4 bytes of the shadow value. And, in the case that we have a misaligned address with an offset of 7 into the word. For example, if we load 25 bytes from 0x1007 - [0x1007,0x101f], then we require the shadow values from 0x1000, 0x1008, 0x1010, and 0x1018 */ + let shadow_bit = self.allocator.shadow_bit(); macro_rules! shadow_check { - ($ops:ident, $bit:expr) => {dynasm!($ops + ($ops:ident, $width:expr) => {dynasm!($ops ; .arch aarch64 - +// ; brk #0xe ; stp x2, x3, [sp, #-0x10]! - ; mov x1, #0 + ; mov x1, xzr // ; add x1, xzr, x1, lsl #shadow_bit ; add x1, x1, x0, lsr #3 ; ubfx x1, x1, #0, #(shadow_bit + 1) ; mov x2, #1 ; add x1, x1, x2, lsl #shadow_bit - ; ldrh w1, [x1, #0] - ; and x0, x0, #7 - ; rev16 w1, w1 + ; ldr w1, [x1, #0] //w1 contains our shadow check + ; and x0, x0, #7 //x0 is the offset for unaligned accesses + ; rev32 x1, x1 ; rbit w1, w1 - ; lsr x1, x1, #16 - ; lsr x1, x1, x0 + ; lsr w1, w1, w0 //x1 now contains our shadow value ; ldp x2, x3, [sp], 0x10 - ; tbnz x1, #$bit, >done - + ; mov w0, #1 + ; add w0, wzr, w0, LSL #$width + ; sub w0, w0, #1 //x0 now contains our bitmask + ; and w1, w0, w1 //and the bitmask and the shadow value + ; cmp w0, w1 //our bitmask and shadow & mask must be the same + ; b.eq >done ; adr x1, >done ; nop // will be replaced by b to report ; done: @@ -1755,38 +2237,44 @@ impl AsanRuntime { } let mut ops = dynasmrt::VecAssembler::::new(0); - shadow_check!(ops, bit); + shadow_check!(ops, width); let ops_vec = ops.finalize().unwrap(); - ops_vec[..ops_vec.len() - 4].to_vec().into_boxed_slice() + ops_vec[..ops_vec.len() - 4].to_vec().into_boxed_slice() //we don't need the last nop so subtract by 4 } #[cfg(target_arch = "aarch64")] #[allow(clippy::unused_self)] - fn generate_shadow_check_exact_blob(&mut self, val: u64) -> Box<[u8]> { + fn generate_shadow_check_large_blob(&mut self, width: u32) -> Box<[u8]> { + //x0 contains the shadow address + //x0 and x1 are saved by the asan_check + //large blobs require 16 byte alignment as they are only possible with vector insns, so just abuse that + + //This is used for checking shadow blobs that are larger than 25 bytes + + assert!(width <= 64, "width must be <= 64"); + let shift = 64 - width; let shadow_bit = self.allocator.shadow_bit(); macro_rules! shadow_check_exact { - ($ops:ident, $val:expr) => {dynasm!($ops + ($ops:ident, $shift:expr) => {dynasm!($ops ; .arch aarch64 ; stp x2, x3, [sp, #-0x10]! - ; mov x1, #0 + ; mov x1, xzr // ; add x1, xzr, x1, lsl #shadow_bit ; add x1, x1, x0, lsr #3 ; ubfx x1, x1, #0, #(shadow_bit + 1) ; mov x2, #1 ; add x1, x1, x2, lsl #shadow_bit - ; ldrh w1, [x1, #0] - ; and x0, x0, #7 - ; rev16 w1, w1 - ; rbit w1, w1 - ; lsr x1, x1, #16 - ; lsr x1, x1, x0 - ; .dword -717536768 // 0xd53b4200 //mrs x0, NZCV - ; mov x2, $val - ; ands x1, x1, x2 + ; ldr x1, [x1, #0] //x1 contains our shadow check + ; rev64 x1, x1 + ; rbit x1, x1 //x1 now contains our shadow value ; ldp x2, x3, [sp], 0x10 - ; b.ne >done - + ; mov x0, xzr + ; sub x0, x0, #1 //gives us all 1s + ; lsr x0, x0, #$shift //x0 now contains our bitmask + ; and x1, x0, x1 //and the bitmask and the shadow value and put it in x1 + ; cmp x0, x1 //our bitmask and shadow & mask must be the same to ensure that the bytes are valid + ; b.eq >done ; adr x1, >done ; nop // will be replaced by b to report ; done: @@ -1794,7 +2282,7 @@ impl AsanRuntime { } let mut ops = dynasmrt::VecAssembler::::new(0); - shadow_check_exact!(ops, val); + shadow_check_exact!(ops, shift); let ops_vec = ops.finalize().unwrap(); ops_vec[..ops_vec.len() - 4].to_vec().into_boxed_slice() } @@ -1812,7 +2300,7 @@ impl AsanRuntime { ; .arch x64 ; report: ; mov rdi, [>self_regs_addr] // load self.regs into rdi - ; mov [rdi + 0x80], rsi // return address is loaded into rsi in generate_shadow_check_blob + ; mov [rdi + 0x80], rsi // return address is loaded into rsi in generate_shadow_check_blob. rsi is the address of done ; mov [rdi + 0x8], rbx ; mov [rdi + 0x20], rbp ; mov [rdi + 0x28], rsp @@ -1875,7 +2363,7 @@ impl AsanRuntime { ;->accessed_address: ; .dword 0x0 ; self_addr: - ; .qword core::ptr::from_mut(self) as *mut c_void as i64 + ; .qword core::ptr::from_mut(self) as *mut c_void as i64 ; self_regs_addr: ; .qword addr_of_mut!(self.regs) as i64 ; trap_func: @@ -1974,7 +2462,7 @@ impl AsanRuntime { ; br x1 // go back to the 'return address' ; self_addr: - ; .qword self as *mut _ as *mut c_void as i64 + ; .qword core::ptr::from_mut(self) as *mut c_void as i64 ; self_regs_addr: ; .qword addr_of_mut!(self.regs) as i64 ; trap_func: @@ -2002,19 +2490,19 @@ impl AsanRuntime { self.blob_report = Some(ops_report.finalize().unwrap().into_boxed_slice()); - self.blob_check_mem_byte = Some(self.generate_shadow_check_blob(0)); - self.blob_check_mem_halfword = Some(self.generate_shadow_check_blob(1)); - self.blob_check_mem_dword = Some(self.generate_shadow_check_blob(2)); - self.blob_check_mem_qword = Some(self.generate_shadow_check_blob(3)); - self.blob_check_mem_16bytes = Some(self.generate_shadow_check_blob(4)); - - self.blob_check_mem_3bytes = Some(self.generate_shadow_check_exact_blob(3)); - self.blob_check_mem_6bytes = Some(self.generate_shadow_check_exact_blob(6)); - self.blob_check_mem_12bytes = Some(self.generate_shadow_check_exact_blob(12)); - self.blob_check_mem_24bytes = Some(self.generate_shadow_check_exact_blob(24)); - self.blob_check_mem_32bytes = Some(self.generate_shadow_check_exact_blob(32)); - self.blob_check_mem_48bytes = Some(self.generate_shadow_check_exact_blob(48)); - self.blob_check_mem_64bytes = Some(self.generate_shadow_check_exact_blob(64)); + self.blob_check_mem_byte = Some(self.generate_shadow_check_blob(1)); + self.blob_check_mem_halfword = Some(self.generate_shadow_check_blob(2)); + self.blob_check_mem_dword = Some(self.generate_shadow_check_blob(4)); + self.blob_check_mem_qword = Some(self.generate_shadow_check_blob(8)); + self.blob_check_mem_16bytes = Some(self.generate_shadow_check_blob(16)); + + self.blob_check_mem_3bytes = Some(self.generate_shadow_check_blob(3)); //the below are all possible with vector intrinsics + self.blob_check_mem_6bytes = Some(self.generate_shadow_check_blob(6)); + self.blob_check_mem_12bytes = Some(self.generate_shadow_check_blob(12)); + self.blob_check_mem_24bytes = Some(self.generate_shadow_check_blob(24)); + self.blob_check_mem_32bytes = Some(self.generate_shadow_check_large_blob(32)); //this is possible with ldp q0, q1, [sp]. This must at least 16 byte aligned + self.blob_check_mem_48bytes = Some(self.generate_shadow_check_large_blob(48)); + self.blob_check_mem_64bytes = Some(self.generate_shadow_check_large_blob(64)); } /// Get the blob which implements the report funclet @@ -2218,7 +2706,15 @@ impl AsanRuntime { _address: u64, instr: &Insn, ) -> Option<(u8, X86Register, X86Register, u8, i32)> { - let cs_instr = frida_to_cs(decoder, instr); + let result = frida_to_cs(decoder, instr); + + if let Err(e) = result { + log::error!("{}", e); + return None; + } + + let cs_instr = result.unwrap(); + let mut operands = vec![]; for operand_idx in 0..cs_instr.operand_count() { operands.push(cs_instr.operand(operand_idx)); @@ -2305,8 +2801,6 @@ impl AsanRuntime { #[cfg(target_arch = "x86_64")] writer.put_jmp_near_label(after_report_impl); - #[cfg(target_arch = "aarch64")] - writer.put_b_label(after_report_impl); self.current_report_impl = writer.pc(); writer.put_bytes(self.blob_report()); @@ -2597,7 +3091,7 @@ impl AsanRuntime { Aarch64Register::X0, Aarch64Register::X0, u64::from(displacement_lo), - ); //sub x0, x0, #[displacement & 4095] + ); //sub x0, x0, #[displacement 496] } } else if displacement > 0 { #[allow(clippy::cast_sign_loss)] @@ -2612,12 +3106,12 @@ impl AsanRuntime { } else { let displacement_hi = displacement / 4096; let displacement_lo = displacement % 4096; - writer.put_bytes(&(0x91400000u32 | (displacement_hi << 10)).to_le_bytes()); + writer.put_bytes(&(0x91400000u32 | (displacement_hi << 10)).to_le_bytes()); //add x0, x0, #[displacement/4096] LSL#12 writer.put_add_reg_reg_imm( Aarch64Register::X0, Aarch64Register::X0, u64::from(displacement_lo), - ); + ); //add x0, x0, #[displacement % 4096] } } // Insert the check_shadow_mem code blob diff --git a/libafl_frida/src/asan/errors.rs b/libafl_frida/src/asan/errors.rs index 2148f73235..e5bd7503d2 100644 --- a/libafl_frida/src/asan/errors.rs +++ b/libafl_frida/src/asan/errors.rs @@ -1,5 +1,11 @@ //! Errors that can be caught by the `libafl_frida` address sanitizer. -use std::{fmt::Debug, io::Write, marker::PhantomData}; +use std::{ + borrow::Cow, + fmt::Debug, + io::Write, + marker::PhantomData, + sync::{Mutex, MutexGuard}, +}; use backtrace::Backtrace; use color_backtrace::{default_output_stream, BacktracePrinter, Verbosity}; @@ -13,10 +19,14 @@ use libafl::{ feedbacks::Feedback, inputs::{HasTargetBytes, UsesInput}, observers::{Observer, ObserversTuple}, - state::{HasMetadata, State}, - Error, + state::State, + Error, HasMetadata, +}; +use libafl_bolts::{ + ownedref::OwnedPtr, + tuples::{Handle, Handler, MatchNameRef}, + Named, SerdeAny, }; -use libafl_bolts::{ownedref::OwnedPtr, Named, SerdeAny}; use serde::{Deserialize, Serialize}; use termcolor::{Color, ColorSpec, WriteColor}; #[cfg(target_arch = "aarch64")] @@ -110,7 +120,7 @@ pub struct AsanErrors { impl AsanErrors { /// Creates a new `AsanErrors` struct #[must_use] - pub fn new(continue_on_error: bool) -> Self { + pub const fn new(continue_on_error: bool) -> Self { Self { errors: Vec::new(), continue_on_error, @@ -135,16 +145,18 @@ impl AsanErrors { } /// Get a mutable reference to the global [`struct@AsanErrors`] object - #[must_use] - pub fn get_mut<'a>() -> &'a mut Self { - unsafe { ASAN_ERRORS.as_mut().unwrap() } + pub fn get_mut_blocking() -> MutexGuard<'static, Self> { + ASAN_ERRORS.lock().unwrap() + } + + /// Sets if this [`AsanErrors`] variable should continue on error, or not. + pub fn set_continue_on_error(&mut self, continue_on_error: bool) { + self.continue_on_error = continue_on_error; } /// Report an error #[allow(clippy::too_many_lines)] pub(crate) fn report_error(&mut self, error: AsanError) { - self.errors.push(error.clone()); - let mut out_stream = default_output_stream(); let output = out_stream.as_mut(); @@ -164,11 +176,11 @@ impl AsanErrors { .set_color(ColorSpec::new().set_fg(Some(Color::Red))) .unwrap(); write!(output, "{}", error.description()).unwrap(); - match error { - AsanError::OobRead(mut error) - | AsanError::OobWrite(mut error) - | AsanError::ReadAfterFree(mut error) - | AsanError::WriteAfterFree(mut error) => { + match &error { + AsanError::OobRead(error) + | AsanError::OobWrite(error) + | AsanError::ReadAfterFree(error) + | AsanError::WriteAfterFree(error) => { let (basereg, indexreg, _displacement, fault_address) = error.fault; if let Some(module_details) = ModuleDetails::with_address(error.pc as u64) { @@ -301,7 +313,11 @@ impl AsanErrors { writeln!(output, "allocation was zero-sized").unwrap(); } - if let Some(backtrace) = error.metadata.allocation_site_backtrace.as_mut() { + let mut allocation_site_backtrace = + error.metadata.allocation_site_backtrace.clone(); + let mut release_site_backtrace = error.metadata.release_site_backtrace.clone(); + + if let Some(backtrace) = &mut allocation_site_backtrace { writeln!(output, "allocation site backtrace:").unwrap(); backtrace.resolve(); backtrace_printer.print_trace(backtrace, output).unwrap(); @@ -310,7 +326,7 @@ impl AsanErrors { if error.metadata.freed { #[allow(clippy::non_ascii_literal)] writeln!(output, "{:━^100}", " FREE INFO ").unwrap(); - if let Some(backtrace) = error.metadata.release_site_backtrace.as_mut() { + if let Some(backtrace) = &mut release_site_backtrace { writeln!(output, "free site backtrace:").unwrap(); backtrace.resolve(); backtrace_printer.print_trace(backtrace, output).unwrap(); @@ -330,7 +346,7 @@ impl AsanErrors { { let invocation = Interceptor::current_invocation(); let cpu_context = invocation.cpu_context(); - if let Some(module_details) = ModuleDetails::with_address(_pc as u64) { + if let Some(module_details) = ModuleDetails::with_address(*_pc as u64) { writeln!( output, " at 0x{:x} ({}@0x{:04x})", @@ -347,7 +363,7 @@ impl AsanErrors { writeln!(output, "{:━^100}", " REGISTERS ").unwrap(); for reg in 0..29 { let val = cpu_context.reg(reg); - if val as usize == address { + if val as usize == *address { output .set_color(ColorSpec::new().set_fg(Some(Color::Red))) .unwrap(); @@ -363,12 +379,12 @@ impl AsanErrors { writeln!(output, "pc : 0x{:016x} ", cpu_context.pc()).unwrap(); } - backtrace_printer.print_trace(&backtrace, output).unwrap(); + backtrace_printer.print_trace(backtrace, output).unwrap(); } - AsanError::DoubleFree((ptr, mut metadata, backtrace)) => { + AsanError::DoubleFree((ptr, metadata, backtrace)) => { writeln!(output, " of {ptr:?}").unwrap(); output.reset().unwrap(); - backtrace_printer.print_trace(&backtrace, output).unwrap(); + backtrace_printer.print_trace(backtrace, output).unwrap(); #[allow(clippy::non_ascii_literal)] writeln!(output, "{:━^100}", " ALLOCATION INFO ").unwrap(); @@ -383,14 +399,17 @@ impl AsanErrors { writeln!(output, "allocation was zero-sized").unwrap(); } - if let Some(backtrace) = metadata.allocation_site_backtrace.as_mut() { + let mut allocation_site_backtrace = metadata.allocation_site_backtrace.clone(); + let mut release_site_backtrace = metadata.release_site_backtrace.clone(); + + if let Some(backtrace) = &mut allocation_site_backtrace { writeln!(output, "allocation site backtrace:").unwrap(); backtrace.resolve(); backtrace_printer.print_trace(backtrace, output).unwrap(); } #[allow(clippy::non_ascii_literal)] writeln!(output, "{:━^100}", " FREE INFO ").unwrap(); - if let Some(backtrace) = metadata.release_site_backtrace.as_mut() { + if let Some(backtrace) = &mut release_site_backtrace { writeln!(output, "previous free site backtrace:").unwrap(); backtrace.resolve(); backtrace_printer.print_trace(backtrace, output).unwrap(); @@ -399,9 +418,9 @@ impl AsanErrors { AsanError::UnallocatedFree((ptr, backtrace)) => { writeln!(output, " of {ptr:#016x}").unwrap(); output.reset().unwrap(); - backtrace_printer.print_trace(&backtrace, output).unwrap(); + backtrace_printer.print_trace(backtrace, output).unwrap(); } - AsanError::Leak((ptr, mut metadata)) => { + AsanError::Leak((ptr, metadata)) => { writeln!(output, " of {ptr:#016x}").unwrap(); output.reset().unwrap(); @@ -418,7 +437,9 @@ impl AsanErrors { writeln!(output, "allocation was zero-sized").unwrap(); } - if let Some(backtrace) = metadata.allocation_site_backtrace.as_mut() { + let mut allocation_site_backtrace = metadata.allocation_site_backtrace.clone(); + + if let Some(backtrace) = &mut allocation_site_backtrace { writeln!(output, "allocation site backtrace:").unwrap(); backtrace.resolve(); backtrace_printer.print_trace(backtrace, output).unwrap(); @@ -429,7 +450,7 @@ impl AsanErrors { | AsanError::StackOobWrite((registers, pc, fault, backtrace)) => { let (basereg, indexreg, _displacement, fault_address) = fault; - if let Some(module_details) = ModuleDetails::with_address(pc as u64) { + if let Some(module_details) = ModuleDetails::with_address(*pc as u64) { writeln!( output, " at 0x{:x} ({}:0x{:04x}), faulting address 0x{:x}", @@ -507,20 +528,20 @@ impl AsanErrors { #[cfg(target_arch = "x86_64")] let insts = disas_count( &decoder, - unsafe { std::slice::from_raw_parts(start_pc as *mut u8, 15 * 11) }, + unsafe { std::slice::from_raw_parts(*start_pc as *mut u8, 15 * 11) }, 11, ); #[cfg(target_arch = "aarch64")] let insts = disas_count( &decoder, - unsafe { std::slice::from_raw_parts(start_pc as *mut u8, 4 * 11) }, + unsafe { std::slice::from_raw_parts(*start_pc as *mut u8, 4 * 11) }, 11, ); - let mut inst_address = start_pc; + let mut inst_address = *start_pc; for insn in insts { - if inst_address == pc { + if inst_address == *pc { output .set_color(ColorSpec::new().set_fg(Some(Color::Red))) .unwrap(); @@ -532,10 +553,12 @@ impl AsanErrors { inst_address += insn.len().to_const() as usize; } - backtrace_printer.print_trace(&backtrace, output).unwrap(); + backtrace_printer.print_trace(backtrace, output).unwrap(); } }; + self.errors.push(error); + #[allow(clippy::manual_assert)] if !self.continue_on_error { panic!("ASAN: Crashing target!"); @@ -544,13 +567,16 @@ impl AsanErrors { } /// static field for `AsanErrors` for a run -pub static mut ASAN_ERRORS: Option = None; +pub static ASAN_ERRORS: Mutex = Mutex::new(AsanErrors::new(true)); -/// An observer for frida address sanitizer `AsanError`s for a frida executor run +/// An observer for frida address sanitizer `AsanError`s for a `Frida` executor run #[derive(Debug, Serialize, Deserialize)] #[allow(clippy::unsafe_derive_deserialize)] -pub struct AsanErrorsObserver { - errors: OwnedPtr>, +pub enum AsanErrorsObserver { + /// Observer referencing a list behind a [`OwnedPtr`] pointer. + Ptr(OwnedPtr), + /// Observer referencing the static [`ASAN_ERRORS`] variable. + Static, } impl Observer for AsanErrorsObserver @@ -558,11 +584,7 @@ where S: UsesInput, { fn pre_exec(&mut self, _state: &mut S, _input: &S::Input) -> Result<(), Error> { - unsafe { - if ASAN_ERRORS.is_some() { - ASAN_ERRORS.as_mut().unwrap().clear(); - } - } + AsanErrors::get_mut_blocking().clear(); Ok(()) } @@ -570,42 +592,52 @@ where impl Named for AsanErrorsObserver { #[inline] - fn name(&self) -> &str { - "AsanErrors" + fn name(&self) -> &Cow<'static, str> { + static ASAN_ERRORS_NAME: Cow<'static, str> = Cow::Borrowed("AsanErrors"); + &ASAN_ERRORS_NAME } } impl AsanErrorsObserver { - /// Creates a new `AsanErrorsObserver`, pointing to a constant `AsanErrors` field + /// Creates a new [`AsanErrorsObserver`], pointing to a constant `AsanErrors` field #[must_use] - pub fn new(errors: *const Option) -> Self { - Self { - errors: OwnedPtr::Ptr(errors), - } + pub fn new(errors: OwnedPtr) -> Self { + Self::Ptr(errors) + } + + /// Creates a new [`AsanErrorsObserver`], pointing to the [`ASAN_ERRORS`] global static field. + /// + /// # Safety + /// The field should not be accessed multiple times at the same time (i.e., from different threads)! + pub fn from_static_asan_errors() -> Self { + Self::Static } /// Creates a new `AsanErrorsObserver`, owning the `AsanErrors` #[must_use] - pub fn owned(errors: Option) -> Self { - Self { - errors: OwnedPtr::Owned(Box::new(errors)), - } + pub fn owned(errors: AsanErrors) -> Self { + Self::Ptr(OwnedPtr::Owned(Box::new(errors))) } /// Creates a new `AsanErrorsObserver` from a raw ptr + /// + /// # Safety + /// Will dereference this pointer at a later point in time. + /// The pointer *must* outlive this [`AsanErrorsObserver`]'s lifetime. #[must_use] - pub fn from_mut_ptr(errors: *const Option) -> Self { - Self { - errors: OwnedPtr::Ptr(errors), - } + pub unsafe fn from_ptr(errors: *const AsanErrors) -> Self { + Self::Ptr(OwnedPtr::Ptr(errors)) } - /// gets the [`struct@AsanErrors`] from the previous run + /// Gets the [`struct@AsanErrors`] from the previous run #[must_use] - pub fn errors(&self) -> Option<&AsanErrors> { - match &self.errors { - OwnedPtr::Ptr(p) => unsafe { p.as_ref().unwrap().as_ref() }, - OwnedPtr::Owned(b) => b.as_ref().as_ref(), + pub fn errors(&self) -> AsanErrors { + match self { + Self::Ptr(errors) => match errors { + OwnedPtr::Ptr(p) => unsafe { p.as_ref().unwrap().clone() }, + OwnedPtr::Owned(b) => b.as_ref().clone(), + }, + Self::Static => AsanErrors::get_mut_blocking().clone(), } } } @@ -614,6 +646,7 @@ impl AsanErrorsObserver { #[derive(Serialize, Deserialize, Clone, Debug)] pub struct AsanErrorsFeedback { errors: Option, + obs_ref: Handle, phantom: PhantomData, } @@ -636,24 +669,21 @@ where OT: ObserversTuple, { let observer = observers - .match_name::("AsanErrors") + .get(&self.obs_ref) .expect("An AsanErrorsFeedback needs an AsanErrorsObserver"); - match observer.errors() { - None => Ok(false), - Some(errors) => { - if errors.errors.is_empty() { - Ok(false) - } else { - self.errors = Some(errors.clone()); - Ok(true) - } - } + let errors = observer.errors(); + if errors.is_empty() { + Ok(false) + } else { + self.errors = Some(errors); + Ok(true) } } - fn append_metadata( + fn append_metadata( &mut self, _state: &mut S, + _manager: &mut EM, _observers: &OT, testcase: &mut Testcase, ) -> Result<(), Error> @@ -675,24 +705,19 @@ where impl Named for AsanErrorsFeedback { #[inline] - fn name(&self) -> &str { - "AsanErrors" + fn name(&self) -> &Cow<'static, str> { + self.obs_ref.name() } } impl AsanErrorsFeedback { /// Create a new `AsanErrorsFeedback` #[must_use] - pub fn new() -> Self { + pub fn new(obs: &AsanErrorsObserver) -> Self { Self { errors: None, + obs_ref: obs.handle(), phantom: PhantomData, } } } - -impl Default for AsanErrorsFeedback { - fn default() -> Self { - Self::new() - } -} diff --git a/libafl_frida/src/asan/hook_funcs.rs b/libafl_frida/src/asan/hook_funcs.rs index 9de091fe99..3c80a503f3 100644 --- a/libafl_frida/src/asan/hook_funcs.rs +++ b/libafl_frida/src/asan/hook_funcs.rs @@ -939,8 +939,8 @@ impl AsanRuntime { extern "system" { fn write(fd: i32, buf: *const c_void, count: usize) -> usize; } - if !self.allocator_mut().check_shadow(buf, count) { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgWrite(( + if !(self.shadow_check_func().unwrap())(buf, count) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgWrite(( "write".to_string(), self.real_address_for_stalked(self.pc()), buf as usize, @@ -961,8 +961,8 @@ impl AsanRuntime { extern "system" { fn read(fd: i32, buf: *mut c_void, count: usize) -> usize; } - if !self.allocator_mut().check_shadow(buf, count) { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(buf, count) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "read".to_string(), self.real_address_for_stalked(self.pc()), buf as usize, @@ -978,8 +978,8 @@ impl AsanRuntime { extern "system" { fn fgets(s: *mut c_void, size: u32, stream: *mut c_void) -> *mut c_void; } - if !self.allocator_mut().check_shadow(s, size as usize) { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(s, size as usize) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "fgets".to_string(), self.real_address_for_stalked(self.pc()), s as usize, @@ -995,8 +995,8 @@ impl AsanRuntime { extern "system" { fn memcmp(s1: *const c_void, s2: *const c_void, n: usize) -> i32; } - if !self.allocator_mut().check_shadow(s1, n) { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(s1, n) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "memcmp".to_string(), self.real_address_for_stalked(self.pc()), s1 as usize, @@ -1004,8 +1004,8 @@ impl AsanRuntime { Backtrace::new(), ))); } - if !self.allocator_mut().check_shadow(s2, n) { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(s2, n) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "memcmp".to_string(), self.real_address_for_stalked(self.pc()), s2 as usize, @@ -1021,18 +1021,18 @@ impl AsanRuntime { extern "system" { fn memcpy(dest: *mut c_void, src: *const c_void, n: usize) -> *mut c_void; } - if !self.allocator_mut().check_shadow(dest, n) { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgWrite(( - "memcpy dest".to_string(), + if !(self.shadow_check_func().unwrap())(dest, n) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgWrite(( + "memcpy".to_string(), self.real_address_for_stalked(self.pc()), dest as usize, n, Backtrace::new(), ))); } - if !self.allocator_mut().check_shadow(src, n) { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( - "memcpy src".to_string(), + if !(self.shadow_check_func().unwrap())(src, n) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( + "memcpy".to_string(), self.real_address_for_stalked(self.pc()), src as usize, n, @@ -1048,8 +1048,8 @@ impl AsanRuntime { extern "system" { fn mempcpy(dest: *mut c_void, src: *const c_void, n: usize) -> *mut c_void; } - if !self.allocator_mut().check_shadow(dest, n) { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgWrite(( + if !(self.shadow_check_func().unwrap())(dest, n) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgWrite(( "mempcpy".to_string(), self.real_address_for_stalked(self.pc()), dest as usize, @@ -1057,8 +1057,8 @@ impl AsanRuntime { Backtrace::new(), ))); } - if !self.allocator_mut().check_shadow(src, n) { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(src, n) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "mempcpy".to_string(), self.real_address_for_stalked(self.pc()), src as usize, @@ -1074,9 +1074,8 @@ impl AsanRuntime { extern "system" { fn memmove(dest: *mut c_void, src: *const c_void, n: usize) -> *mut c_void; } - if !self.allocator_mut().check_shadow(dest, n) { - log::trace!("holy shit!"); - AsanErrors::get_mut().report_error(AsanError::BadFuncArgWrite(( + if !(self.shadow_check_func().unwrap())(dest, n) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgWrite(( "memmove".to_string(), self.real_address_for_stalked(self.pc()), dest as usize, @@ -1084,8 +1083,8 @@ impl AsanRuntime { Backtrace::new(), ))); } - if !self.allocator_mut().check_shadow(src, n) { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(src, n) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "memmove".to_string(), self.real_address_for_stalked(self.pc()), src as usize, @@ -1104,8 +1103,8 @@ impl AsanRuntime { extern "system" { fn memset(dest: *mut c_void, c: i32, n: usize) -> *mut c_void; } - if !self.allocator_mut().check_shadow(dest, n) { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgWrite(( + if !(self.shadow_check_func().unwrap())(dest, n) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgWrite(( "memset".to_string(), self.real_address_for_stalked(self.pc()), dest as usize, @@ -1121,8 +1120,8 @@ impl AsanRuntime { extern "system" { fn memchr(s: *mut c_void, c: i32, n: usize) -> *mut c_void; } - if !self.allocator_mut().check_shadow(s, n) { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(s, n) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "memchr".to_string(), self.real_address_for_stalked(self.pc()), s as usize, @@ -1139,8 +1138,8 @@ impl AsanRuntime { extern "system" { fn memrchr(s: *mut c_void, c: i32, n: usize) -> *mut c_void; } - if !self.allocator_mut().check_shadow(s, n) { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(s, n) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "memrchr".to_string(), self.real_address_for_stalked(self.pc()), s as usize, @@ -1167,8 +1166,8 @@ impl AsanRuntime { needlelen: usize, ) -> *mut c_void; } - if !self.allocator_mut().check_shadow(haystack, haystacklen) { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(haystack, haystacklen) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "memmem".to_string(), self.real_address_for_stalked(self.pc()), haystack as usize, @@ -1176,8 +1175,8 @@ impl AsanRuntime { Backtrace::new(), ))); } - if !self.allocator_mut().check_shadow(needle, needlelen) { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(needle, needlelen) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "memmem".to_string(), self.real_address_for_stalked(self.pc()), needle as usize, @@ -1194,8 +1193,8 @@ impl AsanRuntime { extern "system" { fn bzero(s: *mut c_void, n: usize) -> usize; } - if !self.allocator_mut().check_shadow(s, n) { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgWrite(( + if !(self.shadow_check_func().unwrap())(s, n) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgWrite(( "bzero".to_string(), self.real_address_for_stalked(self.pc()), s as usize, @@ -1212,8 +1211,8 @@ impl AsanRuntime { extern "system" { fn explicit_bzero(s: *mut c_void, n: usize) -> usize; } - if !self.allocator_mut().check_shadow(s, n) { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgWrite(( + if !(self.shadow_check_func().unwrap())(s, n) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgWrite(( "explicit_bzero".to_string(), self.real_address_for_stalked(self.pc()), s as usize, @@ -1230,8 +1229,8 @@ impl AsanRuntime { extern "system" { fn bcmp(s1: *const c_void, s2: *const c_void, n: usize) -> i32; } - if !self.allocator_mut().check_shadow(s1, n) { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(s1, n) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "bcmp".to_string(), self.real_address_for_stalked(self.pc()), s1 as usize, @@ -1239,8 +1238,8 @@ impl AsanRuntime { Backtrace::new(), ))); } - if !self.allocator_mut().check_shadow(s2, n) { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(s2, n) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "bcmp".to_string(), self.real_address_for_stalked(self.pc()), s2 as usize, @@ -1257,11 +1256,8 @@ impl AsanRuntime { fn strchr(s: *mut c_char, c: i32) -> *mut c_char; fn strlen(s: *const c_char) -> usize; } - if !self - .allocator_mut() - .check_shadow(s as *const c_void, unsafe { strlen(s) }) - { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(s as *const c_void, unsafe { strlen(s) }) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "strchr".to_string(), self.real_address_for_stalked(self.pc()), s as usize, @@ -1278,11 +1274,8 @@ impl AsanRuntime { fn strrchr(s: *mut c_char, c: i32) -> *mut c_char; fn strlen(s: *const c_char) -> usize; } - if !self - .allocator_mut() - .check_shadow(s as *const c_void, unsafe { strlen(s) }) - { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(s as *const c_void, unsafe { strlen(s) }) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "strrchr".to_string(), self.real_address_for_stalked(self.pc()), s as usize, @@ -1299,11 +1292,8 @@ impl AsanRuntime { fn strcasecmp(s1: *const c_char, s2: *const c_char) -> i32; fn strlen(s: *const c_char) -> usize; } - if !self - .allocator_mut() - .check_shadow(s1 as *const c_void, unsafe { strlen(s1) }) - { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(s1 as *const c_void, unsafe { strlen(s1) }) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "strcasecmp".to_string(), self.real_address_for_stalked(self.pc()), s1 as usize, @@ -1311,11 +1301,8 @@ impl AsanRuntime { Backtrace::new(), ))); } - if !self - .allocator_mut() - .check_shadow(s2 as *const c_void, unsafe { strlen(s2) }) - { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(s2 as *const c_void, unsafe { strlen(s2) }) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "strcasecmp".to_string(), self.real_address_for_stalked(self.pc()), s2 as usize, @@ -1331,8 +1318,8 @@ impl AsanRuntime { extern "system" { fn strncasecmp(s1: *const c_char, s2: *const c_char, n: usize) -> i32; } - if !self.allocator_mut().check_shadow(s1 as *const c_void, n) { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(s1 as *const c_void, n) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "strncasecmp".to_string(), self.real_address_for_stalked(self.pc()), s1 as usize, @@ -1340,8 +1327,8 @@ impl AsanRuntime { Backtrace::new(), ))); } - if !self.allocator_mut().check_shadow(s2 as *const c_void, n) { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(s2 as *const c_void, n) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "strncasecmp".to_string(), self.real_address_for_stalked(self.pc()), s2 as usize, @@ -1358,11 +1345,8 @@ impl AsanRuntime { fn strcat(s1: *mut c_char, s2: *const c_char) -> *mut c_char; fn strlen(s: *const c_char) -> usize; } - if !self - .allocator_mut() - .check_shadow(s1 as *const c_void, unsafe { strlen(s1) }) - { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(s1 as *const c_void, unsafe { strlen(s1) }) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "strcat".to_string(), self.real_address_for_stalked(self.pc()), s1 as usize, @@ -1370,11 +1354,8 @@ impl AsanRuntime { Backtrace::new(), ))); } - if !self - .allocator_mut() - .check_shadow(s2 as *const c_void, unsafe { strlen(s2) }) - { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(s2 as *const c_void, unsafe { strlen(s2) }) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "strcat".to_string(), self.real_address_for_stalked(self.pc()), s2 as usize, @@ -1391,11 +1372,8 @@ impl AsanRuntime { fn strcmp(s1: *const c_char, s2: *const c_char) -> i32; fn strlen(s: *const c_char) -> usize; } - if !self - .allocator_mut() - .check_shadow(s1 as *const c_void, unsafe { strlen(s1) }) - { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(s1 as *const c_void, unsafe { strlen(s1) }) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "strcmp".to_string(), self.real_address_for_stalked(self.pc()), s1 as usize, @@ -1403,11 +1381,8 @@ impl AsanRuntime { Backtrace::new(), ))); } - if !self - .allocator_mut() - .check_shadow(s2 as *const c_void, unsafe { strlen(s2) }) - { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(s2 as *const c_void, unsafe { strlen(s2) }) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "strcmp".to_string(), self.real_address_for_stalked(self.pc()), s2 as usize, @@ -1424,11 +1399,8 @@ impl AsanRuntime { fn strncmp(s1: *const c_char, s2: *const c_char, n: usize) -> i32; fn strnlen(s: *const c_char, n: usize) -> usize; } - if !self - .allocator_mut() - .check_shadow(s1 as *const c_void, unsafe { strnlen(s1, n) }) - { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(s1 as *const c_void, unsafe { strnlen(s1, n) }) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "strncmp".to_string(), self.real_address_for_stalked(self.pc()), s1 as usize, @@ -1436,11 +1408,8 @@ impl AsanRuntime { Backtrace::new(), ))); } - if !self - .allocator_mut() - .check_shadow(s2 as *const c_void, unsafe { strnlen(s2, n) }) - { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(s2 as *const c_void, unsafe { strnlen(s2, n) }) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "strncmp".to_string(), self.real_address_for_stalked(self.pc()), s2 as usize, @@ -1457,11 +1426,8 @@ impl AsanRuntime { fn strcpy(dest: *mut c_char, src: *const c_char) -> *mut c_char; fn strlen(s: *const c_char) -> usize; } - if !self - .allocator_mut() - .check_shadow(dest as *const c_void, unsafe { strlen(src) }) - { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgWrite(( + if !(self.shadow_check_func().unwrap())(dest as *const c_void, unsafe { strlen(src) }) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgWrite(( "strcpy".to_string(), self.real_address_for_stalked(self.pc()), dest as usize, @@ -1469,11 +1435,8 @@ impl AsanRuntime { Backtrace::new(), ))); } - if !self - .allocator_mut() - .check_shadow(src as *const c_void, unsafe { strlen(src) }) - { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(src as *const c_void, unsafe { strlen(src) }) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "strcpy".to_string(), self.real_address_for_stalked(self.pc()), src as usize, @@ -1489,8 +1452,8 @@ impl AsanRuntime { extern "system" { fn strncpy(dest: *mut c_char, src: *const c_char, n: usize) -> *mut c_char; } - if !self.allocator_mut().check_shadow(dest as *const c_void, n) { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgWrite(( + if !(self.shadow_check_func().unwrap())(dest as *const c_void, n) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgWrite(( "strncpy".to_string(), self.real_address_for_stalked(self.pc()), dest as usize, @@ -1498,8 +1461,8 @@ impl AsanRuntime { Backtrace::new(), ))); } - if !self.allocator_mut().check_shadow(src as *const c_void, n) { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(src as *const c_void, n) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "strncpy".to_string(), self.real_address_for_stalked(self.pc()), src as usize, @@ -1516,11 +1479,8 @@ impl AsanRuntime { fn stpcpy(dest: *mut c_char, src: *const c_char) -> *mut c_char; fn strlen(s: *const c_char) -> usize; } - if !self - .allocator_mut() - .check_shadow(dest as *const c_void, unsafe { strlen(src) }) - { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgWrite(( + if !(self.shadow_check_func().unwrap())(dest as *const c_void, unsafe { strlen(src) }) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgWrite(( "stpcpy".to_string(), self.real_address_for_stalked(self.pc()), dest as usize, @@ -1528,11 +1488,8 @@ impl AsanRuntime { Backtrace::new(), ))); } - if !self - .allocator_mut() - .check_shadow(src as *const c_void, unsafe { strlen(src) }) - { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(src as *const c_void, unsafe { strlen(src) }) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "stpcpy".to_string(), self.real_address_for_stalked(self.pc()), src as usize, @@ -1555,8 +1512,8 @@ impl AsanRuntime { fn strcpy(dest: *mut c_char, src: *const c_char) -> *mut c_char; } let size = unsafe { strlen(s) }; - if !self.allocator_mut().check_shadow(s as *const c_void, size) { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(s as *const c_void, size) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "strdup".to_string(), self.real_address_for_stalked(self.pc()), s as usize, @@ -1578,8 +1535,8 @@ impl AsanRuntime { fn strlen(s: *const c_char) -> usize; } let size = unsafe { strlen(s) }; - if !self.allocator_mut().check_shadow(s as *const c_void, size) { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(s as *const c_void, size) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "strlen".to_string(), self.real_address_for_stalked(self.pc()), s as usize, @@ -1596,8 +1553,8 @@ impl AsanRuntime { fn strnlen(s: *const c_char, n: usize) -> usize; } let size = unsafe { strnlen(s, n) }; - if !self.allocator_mut().check_shadow(s as *const c_void, size) { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(s as *const c_void, size) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "strnlen".to_string(), self.real_address_for_stalked(self.pc()), s as usize, @@ -1614,11 +1571,10 @@ impl AsanRuntime { fn strstr(haystack: *const c_char, needle: *const c_char) -> *mut c_char; fn strlen(s: *const c_char) -> usize; } - if !self - .allocator_mut() - .check_shadow(haystack as *const c_void, unsafe { strlen(haystack) }) - { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(haystack as *const c_void, unsafe { + strlen(haystack) + }) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "strstr".to_string(), self.real_address_for_stalked(self.pc()), haystack as usize, @@ -1630,7 +1586,7 @@ impl AsanRuntime { .allocator_mut() .check_shadow(needle as *const c_void, unsafe { strlen(needle) }) { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "strstr".to_string(), self.real_address_for_stalked(self.pc()), needle as usize, @@ -1651,11 +1607,10 @@ impl AsanRuntime { fn strcasestr(haystack: *const c_char, needle: *const c_char) -> *mut c_char; fn strlen(s: *const c_char) -> usize; } - if !self - .allocator_mut() - .check_shadow(haystack as *const c_void, unsafe { strlen(haystack) }) - { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(haystack as *const c_void, unsafe { + strlen(haystack) + }) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "strcasestr".to_string(), self.real_address_for_stalked(self.pc()), haystack as usize, @@ -1667,7 +1622,7 @@ impl AsanRuntime { .allocator_mut() .check_shadow(needle as *const c_void, unsafe { strlen(needle) }) { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "strcasestr".to_string(), self.real_address_for_stalked(self.pc()), needle as usize, @@ -1684,11 +1639,8 @@ impl AsanRuntime { fn atoi(s: *const c_char) -> i32; fn strlen(s: *const c_char) -> usize; } - if !self - .allocator_mut() - .check_shadow(s as *const c_void, unsafe { strlen(s) }) - { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(s as *const c_void, unsafe { strlen(s) }) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "atoi".to_string(), self.real_address_for_stalked(self.pc()), s as usize, @@ -1706,11 +1658,8 @@ impl AsanRuntime { fn atol(s: *const c_char) -> i32; fn strlen(s: *const c_char) -> usize; } - if !self - .allocator_mut() - .check_shadow(s as *const c_void, unsafe { strlen(s) }) - { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(s as *const c_void, unsafe { strlen(s) }) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "atol".to_string(), self.real_address_for_stalked(self.pc()), s as usize, @@ -1728,11 +1677,8 @@ impl AsanRuntime { fn atoll(s: *const c_char) -> i64; fn strlen(s: *const c_char) -> usize; } - if !self - .allocator_mut() - .check_shadow(s as *const c_void, unsafe { strlen(s) }) - { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(s as *const c_void, unsafe { strlen(s) }) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "atoll".to_string(), self.real_address_for_stalked(self.pc()), s as usize, @@ -1750,11 +1696,8 @@ impl AsanRuntime { fn wcslen(s: *const wchar_t) -> usize; } let size = unsafe { wcslen(s) }; - if !self - .allocator_mut() - .check_shadow(s as *const c_void, (size + 1) * 2) - { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(s as *const c_void, (size + 1) * 2) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "wcslen".to_string(), self.real_address_for_stalked(self.pc()), s as usize, @@ -1772,11 +1715,10 @@ impl AsanRuntime { fn wcscpy(dest: *mut wchar_t, src: *const wchar_t) -> *mut wchar_t; fn wcslen(s: *const wchar_t) -> usize; } - if !self - .allocator_mut() - .check_shadow(dest as *const c_void, unsafe { (wcslen(src) + 1) * 2 }) - { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgWrite(( + if !(self.shadow_check_func().unwrap())(dest as *const c_void, unsafe { + (wcslen(src) + 1) * 2 + }) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgWrite(( "wcscpy".to_string(), self.real_address_for_stalked(self.pc()), dest as usize, @@ -1784,11 +1726,10 @@ impl AsanRuntime { Backtrace::new(), ))); } - if !self - .allocator_mut() - .check_shadow(src as *const c_void, unsafe { (wcslen(src) + 1) * 2 }) - { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(src as *const c_void, unsafe { + (wcslen(src) + 1) * 2 + }) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "wcscpy".to_string(), self.real_address_for_stalked(self.pc()), src as usize, @@ -1806,11 +1747,10 @@ impl AsanRuntime { fn wcscmp(s1: *const wchar_t, s2: *const wchar_t) -> i32; fn wcslen(s: *const wchar_t) -> usize; } - if !self - .allocator_mut() - .check_shadow(s1 as *const c_void, unsafe { (wcslen(s1) + 1) * 2 }) - { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(s1 as *const c_void, unsafe { + (wcslen(s1) + 1) * 2 + }) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "wcscmp".to_string(), self.real_address_for_stalked(self.pc()), s1 as usize, @@ -1818,11 +1758,10 @@ impl AsanRuntime { Backtrace::new(), ))); } - if !self - .allocator_mut() - .check_shadow(s2 as *const c_void, unsafe { (wcslen(s2) + 1) * 2 }) - { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgRead(( + if !(self.shadow_check_func().unwrap())(s2 as *const c_void, unsafe { + (wcslen(s2) + 1) * 2 + }) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgRead(( "wcscmp".to_string(), self.real_address_for_stalked(self.pc()), s2 as usize, @@ -1839,8 +1778,8 @@ impl AsanRuntime { extern "system" { fn memset_pattern4(s: *mut c_void, p4: *const c_void, n: usize); } - if !self.allocator_mut().check_shadow(s, n) { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgWrite(( + if !(self.shadow_check_func().unwrap())(s, n) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgWrite(( "memset_pattern4".to_string(), self.real_address_for_stalked(self.pc()), s as usize, @@ -1848,8 +1787,8 @@ impl AsanRuntime { Backtrace::new(), ))); } - if !self.allocator_mut().check_shadow(p4, n / 4) { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgWrite(( + if !(self.shadow_check_func().unwrap())(p4, n / 4) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgWrite(( "memset_pattern4".to_string(), self.real_address_for_stalked(self.pc()), p4 as usize, @@ -1866,8 +1805,8 @@ impl AsanRuntime { extern "system" { fn memset_pattern8(s: *mut c_void, p8: *const c_void, n: usize); } - if !self.allocator_mut().check_shadow(s, n) { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgWrite(( + if !(self.shadow_check_func().unwrap())(s, n) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgWrite(( "memset_pattern8".to_string(), self.real_address_for_stalked(self.pc()), s as usize, @@ -1875,8 +1814,8 @@ impl AsanRuntime { Backtrace::new(), ))); } - if !self.allocator_mut().check_shadow(p8, n / 8) { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgWrite(( + if !(self.shadow_check_func().unwrap())(p8, n / 8) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgWrite(( "memset_pattern8".to_string(), self.real_address_for_stalked(self.pc()), p8 as usize, @@ -1893,8 +1832,8 @@ impl AsanRuntime { extern "system" { fn memset_pattern16(s: *mut c_void, p16: *const c_void, n: usize); } - if !self.allocator_mut().check_shadow(s, n) { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgWrite(( + if !(self.shadow_check_func().unwrap())(s, n) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgWrite(( "memset_pattern16".to_string(), self.real_address_for_stalked(self.pc()), s as usize, @@ -1902,8 +1841,8 @@ impl AsanRuntime { Backtrace::new(), ))); } - if !self.allocator_mut().check_shadow(p16, n / 16) { - AsanErrors::get_mut().report_error(AsanError::BadFuncArgWrite(( + if !(self.shadow_check_func().unwrap())(p16, n / 16) { + AsanErrors::get_mut_blocking().report_error(AsanError::BadFuncArgWrite(( "memset_pattern16".to_string(), self.real_address_for_stalked(self.pc()), p16 as usize, diff --git a/libafl_frida/src/cmplog_rt.rs b/libafl_frida/src/cmplog_rt.rs index a9e91505bc..3970cb8afe 100644 --- a/libafl_frida/src/cmplog_rt.rs +++ b/libafl_frida/src/cmplog_rt.rs @@ -4,29 +4,15 @@ //! related to the input. //! Read the [`RedQueen`](https://www.ndss-symposium.org/ndss-paper/redqueen-fuzzing-with-input-to-state-correspondence/) paper for the general concepts. +#[cfg(target_arch = "aarch64")] +use core::ffi::c_void; #[cfg(all(feature = "cmplog", target_arch = "x86_64"))] use std::collections::HashMap; +use std::rc::Rc; use dynasmrt::dynasm; #[cfg(target_arch = "aarch64")] use dynasmrt::{DynasmApi, DynasmLabelApi}; -use libafl::{ - inputs::{HasTargetBytes, Input}, - Error, -}; -use libafl_targets::{self, CMPLOG_MAP_W}; -use rangemap::RangeMap; - -use crate::helper::FridaRuntime; -extern "C" { - /// Tracks cmplog instructions - pub fn __libafl_targets_cmplog_instructions(k: u64, shape: u8, arg1: u64, arg2: u64); -} - -#[cfg(target_arch = "aarch64")] -use core::ffi::c_void; -use std::rc::Rc; - use frida_gum::ModuleMap; #[cfg(target_arch = "x86_64")] use frida_gum::{instruction_writer::InstructionWriter, stalker::StalkerOutput}; @@ -41,7 +27,14 @@ use iced_x86::{ BlockEncoder, Code, DecoderOptions, Instruction, InstructionBlock, MemoryOperand, MemorySize, OpKind, Register, }; +use libafl::{ + inputs::{HasTargetBytes, Input}, + Error, +}; +use libafl_targets::{cmps::__libafl_targets_cmplog_instructions, CMPLOG_MAP_W}; +use rangemap::RangeMap; +use crate::helper::FridaRuntime; #[cfg(all(feature = "cmplog", target_arch = "aarch64"))] use crate::utils::{disas_count, writer_register}; @@ -179,7 +172,7 @@ impl CmpLogRuntime { k &= (CMPLOG_MAP_W as u64) - 1; unsafe { - __libafl_targets_cmplog_instructions(k, 8, op1, op2); + __libafl_targets_cmplog_instructions(k as usize, 8, op1, op2); } } @@ -195,7 +188,7 @@ impl CmpLogRuntime { k &= (CMPLOG_MAP_W as u64) - 1; unsafe { - __libafl_targets_cmplog_instructions(k, size, op1, op2); + __libafl_targets_cmplog_instructions(k as usize, size, op1, op2); } } @@ -247,7 +240,7 @@ impl CmpLogRuntime { ; ldp x2, x3, [sp], #0x10 ; b >done ; self_addr: - ; .qword self as *mut _ as *mut c_void as i64 + ; .qword core::ptr::from_mut(self) as *mut c_void as i64 ; populate_lists: ; .qword CmpLogRuntime::populate_lists as *mut c_void as i64 ; done: @@ -661,11 +654,11 @@ impl CmpLogRuntime { )> { let bytes = instr.bytes(); let mut decoder = - iced_x86::Decoder::with_ip(64, bytes, instr.address(), iced_x86::DecoderOptions::NONE); + iced_x86::Decoder::with_ip(64, bytes, instr.address(), DecoderOptions::NONE); if !decoder.can_decode() { return None; } - let mut instruction = iced_x86::Instruction::default(); + let mut instruction = Instruction::default(); decoder.decode_out(&mut instruction); match instruction.mnemonic() { iced_x86::Mnemonic::Cmp | iced_x86::Mnemonic::Sub => {} // continue @@ -753,7 +746,7 @@ impl CmpLogRuntime { } #[cfg(all(feature = "cmplog", target_arch = "aarch64"))] - #[allow(clippy::similar_names)] + #[allow(clippy::similar_names, clippy::type_complexity)] #[inline] /// Check if the current instruction is cmplog relevant one(any opcode which sets the flags) #[must_use] @@ -772,7 +765,7 @@ impl CmpLogRuntime { .operands .iter() .position(|item| *item == Operand::Nothing) - .unwrap_or_else(|| 4); + .unwrap_or(4); // "cmp" | "ands" | "subs" | "adds" | "negs" | "ngcs" | "sbcs" | "bics" | "cbz" // | "cbnz" | "tbz" | "tbnz" | "adcs" - yaxpeax aliases insns (i.e., cmp -> subs) // We only care for compare instructions - aka instructions which set the flags @@ -853,14 +846,14 @@ impl CmpLogRuntime { None, )), Operand::ImmShift(imm, shift) => { - Some((CmplogOperandType::Imm((imm as u64) << shift), None)) + Some((CmplogOperandType::Imm(u64::from(imm) << shift), None)) } //precalculate the shift Operand::RegShift(shiftstyle, amount, regsize, reg) => { let reg = CmplogOperandType::Regid(writer_register(reg, regsize, true)); let shift = (shiftstyle, amount); Some((reg, Some(shift))) } - Operand::Immediate(imm) => Some((CmplogOperandType::Imm(imm as u64), None)), + Operand::Immediate(imm) => Some((CmplogOperandType::Imm(u64::from(imm)), None)), _ => panic!("Second argument could not be decoded"), } }; diff --git a/libafl_frida/src/executor.rs b/libafl_frida/src/executor.rs index 56e60ee896..197a783eae 100644 --- a/libafl_frida/src/executor.rs +++ b/libafl_frida/src/executor.rs @@ -1,5 +1,9 @@ +#[cfg(all(unix, not(test)))] +use core::borrow::Borrow; use core::fmt::{self, Debug, Formatter}; -use std::{ffi::c_void, marker::PhantomData, process::abort}; +#[cfg(windows)] +use std::process::abort; +use std::{ffi::c_void, marker::PhantomData}; use frida_gum::{ stalker::{NoneEventSink, Stalker}, @@ -17,9 +21,10 @@ use libafl::{ state::{HasExecutions, State, UsesState}, Error, }; +use libafl_bolts::tuples::RefIndexable; #[cfg(not(test))] -use crate::asan::errors::ASAN_ERRORS; +use crate::asan::errors::AsanErrors; use crate::helper::{FridaInstrumentationHelper, FridaRuntimeTuple}; #[cfg(windows)] use crate::windows_hooks::initialize; @@ -103,11 +108,10 @@ where self.stalker.deactivate(); } - #[cfg(not(test))] + #[cfg(all(unix, not(test)))] unsafe { - if ASAN_ERRORS.is_some() && !ASAN_ERRORS.as_ref().unwrap().is_empty() { - log::error!("Crashing target as it had ASAN errors"); - #[cfg(unix)] + if !AsanErrors::get_mut_blocking().borrow().is_empty() { + log::error!("Crashing target as it had ASan errors"); libc::raise(libc::SIGABRT); #[cfg(windows)] abort(); @@ -146,12 +150,12 @@ where OT: ObserversTuple, { #[inline] - fn observers(&self) -> &OT { + fn observers(&self) -> RefIndexable<&Self::Observers, Self::Observers> { self.base.observers() } #[inline] - fn observers_mut(&mut self) -> &mut OT { + fn observers_mut(&mut self) -> RefIndexable<&mut Self::Observers, Self::Observers> { self.base.observers_mut() } } @@ -232,7 +236,7 @@ where } #[cfg(windows)] -impl<'a, 'b, 'c, H, OT, RT, S> HasInProcessHooks +impl<'a, 'b, 'c, H, OT, RT, S> HasInProcessHooks for FridaInProcessExecutor<'a, 'b, 'c, H, OT, RT, S> where H: FnMut(&S::Input) -> ExitKind, @@ -243,13 +247,13 @@ where { /// the timeout handler #[inline] - fn inprocess_hooks(&self) -> &InProcessHooks { + fn inprocess_hooks(&self) -> &InProcessHooks { &self.base.hooks().0 } /// the timeout handler #[inline] - fn inprocess_hooks_mut(&mut self) -> &mut InProcessHooks { + fn inprocess_hooks_mut(&mut self) -> &mut InProcessHooks { &mut self.base.hooks_mut().0 } } diff --git a/libafl_frida/src/helper.rs b/libafl_frida/src/helper.rs index 14e0937689..190d6e30e7 100644 --- a/libafl_frida/src/helper.rs +++ b/libafl_frida/src/helper.rs @@ -131,7 +131,7 @@ pub enum SkipRange { }, } -/// Builder for [`FridaInstrumentationHelper`](FridaInstrumentationHelper) +/// Builder for [`FridaInstrumentationHelper`] pub struct FridaInstrumentationHelperBuilder { stalker_enabled: bool, disable_excludes: bool, @@ -142,14 +142,14 @@ pub struct FridaInstrumentationHelperBuilder { } impl FridaInstrumentationHelperBuilder { - /// Create a new `FridaInstrumentationHelperBuilder` + /// Create a new [`FridaInstrumentationHelperBuilder`] pub fn new() -> Self { Self::default() } - /// Enable or disable the Stalker + /// Enable or disable the [`Stalker`](https://frida.re/docs/stalker/) /// - /// Required for coverage collection, ASAN, and `CmpLog`. + /// Required for all instrumentation, such as coverage collection, `ASan`, and `CmpLog`. /// Enabled by default. #[must_use] pub fn enable_stalker(self, enabled: bool) -> Self { @@ -239,7 +239,7 @@ impl FridaInstrumentationHelperBuilder { self } - /// Build a `FridaInstrumentationHelper` + /// Build a [`FridaInstrumentationHelper`] pub fn build( self, gum: &Gum, @@ -411,9 +411,9 @@ where } impl<'a> FridaInstrumentationHelper<'a, ()> { - /// Create a builder to initialize a `FridaInstrumentationHelper`. + /// Create a builder to initialize a [`FridaInstrumentationHelper`]. /// - /// See the documentation of [`FridaInstrumentationHelperBuilder`](FridaInstrumentationHelperBuilder) + /// See the documentation of [`FridaInstrumentationHelperBuilder`] /// for more details. pub fn builder() -> FridaInstrumentationHelperBuilder { FridaInstrumentationHelperBuilder::default() @@ -486,7 +486,7 @@ where let address = instr.address(); let mut keep_instr = true; // log::trace!("x - block @ {:x} transformed to {:x}", address, output.writer().pc()); - + //the ASAN check needs to be done before the hook_rt check due to x86 insns such as call [mem] if ranges.borrow().contains_key(&(address as usize)) { let mut runtimes = (*runtimes_unborrowed).borrow_mut(); if first { @@ -504,24 +504,24 @@ where } } - if let Some(rt) = runtimes.match_first_type_mut::() { - if let Some((call_target, needs_return)) = rt.is_interesting(decoder, instr) { - rt.emit_callout(call_target, &instruction, needs_return, runtimes_unborrowed.clone()); - keep_instr = false; - } - } - let res = if let Some(_rt) = runtimes.match_first_type_mut::() { AsanRuntime::asan_is_interesting_instruction(decoder, address, instr) } else { None }; - + #[cfg(target_arch = "x86_64")] if let Some(details) = res { if let Some(rt) = runtimes.match_first_type_mut::() { rt.emit_shadow_check( - address, output, instr.bytes().len(), details.0, details.1, details.2, details.3, details.4, + address, + output, + instr.bytes().len(), + details.0, + details.1, + details.2, + details.3, + details.4, ); } } @@ -541,6 +541,33 @@ where } } + #[cfg(target_arch = "x86_64")] + if let Some(rt) = runtimes.match_first_type_mut::() { + if let Some(call_target) = rt.is_interesting(decoder, instr) { + rt.emit_callout( + call_target, + &instruction, + output.writer(), + runtimes_unborrowed.clone(), + ); + keep_instr = false; + } + } + + #[cfg(target_arch = "aarch64")] + if let Some(rt) = runtimes.match_first_type_mut::() { + if let Some((call_target, is_reg)) = rt.is_interesting(decoder, instr) { + rt.emit_callout( + call_target, + &instruction, + is_reg, + output.writer(), + runtimes_unborrowed.clone(), + ); + keep_instr = false; //we keep the instruction in the emit if needed + } + } + #[cfg(all( feature = "cmplog", any(target_arch = "aarch64", target_arch = "x86_64") diff --git a/libafl_frida/src/hook_rt.rs b/libafl_frida/src/hook_rt.rs index 3a33c14f5c..11bd60c766 100644 --- a/libafl_frida/src/hook_rt.rs +++ b/libafl_frida/src/hook_rt.rs @@ -1,25 +1,55 @@ //! Functionality implementing hooks for instrumented code -use std::{cell::RefCell, collections::HashMap, rc::Rc}; +use std::{ + cell::RefCell, + collections::HashMap, + ptr::addr_of, + rc::Rc, +}; -use frida_gum::{ - instruction_writer::X86Register, - stalker::Instruction, - CpuContext, ModuleMap, +#[cfg(target_arch = "aarch64")] +use frida_gum::instruction_writer::{ + Aarch64InstructionWriter, Aarch64Register, IndexMode, InstructionWriter, +}; +#[cfg(target_arch = "x86_64")] +use frida_gum::instruction_writer::{ + InstructionWriter, X86BranchCondition, X86InstructionWriter, X86Register, }; +use frida_gum::{stalker::Instruction, CpuContext, ModuleMap}; use frida_gum_sys::Insn; use rangemap::RangeMap; -use yaxpeax_arch::LengthedInstruction; -use yaxpeax_x86::long_mode::{InstDecoder, Opcode}; +#[cfg(target_arch = "aarch64")] +use yaxpeax_arm::armv8::a64::{InstDecoder, Opcode, Operand}; +#[cfg(target_arch = "x86_64")] +use yaxpeax_x86::long_mode::{InstDecoder, Opcode, Operand}; +#[cfg(target_arch = "x86_64")] +use crate::utils::{get_register, operand_details, writer_register}; use crate::{ asan::asan_rt::AsanRuntime, helper::{FridaRuntime, FridaRuntimeTuple}, - utils::{frida_to_cs, immediate_value, operand_details}, + utils::frida_to_cs, }; +#[cfg(target_arch = "x86_64")] +use std::ptr::read_unaligned; + +/* +LibAFL hook_rt design: + +The objective of this runtime is to move away from using Interceptor for hooking and move to something that hooks during the stalk. The way this does this is different for direct and indirect branches. + +For direct branches, the hooking is easy. We simply check if the branch target is hooked. If it is, run the hooked function. If it is not, then continue as per normal. If it is hooked, we chaining return to return the caller. + +For indirect branches (i.e., jmp rax/blr x16), it is harder as the branch target is difficult to know at block-compile time. In the case of indirect branches, we check the register during runtime. If the value of the register is a hooked function then run the hooked function in the callout and set HookRuntime::hooked = 1. If it is not then set HookRuntime::hooked = 0. + +From here, we either chaining return if HookRuntime::hooked == 1 or continue on to the next block via a keeping the instruction if HookRuntime::hooked = 0 + +*/ + /// Frida hooks for instrumented code pub struct HookRuntime { hooks: HashMap) + 'static>>, + hooked: u64, //Runtimes are wrapped in a RefCell, so in theory we shouldn't need to pin this } impl Default for HookRuntime { @@ -60,13 +90,22 @@ impl FridaRuntime for HookRuntime { } } +#[derive(Debug)] +#[cfg(target_arch = "x86_64")] +pub enum CallType { + Imm(usize), + Reg(X86Register), + Mem((X86Register, X86Register, u8, i32)), //this is the return type from operand_details +} + impl HookRuntime { /// Create a new hook runtime #[must_use] pub fn new() -> Self { - Self { + return Self { hooks: HashMap::new(), - } + hooked: 0, + }; } /// Register a hook with the runtime @@ -79,121 +118,287 @@ impl HookRuntime { self.hooks.insert(address, Box::new(callback)); } - fn resolve_jump_target(&self, decoder: InstDecoder, address: usize) -> Option { - let slice = unsafe { std::slice::from_raw_parts(address as *const u8, 32) }; - if let Ok(instruction) = decoder.decode_slice(slice) { - if instruction.opcode() == Opcode::JMP || instruction.opcode() == Opcode::JMPF { - let operand = instruction.operand(0); - if operand.is_memory() { - if let Some((basereg, _indexreg, _scale, disp)) = operand_details(&operand) { - if basereg == X86Register::Rip { - let target_address = unsafe { - (((address as u64 + instruction.len()) as i64 + disp as i64) - as *const usize) - .read() - }; - - return if let Some(address) = - self.resolve_jump_target(decoder, target_address) - { - Some(address) - } else { - Some(target_address) - }; - } - } - } else { - if let Some(immediate) = immediate_value(&instruction.operand(0)) { - let inner_address = (address as u64 + instruction.len()) as i64 + immediate; - return if let Some(inner_address) = - self.resolve_jump_target(decoder, inner_address as usize) - { - Some(inner_address) - } else { - Some(address) - }; - } - } - } - } - None - } - /// Determine if this instruction is interesting for the purposes of hooking #[inline] - pub fn is_interesting(&self, decoder: InstDecoder, instr: &Insn) -> Option<(usize, bool)> { - let instruction = frida_to_cs(decoder, instr); + #[cfg(target_arch = "x86_64")] + pub fn is_interesting(&self, decoder: InstDecoder, instr: &Insn) -> Option { + let result = frida_to_cs(decoder, instr); + + if let Err(e) = result { + log::error!("{}", e); + return None; + } + let instruction = result.unwrap(); + + //there are 3 seperate cases we need to handle: loads, immediates, and registers + //we need to deal with all cases in case of dlsym if instruction.opcode() == Opcode::CALL || instruction.opcode() == Opcode::JMP { + //if its a memory op, we can't resolve it yet as it may not be resolved yet if instruction.operand(0).is_memory() { - log::trace!("{:x}: instruction: {}",instr.address(), instruction); - if let Some((basereg, _indexreg, _scale, disp)) = - operand_details(&instruction.operand(0)) - { - if basereg == X86Register::Rip { - let target_address = unsafe { - (((instr.address() + instruction.len()) as i64 + disp as i64) - as *const usize) - .read() - }; - log::trace!("- {:x} : {:x}", ((instr.address() + instruction.len()) as i64 + disp as i64), target_address); - - let (address, needs_return) = if let Some(address) = - self.resolve_jump_target(decoder, target_address) - { - (address, false) - } else { - (target_address, true) - }; - if self.hooks.contains_key(&address) { - return Some(( - address, - needs_return && instruction.opcode() == Opcode::JMP, - )); - }; + log::trace!("{:x}: instruction: {}", instr.address(), instruction); + let mem_details = operand_details(&instruction.operand(0)); + + if let Some((reg, index_reg, scale, disp)) = mem_details { + if reg == X86Register::Rip { + //rip relative loads are from the end of the instruction + return Some(CallType::Mem(( + reg, + index_reg, + scale, + disp + instr.len() as i32, + ))); } + return Some(CallType::Mem((reg, index_reg, scale, disp))); } } else { - if let Some(immediate) = immediate_value(&instruction.operand(0)) { - let inner_address = - (instr.address() as i64 + instr.bytes().len() as i64 + immediate) as usize; - if self.hooks.contains_key(&inner_address) { - return Some((inner_address, instruction.opcode() == Opcode::JMP)); + match instruction.operand(0) { + Operand::Register(reg_spec) => { + return Some(CallType::Reg(writer_register(reg_spec))); } - - if let Some(target_address) = - self.resolve_jump_target(decoder, inner_address) - { - if self.hooks.contains_key(&target_address) { - return Some((target_address, false)); + Operand::ImmediateI32(imm) => { + //https://www.felixcloutier.com/x86/call + let target = (instr.address() as i64 + imm as i64) as usize; + if !self.hooks.contains_key(&target) { + return None; } + return Some(CallType::Imm(target)); } + _ => panic!("Invalid call/jmp instructions"), } } } None } + #[inline] + #[cfg(target_arch = "aarch64")] + pub fn is_interesting(&self, decoder: InstDecoder, instr: &Insn) -> Option<(usize, bool)> { + let result = frida_to_cs(decoder, instr); + + if let Err(e) = result { + log::error!("{}", e); + return None; + } + + let instruction = result.unwrap(); + + match instruction.opcode { + Opcode::BR | Opcode::BLR => { + let reg_op = instruction.operands[0]; + let reg_num = if let Operand::Register(_, num) = reg_op { + num + } else { + panic!( + "Invalid instruction - opcode: {:?}, operands: {:?}", + instruction.opcode, instruction.operands + ); + }; + + //we could probably introduce some kind of speculative backpatching as it is unlikely that if it is hooked the first time that it ever hooks again + + return Some((reg_num as usize, true)); //the reg should always be checked + } + Opcode::BL | Opcode::B => { + let call_address = if let Operand::PCOffset(off) = instruction.operands[0] { + (instr.address() as i64 + off) as usize + } else { + panic!( + "Invalid instruction - opcode: {:?}, operands: {:?}", + instruction.opcode, instruction.operands + ); //impossible to have b/bl with a PCOffset + }; + + if !self.hooks.contains_key(&call_address) { + return None; + } + + return Some((call_address, false)); + } + + _ => { + return None; + } + } + } + /// Emits a callout to the hook #[inline] + #[cfg(target_arch = "x86_64")] pub fn emit_callout( &mut self, - address: usize, + call_type: CallType, insn: &Instruction, - needs_return: bool, + writer: X86InstructionWriter, runtimes: Rc>, ) { - log::trace!("emit_callout: {:x}", address); + log::trace!("emit_callout: {:#x}", insn.instr().address()); + log::trace!("call: {:?}", call_type); + let hooked_address = addr_of!(self.hooked) as u64; + let rip = insn.instr().address(); + let is_imm = if let CallType::Imm(_) = call_type { + //log::trace!("needs return at {:x}", address); + true + } else { + false + }; + + // writer.put_bytes(&[0xcc]); //put int3 + insn.put_callout(move |context| { - (self.hooks.get_mut(&address).unwrap())( - address, - context, - runtimes.borrow_mut().match_first_type_mut::(), - ) + let address = match call_type { + CallType::Mem((reg, index_reg, scale, disp)) => { + let base = if let X86Register::Rip = reg { + rip + } else { + get_register(&context, reg) + }; + + let index = get_register(&context, index_reg); + let addr = (base.wrapping_add(index.wrapping_mul(scale as u64)) as i64 + + disp as i64) as *const u64; //disp already has the offset applied if we are doing an rip relative load + + log::trace!("Call dereference address: {:#x}", addr as u64); + + let value = unsafe { read_unaligned(addr) }; + log::trace!("call value: {:#x}", value); + value as usize + } + CallType::Imm(address) => address, + CallType::Reg(reg) => get_register(&context, reg) as usize, + }; + + if let Some(f) = self.hooks.get_mut(&address) { + f( + address, + context, + runtimes.borrow_mut().match_first_type_mut::(), + ); + self.hooked = 1; + } else { + self.hooked = 0; + } }); + // + + if !is_imm { + let not_hooked_label_id = insn.instr().address() | 0xfaded; //this is label id for the hooked + writer.put_sub_reg_imm(X86Register::Rsp, frida_gum_sys::GUM_RED_ZONE_SIZE as isize); + writer.put_push_reg(X86Register::Rdi); + writer.put_mov_reg_u64(X86Register::Rdi, hooked_address); //hooked address is in RDI + writer.put_mov_reg_reg_ptr(X86Register::Rdi, X86Register::Rdi); //mov rdi, [rdi] + + //sub is the same as cmp. rdi is 0 if we hooked + writer.put_sub_reg_imm(X86Register::Rdi, 1); + //jne is the same as jnz. if the result is not 0 then we did not hook + writer.put_jcc_near_label(X86BranchCondition::Jne, not_hooked_label_id, 0); + + //if we are here we did not hook + writer.put_pop_reg(X86Register::Rdi); + writer.put_add_reg_imm(X86Register::Rsp, frida_gum_sys::GUM_RED_ZONE_SIZE as isize); + insn.put_chaining_return(); //we hooked, run the chaining return + + writer.put_label(not_hooked_label_id); + //we did not hook, normally run the function + writer.put_pop_reg(X86Register::Rdi); + writer.put_add_reg_imm(X86Register::Rsp, frida_gum_sys::GUM_RED_ZONE_SIZE as isize); + + insn.keep(); + } else { + insn.put_chaining_return(); + } + } + + #[inline] + #[cfg(target_arch = "aarch64")] + pub fn emit_callout( + &mut self, + address_or_reg: usize, + insn: &Instruction, + is_reg: bool, + writer: Aarch64InstructionWriter, + runtimes: Rc>, + ) { + let hooked_address = addr_of!(self.hooked) as u64; + log::trace!("emit_callout: {:x}", address_or_reg); + insn.put_callout(move |context| { + if !is_reg { + //if we are not in a register, address_or_reg is the actual address + //safe to unwrap because we check in is_interesting to see if we should hook + (self.hooks.get_mut(&address_or_reg).unwrap())( + address_or_reg, + context, + runtimes.borrow_mut().match_first_type_mut::(), + ) + } else { + //we are a register + let address = match address_or_reg { + 0..=28 => context.reg(address_or_reg), + 29 => context.fp(), + 30 => context.lr(), + _ => { + panic!("Invalid register: {:#x}", address_or_reg); + } + } as usize; + + if let Some(f) = self.hooks.get_mut(&address) { + //the hook sets the return value for us, so we have nothing to do + f( + address, + context, + runtimes.borrow_mut().match_first_type_mut::(), + ); + self.hooked = 1; + } else { + self.hooked = 0; + } + } + }); + + if is_reg { + //Opcode::BR/Opcode::BLR + //write load from self.hooked, cbz to end, + let redzone_size = frida_gum_sys::GUM_RED_ZONE_SIZE as i32; + let not_hooked_label_id = insn.instr().address() | 0xfaded; //this is label id for the hooked + + //stp x16, x17, [sp, #-0x90]! + writer.put_stp_reg_reg_reg_offset( + Aarch64Register::X16, + Aarch64Register::X17, + Aarch64Register::Sp, + i64::from(-(16 + redzone_size)), + IndexMode::PreAdjust, + ); + //mov &self->hooked into x16 + writer.put_ldr_reg_u64(Aarch64Register::X16, hooked_address); + //move self->hooked into x16 + writer.put_ldr_reg_reg(Aarch64Register::X16, Aarch64Register::X16); + //if hooked is 0 then we want to continue as if nothing happened + writer.put_cbz_reg_label(Aarch64Register::X16, not_hooked_label_id); + //this branch we have a hook + writer.put_ldp_reg_reg_reg_offset( + Aarch64Register::X16, + Aarch64Register::X17, + Aarch64Register::Sp, + 16 + i64::from(redzone_size), + IndexMode::PostAdjust, + ); + //then we chaining return because we hooked + insn.put_chaining_return(); + + writer.put_label(not_hooked_label_id); + + writer.put_ldp_reg_reg_reg_offset( + Aarch64Register::X16, + Aarch64Register::X17, + Aarch64Register::Sp, + 16 + i64::from(redzone_size), + IndexMode::PostAdjust, + ); - if needs_return { - log::trace!("needs return at {:x}", address); + insn.keep(); //the keep will dispatch to the next block + } else { + //Opcode::B/Opcode::BL insn.put_chaining_return(); } } diff --git a/libafl_frida/src/lib.rs b/libafl_frida/src/lib.rs index c78941a0a3..81d5373287 100644 --- a/libafl_frida/src/lib.rs +++ b/libafl_frida/src/lib.rs @@ -1,6 +1,6 @@ /*! -The frida executor is a binary-only mode for `LibAFL`. -It can report coverage and, on supported architecutres, even reports memory access errors. +The [`Frida`](https://frida.re) executor is a binary-only mode for `LibAFL`. +It can report coverage and, on supported architectures, even reports memory access errors. Additional documentation is available in [the `LibAFL` book](https://aflplus.plus/libafl-book/advanced_features/frida.html). */ @@ -34,14 +34,12 @@ Additional documentation is available in [the `LibAFL` book](https://aflplus.plu ))] #![cfg_attr(test, deny( missing_debug_implementations, - missing_docs, //trivial_casts, trivial_numeric_casts, unused_extern_crates, unused_import_braces, unused_qualifications, unused_must_use, - missing_docs, //unused_results ))] #![cfg_attr( @@ -347,7 +345,7 @@ impl Default for FridaOptions { #[cfg(test)] mod tests { - use std::{ptr::addr_of, sync::OnceLock}; + use std::sync::OnceLock; use clap::Parser; use frida_gum::Gum; @@ -371,7 +369,7 @@ mod tests { use crate::{ asan::{ asan_rt::AsanRuntime, - errors::{AsanErrorsFeedback, AsanErrorsObserver, ASAN_ERRORS}, + errors::{AsanErrorsFeedback, AsanErrorsObserver, AsanErrors}, }, coverage_rt::CoverageRuntime, executor::FridaInProcessExecutor, @@ -391,17 +389,45 @@ mod tests { ("heap_uaf_read", Some("heap use-after-free read")), ("malloc_heap_oob_read", Some("heap out-of-bounds read")), ("malloc_heap_oob_write", Some("heap out-of-bounds write")), - ("malloc_heap_oob_write_0x12", Some("heap out-of-bounds write")), - ("malloc_heap_oob_write_0x14", Some("heap out-of-bounds write")), - ("malloc_heap_oob_write_0x17", Some("heap out-of-bounds write")), - ("malloc_heap_oob_write_0x17_int_at_0x16", Some("heap out-of-bounds write")), - ("malloc_heap_oob_write_0x17_int_at_0x15", Some("heap out-of-bounds write")), + ( + "malloc_heap_oob_write_0x12", + Some("heap out-of-bounds write"), + ), + ( + "malloc_heap_oob_write_0x14", + Some("heap out-of-bounds write"), + ), + ( + "malloc_heap_oob_write_0x17", + Some("heap out-of-bounds write"), + ), + ( + "malloc_heap_oob_write_0x17_int_at_0x16", + Some("heap out-of-bounds write"), + ), + ( + "malloc_heap_oob_write_0x17_int_at_0x15", + Some("heap out-of-bounds write"), + ), ("malloc_heap_oob_write_0x17_int_at_0x13", None), - ("malloc_heap_oob_write_0x17_int_at_0x14", Some("heap out-of-bounds write")), + ( + "malloc_heap_oob_write_0x17_int_at_0x14", + Some("heap out-of-bounds write"), + ), ("malloc_heap_uaf_write", Some("heap use-after-free write")), ("malloc_heap_uaf_read", Some("heap use-after-free read")), ]; + //NOTE: RTLD_NOW is required on linux as otherwise the hooks will NOT work + + #[cfg(target_os = "linux")] + let lib = libloading::os::unix::Library::open( + Some(options.clone().harness.unwrap()), + libloading::os::unix::RTLD_NOW, + ) + .unwrap(); + + #[cfg(not(target_os = "linux"))] let lib = libloading::Library::new(options.clone().harness.unwrap()).unwrap(); let coverage = CoverageRuntime::new(); @@ -427,10 +453,15 @@ mod tests { let mut feedback = ConstFeedback::new(true); + let asan_obs = AsanErrorsObserver::from_static_asan_errors(); + // Feedbacks to recognize an input as solution let mut objective = feedback_or_fast!( // true enables the AsanErrorFeedback - feedback_and_fast!(ConstFeedback::from(true), AsanErrorsFeedback::new()) + feedback_and_fast!( + ConstFeedback::from(true), + AsanErrorsFeedback::new(&asan_obs) + ) ); let mut state = StdState::new( @@ -447,10 +478,16 @@ mod tests { let mut fuzzer = StdFuzzer::new(StdScheduler::new(), feedback, objective); let observers = tuple_list!( - AsanErrorsObserver::new(addr_of!(ASAN_ERRORS)) //, + asan_obs //, ); { + #[cfg(target_os = "linux")] + let target_func: libloading::os::unix::Symbol< + unsafe extern "C" fn(data: *const u8, size: usize) -> i32, + > = lib.get(function_name.as_bytes()).unwrap(); + + #[cfg(not(target_os = "linux"))] let target_func: libloading::Symbol< unsafe extern "C" fn(data: *const u8, size: usize) -> i32, > = lib.get(function_name.as_bytes()).unwrap(); @@ -486,7 +523,7 @@ mod tests { log::info!("Done fuzzing! Got {} solutions", state.solutions().count()); if let Some(expected_error) = expected_error { assert_eq!(state.solutions().count(), 1); - if let Some(error) = unsafe { ASAN_ERRORS.as_ref().unwrap() }.errors.first() { + if let Some(error) = AsanErrors::get_mut_blocking().errors.first() { assert_eq!(error.description(), expected_error); } } else { diff --git a/libafl_frida/src/pthread_hook.rs b/libafl_frida/src/pthread_hook.rs index de1e4ca421..c436251e8b 100644 --- a/libafl_frida/src/pthread_hook.rs +++ b/libafl_frida/src/pthread_hook.rs @@ -1,11 +1,5 @@ -use std::{ - convert::{TryFrom, TryInto}, - sync::RwLock, -}; - /// Rust bindings for Apple's [`pthread_introspection`](https://opensource.apple.com/source/libpthread/libpthread-218.20.1/pthread/introspection.h.auto.html) hooks. -use libc; - +use std::sync::RwLock; const PTHREAD_INTROSPECTION_THREAD_CREATE: libc::c_uint = 1; const PTHREAD_INTROSPECTION_THREAD_START: libc::c_uint = 2; const PTHREAD_INTROSPECTION_THREAD_TERMINATE: libc::c_uint = 3; diff --git a/libafl_frida/src/utils.rs b/libafl_frida/src/utils.rs index 55909c1695..459cfcd1de 100644 --- a/libafl_frida/src/utils.rs +++ b/libafl_frida/src/utils.rs @@ -1,7 +1,8 @@ #[cfg(target_arch = "aarch64")] use frida_gum::instruction_writer::Aarch64Register; #[cfg(target_arch = "x86_64")] -use frida_gum::instruction_writer::X86Register; +use frida_gum::{instruction_writer::X86Register, CpuContext}; +use libafl::Error; #[cfg(target_arch = "aarch64")] use num_traits::cast::FromPrimitive; #[cfg(target_arch = "x86_64")] @@ -158,9 +159,35 @@ const X86_64_REGS: [(RegSpec, X86Register); 34] = [ (RegSpec::rip(), X86Register::Rip), ]; + +/// Get the value of a register given a context +#[cfg(target_arch = "x86_64")] +pub fn get_register(context: &CpuContext, reg: X86Register) -> u64 +{ + match reg { + X86Register::Rax => context.rax(), + X86Register::Rbx => context.rbx(), + X86Register::Rcx => context.rcx(), + X86Register::Rdx => context.rdx(), + X86Register::Rdi => context.rdi(), + X86Register::Rsi => context.rsi(), + X86Register::Rsp => context.rsp(), + X86Register::Rbp => context.rbp(), + X86Register::R8 => context.r8(), + X86Register::R9 => context.r9(), + X86Register::R10 => context.r10(), + X86Register::R11 => context.r11(), + X86Register::R12 => context.r12(), + X86Register::R13 => context.r13(), + X86Register::R14 => context.r14(), + X86Register::R15 => context.r15(), + _ => 0, + } +} + /// The writer registers -/// frida registers: -/// capstone registers: +/// frida registers: +/// capstone registers: #[cfg(target_arch = "x86_64")] #[must_use] #[inline] @@ -177,17 +204,46 @@ pub fn writer_register(reg: RegSpec) -> X86Register { /// Translates a frida instruction to a disassembled instruction. #[cfg(all(target_arch = "x86_64"))] -pub(crate) fn frida_to_cs(decoder: InstDecoder, frida_insn: &frida_gum_sys::Insn) -> Instruction { +pub(crate) fn frida_to_cs( + decoder: InstDecoder, + frida_insn: &frida_gum_sys::Insn, +) -> Result { match decoder.decode_slice(frida_insn.bytes()) { - Ok(result) => return result, + Ok(result) => return Ok(result), Err(error) => { - log::error!("{:?}: {:x}: {:?}", error, frida_insn.address(), frida_insn.bytes()); - panic!("FAILED"); + log::error!( + "{:?}: {:x}: {:?}", + error, + frida_insn.address(), + frida_insn.bytes() + ); + return Err(Error::illegal_state( + "Instruction did not diassemble properly", + )); } - }; } +#[cfg(all(target_arch = "aarch64"))] +pub(crate) fn frida_to_cs( + decoder: InstDecoder, + frida_insn: &frida_gum_sys::Insn, +) -> Result { + let insn = disas_count(&decoder, frida_insn.bytes(), 4); + + if insn.len() < 1 { + log::error!( + "Failed to disassemble: {:#x}: {:?}", + frida_insn.address(), + frida_insn.bytes() + ); + return Err(Error::illegal_state( + "Instruction did not diassemble properly", + )); + } + return Ok(insn[0]); +} + #[cfg(target_arch = "x86_64")] /// Get the base, idx, scale, disp for each operand pub fn operand_details(operand: &Operand) -> Option<(X86Register, X86Register, u8, i32)> { diff --git a/libafl_libfuzzer/README.md b/libafl_libfuzzer/README.md index 6fcd0daba8..cff72c08a6 100644 --- a/libafl_libfuzzer/README.md +++ b/libafl_libfuzzer/README.md @@ -15,9 +15,7 @@ fuzzing environment or updating their harnesses. ## Usage -`libafl_libfuzzer` currently has known support for Rust, C, and C++ targets on Linux. -macOS has experimental support, but requires patching of LibAFL which leads to breaking changes elsewhere (ask in the -Discord for a patch file -- and [let us know what problems you face](https://github.com/AFLplusplus/LibAFL/issues/1564)). +`libafl_libfuzzer` currently has known support for Rust, C, and C++ targets on Linux and macOS. Windows is not currently supported, as we do not currently test or develop for Windows machines, but [we will happily hear what issues you face and patch them as possible](https://github.com/AFLplusplus/LibAFL/issues/1563). @@ -51,6 +49,22 @@ As this branch generally offers the highest performance version of `libafl_libfu Remember to `cargo update` often if using the experimental changes, and please [submit an issue] if you encounter problems while using `libfuzzer-best`! +#### macOS + +On macOS, you will need to add weak linking for some functions in a `build.rs` file: + +```rust +fn main() { + for func in [ + "_libafl_main", + "_LLVMFuzzerCustomMutator", + "_LLVMFuzzerCustomCrossOver", + ] { + println!("cargo:rustc-link-arg=-Wl,-U,{func}"); + } +} +``` + #### Caveats Like harnesses built with `libfuzzer-sys`, Rust targets which build other libraries (e.g. C/C++ FFI) may not diff --git a/libafl_libfuzzer/build.rs b/libafl_libfuzzer/build.rs index face269243..d6022f62ff 100644 --- a/libafl_libfuzzer/build.rs +++ b/libafl_libfuzzer/build.rs @@ -78,7 +78,7 @@ fn main() { assert!( command.status().map_or(false, |s| s.success()), - "Couldn't build runtime crate! Did you remember to use nightly? (`rustup default nightly` to install) Or, did you remember to install ucd-generate? (`cargo install ucd-generate` to install)" + "Couldn't build runtime crate! Did you remember to use nightly? (`rustup default nightly` to install)" ); let mut archive_path = custom_lib_dir.join(std::env::var_os("TARGET").unwrap()); diff --git a/libafl_libfuzzer/libafl_libfuzzer_runtime/Cargo.toml b/libafl_libfuzzer/libafl_libfuzzer_runtime/Cargo.toml index c4560bc7e8..a2a51f0871 100644 --- a/libafl_libfuzzer/libafl_libfuzzer_runtime/Cargo.toml +++ b/libafl_libfuzzer/libafl_libfuzzer_runtime/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "libafl_libfuzzer_runtime" -version = "0.11.2" +version = "0.12.0" edition = "2021" publish = false @@ -32,7 +32,7 @@ crate-type = ["staticlib", "rlib"] [dependencies] libafl = { path = "../../libafl", default-features = false, features = ["std", "derive", "llmp_compression", "rand_trait", "regex", "errors_backtrace", "serdeany_autoreg", "tui_monitor", "unicode"] } libafl_bolts = { path = "../../libafl_bolts", default-features = false, features = ["std", "derive", "llmp_compression", "rand_trait", "serdeany_autoreg", "errors_backtrace"] } -libafl_targets = { path = "../../libafl_targets", features = ["sancov_8bit", "sancov_cmplog", "libfuzzer", "libfuzzer_oom", "libfuzzer_define_run_driver", "libfuzzer_interceptors", "sanitizers_flags", "whole_archive"] } +libafl_targets = { path = "../../libafl_targets", features = ["sancov_8bit", "sancov_cmplog", "sancov_pcguard", "libfuzzer", "libfuzzer_oom", "libfuzzer_define_run_driver", "libfuzzer_interceptors", "sanitizers_flags", "whole_archive", "sanitizer_interfaces"] } ahash = { version = "0.8.3", default-features = false } libc = "0.2.139" @@ -48,7 +48,7 @@ utf8-chars = "3.0.1" env_logger = "0.10" [build-dependencies] -bindgen = "0.68.1" +bindgen = "0.69.4" cc = { version = "1.0", features = ["parallel"] } [workspace] diff --git a/libafl_libfuzzer/libafl_libfuzzer_runtime/build.rs b/libafl_libfuzzer/libafl_libfuzzer_runtime/build.rs index c2566b847f..f3926213dc 100644 --- a/libafl_libfuzzer/libafl_libfuzzer_runtime/build.rs +++ b/libafl_libfuzzer/libafl_libfuzzer_runtime/build.rs @@ -10,7 +10,7 @@ fn main() { let build = bindgen::builder() .header("src/harness_wrap.h") .generate_comments(true) - .parse_callbacks(Box::new(bindgen::CargoCallbacks)) + .parse_callbacks(Box::new(bindgen::CargoCallbacks::new())) .generate() .expect("Couldn't generate the harness wrapper!"); diff --git a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/corpus.rs b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/corpus.rs index 9f473bf22c..20f0ad3092 100644 --- a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/corpus.rs +++ b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/corpus.rs @@ -7,7 +7,10 @@ use std::{ }; use libafl::{ - corpus::{inmemory::TestcaseStorage, Corpus, CorpusId, Testcase}, + corpus::{ + inmemory::{TestcaseStorage, TestcaseStorageMap}, + Corpus, CorpusId, Testcase, + }, inputs::{Input, UsesInput}, }; use libafl_bolts::Error; @@ -51,7 +54,7 @@ where } /// Touch this index and maybe evict an entry if we have touched an input which was unloaded. - fn touch(&self, idx: CorpusId) -> Result<(), Error> { + fn touch(&self, idx: CorpusId, corpus: &TestcaseStorageMap) -> Result<(), Error> { let mut loaded_mapping = self.loaded_mapping.borrow_mut(); let mut loaded_entries = self.loaded_entries.borrow_mut(); match loaded_mapping.entry(idx) { @@ -71,7 +74,7 @@ where } if loaded_entries.len() > self.max_len { let idx = loaded_entries.pop_first().unwrap().1; // cannot panic - let cell = self.mapping.get(idx).ok_or_else(|| { + let cell = corpus.get(idx).ok_or_else(|| { Error::key_not_found(format!("Tried to evict non-existent entry {idx}")) })?; let mut tc = cell.try_borrow_mut()?; @@ -79,27 +82,32 @@ where } Ok(()) } -} - -impl UsesInput for LibfuzzerCorpus -where - I: Input + Serialize + for<'de> Deserialize<'de>, -{ - type Input = I; -} - -impl Corpus for LibfuzzerCorpus -where - I: Input + Serialize + for<'de> Deserialize<'de>, -{ - fn count(&self) -> usize { - self.mapping.map.len() + #[inline] + fn _get<'a>( + &'a self, + id: CorpusId, + corpus: &'a TestcaseStorageMap, + ) -> Result<&RefCell>, Error> { + self.touch(id, corpus)?; + corpus.map.get(&id).map(|item| &item.testcase).ok_or_else(|| Error::illegal_state("Nonexistent corpus entry {id} requested (present in loaded entries, but not the mapping?)")) } - fn add(&mut self, testcase: Testcase) -> Result { - let idx = self.mapping.insert(RefCell::new(testcase)); - let mut testcase = self.mapping.get(idx).unwrap().borrow_mut(); - + fn _add( + &mut self, + testcase: RefCell>, + is_disabled: bool, + ) -> Result { + let idx = if is_disabled { + self.mapping.insert_disabled(testcase) + } else { + self.mapping.insert(testcase) + }; + let corpus = if is_disabled { + &self.mapping.disabled + } else { + &self.mapping.enabled + }; + let mut testcase = corpus.get(idx).unwrap().borrow_mut(); match testcase.file_path() { Some(path) if path.canonicalize()?.starts_with(&self.corpus_dir) => { // if it's already in the correct dir, we retain it @@ -114,7 +122,7 @@ where let path = self.corpus_dir.join(&name); match input.to_file(&path) { - Err(Error::File(e, _)) if e.kind() == ErrorKind::AlreadyExists => { + Err(Error::OsError(e, ..)) if e.kind() == ErrorKind::AlreadyExists => { // we do not care if the file already exists; in this case, we assume it is equal } res => res?, @@ -126,10 +134,40 @@ where testcase.file_path_mut().replace(path); } }; - - self.touch(idx)?; + self.touch(idx, corpus)?; Ok(idx) } +} + +impl UsesInput for LibfuzzerCorpus +where + I: Input + Serialize + for<'de> Deserialize<'de>, +{ + type Input = I; +} + +impl Corpus for LibfuzzerCorpus +where + I: Input + Serialize + for<'de> Deserialize<'de>, +{ + #[inline] + fn count(&self) -> usize { + self.mapping.enabled.map.len() + } + #[inline] + fn count_disabled(&self) -> usize { + self.mapping.disabled.map.len() + } + #[inline] + fn count_all(&self) -> usize { + self.count_disabled().saturating_add(self.count_disabled()) + } + fn add(&mut self, testcase: Testcase) -> Result { + self._add(RefCell::new(testcase), false) + } + fn add_disabled(&mut self, testcase: Testcase) -> Result { + self._add(RefCell::new(testcase), true) + } fn replace( &mut self, @@ -144,10 +182,16 @@ where } fn get(&self, id: CorpusId) -> Result<&RefCell>, Error> { - self.touch(id)?; - self.mapping.map.get(&id).map(|item| &item.testcase).ok_or_else(|| Error::illegal_state("Nonexistent corpus entry {id} requested (present in loaded entries, but not the mapping?)")) + self._get(id, &self.mapping.enabled) } + fn get_from_all(&self, id: CorpusId) -> Result<&RefCell>, Error> { + match self._get(id, &self.mapping.enabled) { + Ok(input) => Ok(input), + Err(Error::KeyNotFound(..)) => return self._get(id, &self.mapping.disabled), + Err(e) => Err(e), + } + } fn current(&self) -> &Option { &self.current } @@ -157,19 +201,29 @@ where } fn next(&self, id: CorpusId) -> Option { - self.mapping.next(id) + self.mapping.enabled.next(id) } fn prev(&self, id: CorpusId) -> Option { - self.mapping.prev(id) + self.mapping.enabled.prev(id) } fn first(&self) -> Option { - self.mapping.first() + self.mapping.enabled.first() } fn last(&self) -> Option { - self.mapping.last() + self.mapping.enabled.last() + } + + /// Get the nth corpus id; considers both enabled and disabled testcases + #[inline] + fn nth_from_all(&self, nth: usize) -> CorpusId { + let enabled_count = self.count(); + if nth >= enabled_count { + return self.mapping.disabled.keys[nth.saturating_sub(enabled_count)]; + } + self.mapping.enabled.keys[nth] } fn load_input_into(&self, testcase: &mut Testcase) -> Result<(), Error> { @@ -192,7 +246,7 @@ where Error::empty("The testcase, when being saved, must have a file path!") })?; match input.to_file(path) { - Err(Error::File(e, _)) if e.kind() == ErrorKind::AlreadyExists => { + Err(Error::OsError(e, ..)) if e.kind() == ErrorKind::AlreadyExists => { // we do not care if the file already exists; in this case, we assume it is equal Ok(()) } @@ -239,6 +293,16 @@ where self.count } + // ArtifactCorpus disregards disabled entries + fn count_disabled(&self) -> usize { + 0 + } + + fn count_all(&self) -> usize { + // count_disabled will always return 0 + self.count() + self.count_disabled() + } + fn add(&mut self, testcase: Testcase) -> Result { let idx = self.count; self.count += 1; @@ -250,7 +314,7 @@ where Error::illegal_state("Should have set the path in the LibfuzzerCrashCauseFeedback.") })?; match input.to_file(path) { - Err(Error::File(e, _)) if e.kind() == ErrorKind::AlreadyExists => { + Err(Error::OsError(e, ..)) if e.kind() == ErrorKind::AlreadyExists => { // we do not care if the file already exists; in this case, we assume it is equal } res => res?, @@ -262,6 +326,10 @@ where Ok(CorpusId::from(idx)) } + fn add_disabled(&mut self, _testcase: Testcase) -> Result { + unimplemented!("ArtifactCorpus disregards disabled inputs") + } + fn replace( &mut self, _idx: CorpusId, @@ -288,6 +356,16 @@ where maybe_last.ok_or_else(|| Error::illegal_argument("Can only get the last corpus ID.")) } + // This just calls Self::get as ArtifactCorpus disregards disabled entries + fn get_from_all(&self, id: CorpusId) -> Result<&RefCell>, Error> { + self.get(id) + } + + // This just calls Self::nth as ArtifactCorpus disregards disabled entries + fn nth_from_all(&self, nth: usize) -> CorpusId { + self.nth(nth) + } + fn current(&self) -> &Option { unimplemented!("Artifact prefix is thin and cannot get, replace, or remove.") } diff --git a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/feedbacks.rs b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/feedbacks.rs index 9b7d5dbcb1..72ad044443 100644 --- a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/feedbacks.rs +++ b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/feedbacks.rs @@ -1,5 +1,6 @@ use alloc::rc::Rc; use core::{cell::RefCell, fmt::Debug}; +use std::borrow::Cow; use libafl::{ alloc, @@ -9,8 +10,8 @@ use libafl::{ feedbacks::{Feedback, MinMapFeedback}, inputs::{BytesInput, Input}, observers::ObserversTuple, - state::{HasMetadata, State}, - Error, + state::State, + Error, HasMetadata, }; use libafl_bolts::{impl_serdeany, Named}; use libafl_targets::OomFeedback; @@ -36,8 +37,9 @@ impl LibfuzzerKeepFeedback { } impl Named for LibfuzzerKeepFeedback { - fn name(&self) -> &str { - "libfuzzer-keep" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("libfuzzer-keep"); + &NAME } } @@ -90,8 +92,9 @@ impl LibfuzzerCrashCauseFeedback { } impl Named for LibfuzzerCrashCauseFeedback { - fn name(&self) -> &str { - "crash-cause" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("crash-cause"); + &NAME } } @@ -131,9 +134,10 @@ where Ok(false) } - fn append_metadata( + fn append_metadata( &mut self, _state: &mut S, + _manager: &mut EM, _observers: &OT, testcase: &mut Testcase, ) -> Result<(), Error> @@ -170,4 +174,4 @@ where } } -pub type ShrinkMapFeedback = MinMapFeedback, S, usize>; +pub type ShrinkMapFeedback = MinMapFeedback, usize>; diff --git a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/fuzz.rs b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/fuzz.rs index 07e1551794..00564631d1 100644 --- a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/fuzz.rs +++ b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/fuzz.rs @@ -23,8 +23,8 @@ use libafl::{ Monitor, MultiMonitor, SimpleMonitor, }, stages::{HasCurrentStage, StagesTuple}, - state::{HasExecutions, HasLastReportTime, HasMetadata, HasSolutions, UsesState}, - Error, Fuzzer, + state::{HasExecutions, HasLastReportTime, HasSolutions, UsesState}, + Error, Fuzzer, HasMetadata, }; use libafl_bolts::{ core_affinity::Cores, @@ -175,7 +175,7 @@ where }) } -fn create_monitor_closure() -> impl Fn(String) + Clone { +fn create_monitor_closure() -> impl Fn(&str) + Clone { #[cfg(unix)] let stderr_fd = std::os::fd::RawFd::from_str(&std::env::var(crate::STDERR_FD_VAR).unwrap()).unwrap(); // set in main diff --git a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/lib.rs b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/lib.rs index 1c790ee4c5..9e465bab4c 100644 --- a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/lib.rs +++ b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/lib.rs @@ -105,6 +105,7 @@ mod harness_wrap { #![allow(improper_ctypes)] #![allow(clippy::unreadable_literal)] #![allow(missing_docs)] + #![allow(unused_qualifications)] include!(concat!(env!("OUT_DIR"), "/harness_wrap.rs")); } @@ -149,7 +150,6 @@ impl CustomMutationStatus { macro_rules! fuzz_with { ($options:ident, $harness:ident, $operation:expr, $and_then:expr, $edge_maker:expr) => {{ use libafl_bolts::{ - current_nanos, rands::StdRand, tuples::{Merge, tuple_list}, AsSlice, @@ -167,7 +167,7 @@ macro_rules! fuzz_with { I2SRandReplace, StdScheduledMutator, StringCategoryRandMutator, StringSubcategoryRandMutator, StringCategoryTokenReplaceMutator, StringSubcategoryTokenReplaceMutator, Tokens, tokens_mutations }, - observers::{stacktrace::BacktraceObserver, TimeObserver}, + observers::{stacktrace::BacktraceObserver, TimeObserver, CanTrack}, schedulers::{ IndexesLenTimeMinimizerScheduler, powersched::PowerSchedule, PowerQueueScheduler, }, @@ -197,7 +197,7 @@ macro_rules! fuzz_with { let grimoire_metadata = should_use_grimoire(&mut state, &$options, &mutator_status)?; let grimoire = grimoire_metadata.should(); - let edges_observer = edge_maker(); + let edges_observer = edge_maker().track_indices().track_novelties(); let size_edges_observer = MappedEdgeMapObserver::new(edge_maker(), SizeValueObserver::default()); let keep_observer = LibfuzzerKeepFeedback::new(); @@ -219,8 +219,8 @@ macro_rules! fuzz_with { ); // New maximization map feedback linked to the edges observer - let map_feedback = MaxMapFeedback::tracking(&edges_observer, true, true); - let shrinking_map_feedback = ShrinkMapFeedback::tracking(&size_edges_observer, false, false); + let map_feedback = MaxMapFeedback::new(&edges_observer); + let shrinking_map_feedback = ShrinkMapFeedback::new(&size_edges_observer); // Set up a generalization stage for grimoire let generalization = GeneralizationStage::new(&edges_observer); @@ -243,7 +243,7 @@ macro_rules! fuzz_with { map_feedback, feedback_and_fast!(ConstFeedback::new($options.shrink()), shrinking_map_feedback), // Time feedback, this one does not need a feedback state - TimeFeedback::with_observer(&time_observer) + TimeFeedback::new(&time_observer) ) ); @@ -281,7 +281,7 @@ macro_rules! fuzz_with { let mut state = state.unwrap_or_else(|| { StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance LibfuzzerCorpus::new(corpus_dir.clone(), 4096), // Corpus in which we store solutions (crashes in this example), @@ -411,7 +411,7 @@ macro_rules! fuzz_with { let grimoire = IfStage::new(|_, _, _, _| Ok(grimoire.into()), (StdMutationalStage::transforming(grimoire_mutator), ())); // A minimization+queue policy to get testcasess from the corpus - let scheduler = IndexesLenTimeMinimizerScheduler::new(PowerQueueScheduler::new(&mut state, &edges_observer, PowerSchedule::FAST)); + let scheduler = IndexesLenTimeMinimizerScheduler::new(&edges_observer, PowerQueueScheduler::new(&mut state, &edges_observer, PowerSchedule::FAST)); // A fuzzer with feedbacks and a corpus scheduler let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); diff --git a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/merge.rs b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/merge.rs index 569255ac08..80f288821d 100644 --- a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/merge.rs +++ b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/merge.rs @@ -4,7 +4,8 @@ use std::{ fs::{rename, File}, io::Write, os::fd::{AsRawFd, FromRawFd}, - time::{SystemTime, UNIX_EPOCH}, ptr::addr_of_mut, + ptr::addr_of_mut, + time::{SystemTime, UNIX_EPOCH}, }; use libafl::{ @@ -21,7 +22,7 @@ use libafl::{ Error, HasScheduler, StdFuzzer, }; use libafl_bolts::{ - rands::{Rand, RandomSeed, StdRand}, + rands::{Rand, StdRand}, shmem::{ShMemProvider, StdShMemProvider}, tuples::tuple_list, AsSlice, @@ -104,7 +105,7 @@ pub fn merge( let edges_observer = MappedEdgeMapObserver::new(edges_observer, SizeTimeValueObserver::new(time)); - let map_feedback = MinMapFeedback::tracking(&edges_observer, false, true); + let map_feedback = MinMapFeedback::new(&edges_observer); // Create an OOM observer to monitor if an OOM has occurred let oom_observer = OomObserver::new(options.rss_limit(), options.malloc_limit()); diff --git a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/misc.rs b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/misc.rs index 7b496a8283..89e7c32999 100644 --- a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/misc.rs +++ b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/misc.rs @@ -3,7 +3,7 @@ use std::{ path::PathBuf, }; -use libafl::{state::HasMetadata, Error}; +use libafl::{Error, HasMetadata}; use libafl_bolts::impl_serdeany; use serde::{Deserialize, Serialize}; use utf8_chars::BufReadCharsExt; diff --git a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/observers.rs b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/observers.rs index 22ada45c5a..a3f95e69a4 100644 --- a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/observers.rs +++ b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/observers.rs @@ -1,6 +1,8 @@ use std::{ + borrow::Cow, fmt::Debug, hash::{Hash, Hasher}, + ops::Deref, }; use ahash::AHasher; @@ -37,7 +39,7 @@ pub trait ValueObserver: for<'de> Deserialize<'de> + Serialize + Debug + Named { #[derive(Deserialize, Serialize, Debug)] pub struct MappedEdgeMapObserver { inner: M, - name: String, + name: Cow<'static, str>, value_observer: O, } @@ -48,13 +50,25 @@ where { pub fn new(obs: M, value_obs: O) -> Self { Self { - name: format!("{}_{}", value_obs.name(), obs.name()), + name: Cow::from(format!("{}_{}", value_obs.name(), obs.name())), inner: obs, value_observer: value_obs, } } } +impl AsRef for MappedEdgeMapObserver { + fn as_ref(&self) -> &Self { + self + } +} + +impl AsMut for MappedEdgeMapObserver { + fn as_mut(&mut self) -> &mut Self { + self + } +} + impl HasLen for MappedEdgeMapObserver where M: HasLen, @@ -65,11 +79,28 @@ where } impl Named for MappedEdgeMapObserver { - fn name(&self) -> &str { + fn name(&self) -> &Cow<'static, str> { &self.name } } +impl Hash for MappedEdgeMapObserver +where + M: MapObserver + for<'it> AsIter<'it, Item = M::Entry>, + O: ValueObserver, +{ + fn hash(&self, hasher: &mut H) { + let initial = self.inner.initial(); + for e in self.inner.as_iter() { + if *e == initial { + self.value_observer.default_value().hash(hasher); + } else { + self.value_observer.value().hash(hasher); + } + } + } +} + impl MapObserver for MappedEdgeMapObserver where M: MapObserver + for<'it> AsIter<'it, Item = M::Entry>, @@ -77,16 +108,16 @@ where { type Entry = O::ValueType; - fn get(&self, idx: usize) -> &Self::Entry { + fn get(&self, idx: usize) -> Self::Entry { let initial = self.inner.initial(); - if *self.inner.get(idx) == initial { - self.value_observer.default_value() + if self.inner.get(idx) == initial { + *self.value_observer.default_value() } else { - self.value_observer.value() + *self.value_observer.value() } } - fn get_mut(&mut self, _idx: usize) -> &mut Self::Entry { + fn set(&mut self, _idx: usize, _val: Self::Entry) { unimplemented!("Impossible to implement for a proxy map.") } @@ -98,16 +129,9 @@ where self.inner.count_bytes() } - fn hash(&self) -> u64 { + fn hash_simple(&self) -> u64 { let mut hasher = AHasher::default(); - let initial = self.inner.initial(); - for e in self.inner.as_iter() { - if *e == initial { - self.value_observer.default_value().hash(&mut hasher); - } else { - self.value_observer.value().hash(&mut hasher); - } - } + self.hash(&mut hasher); hasher.finish() } @@ -125,7 +149,7 @@ where let value = *self.value_observer.value(); self.inner .as_iter() - .map(|&e| if e == initial { default } else { value }) + .map(|e| if *e == initial { default } else { value }) .collect() } @@ -179,9 +203,10 @@ impl<'it, I, O, T> MappedEdgeMapIter<'it, I, O, T> { } } -impl<'it, I, O, T> Iterator for MappedEdgeMapIter<'it, I, O, T> +impl<'it, I, O, R, T> Iterator for MappedEdgeMapIter<'it, I, O, T> where - I: Iterator, + I: Iterator, + R: Deref, T: PartialEq + 'it, O: ValueObserver, { @@ -202,6 +227,7 @@ where O: ValueObserver + 'it, { type Item = O::ValueType; + type Ref = &'it Self::Item; type IntoIter = MappedEdgeMapIter<'it, >::IntoIter, O, M::Entry>; fn as_iter(&'it self) -> Self::IntoIter { @@ -229,8 +255,9 @@ impl ValueObserver for SizeValueObserver { } impl Named for SizeValueObserver { - fn name(&self) -> &str { - "size" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("size"); + &NAME } } @@ -273,7 +300,7 @@ impl ValueObserver for TimeValueObserver { } impl Named for TimeValueObserver { - fn name(&self) -> &str { + fn name(&self) -> &Cow<'static, str> { self.time_obs.name() } } @@ -334,8 +361,9 @@ impl ValueObserver for SizeTimeValueObserver { } impl Named for SizeTimeValueObserver { - fn name(&self) -> &str { - "size_time" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("size_time"); + &NAME } } diff --git a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/options.rs b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/options.rs index 67965947b5..98d9172234 100644 --- a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/options.rs +++ b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/options.rs @@ -131,7 +131,7 @@ impl LibfuzzerOptions { let name = if let Some(executable) = std::env::current_exe().ok().and_then(|path| { path.file_name() .and_then(std::ffi::OsStr::to_str) - .map(std::string::ToString::to_string) + .map(ToString::to_string) }) { executable } else { @@ -394,11 +394,7 @@ impl<'a> LibfuzzerOptionsBuilder<'a> { tui: self.tui, runs: self.runs, close_fd_mask: self.close_fd_mask, - unknown: self - .unknown - .into_iter() - .map(std::string::ToString::to_string) - .collect(), + unknown: self.unknown.into_iter().map(ToString::to_string).collect(), } } } diff --git a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/report.rs b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/report.rs index ed50a571a6..7e4353ee98 100644 --- a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/report.rs +++ b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/report.rs @@ -3,12 +3,12 @@ use std::ffi::c_int; use libafl::{ events::{ProgressReporter, SimpleEventManager}, executors::HasObservers, - feedbacks::{MapFeedbackMetadata, MAPFEEDBACK_PREFIX}, + feedbacks::MapFeedbackMetadata, inputs::UsesInput, monitors::SimpleMonitor, stages::{HasCurrentStage, StagesTuple}, - state::{HasExecutions, HasLastReportTime, HasMetadata, HasNamedMetadata}, - Error, Fuzzer, + state::{HasExecutions, HasLastReportTime}, + Error, Fuzzer, HasMetadata, HasNamedMetadata, }; use crate::{fuzz_with, options::LibfuzzerOptions}; @@ -35,7 +35,7 @@ where ST: StagesTuple, { let meta = state - .named_metadata::>(&(MAPFEEDBACK_PREFIX.to_string() + "edges")) + .named_metadata::>("edges") .unwrap(); let observed = meta.history_map.iter().filter(|&&e| e != 0).count(); let total = meta.history_map.len(); diff --git a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/schedulers.rs b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/schedulers.rs index 30b45f0f74..8929671061 100644 --- a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/schedulers.rs +++ b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/schedulers.rs @@ -8,8 +8,8 @@ use libafl::{ feedbacks::MapNoveltiesMetadata, inputs::UsesInput, schedulers::{RemovableScheduler, Scheduler}, - state::{HasCorpus, HasMetadata, State, UsesState}, - Error, + state::{HasCorpus, State, UsesState}, + Error, HasMetadata, }; #[derive(Clone, Debug)] diff --git a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/tmin.rs b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/tmin.rs index aae4d56a83..6dfb4f1d9d 100644 --- a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/tmin.rs +++ b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/tmin.rs @@ -7,7 +7,7 @@ use libafl::{ corpus::{Corpus, HasTestcase, InMemoryCorpus, Testcase}, events::SimpleEventManager, executors::{inprocess_fork::InProcessForkExecutor, ExitKind}, - feedbacks::{CrashFeedbackFactory, TimeoutFeedbackFactory}, + feedbacks::{CrashFeedback, TimeoutFeedbackFactory}, inputs::{BytesInput, HasBytesVec, HasTargetBytes}, mutators::{havoc_mutations_no_crossover, Mutator, StdScheduledMutator}, schedulers::QueueScheduler, @@ -16,7 +16,7 @@ use libafl::{ Error, Fuzzer, StdFuzzer, }; use libafl_bolts::{ - rands::{RandomSeed, RomuDuoJrRand, StdRand}, + rands::{RomuDuoJrRand, StdRand}, shmem::{ShMemProvider, StdShMemProvider}, tuples::tuple_list, AsSlice, HasLen, @@ -79,7 +79,7 @@ fn minimize_crash_with_mutator>( match exit_kind { ExitKind::Crash => { - let factory = CrashFeedbackFactory::default(); + let factory = CrashFeedback::new(); let tmin = StdTMinMutationalStage::new( mutator, factory, diff --git a/libafl_libfuzzer/src/lib.rs b/libafl_libfuzzer/src/lib.rs index b3ad2c48bd..b995ece126 100644 --- a/libafl_libfuzzer/src/lib.rs +++ b/libafl_libfuzzer/src/lib.rs @@ -97,14 +97,14 @@ extern "C" { target_family = "unix", // Disable when building with clippy, as it will complain about the missing environment // variable which is set by the build script, which is not run under clippy. - not(feature = "cargo-clippy") + not(clippy) ))] pub const LIBAFL_LIBFUZZER_RUNTIME_LIBRARY: &'static [u8] = include_bytes!(env!("LIBAFL_LIBFUZZER_RUNTIME_PATH")); #[cfg(test)] mod tests { - #[cfg(all(feature = "embed-runtime", not(feature = "cargo-clippy")))] + #[cfg(all(feature = "embed-runtime", not(clippy)))] #[test] fn test_embed_runtime_sized() { use crate::LIBAFL_LIBFUZZER_RUNTIME_LIBRARY; diff --git a/libafl_nyx/Cargo.toml b/libafl_nyx/Cargo.toml index c4fa0746b9..89a2e50592 100644 --- a/libafl_nyx/Cargo.toml +++ b/libafl_nyx/Cargo.toml @@ -14,7 +14,10 @@ categories = ["development-tools::testing", "emulators", "embedded", "os", "no-s # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [target.'cfg(target_os = "linux")'.dependencies] -libnyx = {git = "https://github.com/nyx-fuzz/libnyx.git",rev = "acaf7f6"} -libafl = { path = "../libafl", version = "0.11.2", features = ["std", "libafl_derive", "frida_cli" ]} -libafl_bolts = { path = "../libafl_bolts", version = "0.11.2", features = ["std", "libafl_derive", "frida_cli" ]} -libafl_targets = { path = "../libafl_targets", version = "0.11.2", features = ["std", "sancov_cmplog"] } +libnyx = { git = "https://github.com/nyx-fuzz/libnyx.git", rev = "6833d23" } +libafl = { path = "../libafl", version = "0.12.0", features = ["std", "libafl_derive", "frida_cli" ]} +libafl_bolts = { path = "../libafl_bolts", version = "0.12.0", features = ["std", "libafl_derive", "frida_cli" ]} +libafl_targets = { path = "../libafl_targets", version = "0.12.0", features = ["std", "sancov_cmplog"] } + +nix = { version = "0.28.0", features = ["fs"] } +typed-builder = "0.18.1" diff --git a/libafl_nyx/Makefile.libxdc b/libafl_nyx/Makefile.libxdc new file mode 100644 index 0000000000..26ee37696c --- /dev/null +++ b/libafl_nyx/Makefile.libxdc @@ -0,0 +1,54 @@ +CC ?= gcc +CFLAGS += -Ofast -fPIC -fvisibility=hidden -finline-functions +LDFLAGS = + +ifneq ($(origin NO_LTO), environment) + CFLAGS += -flto + LDFLAGS += -flto +endif + +PREFIX ?= /usr + +ODIR=build +SDIR=src + +_OBJ = cfg.o disassembler.o tnt_cache.o decoder.o libxdc.o mmh3.o trace_cache.o +OBJ = $(patsubst %,$(ODIR)/%,$(_OBJ)) + +default: libxdc.so libxdc.a ptdump ptdump_static + +$(ODIR)/%.o: $(SDIR)/%.c $(SDIR)/*.h libxdc.h + mkdir -p build + $(CC) -c -o $@ $< $(CFLAGS) + +libxdc.so: $(OBJ) + $(CC) $^ -o $@ -shared $(CFLAGS) $(LDFLAGS) -L../capstone_v4/ -l:libcapstone.so.4 + +libxdc.a: $(OBJ) + $(AR) rcs $@ $^ + +ptdump: libxdc.so test/*.c test/*.h + $(CC) test/ptdump.c test/page_cache.c test/helper.c -o build/$@ -Itest/ -I./ -Lbuild/ $(CFLAGS) $(LDFLAGS) -L. -lxdc -L../capstone_v4/ -l:libcapstone.so.4 + +ptdump_static: libxdc.a test/*.c test/*.h + $(CC) test/ptdump.c test/page_cache.c test/helper.c -o build/$@ -Itest/ -I./ $(CFLAGS) $(LDFLAGS) -L. -l:libxdc.a -L../capstone_v4/ -l:libcapstone.a + +tester_dyn: libxdc.so test/*.c test/*.h + $(CC) test/tester.c test/page_cache.c test/helper.c -o $@ -Itest/ -I./ $(CFLAGS) $(LDFLAGS) -L. -lxdc -L../capstone_v4/ -l:libcapstone.so.4 + +tester_static: libxdc.a test/*.c test/*.h + $(CC) test/tester.c test/page_cache.c test/helper.c -o $@ -Itest/ -I./ $(CFLAGS) $(LDFLAGS) -L. -l:libxdc.a -L../capstone_v4/ -l:libcapstone.a + +install: libxdc.so libxdc.a ptdump + mkdir -p $(PREFIX)/include $(PREFIX)/lib + install -m0644 libxdc.h $(PREFIX)/include/ + install -m0755 libxdc.so $(PREFIX)/lib/ + install -m0755 libxdc.a $(PREFIX)/lib/ + install -m0755 build/ptdump $(PREFIX)/bin/ + +.PHONY: clean install + +clean: + rm -f $(ODIR)/*.o build/* + rm -f libxdc.so + rm -f libxdc.a diff --git a/libafl_nyx/build_nyx_support.sh b/libafl_nyx/build_nyx_support.sh index d187068fe2..311436603c 100755 --- a/libafl_nyx/build_nyx_support.sh +++ b/libafl_nyx/build_nyx_support.sh @@ -13,21 +13,23 @@ if [ ! -e ./QEMU-Nyx/.git ]; then rm -rf ./QEMU-Nyx git clone https://github.com/nyx-fuzz/QEMU-Nyx.git || exit 1 pushd QEMU-Nyx - git reset --hard 80f22f77d6aab14e62bf11c80db4e210bbca5fb5 + git reset --hard e5e1c4c21ff9c4dc80e6409d4eab47146c6024cd popd fi if [ ! -e ./packer/.git ]; then rm -rf ./packer - git clone https://github.com/syheliel/packer.git || exit 1 + git clone https://github.com/nyx-fuzz/packer || exit 1 pushd packer - git reset --hard 86b159bafc0b2ba8feeaa8761a45b6201d34084f + git reset --hard bcf3e248b660764f48af54232a3388389a2dfc22 popd fi git submodule init || exit 1 echo "[*] initializing QEMU-Nyx submodule" -git submodule update ./QEMU-Nyx 2>/dev/null # ignore errors +cd QEMU-Nyx/ || return +git submodule update --init . +cd .. echo "[*] initializing packer submodule" git submodule update ./packer 2>/dev/null # ignore errors @@ -35,21 +37,21 @@ git submodule update ./packer 2>/dev/null # ignore errors test -e packer/.git || { echo "[-] packer not checked out, please install git or check your internet connection." ; exit 1 ; } test -e QEMU-Nyx/.git || { echo "[-] QEMU-Nyx not checked out, please install git or check your internet connection." ; exit 1 ; } -echo "[*] checking packer init.cpio.gz ..." -if [ ! -f "packer/linux_initramfs/init.cpio.gz" ]; then - cd packer/linux_initramfs/ || return - sh pack.sh || exit 1 - cd ../../ -fi - - echo "[*] Checking QEMU-Nyx ..." if [ ! -f "QEMU-Nyx/x86_64-softmmu/qemu-system-x86_64" ]; then cd QEMU-Nyx/ || return + cp ../Makefile.libxdc ./libxdc/Makefile || exit 1 ./compile_qemu_nyx.sh lto || exit 1 cd .. fi +echo "[*] checking packer init.cpio.gz ..." +if [ ! -f "packer/linux_initramfs/init.cpio.gz" ]; then + cd packer/linux_initramfs/ || return + sh pack.sh || exit 1 + cd ../../ +fi + echo "[+] All done for nyx_mode, enjoy!" exit 0 diff --git a/libafl_nyx/src/executor.rs b/libafl_nyx/src/executor.rs index e8ca7282a0..334082d131 100644 --- a/libafl_nyx/src/executor.rs +++ b/libafl_nyx/src/executor.rs @@ -1,4 +1,8 @@ -use std::{fmt::Debug, marker::PhantomData}; +use std::{ + io::{Read, Seek}, + marker::PhantomData, + os::fd::AsRawFd, +}; use libafl::{ executors::{Executor, ExitKind, HasObservers}, @@ -7,37 +11,29 @@ use libafl::{ state::{HasExecutions, State, UsesState}, Error, }; -use libafl_bolts::AsSlice; +use libafl_bolts::{tuples::RefIndexable, AsSlice}; use libnyx::NyxReturnValue; use crate::helper::NyxHelper; /// executor for nyx standalone mode -pub struct NyxExecutor<'a, S, OT> { +pub struct NyxExecutor { /// implement nyx function - pub helper: &'a mut NyxHelper, + pub helper: NyxHelper, /// observers observers: OT, /// phantom data to keep generic type phantom: PhantomData, } -impl<'a, S, OT> Debug for NyxExecutor<'a, S, OT> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("NyxInprocessExecutor") - .field("helper", &self.helper) - .finish() - } -} - -impl<'a, S, OT> UsesState for NyxExecutor<'a, S, OT> +impl UsesState for NyxExecutor where S: State, { type State = S; } -impl<'a, S, OT> UsesObservers for NyxExecutor<'a, S, OT> +impl UsesObservers for NyxExecutor where OT: ObserversTuple, S: State, @@ -45,12 +41,13 @@ where type Observers = OT; } -impl<'a, EM, S, Z, OT> Executor for NyxExecutor<'a, S, OT> +impl Executor for NyxExecutor where EM: UsesState, S: State + HasExecutions, S::Input: HasTargetBytes, Z: UsesState, + OT: ObserversTuple, { fn run_target( &mut self, @@ -60,65 +57,100 @@ where input: &Self::Input, ) -> Result { *state.executions_mut() += 1; - let input_owned = input.target_bytes(); - let input = input_owned.as_slice(); - self.helper.nyx_process.set_input( - input, - input - .len() - .try_into() - .expect("Inputs larger than 4GB not supported"), - ); + + let bytes = input.target_bytes(); + let buffer = bytes.as_slice(); + + if buffer.len() > self.helper.nyx_process.input_buffer_size() { + return Err(Error::illegal_state(format!( + "Input does not fit in the Nyx input buffer.\ + You may want to increase the Nyx input buffer size: {} > {}", + buffer.len(), + self.helper.nyx_process.input_buffer_size() + ))); + } + + self.helper + .nyx_stdout + .set_len(0) + .map_err(|e| Error::illegal_state(format!("Failed to clear Nyx stdout: {e}")))?; + + let size = u32::try_from(buffer.len()) + .map_err(|_| Error::unsupported("Inputs larger than 4GB are not supported"))?; + // Duplicate the file descriptor since QEMU(?) closes it and we + // want to keep |self.helper.nyx_stdout| open. + let hprintf_fd = nix::unistd::dup(self.helper.nyx_stdout.as_raw_fd()) + .map_err(|e| Error::illegal_state(format!("Failed to duplicate Nyx stdout fd: {e}")))?; + + self.helper.nyx_process.set_input(buffer, size); + self.helper.nyx_process.set_hprintf_fd(hprintf_fd); // exec will take care of trace_bits, so no need to reset - let ret_val = self.helper.nyx_process.exec(); - match ret_val { - NyxReturnValue::Normal => Ok(ExitKind::Ok), - NyxReturnValue::Crash | NyxReturnValue::Asan => Ok(ExitKind::Crash), - NyxReturnValue::Timeout => Ok(ExitKind::Timeout), - NyxReturnValue::InvalidWriteToPayload => Err(libafl::Error::illegal_state( - "FixMe: Nyx InvalidWriteToPayload handler is missing", - )), - NyxReturnValue::Error => Err(libafl::Error::illegal_state( - "Error: Nyx runtime error has occurred...", - )), + let exit_kind = match self.helper.nyx_process.exec() { + NyxReturnValue::Normal => ExitKind::Ok, + NyxReturnValue::Crash | NyxReturnValue::Asan => ExitKind::Crash, + NyxReturnValue::Timeout => ExitKind::Timeout, + NyxReturnValue::InvalidWriteToPayload => { + self.helper.nyx_process.shutdown(); + return Err(Error::illegal_state( + "FixMe: Nyx InvalidWriteToPayload handler is missing", + )); + } + NyxReturnValue::Error => { + self.helper.nyx_process.shutdown(); + return Err(Error::illegal_state("Nyx runtime error has occurred")); + } NyxReturnValue::IoError => { - // todo! *stop_soon_p = 0 - Err(libafl::Error::unknown("Error: QEMU-nyx died...")) + self.helper.nyx_process.shutdown(); + return Err(Error::unknown("QEMU-nyx died")); } NyxReturnValue::Abort => { self.helper.nyx_process.shutdown(); - Err(libafl::Error::shutting_down()) + return Err(Error::shutting_down()); } + }; + + if self.observers.observes_stdout() { + let mut stdout = Vec::new(); + self.helper.nyx_stdout.rewind()?; + self.helper + .nyx_stdout + .read_to_end(&mut stdout) + .map_err(|e| Error::illegal_state(format!("Failed to read Nyx stdout: {e}")))?; + self.observers.observe_stdout(&stdout); } + + Ok(exit_kind) } } -impl<'a, S, OT> NyxExecutor<'a, S, OT> { - pub fn new(helper: &'a mut NyxHelper, observers: OT) -> Result { - Ok(Self { +impl NyxExecutor { + pub fn new(helper: NyxHelper, observers: OT) -> Self { + Self { helper, observers, phantom: PhantomData, - }) + } } /// convert `trace_bits` ptr into real trace map pub fn trace_bits(self) -> &'static mut [u8] { - unsafe { std::slice::from_raw_parts_mut(self.helper.trace_bits, self.helper.real_map_size) } + unsafe { + std::slice::from_raw_parts_mut(self.helper.bitmap_buffer, self.helper.bitmap_size) + } } } -impl<'a, S, OT> HasObservers for NyxExecutor<'a, S, OT> +impl HasObservers for NyxExecutor where S: State, OT: ObserversTuple, { - fn observers(&self) -> &OT { - &self.observers + fn observers(&self) -> RefIndexable<&Self::Observers, Self::Observers> { + RefIndexable::from(&self.observers) } - fn observers_mut(&mut self) -> &mut OT { - &mut self.observers + fn observers_mut(&mut self) -> RefIndexable<&mut Self::Observers, Self::Observers> { + RefIndexable::from(&mut self.observers) } } diff --git a/libafl_nyx/src/helper.rs b/libafl_nyx/src/helper.rs index 2184080997..fc7b2163de 100644 --- a/libafl_nyx/src/helper.rs +++ b/libafl_nyx/src/helper.rs @@ -1,25 +1,19 @@ /// [`NyxHelper`] is used to wrap `NyxProcess` -use std::{ - fmt::{self, Debug}, - path::Path, - time::Duration, -}; +use std::{fmt::Debug, fs::File, path::Path}; use libafl::Error; -use libnyx::{NyxProcess, NyxReturnValue}; +use libnyx::{NyxConfig, NyxProcess, NyxProcessRole}; + +use crate::settings::NyxSettings; -const INIT_TIMEOUT: Duration = Duration::new(2, 0); pub struct NyxHelper { pub nyx_process: NyxProcess, - /// real size of trace_bits - pub real_map_size: usize, - // real size of the trace_bits - pub map_size: usize, - /// shared memory with instruction bitmaps - pub trace_bits: *mut u8, + pub nyx_stdout: File, + + pub bitmap_size: usize, + pub bitmap_buffer: *mut u8, } -const MAX_FILE: u32 = 1024 * 1024; #[derive(Clone, Copy, Debug)] pub enum NyxProcessType { /// stand alone mode @@ -29,125 +23,60 @@ pub enum NyxProcessType { /// parallel mode's child, consume snapshot and execute CHILD, } -impl NyxHelper { - /// Create [`NyxProcess`] and do basic settings - /// It will convert instance to parent or child using `parent_cpu_id` when set`parallel_mode` - /// will fail if initial connection takes more than 2 seconds - pub fn new( - target_dir: &Path, - cpu_id: u32, - snap_mode: bool, - parallel_mode: bool, - parent_cpu_id: Option, - ) -> Result { - NyxHelper::with_initial_timeout( - target_dir, - cpu_id, - snap_mode, - parallel_mode, - parent_cpu_id, - INIT_TIMEOUT, - ) - } - /// Create [`NyxProcess`] and do basic settings - /// It will convert instance to parent or child using `parent_cpu_id` when set`parallel_mode` - /// will fail if initial connection takes more than `initial_timeout` seconds - pub fn with_initial_timeout( - target_dir: &Path, - cpu_id: u32, - snap_mode: bool, - parallel_mode: bool, - parent_cpu_id: Option, - initial_timeout: Duration, - ) -> Result { - let Some(sharedir) = target_dir.to_str() else { - return Err(Error::illegal_argument("can't convert sharedir to str")); - }; - let work_dir = target_dir.join("workdir"); - let work_dir = work_dir.to_str().expect("unable to convert workdir to str"); - let nyx_type = if parallel_mode { - let Some(parent_cpu_id) = parent_cpu_id else { - return Err(Error::illegal_argument( - "please set parent_cpu_id in nyx parallel mode", - )); - }; - if cpu_id == parent_cpu_id { - NyxProcessType::PARENT - } else { - NyxProcessType::CHILD - } - } else { - NyxProcessType::ALONE - }; - let nyx_process = match nyx_type { - NyxProcessType::ALONE => NyxProcess::new(sharedir, work_dir, cpu_id, MAX_FILE, true), - NyxProcessType::PARENT => { - NyxProcess::new_parent(sharedir, work_dir, cpu_id, MAX_FILE, true) - } - NyxProcessType::CHILD => NyxProcess::new_child(sharedir, work_dir, cpu_id, cpu_id), - }; +impl NyxHelper { + /// Create [`NyxProcess`] and do basic settings. It will convert the + /// instance to a parent or child using `parent_cpu_id` when + /// `parallel_mode` is set. + pub fn new

(share_dir: P, settings: NyxSettings) -> Result + where + P: AsRef, + { + let share_dir_str = share_dir.as_ref().to_str().ok_or(Error::illegal_argument( + "`share_dir` contains invalid UTF-8", + ))?; - let mut nyx_process = - nyx_process.map_err(|msg: String| -> Error { Error::illegal_argument(msg) })?; + let mut nyx_config = NyxConfig::load(share_dir_str).map_err(|e| { + Error::illegal_argument(format!("Failed to load Nyx config from share dir: {e}")) + })?; + nyx_config.set_input_buffer_size(settings.input_buffer_size); + nyx_config.set_process_role(match settings.parent_cpu_id { + None => NyxProcessRole::StandAlone, + Some(parent_cpu_id) if parent_cpu_id == settings.cpu_id => NyxProcessRole::Parent, + _ => NyxProcessRole::Child, + }); + nyx_config.set_worker_id(settings.cpu_id); - let real_map_size = nyx_process.bitmap_buffer_size(); - let map_size = ((real_map_size + 63) >> 6) << 6; - let trace_bits = nyx_process.bitmap_buffer_mut().as_mut_ptr(); - nyx_process.option_set_reload_mode(snap_mode); + let mut nyx_process = NyxProcess::new(&mut nyx_config, settings.cpu_id) + .map_err(|e| Error::illegal_state(format!("Failed to create Nyx process: {e}")))?; + nyx_process.option_set_reload_mode(settings.snap_mode); + nyx_process.option_set_timeout(settings.timeout_secs, settings.timeout_micro_secs); nyx_process.option_apply(); - // default timeout for initial dry-run - let sec = initial_timeout - .as_secs() - .try_into() - .map_err(|_| -> Error { Error::illegal_argument("can't cast time's sec to u8") })?; + let path = Path::new(nyx_config.workdir_path()) + .join(format!("hprintf_{}", nyx_config.worker_id())); + let nyx_stdout = File::options() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open(path) + .map_err(|e| Error::illegal_state(format!("Failed to create Nyx stdout file: {e}")))?; - let micro_sec: u32 = initial_timeout.subsec_micros(); - nyx_process.option_set_timeout(sec, micro_sec); - nyx_process.option_apply(); + let bitmap_size = nyx_process.bitmap_buffer_size(); + let bitmap_buffer = nyx_process.bitmap_buffer_mut().as_mut_ptr(); - // dry run to check if qemu is spawned - nyx_process.set_input(b"INIT", 4); - match nyx_process.exec() { - NyxReturnValue::Error => { - nyx_process.shutdown(); - let msg = "Error: Nyx runtime error has occurred..."; - return Err(Error::illegal_state(msg)); - } - NyxReturnValue::IoError => { - let msg = "Error: QEMU-nyx died..."; - return Err(Error::illegal_state(msg)); - } - NyxReturnValue::Abort => { - nyx_process.shutdown(); - let msg = "Error: Nyx abort occurred..."; - return Err(Error::illegal_state(msg)); - } - _ => {} - } Ok(Self { nyx_process, - real_map_size, - map_size, - trace_bits, + nyx_stdout, + bitmap_size, + bitmap_buffer, }) } - /// Set a timeout for Nyx - pub fn set_timeout(&mut self, time: Duration) { - let sec: u8 = time - .as_secs() - .try_into() - .expect("can't cast time's sec to u8"); - let micro_sec: u32 = time.subsec_micros(); - self.nyx_process.option_set_timeout(sec, micro_sec); + /// Set a timeout for Nyx. + pub fn set_timeout(&mut self, secs: u8, micro_secs: u32) { + self.nyx_process.option_set_timeout(secs, micro_secs); self.nyx_process.option_apply(); } } - -impl Debug for NyxHelper { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("NyxInprocessHelper").finish() - } -} diff --git a/libafl_nyx/src/lib.rs b/libafl_nyx/src/lib.rs index c08942e2a5..069d556f60 100644 --- a/libafl_nyx/src/lib.rs +++ b/libafl_nyx/src/lib.rs @@ -3,3 +3,5 @@ pub mod executor; #[cfg(target_os = "linux")] pub mod helper; +#[cfg(target_os = "linux")] +pub mod settings; diff --git a/libafl_nyx/src/settings.rs b/libafl_nyx/src/settings.rs new file mode 100644 index 0000000000..51e9eef051 --- /dev/null +++ b/libafl_nyx/src/settings.rs @@ -0,0 +1,47 @@ +use typed_builder::TypedBuilder; + +const DEFAULT_INPUT_BUFFER_SIZE: usize = 1024 * 1024; +const DEFAULT_TIMEOUT_SECS: u8 = 2; +const DEFAULT_TIMEOUT_MICRO_SECS: u32 = 0; +const DEFAULT_SNAP_MODE: bool = true; + +#[derive(Debug, Clone, Copy, TypedBuilder)] +pub struct NyxSettings { + /// The CPU core for the Nyx process. + /// + /// Depending on the value of `parent_cpu_id`, the created Nyx process + /// will be one of the following types: + /// * Standalone: `parent_cpu_id.is_none()`. + /// * Parent: `parent_cpu_id.is_some_and(|parent_cpu_id| parent_cpu_id == cpu_id)`. + /// * Child: `parent_cpu_id.is_some_and(|parent_cpu_id| parent_cpu_id != cpu_id)`. + pub cpu_id: usize, + + /// The CPU core for the Nyx parent process. The parent process + /// creates the fuzzing snapshot that can then be used by the child + /// processes. + /// + /// Not specifying this will start the Nyx process in standalone mode. + pub parent_cpu_id: Option, + + /// Reload the VM by using the fuzzing snapshot. You probably want + /// this to be `true`. + #[builder(default = DEFAULT_SNAP_MODE)] + pub snap_mode: bool, + + /// The input buffer size (in bytes) used to pass the input to the + /// QEMU-Nyx VM. + /// + /// Default is `1MB`. + #[builder(default = DEFAULT_INPUT_BUFFER_SIZE)] + pub input_buffer_size: usize, + + /// The timeout for a single execution in seconds (until the + /// hypervisor restore snapshot call). + #[builder(default = DEFAULT_TIMEOUT_SECS)] + pub timeout_secs: u8, + + /// Additional timeout in microseconds that gets added to + /// `timeout_secs`. + #[builder(default = DEFAULT_TIMEOUT_MICRO_SECS)] + pub timeout_micro_secs: u32, +} diff --git a/libafl_qemu/Cargo.toml b/libafl_qemu/Cargo.toml index 8dcc30d4e7..3042f4537e 100644 --- a/libafl_qemu/Cargo.toml +++ b/libafl_qemu/Cargo.toml @@ -12,24 +12,25 @@ edition = "2021" categories = ["development-tools::testing", "emulators", "embedded", "os", "no-std"] [package.metadata.docs.rs] -features = ["document-features"] -all-features = true +features = ["document-features", "default", "python", "x86_64", "usermode"] rustdoc-args = ["--cfg", "docsrs"] [features] -default = ["fork", "build_libqasan", "serdeany_autoreg", "injections"] +default = ["fork", "build_libgasan", "build_libqasan", "serdeany_autoreg", "injections"] clippy = [] # special feature for clippy, don't use in normal projects§ document-features = ["dep:document-features"] +paranoid_debug = ["libafl_qemu_sys/paranoid_debug"] # Will perform as many checks as possible. The target will be greatly slowed down. #! # Feature Flags #! ### General Features ## Find injections during fuzzing injections = ["serde_yaml", "toml"] ## Python bindings support -python = ["pyo3", "pyo3-build-config"] +python = ["pyo3", "pyo3-build-config", "libafl_qemu_sys/python"] ## Fork support fork = ["libafl/fork"] ## Build libqasan for address sanitization +build_libgasan = [] build_libqasan = [] #! ## The following architecture features are mutually exclusive. @@ -58,14 +59,14 @@ serdeany_autoreg = ["libafl_bolts/serdeany_autoreg"] slirp = [ "systemmode", "libafl_qemu_sys/slirp" ] # build qemu with host libslirp (for user networking) -# disabled atm, enabled when fixed with dynamic list -# shared = [ "libafl_qemu_sys/shared" ] +# Requires the binary's build.rs to call `build_libafl_qemu` +shared = [ "libafl_qemu_sys/shared" ] [dependencies] -libafl = { path = "../libafl", version = "0.11.2", default-features = false, features = ["std", "derive", "regex"] } -libafl_bolts = { path = "../libafl_bolts", version = "0.11.2", default-features = false, features = ["std", "derive"] } -libafl_targets = { path = "../libafl_targets", version = "0.11.2" } -libafl_qemu_sys = { path = "./libafl_qemu_sys", version = "0.11.2" } +libafl = { path = "../libafl", version = "0.12.0", default-features = false, features = ["std", "derive", "regex"] } +libafl_bolts = { path = "../libafl_bolts", version = "0.12.0", default-features = false, features = ["std", "derive"] } +libafl_targets = { path = "../libafl_targets", version = "0.12.0" } +libafl_qemu_sys = { path = "./libafl_qemu_sys", version = "0.12.0" } serde = { version = "1.0", default-features = false, features = ["alloc"] } # serialization lib hashbrown = { version = "0.14", features = ["serde"] } # A faster hashmap, nostd compatible @@ -93,8 +94,10 @@ pyo3 = { version = "0.18", optional = true } document-features = { version = "0.2", optional = true } [build-dependencies] +libafl_qemu_build = { path = "./libafl_qemu_build", version = "0.12.0" } pyo3-build-config = { version = "0.18", optional = true } rustversion = "1.0" +bindgen = "0.69" [lib] name = "libafl_qemu" diff --git a/libafl_qemu/build_linux.rs b/libafl_qemu/build_linux.rs index f6dcbd0b60..973c519dbe 100644 --- a/libafl_qemu/build_linux.rs +++ b/libafl_qemu/build_linux.rs @@ -1,5 +1,6 @@ -use std::{env, fs, path::Path, process::Command}; +use std::{env, fs, path::{Path, PathBuf}, process::Command}; +#[allow(clippy::too_many_lines)] pub fn build() { // Note: Unique features are checked in libafl_qemu_sys @@ -13,13 +14,35 @@ pub fn build() { }) }; - let build_libqasan = cfg!(all(feature = "build_libqasan", not(feature = "hexagon"))); + let src_dir = env::var("CARGO_MANIFEST_DIR").unwrap(); + let src_dir = PathBuf::from(src_dir); + + let out_dir = env::var("OUT_DIR").unwrap(); + let out_dir = PathBuf::from(&out_dir); + + let mut target_dir = out_dir.clone(); + target_dir.pop(); + target_dir.pop(); + target_dir.pop(); + let include_dir = target_dir.join("include"); + + let qemu_asan_guest = cfg!(all(feature = "build_libgasan", not(feature = "hexagon"))); + let qemu_asan = cfg!(all(feature = "build_libqasan", not(feature = "hexagon"))); + + let libafl_qemu_hdr_name = "libafl_qemu.h"; + + let libafl_runtime_dir = src_dir.join("runtime"); + let libafl_qemu_hdr = libafl_runtime_dir.join(libafl_qemu_hdr_name); + + let runtime_bindings_file = out_dir.join("libafl_qemu_bindings.rs"); + let stub_runtime_bindings_file = src_dir.join("runtime/libafl_qemu_stub_bindings.rs"); println!("cargo:rustc-cfg=emulation_mode=\"{emulation_mode}\""); println!("cargo:rerun-if-env-changed=EMULATION_MODE"); println!("cargo:rerun-if-changed=build.rs"); println!("cargo:rerun-if-changed=build_linux.rs"); + println!("cargo:rerun-if-changed={}", libafl_runtime_dir.display()); let cpu_target = if cfg!(feature = "x86_64") { "x86_64".to_string() @@ -43,7 +66,7 @@ pub fn build() { println!("cargo:rerun-if-env-changed=CPU_TARGET"); println!("cargo:rustc-cfg=cpu_target=\"{cpu_target}\""); - let cross_cc = if (emulation_mode == "usermode") && build_libqasan { + let cross_cc = if (emulation_mode == "usermode") && (qemu_asan || qemu_asan_guest) { // TODO try to autodetect a cross compiler with the arch name (e.g. aarch64-linux-gnu-gcc) let cross_cc = env::var("CROSS_CC").unwrap_or_else(|_| { println!("cargo:warning=CROSS_CC is not set, default to cc (things can go wrong if the selected cpu target ({cpu_target}) is not the host arch ({}))", env::consts::ARCH); @@ -56,18 +79,33 @@ pub fn build() { String::new() }; - if std::env::var("DOCS_RS").is_ok() { + if env::var("DOCS_RS").is_ok() || cfg!(feature = "clippy") { + fs::copy(&stub_runtime_bindings_file, &runtime_bindings_file).expect("Could not copy stub bindings file"); return; // only build when we're not generating docs } - let out_dir = env::var("OUT_DIR").unwrap(); - let out_dir_path = Path::new(&out_dir); - let mut target_dir = out_dir_path.to_path_buf(); - target_dir.pop(); - target_dir.pop(); - target_dir.pop(); + fs::create_dir_all(&include_dir).expect("Could not create include dir"); + + fs::copy(libafl_qemu_hdr.clone(), include_dir.join(libafl_qemu_hdr_name)).expect("Could not copy libafl_qemu.h to out directory."); + + bindgen::Builder::default() + .derive_debug(true) + .derive_default(true) + .impl_debug(true) + .generate_comments(true) + .default_enum_style(bindgen::EnumVariation::NewType { + is_global: true, + is_bitfield: true, + }) + .header(libafl_qemu_hdr.display().to_string()) + .generate() + .expect("Exit bindings generation failed.") + .write_to_file(&runtime_bindings_file) + .expect("Could not write bindings."); + + libafl_qemu_build::store_generated_content_if_different(&stub_runtime_bindings_file, fs::read(&runtime_bindings_file).expect("Could not read generated bindings file").as_slice()); - if (emulation_mode == "usermode") && build_libqasan { + if (emulation_mode == "usermode") && (qemu_asan || qemu_asan_guest) { let qasan_dir = Path::new("libqasan"); let qasan_dir = fs::canonicalize(qasan_dir).unwrap(); println!("cargo:rerun-if-changed={}", qasan_dir.display()); @@ -77,7 +115,7 @@ pub fn build() { make.env("CFLAGS", "-DDEBUG=1"); } assert!(make - .current_dir(out_dir_path) + .current_dir(&out_dir) .env("CC", &cross_cc) .env("OUT_DIR", &target_dir) .arg("-C") @@ -85,6 +123,5 @@ pub fn build() { .status() .expect("make failed") .success()); - // println!("cargo:rerun-if-changed={}/libqasan.so", target_dir.display()); } } diff --git a/libafl_qemu/libafl_qemu_build/Cargo.toml b/libafl_qemu/libafl_qemu_build/Cargo.toml index 6a3b1aa51a..e864e4c0d3 100644 --- a/libafl_qemu/libafl_qemu_build/Cargo.toml +++ b/libafl_qemu/libafl_qemu_build/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "libafl_qemu_build" -version = "0.11.2" +version = "0.12.0" authors = ["Andrea Fioraldi "] description = "Builder for LibAFL QEMU" documentation = "https://docs.rs/libafl_qemu_build" @@ -26,8 +26,10 @@ slirp = [] # build qemu with host libslirp (for user networking) clippy = [] # special feature for clippy, don't use in normal projects§ +paranoid_debug = [] # Will perform as many checks as possible. The target will be greatly slowed down. + [dependencies] -bindgen = "0.68" +bindgen = "0.69.4" which = "4.4" json = "0.12" shell-words = "1.1" diff --git a/libafl_qemu/libafl_qemu_build/src/bindings.rs b/libafl_qemu/libafl_qemu_build/src/bindings.rs index 26d6eb4871..bf676273c5 100644 --- a/libafl_qemu/libafl_qemu_build/src/bindings.rs +++ b/libafl_qemu/libafl_qemu_build/src/bindings.rs @@ -1,7 +1,9 @@ -use std::{fs, path::Path}; +use std::path::Path; use bindgen::{BindgenError, Bindings}; +use crate::store_generated_content_if_different; + const WRAPPER_HEADER: &str = r#" // https://github.com/rust-lang/rust-bindgen/issues/2500 @@ -45,6 +47,9 @@ const WRAPPER_HEADER: &str = r#" #include "user/safe-syscall.h" #include "qemu/selfmap.h" #include "cpu_loop-common.h" +#include "qemu/selfmap.h" + +#include "libafl/user.h" #else @@ -55,8 +60,8 @@ const WRAPPER_HEADER: &str = r#" #include "sysemu/tcg.h" #include "sysemu/replay.h" -#include "libafl_extras/syx-snapshot/device-save.h" -#include "libafl_extras/syx-snapshot/syx-snapshot.h" +#include "libafl/syx-snapshot/device-save.h" +#include "libafl/syx-snapshot/syx-snapshot.h" #endif @@ -76,9 +81,9 @@ const WRAPPER_HEADER: &str = r#" #include "qemu/plugin-memory.h" -#include "libafl_extras/exit.h" -#include "libafl_extras/hook.h" -#include "libafl_extras/jit.h" +#include "libafl/exit.h" +#include "libafl/hook.h" +#include "libafl/jit.h" "#; @@ -88,7 +93,8 @@ pub fn generate( clang_args: Vec, ) -> Result { let wrapper_h = build_dir.join("wrapper.h"); - fs::write(&wrapper_h, WRAPPER_HEADER).expect("Unable to write wrapper.h"); + + store_generated_content_if_different(&wrapper_h, WRAPPER_HEADER.as_bytes()); let bindings = bindgen::Builder::default() .derive_debug(true) @@ -112,11 +118,14 @@ pub fn generate( .allowlist_type("MemOpIdx") .allowlist_type("MemOp") .allowlist_type("DeviceSnapshotKind") + .allowlist_type("ShutdownCause") .allowlist_type("libafl_exit_reason") .allowlist_type("libafl_exit_reason_kind") .allowlist_type("libafl_exit_reason_sync_backdoor") .allowlist_type("libafl_exit_reason_breakpoint") .allowlist_type("Syx.*") + .allowlist_type("libafl_mapinfo") + .allowlist_type("IntervalTreeRoot") .allowlist_function("qemu_user_init") .allowlist_function("target_mmap") .allowlist_function("target_mprotect") @@ -134,8 +143,11 @@ pub fn generate( .allowlist_function("syx_.*") .allowlist_function("device_list_all") .allowlist_function("libafl_.*") + .allowlist_function("read_self_maps") + .allowlist_function("free_self_maps") + .allowlist_function("pageflags_get_root") .blocklist_function("main_loop_wait") // bindgen issue #1313 - .parse_callbacks(Box::new(bindgen::CargoCallbacks)); + .parse_callbacks(Box::new(bindgen::CargoCallbacks::new())); // arch specific functions let bindings = if cpu_target == "i386" || cpu_target == "x86_64" { diff --git a/libafl_qemu/libafl_qemu_build/src/build.rs b/libafl_qemu/libafl_qemu_build/src/build.rs index cec7203f40..62e7afd48c 100644 --- a/libafl_qemu/libafl_qemu_build/src/build.rs +++ b/libafl_qemu/libafl_qemu_build/src/build.rs @@ -2,14 +2,18 @@ use std::{ env, fs, path::{Path, PathBuf}, process::Command, + str::FromStr, }; use which::which; +use crate::cargo_add_rpath; + const QEMU_URL: &str = "https://github.com/AFLplusplus/qemu-libafl-bridge"; const QEMU_DIRNAME: &str = "qemu-libafl-bridge"; -const QEMU_REVISION: &str = "75d15d54f4417a4766d2dcb493982d9df0e8eac4"; +const QEMU_REVISION: &str = "538e6b02c36838e0de469b39dd03fef05678444f"; +#[allow(clippy::module_name_repetitions)] pub struct BuildResult { pub qemu_path: PathBuf, pub build_dir: PathBuf, @@ -21,6 +25,224 @@ fn build_dep_check(tools: &[&str]) { } } +fn get_config_signature(config_cmd: &Command) -> String { + let mut signature_string = String::new(); + + // Command env + let config_env: String = config_cmd + .get_envs() + .map(|(key, value)| { + format!( + "\t{}={}", + key.to_str().expect("Couldn't convert OsStr to str"), + if let Some(v) = value { + v.to_str().expect("Could't convert OsStr to str") + } else { + "" + } + ) + }) + .reduce(|acc, elt| format!("{acc}\n{elt}")) + .into_iter() + .collect(); + signature_string += format!("Environment:\n{config_env}").as_str(); + + // Command args + let config_args: String = config_cmd + .get_args() + .map(|arg| format!("\t{}", arg.to_str().expect("Couldn't convert OsStr to str"))) + .reduce(|acc, arg| format!("{acc}\n{arg}")) + .into_iter() + .collect(); + signature_string += format!("\n\nArguments:\n{config_args}").as_str(); + + signature_string +} + +#[allow(clippy::too_many_lines)] +fn configure_qemu( + cc_compiler: &cc::Tool, + cpp_compiler: &cc::Tool, + qemu_path: &PathBuf, + build_dir: &Path, + is_usermode: bool, + cpu_target: &String, + target_suffix: &String, +) -> Command { + let mut cmd = Command::new("./configure"); + + let linker_interceptor = qemu_path.join("linker_interceptor.py"); + let linker_interceptor_plus_plus = qemu_path.join("linker_interceptor++.py"); + + println!("cargo:rerun-if-changed={}", linker_interceptor.display()); + println!( + "cargo:rerun-if-changed={}", + linker_interceptor_plus_plus.display() + ); + + // Set common options for usermode and systemmode + cmd.current_dir(qemu_path) + .env("__LIBAFL_QEMU_CONFIGURE", "") + .env("__LIBAFL_QEMU_BUILD_OUT", build_dir.join("linkinfo.json")) + .env("__LIBAFL_QEMU_BUILD_CC", cc_compiler.path()) + .env("__LIBAFL_QEMU_BUILD_CXX", cpp_compiler.path()) + .arg(&format!("--cc={}", linker_interceptor.display())) + .arg(&format!("--cxx={}", linker_interceptor_plus_plus.display())) + .arg("--as-shared-lib") + .arg(&format!("--target-list={cpu_target}-{target_suffix}")) + // .arg("--disable-capstone") + .arg("--disable-bsd-user"); + + if cfg!(feature = "paranoid_debug") { + cmd.arg("--enable-debug") + .arg("--enable-debug-tcg") + .arg("--enable-sanitizers"); + } + + if is_usermode { + // Usermode options + cmd.args(["--disable-fdt", "--disable-system"]); + } else { + // Systemmode options + cmd.arg(if cfg!(feature = "slirp") { + "--enable-slirp" + } else { + "--disable-slirp" + }) + .arg("--enable-fdt=internal") + .arg("--audio-drv-list=") + .arg("--disable-af-xdp") + .arg("--disable-alsa") + .arg("--disable-attr") + .arg("--disable-auth-pam") + .arg("--disable-dbus-display") + .arg("--disable-bochs") + .arg("--disable-bpf") + .arg("--disable-brlapi") + .arg("--disable-bzip2") + .arg("--disable-cap-ng") + .arg("--disable-canokey") + .arg("--disable-cloop") + .arg("--disable-cocoa") + .arg("--disable-coreaudio") + .arg("--disable-curl") + .arg("--disable-curses") + .arg("--disable-dmg") + .arg("--disable-docs") + .arg("--disable-dsound") + .arg("--disable-fuse") + .arg("--disable-fuse-lseek") + .arg("--disable-gcrypt") + .arg("--disable-gettext") + .arg("--disable-gio") + .arg("--disable-glusterfs") + .arg("--disable-gnutls") + .arg("--disable-gtk") + .arg("--disable-guest-agent") + .arg("--disable-guest-agent-msi") + .arg("--disable-hvf") + .arg("--disable-iconv") + .arg("--disable-jack") + .arg("--disable-keyring") + .arg("--disable-kvm") + .arg("--disable-libdaxctl") + .arg("--disable-libiscsi") + .arg("--disable-libnfs") + .arg("--disable-libpmem") + .arg("--disable-libssh") + .arg("--disable-libudev") + .arg("--disable-libusb") + .arg("--disable-linux-aio") + .arg("--disable-linux-io-uring") + .arg("--disable-linux-user") + .arg("--disable-live-block-migration") + .arg("--disable-lzfse") + .arg("--disable-lzo") + .arg("--disable-l2tpv3") + .arg("--disable-malloc-trim") + .arg("--disable-mpath") + .arg("--disable-multiprocess") + .arg("--disable-netmap") + .arg("--disable-nettle") + .arg("--disable-numa") + .arg("--disable-nvmm") + .arg("--disable-opengl") + .arg("--disable-oss") + .arg("--disable-pa") + .arg("--disable-parallels") + .arg("--disable-png") + .arg("--disable-pvrdma") + .arg("--disable-qcow1") + .arg("--disable-qed") + .arg("--disable-qga-vss") + .arg("--disable-rbd") + .arg("--disable-rdma") + .arg("--disable-replication") + .arg("--disable-sdl") + .arg("--disable-sdl-image") + .arg("--disable-seccomp") + .arg("--disable-selinux") + .arg("--disable-slirp-smbd") + .arg("--disable-smartcard") + .arg("--disable-snappy") + .arg("--disable-sndio") + .arg("--disable-sparse") + .arg("--disable-spice") + .arg("--disable-spice-protocol") + .arg("--disable-tools") + .arg("--disable-tpm") + .arg("--disable-usb-redir") + .arg("--disable-user") + .arg("--disable-u2f") + .arg("--disable-vde") + .arg("--disable-vdi") + .arg("--disable-vduse-blk-export") + .arg("--disable-vhost-crypto") + .arg("--disable-vhost-kernel") + .arg("--disable-vhost-net") + .arg("--disable-vhost-user-blk-server") + .arg("--disable-vhost-vdpa") + .arg("--disable-virglrenderer") + .arg("--disable-virtfs") + .arg("--disable-vmnet") + .arg("--disable-vnc") + .arg("--disable-vnc-jpeg") + .arg("--disable-vnc-sasl") + .arg("--disable-vte") + .arg("--disable-vvfat") + .arg("--disable-whpx") + .arg("--disable-xen") + .arg("--disable-xen-pci-passthrough") + .arg("--disable-xkbcommon") + .arg("--disable-zstd") + .arg("--disable-tests"); + } + + cmd +} + +fn build_qemu( + cc_compiler: &cc::Tool, + cpp_compiler: &cc::Tool, + build_dir: &PathBuf, + jobs: Option, +) -> Command { + let mut cmd = Command::new("make"); + + cmd.current_dir(build_dir) + .env("__LIBAFL_QEMU_CONFIGURE", "") + .env("__LIBAFL_QEMU_BUILD_OUT", build_dir.join("linkinfo.json")) + .env("__LIBAFL_QEMU_BUILD_CC", cc_compiler.path()) + .env("__LIBAFL_QEMU_BUILD_CXX", cpp_compiler.path()) + .arg("-j"); + + if let Some(j) = jobs { + cmd.arg(&format!("{j}")).env("V", "1"); + } + + cmd +} + #[allow(clippy::too_many_lines, clippy::missing_panics_doc)] #[must_use] pub fn build( @@ -31,7 +253,7 @@ pub fn build( ) -> BuildResult { let mut cpu_target = cpu_target.to_string(); // qemu-system-arm supports both big and little endian configurations and so - // therefore the "be" feature should ignored in this configuration. Also + // the "be" feature should be ignored in this configuration. Also // ignore the feature if we are running in clippy which enables all the // features at once (disabling the check for mutually exclusive options) // resulting in cpu_target being set to 'x86_64' above which obviously @@ -48,12 +270,17 @@ pub fn build( cpu_target += "el"; } - let custom_qemu_dir = env::var_os("CUSTOM_QEMU_DIR").map(|x| x.to_string_lossy().to_string()); - let custom_qemu_no_build = env::var("CUSTOM_QEMU_NO_BUILD").is_ok(); - let custom_qemu_no_configure = env::var("CUSTOM_QEMU_NO_CONFIGURE").is_ok(); - println!("cargo:rerun-if-env-changed=CUSTOM_QEMU_DIR"); - println!("cargo:rerun-if-env-changed=CUSTOM_QEMU_NO_BUILD"); - println!("cargo:rerun-if-env-changed=CUSTOM_QEMU_NO_CONFIGURE"); + let libafl_qemu_dir = env::var_os("LIBAFL_QEMU_DIR").map(|x| x.to_string_lossy().to_string()); + let libafl_qemu_clone_dir = + env::var_os("LIBAFL_QEMU_CLONE_DIR").map(|x| x.to_string_lossy().to_string()); + let libafl_qemu_force_configure = env::var("LIBAFL_QEMU_FORCE_CONFIGURE").is_ok(); + let libafl_qemu_no_build = env::var("LIBAFL_QEMU_NO_BUILD").is_ok(); + + println!("cargo:rerun-if-env-changed=LIBAFL_QEMU_DIR"); + println!("cargo:rerun-if-env-changed=LIBAFL_QEMU_CLONE_DIR"); + println!("cargo:rerun-if-env-changed=LIBAFL_QEMU_FORCE_BUILD"); + println!("cargo:rerun-if-env-changed=LIBAFL_QEMU_FORCE_CONFIGURE"); + println!("cargo:rerun-if-env-changed=LIBAFL_QEMU_NO_BUILD"); let out_dir = env::var_os("OUT_DIR").unwrap(); let out_dir = out_dir.to_string_lossy().to_string(); @@ -68,10 +295,18 @@ pub fn build( let cc_compiler = cc::Build::new().cpp(false).get_compiler(); let cpp_compiler = cc::Build::new().cpp(true).get_compiler(); - let qemu_path = if let Some(qemu_dir) = custom_qemu_dir.as_ref() { + let libafl_qemu_dir = if let Some(qemu_dir) = libafl_qemu_dir.as_ref() { + if libafl_qemu_clone_dir.is_some() { + println!("cargo:warning=LIBAFL_QEMU_DIR and LIBAFL_QEMU_CLONE_DIR are both set. LIBAFL_QEMU_DIR will be considered in priority"); + } + Path::new(&qemu_dir).to_path_buf() } else { - let qemu_path = target_dir.join(QEMU_DIRNAME); + let qemu_path = if let Some(clone_dir) = &libafl_qemu_clone_dir { + PathBuf::from(clone_dir) + } else { + target_dir.join(QEMU_DIRNAME) + }; let qemu_rev = target_dir.join("QEMU_REVISION"); if qemu_rev.exists() @@ -83,20 +318,22 @@ pub fn build( if !qemu_path.is_dir() { println!("cargo:warning=Qemu not found, cloning with git ({QEMU_REVISION})..."); fs::create_dir_all(&qemu_path).unwrap(); - Command::new("git") + assert!(Command::new("git") .current_dir(&qemu_path) .arg("init") .status() - .unwrap(); - Command::new("git") + .unwrap() + .success()); + assert!(Command::new("git") .current_dir(&qemu_path) .arg("remote") .arg("add") .arg("origin") .arg(QEMU_URL) .status() - .unwrap(); - Command::new("git") + .unwrap() + .success()); + assert!(Command::new("git") .current_dir(&qemu_path) .arg("fetch") .arg("--depth") @@ -104,20 +341,23 @@ pub fn build( .arg("origin") .arg(QEMU_REVISION) .status() - .unwrap(); - Command::new("git") + .unwrap() + .success()); + assert!(Command::new("git") .current_dir(&qemu_path) .arg("checkout") .arg("FETCH_HEAD") .status() - .unwrap(); + .unwrap() + .success()); fs::write(&qemu_rev, QEMU_REVISION).unwrap(); } qemu_path }; - let build_dir = qemu_path.join("build"); + let libafl_qemu_build_dir = libafl_qemu_dir.join("build"); + let config_signature_path = libafl_qemu_build_dir.join("libafl_config"); let target_suffix = if is_usermode { "linux-user".to_string() @@ -127,204 +367,71 @@ pub fn build( let (output_lib, output_lib_link) = if is_usermode { ( - build_dir.join(format!("libqemu-{cpu_target}.so")), + libafl_qemu_build_dir.join(format!("libqemu-{cpu_target}.so")), format!("qemu-{cpu_target}"), ) } else { ( - build_dir.join(format!("libqemu-system-{cpu_target}.so")), + libafl_qemu_build_dir.join(format!("libqemu-system-{cpu_target}.so")), format!("qemu-system-{cpu_target}"), ) }; - // println!("cargo:rerun-if-changed={}", output_lib.to_string_lossy()); - - if !output_lib.is_file() || (custom_qemu_dir.is_some() && !custom_qemu_no_build) { - /*drop( - Command::new("make") - .current_dir(&qemu_path) - .arg("distclean") - .status(), - );*/ - if is_usermode && !custom_qemu_no_configure { - let mut cmd = Command::new("./configure"); - cmd.current_dir(&qemu_path) - //.arg("--as-static-lib") - .env("__LIBAFL_QEMU_BUILD_OUT", build_dir.join("linkinfo.json")) - .env("__LIBAFL_QEMU_BUILD_CC", cc_compiler.path()) - .env("__LIBAFL_QEMU_BUILD_CXX", cpp_compiler.path()) - .arg(&format!( - "--cc={}", - qemu_path.join("linker_interceptor.py").display() - )) - .arg(&format!( - "--cxx={}", - qemu_path.join("linker_interceptor++.py").display() - )) - .arg("--as-shared-lib") - .arg(&format!("--target-list={cpu_target}-{target_suffix}")) - .args([ - "--disable-bsd-user", - "--disable-fdt", - "--disable-system", - "--disable-capstone", - ]); - if cfg!(feature = "debug_assertions") { - cmd.arg("--enable-debug"); - } - cmd.status().expect("Configure failed"); - } else if !custom_qemu_no_configure { - let mut cmd = Command::new("./configure"); - cmd.current_dir(&qemu_path) - //.arg("--as-static-lib") - .env("__LIBAFL_QEMU_BUILD_OUT", build_dir.join("linkinfo.json")) - .env("__LIBAFL_QEMU_BUILD_CC", cc_compiler.path()) - .env("__LIBAFL_QEMU_BUILD_CXX", cpp_compiler.path()) - .arg(&format!( - "--cc={}", - qemu_path.join("linker_interceptor.py").display() - )) - .arg(&format!( - "--cxx={}", - qemu_path.join("linker_interceptor++.py").display() - )) - .arg("--as-shared-lib") - .arg(&format!("--target-list={cpu_target}-{target_suffix}")) - .arg(if cfg!(feature = "slirp") { - "--enable-slirp" - } else { - "--disable-slirp" - }) - .arg("--enable-fdt=internal") - .arg("--audio-drv-list=") - .arg("--disable-alsa") - .arg("--disable-attr") - .arg("--disable-auth-pam") - .arg("--disable-dbus-display") - .arg("--disable-bochs") - .arg("--disable-bpf") - .arg("--disable-brlapi") - .arg("--disable-bsd-user") - .arg("--disable-bzip2") - .arg("--disable-capstone") - .arg("--disable-cap-ng") - .arg("--disable-canokey") - .arg("--disable-cloop") - .arg("--disable-cocoa") - .arg("--disable-coreaudio") - .arg("--disable-curl") - .arg("--disable-curses") - .arg("--disable-dmg") - .arg("--disable-docs") - .arg("--disable-dsound") - .arg("--disable-fuse") - .arg("--disable-fuse-lseek") - .arg("--disable-gcrypt") - .arg("--disable-gettext") - .arg("--disable-gio") - .arg("--disable-glusterfs") - .arg("--disable-gnutls") - .arg("--disable-gtk") - .arg("--disable-guest-agent") - .arg("--disable-guest-agent-msi") - .arg("--disable-hvf") - .arg("--disable-iconv") - .arg("--disable-jack") - .arg("--disable-keyring") - .arg("--disable-kvm") - .arg("--disable-libdaxctl") - .arg("--disable-libiscsi") - .arg("--disable-libnfs") - .arg("--disable-libpmem") - .arg("--disable-libssh") - .arg("--disable-libudev") - .arg("--disable-libusb") - .arg("--disable-linux-aio") - .arg("--disable-linux-io-uring") - .arg("--disable-linux-user") - .arg("--disable-live-block-migration") - .arg("--disable-lzfse") - .arg("--disable-lzo") - .arg("--disable-l2tpv3") - .arg("--disable-malloc-trim") - .arg("--disable-mpath") - .arg("--disable-multiprocess") - .arg("--disable-netmap") - .arg("--disable-nettle") - .arg("--disable-numa") - .arg("--disable-nvmm") - .arg("--disable-opengl") - .arg("--disable-oss") - .arg("--disable-pa") - .arg("--disable-parallels") - .arg("--disable-png") - .arg("--disable-pvrdma") - .arg("--disable-qcow1") - .arg("--disable-qed") - .arg("--disable-qga-vss") - .arg("--disable-rbd") - .arg("--disable-rdma") - .arg("--disable-replication") - .arg("--disable-sdl") - .arg("--disable-sdl-image") - .arg("--disable-seccomp") - .arg("--disable-selinux") - .arg("--disable-slirp-smbd") - .arg("--disable-smartcard") - .arg("--disable-snappy") - .arg("--disable-sndio") - .arg("--disable-sparse") - .arg("--disable-spice") - .arg("--disable-spice-protocol") - .arg("--disable-tools") - .arg("--disable-tpm") - .arg("--disable-usb-redir") - .arg("--disable-user") - .arg("--disable-u2f") - .arg("--disable-vde") - .arg("--disable-vdi") - .arg("--disable-vduse-blk-export") - .arg("--disable-vhost-crypto") - .arg("--disable-vhost-kernel") - .arg("--disable-vhost-net") - .arg("--disable-vhost-user-blk-server") - .arg("--disable-vhost-vdpa") - .arg("--disable-virglrenderer") - .arg("--disable-virtfs") - .arg("--disable-vmnet") - .arg("--disable-vnc") - .arg("--disable-vnc-jpeg") - .arg("--disable-vnc-sasl") - .arg("--disable-vte") - .arg("--disable-vvfat") - .arg("--disable-whpx") - .arg("--disable-xen") - .arg("--disable-xen-pci-passthrough") - .arg("--disable-xkbcommon") - .arg("--disable-zstd") - .arg("--disable-tests"); - if cfg!(feature = "debug_assertions") { - cmd.arg("--enable-debug"); - } - cmd.status().expect("Configure failed"); - } - if let Some(j) = jobs { - Command::new("make") - .current_dir(&build_dir) - .arg("-j") - .arg(&format!("{j}")) - .env("V", "1") - .status() - .expect("Make failed"); + let libafl_config_old_signature = fs::read_to_string(&config_signature_path); + + let mut config_cmd = configure_qemu( + &cc_compiler, + &cpp_compiler, + &libafl_qemu_dir, + &libafl_qemu_build_dir, + is_usermode, + &cpu_target, + &target_suffix, + ); + + let current_config_signature = get_config_signature(&config_cmd); + let must_reconfigure = if libafl_qemu_force_configure { + // If the user asked to reconfigure, do so + true + } else if let Ok(libafl_config_old_signature) = libafl_config_old_signature { + if libafl_config_old_signature == current_config_signature { + // Signature match, do not reconfigure + false } else { - Command::new("make") - .current_dir(&build_dir) - .arg("-j") - .status() - .expect("Make failed"); + println!("cargo:warning=QEMU configuration is outdated. Reconfiguring..."); + true } + } else { + // In worst scenario, reconfigure + true + }; + + if must_reconfigure { + assert!( + config_cmd + .status() + .expect("Invoking Configure failed") + .success(), + "Configure didn't finish successfully" + ); + + // Config succeeded at this point, (over)write the signature file + fs::write(config_signature_path, current_config_signature) + .expect("Couldn't write config signature file."); } + // Always build by default, make will detect if it is necessary to rebuild qemu + if !libafl_qemu_no_build { + let mut build_cmd = build_qemu(&cc_compiler, &cpp_compiler, &libafl_qemu_build_dir, jobs); + + assert!( + build_cmd.status().expect("Invoking Make Failed").success(), + "Make didn't finish successfully" + ); + } + + assert!(output_lib.is_file()); // Sanity check + /* let mut objects = vec![]; for dir in &[ @@ -348,18 +455,31 @@ pub fn build( } */ + let compile_commands_string = &fs::read_to_string(libafl_qemu_build_dir.join("linkinfo.json")) + .expect("Failed to read linkinfo.json"); + + let linkinfo = json::parse(compile_commands_string).expect("Failed to parse linkinfo.json"); + + for source in linkinfo["sources"].members() { + let source_path = PathBuf::from_str(source.as_str().unwrap()).unwrap(); + + let source_path = if source_path.is_relative() { + libafl_qemu_build_dir.join(source_path) + } else { + source_path + }; + + println!("cargo:rerun-if-changed={}", source_path.display()); + } + if cfg!(feature = "shared") { - println!( - "cargo:rustc-link-search=native={}", - build_dir.to_string_lossy() - ); + let qemu_build_dir_str = libafl_qemu_build_dir + .to_str() + .expect("Could not convert to str"); + println!("cargo:rustc-link-search=native={qemu_build_dir_str}"); println!("cargo:rustc-link-lib=dylib={output_lib_link}"); + cargo_add_rpath(qemu_build_dir_str); } else { - let compile_commands_string = &fs::read_to_string(build_dir.join("linkinfo.json")) - .expect("Failed to read linkinfo.json"); - - let linkinfo = json::parse(compile_commands_string).expect("Failed to parse linkinfo.json"); - let mut cmd = vec![]; for arg in linkinfo["cmd"].members() { cmd.push( @@ -368,16 +488,28 @@ pub fn build( ); } - assert!(cpp_compiler - .to_command() - .current_dir(&build_dir) + let mut link_command = cpp_compiler.to_command(); + + link_command + .current_dir(&libafl_qemu_build_dir) .arg("-o") .arg("libqemu-partially-linked.o") .arg("-r") - .args(cmd) - .status() - .expect("Partial linked failure") - .success()); + .args(cmd); + + let link_str = format!("{link_command:?}"); + + let output = link_command.output().expect("Partial linked failure"); + + if !output.status.success() { + fs::write(libafl_qemu_build_dir.join("link.command"), link_str) + .expect("Link command failed."); + fs::write(libafl_qemu_build_dir.join("link.stdout"), &output.stdout) + .expect("Link stdout failed."); + fs::write(libafl_qemu_build_dir.join("link.stderr"), &output.stderr) + .expect("Link stderr failed."); + panic!("Linking failed."); + } /* // Old manual linking, kept here for reference and debugging if is_usermode { @@ -467,7 +599,7 @@ pub fn build( .current_dir(out_dir_path) .arg("crs") .arg("libqemu-partially-linked.a") - .arg(build_dir.join("libqemu-partially-linked.o")) + .arg(libafl_qemu_build_dir.join("libqemu-partially-linked.o")) .status() .expect("Ar creation"); @@ -487,6 +619,26 @@ pub fn build( .expect("linkinfo.json `libs` values must be strings"); println!("cargo:rustc-link-lib={val}"); } + + for arg in linkinfo["rpath"].members() { + let val = arg + .as_str() + .expect("linkinfo.json `libs` values must be strings") + .to_string() + .replace( + "$ORIGIN", + libafl_qemu_build_dir + .as_os_str() + .to_str() + .expect("Could not convert OsStr to str"), + ); + cargo_add_rpath(&val); + } + } + + if cfg!(feature = "paranoid_debug") { + println!("cargo:rustc-link-lib=ubsan"); + println!("cargo:rustc-link-lib=asan"); } /* @@ -508,7 +660,7 @@ pub fn build( //} fs::create_dir_all(target_dir.join("pc-bios")).unwrap(); - for path in fs::read_dir(build_dir.join("pc-bios")).unwrap() { + for path in fs::read_dir(libafl_qemu_build_dir.join("pc-bios")).unwrap() { let path = path.unwrap().path(); if path.is_file() { if let Some(name) = path.file_name() { @@ -520,7 +672,7 @@ pub fn build( } BuildResult { - qemu_path, - build_dir, + qemu_path: libafl_qemu_dir, + build_dir: libafl_qemu_build_dir, } } diff --git a/libafl_qemu/libafl_qemu_build/src/lib.rs b/libafl_qemu/libafl_qemu_build/src/lib.rs index c112adf678..0bb58d9443 100644 --- a/libafl_qemu/libafl_qemu_build/src/lib.rs +++ b/libafl_qemu/libafl_qemu_build/src/lib.rs @@ -1,8 +1,13 @@ #![allow(clippy::missing_panics_doc)] use std::{ + collections::hash_map, env, fs, + fs::File, + hash::Hasher, + io::{Read, Seek, SeekFrom, Write}, path::{Path, PathBuf}, process::Command, + ptr::addr_of_mut, }; use regex::Regex; @@ -15,6 +20,43 @@ pub use build::build; const LLVM_VERSION_MAX: i32 = 33; +static mut CARGO_RPATH: Option> = None; +static CARGO_RPATH_SEPARATOR: &str = "|"; + +pub fn cargo_add_rpath(rpath: &str) { + unsafe { + if let Some(rpaths) = &mut *addr_of_mut!(CARGO_RPATH) { + rpaths.push(rpath.to_string()); + } else { + CARGO_RPATH = Some(vec![rpath.to_string()]); + } + } +} + +pub fn cargo_propagate_rpath() { + unsafe { + if let Some(cargo_cmds) = &mut *addr_of_mut!(CARGO_RPATH) { + let rpath = cargo_cmds.join(CARGO_RPATH_SEPARATOR); + println!("cargo:rpath={rpath}"); + } + } +} + +/// Must be called from final binary crates +pub fn build_libafl_qemu() { + // Add rpath if there are some + if let Some(rpaths) = env::var_os("DEP_QEMU_RPATH") { + let rpaths: Vec<&str> = rpaths + .to_str() + .expect("Cannot convert OsString to str") + .split(CARGO_RPATH_SEPARATOR) + .collect(); + for rpath in rpaths { + println!("cargo:rustc-link-arg-bins=-Wl,-rpath,{rpath}"); + } + } +} + pub fn build_with_bindings( cpu_target: &str, is_big_endian: bool, @@ -42,6 +84,8 @@ pub fn build_with_bindings( let re = Regex::new("(Option<\\s*)unsafe( extern \"C\" fn\\(data: u64)").unwrap(); let replaced = re.replace_all(&contents, "$1$2"); fs::write(bindings_file, replaced.as_bytes()).expect("Unable to write file"); + + cargo_propagate_rpath(); } // For bindgen, the llvm version must be >= of the rust llvm version @@ -203,3 +247,46 @@ fn include_path(build_dir: &Path, path: &str) -> String { build_dir.join(include_path).display().to_string() } } + +pub fn store_generated_content_if_different(file_to_update: &PathBuf, fresh_content: &[u8]) { + let mut must_rewrite_file = true; + + // Check if equivalent file already exists without relying on filesystem timestamp. + let mut file_to_check = + if let Ok(mut wrapper_file) = File::options().read(true).write(true).open(file_to_update) { + let mut existing_file_content = Vec::with_capacity(fresh_content.len()); + wrapper_file + .read_to_end(existing_file_content.as_mut()) + .unwrap(); + + let mut existing_wrapper_hasher = hash_map::DefaultHasher::new(); + existing_wrapper_hasher.write(existing_file_content.as_ref()); + + let mut wrapper_h_hasher = hash_map::DefaultHasher::new(); + wrapper_h_hasher.write(fresh_content); + + // Check if wrappers are the same + if existing_wrapper_hasher.finish() == wrapper_h_hasher.finish() { + must_rewrite_file = false; + } + + // Reset file cursor if it's going to be rewritten + if must_rewrite_file { + wrapper_file.set_len(0).expect("Could not set file len"); + wrapper_file + .seek(SeekFrom::Start(0)) + .expect("Could not seek file to beginning"); + } + + wrapper_file + } else { + File::create(file_to_update) + .unwrap_or_else(|_| panic!("Could not create {}", file_to_update.display())) + }; + + if must_rewrite_file { + file_to_check + .write_all(fresh_content) + .unwrap_or_else(|_| panic!("Unable to write in {}", file_to_update.display())); + } +} diff --git a/libafl_qemu/libafl_qemu_sys/Cargo.toml b/libafl_qemu/libafl_qemu_sys/Cargo.toml index d04c6a69ab..b947303b8f 100644 --- a/libafl_qemu/libafl_qemu_sys/Cargo.toml +++ b/libafl_qemu/libafl_qemu_sys/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "libafl_qemu_sys" -version = "0.11.2" +version = "0.12.0" authors = ["Andrea Fioraldi "] description = "C to Rust bindings for the LibAFL QEMU bridge" documentation = "https://docs.rs/libafl_qemu_sys" @@ -10,9 +10,10 @@ license = "MIT OR Apache-2.0" keywords = ["fuzzing", "qemu", "instrumentation"] edition = "2021" categories = ["development-tools::testing", "emulators", "embedded", "os", "no-std"] +links = "qemu" [package.metadata.docs.rs] -all-features = true +features = ["x86_64", "usermode"] rustdoc-args = ["--cfg", "docsrs"] [features] @@ -30,12 +31,24 @@ be = [] usermode = [] systemmode = [] +python = ["pyo3", "pyo3-build-config"] + slirp = [ "systemmode", "libafl_qemu_build/slirp" ] # build qemu with host libslirp (for user networking) shared = [ "libafl_qemu_build/shared" ] -clippy = [ "libafl_qemu_build/clippy" ] # special feature for clippy, don't use in normal projects§ +clippy = [ "libafl_qemu_build/clippy" ] # special feature for clippy, don't use in normal projects + +paranoid_debug = ["libafl_qemu_build/paranoid_debug"] # Will perform as many checks as possible. The target will be greatly slowed down. [dependencies] +paste = "1" +num_enum = "0.7" +libc = "0.2" +strum = "0.25" +strum_macros = "0.25" +pyo3 = { version = "0.18", optional = true } [build-dependencies] -libafl_qemu_build = { path = "../libafl_qemu_build", version = "0.11.2" } +libafl_qemu_build = { path = "../libafl_qemu_build", version = "0.12.0" } +pyo3-build-config = { version = "0.18", optional = true } +rustversion = "1.0" diff --git a/libafl_qemu/libafl_qemu_sys/build_linux.rs b/libafl_qemu/libafl_qemu_sys/build_linux.rs index a6d09da552..e59480c6b1 100644 --- a/libafl_qemu/libafl_qemu_sys/build_linux.rs +++ b/libafl_qemu/libafl_qemu_sys/build_linux.rs @@ -2,6 +2,11 @@ use std::{env, fs::copy, path::PathBuf}; use libafl_qemu_build::build_with_bindings; +#[rustversion::nightly] +use std::fs; +#[rustversion::nightly] +use libafl_qemu_build::store_generated_content_if_different; + #[macro_export] macro_rules! assert_unique_feature { () => {}; @@ -14,6 +19,18 @@ macro_rules! assert_unique_feature { } } +#[rustversion::nightly] +fn maybe_generate_stub_bindings(cpu_target: &str, emulation_mode: &str, stub_bindings_file: &PathBuf, bindings_file: &PathBuf) { + if cpu_target == "x86_64" && emulation_mode == "usermode" { + store_generated_content_if_different(stub_bindings_file, fs::read(bindings_file).expect("Could not read generated bindings file").as_slice()); + } +} + +#[rustversion::not(nightly)] +fn maybe_generate_stub_bindings(_cpu_target: &str, _emulation_mode: &str, _stub_bindings_file: &PathBuf, _bindings_file: &PathBuf) { + // Do nothing +} + pub fn build() { // Make sure that exactly one qemu mode is set assert_unique_feature!("usermode", "systemmode"); @@ -32,9 +49,6 @@ pub fn build() { println!("cargo:rustc-cfg=emulation_mode=\"{emulation_mode}\""); println!("cargo:rerun-if-env-changed=EMULATION_MODE"); - println!("cargo:rerun-if-changed=build.rs"); - println!("cargo:rerun-if-changed=build_linux.rs"); - // Make sure we have at most one architecutre feature set // Else, we default to `x86_64` - having a default makes CI easier :) assert_unique_feature!("arm", "aarch64", "i386", "i86_64", "mips", "ppc", "hexagon"); @@ -76,10 +90,14 @@ pub fn build() { let out_dir = env::var("OUT_DIR").unwrap(); let out_dir = PathBuf::from(out_dir); let bindings_file = out_dir.join("bindings.rs"); + + let src_dir = env::var("CARGO_MANIFEST_DIR").unwrap(); + let src_dir = PathBuf::from(src_dir); + let stub_bindings_file = src_dir.join("src/x86_64_stub_bindings.rs"); - if std::env::var("DOCS_RS").is_ok() || cfg!(feature = "clippy") { + if env::var("DOCS_RS").is_ok() || cfg!(feature = "clippy") { // Only build when we're not generating docs and not in clippy - copy("src/x86_64_stub_bindings.rs", bindings_file).expect("Failed to copy the bindings stub"); + copy(stub_bindings_file, bindings_file).expect("Failed to copy the bindings stub"); return; } @@ -90,4 +108,9 @@ pub fn build() { jobs, &bindings_file, ); -} + + println!("cargo:rerun-if-changed={}", stub_bindings_file.display()); + + // If the bindings are built and differ from the current stub, replace it with the freshly generated bindings + maybe_generate_stub_bindings(&cpu_target, &emulation_mode, &stub_bindings_file, &bindings_file); +} \ No newline at end of file diff --git a/libafl_qemu/libafl_qemu_sys/src/lib.rs b/libafl_qemu/libafl_qemu_sys/src/lib.rs index 5af24d4479..63722c5c33 100644 --- a/libafl_qemu/libafl_qemu_sys/src/lib.rs +++ b/libafl_qemu/libafl_qemu_sys/src/lib.rs @@ -1,3 +1,10 @@ +/*! +`libafl_qemu_sys` is the crate exporting C symbols from QEMU. +Have a look at `libafl_qemu` for higher-level abstractions. + +__Warning__: The documentation is built by default for `x86_64` in `usermode`. To access the documentation of other architectures or systemmode, the documentation must be rebuilt with the right features. +*/ + #![allow(non_upper_case_globals)] #![allow(non_camel_case_types)] #![allow(non_snake_case)] @@ -7,17 +14,142 @@ #![allow(clippy::pedantic)] #[cfg(all(not(feature = "clippy"), target_os = "linux"))] -include!(concat!(env!("OUT_DIR"), "/bindings.rs")); +mod bindings { + include!(concat!(env!("OUT_DIR"), "/bindings.rs")); +} +#[cfg(all(not(feature = "clippy"), target_os = "linux"))] +pub use bindings::*; -#[cfg(all(feature = "clippy", target_os = "linux"))] +#[cfg(any(feature = "clippy", not(target_os = "linux")))] +#[rustfmt::skip] mod x86_64_stub_bindings; +#[cfg(emulation_mode = "usermode")] +mod usermode; +#[cfg(emulation_mode = "usermode")] +pub use usermode::*; + +#[cfg(emulation_mode = "systemmode")] +mod systemmode; +use num_enum::{IntoPrimitive, TryFromPrimitive}; +use paste::paste; +use strum_macros::EnumIter; +#[cfg(emulation_mode = "systemmode")] +pub use systemmode::*; + +/// Safe linking with of extern "C" functions. +/// This macro makes sure the declared symbol is defined *at link time*, avoiding declaring non-existant symbols +/// that could be silently ignored during linking if unused. +/// +/// This macro relies on a nightly feature, and can only be used in this mode +/// It is (nearly) a drop-in replacement for extern "C" { } blocks containing function and static declarations, and will have the same effect in practice. +#[macro_export] +macro_rules! extern_c_checked { + () => {}; + + ($visibility:vis fn $c_fn:ident($($param_ident:ident : $param_ty:ty),*) $( -> $ret_ty:ty )?; $($tail:tt)*) => { + paste! { + #[cfg_attr(nightly, used(linker))] + static [<__ $c_fn:upper __>]: unsafe extern "C" fn($($param_ty),*) $( -> $ret_ty )? = $c_fn; + } + + extern "C" { + $visibility fn $c_fn($($param_ident : $param_ty),*) $( -> $ret_ty )?; + } + + extern_c_checked!($($tail)*); + }; + + ($visibility:vis static $c_var:ident : $c_var_ty:ty; $($tail:tt)*) => { + paste! { + #[allow(non_camel_case_types)] + #[allow(unused)] + struct [<__ $c_var:upper _STRUCT__>] { member: *const $c_var_ty } + + unsafe impl Sync for [<__ $c_var:upper _STRUCT__>] {} + + #[cfg_attr(nightly, used(linker))] + static [<__ $c_var:upper __>]: [<__ $c_var:upper _STRUCT__>] = unsafe { [<__ $c_var:upper _STRUCT__>] { member: core::ptr::addr_of!($c_var) } }; + } + + extern "C" { + $visibility static $c_var: $c_var_ty; + } + + extern_c_checked!($($tail)*); + }; + + ($visibility:vis static mut $c_var:ident : $c_var_ty:ty; $($tail:tt)*) => { + paste! { + #[allow(non_camel_case_types)] + #[allow(unused)] + struct [<__ $c_var:upper _STRUCT__>] { member: *const $c_var_ty } + + unsafe impl Sync for [<__ $c_var:upper _STRUCT__>] {} + + #[cfg_attr(nightly, used(linker))] + static mut [<__ $c_var:upper __>]: [<__ $c_var:upper _STRUCT__>] = unsafe { [<__ $c_var:upper _STRUCT__>] { member: core::ptr::addr_of!($c_var) } }; + } + + extern "C" { + $visibility static mut $c_var: $c_var_ty; + } + + extern_c_checked!($($tail)*); + }; +} + #[cfg(target_os = "linux")] use core::ops::BitAnd; +use std::ffi::c_void; -#[cfg(all(feature = "clippy", target_os = "linux"))] +#[cfg(feature = "python")] +use pyo3::{pyclass, pymethods, IntoPy, PyObject, Python}; +#[cfg(any(feature = "clippy", not(target_os = "linux")))] pub use x86_64_stub_bindings::*; +pub type CPUStatePtr = *mut crate::CPUState; +pub type CPUArchStatePtr = *mut crate::CPUArchState; +pub type ExitReasonPtr = *mut crate::libafl_exit_reason; + +pub type GuestUsize = crate::target_ulong; +pub type GuestIsize = crate::target_long; + +pub type GuestAddr = crate::target_ulong; +pub type GuestPhysAddr = crate::hwaddr; +pub type GuestVirtAddr = crate::vaddr; + +pub type GuestHwAddrInfo = crate::qemu_plugin_hwaddr; + +#[derive(Debug)] +#[repr(C)] +#[cfg_attr(feature = "python", pyclass(unsendable))] +pub struct MapInfo { + start: GuestAddr, + end: GuestAddr, + offset: GuestAddr, + path: Option, + flags: i32, + is_priv: i32, +} + +#[repr(C)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FatPtr(pub *const c_void, pub *const c_void); + +#[derive(IntoPrimitive, TryFromPrimitive, Debug, Clone, Copy, EnumIter, PartialEq, Eq)] +#[repr(i32)] +pub enum MmapPerms { + None = 0, + Read = libc::PROT_READ, + Write = libc::PROT_WRITE, + Execute = libc::PROT_EXEC, + ReadWrite = libc::PROT_READ | libc::PROT_WRITE, + ReadExecute = libc::PROT_READ | libc::PROT_EXEC, + WriteExecute = libc::PROT_WRITE | libc::PROT_EXEC, + ReadWriteExecute = libc::PROT_READ | libc::PROT_WRITE | libc::PROT_EXEC, +} + // from include/exec/memop.h #[cfg(target_os = "linux")] @@ -43,3 +175,113 @@ pub fn make_plugin_meminfo(oi: MemOpIdx, rw: qemu_plugin_mem_rw) -> qemu_plugin_ pub fn cpu_env(cpu: *mut CPUState) -> *mut CPUArchState { unsafe { cpu.add(1) as *mut CPUArchState } } + +extern_c_checked! { + //static libafl_page_size: GuestUsize; + pub fn libafl_page_from_addr(addr: GuestAddr) -> GuestAddr; + + // CPUState* libafl_qemu_get_cpu(int cpu_index); + pub fn libafl_qemu_get_cpu(cpu_index: i32) -> CPUStatePtr; + // int libafl_qemu_num_cpus(void); + pub fn libafl_qemu_num_cpus() -> i32; + // CPUState* libafl_qemu_current_cpu(void); + pub fn libafl_qemu_current_cpu() -> CPUStatePtr; + + // struct libafl_exit_reason* libafl_get_exit_reason(void); + // fn libafl_get_exit_reason() -> ExitReasonPtr; + + pub fn libafl_qemu_cpu_index(cpu: CPUStatePtr) -> i32; + + pub fn libafl_qemu_write_reg(cpu: CPUStatePtr, reg: i32, val: *const u8) -> i32; + pub fn libafl_qemu_read_reg(cpu: CPUStatePtr, reg: i32, val: *mut u8) -> i32; + pub fn libafl_qemu_num_regs(cpu: CPUStatePtr) -> i32; + + // fn libafl_qemu_set_breakpoint(addr: u64) -> i32; + // fn libafl_qemu_remove_breakpoint(addr: u64) -> i32; + pub fn libafl_flush_jit(); + // fn libafl_qemu_trigger_breakpoint(cpu: CPUStatePtr); + + pub fn strlen(s: *const u8) -> usize; + + pub fn libafl_qemu_add_gdb_cmd( + callback: extern "C" fn(*const (), *const u8, usize) -> i32, + data: *const () + ); + pub fn libafl_qemu_gdb_reply(buf: *const u8, len: usize); +} + +#[cfg_attr(feature = "python", pymethods)] +impl MapInfo { + #[must_use] + pub fn start(&self) -> GuestAddr { + self.start + } + + #[must_use] + pub fn end(&self) -> GuestAddr { + self.end + } + + #[must_use] + pub fn offset(&self) -> GuestAddr { + self.offset + } + + #[must_use] + pub fn path(&self) -> Option<&String> { + self.path.as_ref() + } + + #[must_use] + pub fn flags(&self) -> MmapPerms { + MmapPerms::try_from(self.flags).unwrap() + } + + #[must_use] + pub fn is_priv(&self) -> bool { + self.is_priv != 0 + } +} + +impl MmapPerms { + #[must_use] + pub fn readable(&self) -> bool { + matches!( + self, + MmapPerms::Read + | MmapPerms::ReadWrite + | MmapPerms::ReadExecute + | MmapPerms::ReadWriteExecute + ) + } + + #[must_use] + pub fn writable(&self) -> bool { + matches!( + self, + MmapPerms::Write + | MmapPerms::ReadWrite + | MmapPerms::WriteExecute + | MmapPerms::ReadWriteExecute + ) + } + + #[must_use] + pub fn executable(&self) -> bool { + matches!( + self, + MmapPerms::Execute + | MmapPerms::ReadExecute + | MmapPerms::WriteExecute + | MmapPerms::ReadWriteExecute + ) + } +} + +#[cfg(feature = "python")] +impl IntoPy for MmapPerms { + fn into_py(self, py: Python) -> PyObject { + let n: i32 = self.into(); + n.into_py(py) + } +} diff --git a/libafl_qemu/libafl_qemu_sys/src/systemmode.rs b/libafl_qemu/libafl_qemu_sys/src/systemmode.rs new file mode 100644 index 0000000000..85b02d58d4 --- /dev/null +++ b/libafl_qemu/libafl_qemu_sys/src/systemmode.rs @@ -0,0 +1,16 @@ +use paste::paste; + +use crate::{extern_c_checked, CPUStatePtr, GuestPhysAddr}; + +extern_c_checked! { + pub fn qemu_init(argc: i32, argv: *const *const u8, envp: *const *const u8); + + pub fn vm_start(); + pub fn qemu_main_loop(); + pub fn qemu_cleanup(); + + pub fn libafl_save_qemu_snapshot(name: *const u8, sync: bool); + pub fn libafl_load_qemu_snapshot(name: *const u8, sync: bool); + + pub fn libafl_qemu_current_paging_id(cpu: CPUStatePtr) -> GuestPhysAddr; +} diff --git a/libafl_qemu/libafl_qemu_sys/src/usermode.rs b/libafl_qemu/libafl_qemu_sys/src/usermode.rs new file mode 100644 index 0000000000..a378a96f5c --- /dev/null +++ b/libafl_qemu/libafl_qemu_sys/src/usermode.rs @@ -0,0 +1,58 @@ +use core::{slice::from_raw_parts, str::from_utf8_unchecked}; + +use num_enum::{IntoPrimitive, TryFromPrimitive}; +use paste::paste; +use strum_macros::EnumIter; + +use crate::{extern_c_checked, libafl_mapinfo, strlen, GuestAddr, MapInfo}; + +extern_c_checked! { + pub fn qemu_user_init(argc: i32, argv: *const *const u8, envp: *const *const u8) -> i32; + + pub fn libafl_qemu_run() -> i32; + + pub fn libafl_load_addr() -> u64; + pub fn libafl_get_brk() -> u64; + pub fn libafl_set_brk(brk: u64) -> u64; + + pub static exec_path: *const u8; + pub static guest_base: usize; + pub static mut mmap_next_start: GuestAddr; + + pub static mut libafl_dump_core_hook: unsafe extern "C" fn(i32); + pub static mut libafl_force_dfl: i32; +} + +#[derive(IntoPrimitive, TryFromPrimitive, Debug, Clone, Copy, EnumIter, PartialEq, Eq)] +#[repr(i32)] +pub enum VerifyAccess { + Read = libc::PROT_READ, + Write = libc::PROT_READ | libc::PROT_WRITE, +} + +impl From for MapInfo { + fn from(map_info: libafl_mapinfo) -> Self { + let path: Option = if map_info.path.is_null() { + None + } else { + unsafe { + Some( + from_utf8_unchecked(from_raw_parts( + map_info.path as *const u8, + strlen(map_info.path as *const u8), + )) + .to_string(), + ) + } + }; + + MapInfo { + start: map_info.start, + end: map_info.end, + offset: map_info.offset, + path, + flags: map_info.flags, + is_priv: map_info.is_priv, + } + } +} diff --git a/libafl_qemu/libafl_qemu_sys/src/x86_64_stub_bindings.rs b/libafl_qemu/libafl_qemu_sys/src/x86_64_stub_bindings.rs index f7f904f13e..be87633126 100644 --- a/libafl_qemu/libafl_qemu_sys/src/x86_64_stub_bindings.rs +++ b/libafl_qemu/libafl_qemu_sys/src/x86_64_stub_bindings.rs @@ -1,4 +1,4 @@ -/* automatically generated by rust-bindgen 0.68.1 */ +/* automatically generated by rust-bindgen 0.69.4 */ #[repr(C)] #[derive(Copy, Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] @@ -1293,6 +1293,11 @@ pub struct RAMBlock { } #[repr(C)] #[derive(Debug, Copy, Clone)] +pub struct TCGCPUOps { + _unused: [u8; 0], +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] pub struct Visitor { _unused: [u8; 0], } @@ -1589,7 +1594,7 @@ fn bindgen_test_layout_QemuLockCnt() { #[derive(Debug, Default, Copy, Clone)] pub struct MemTxAttrs { pub _bitfield_align_1: [u16; 0], - pub _bitfield_1: __BindgenBitfieldUnit<[u8; 4usize]>, + pub _bitfield_1: __BindgenBitfieldUnit<[u8; 3usize]>, } #[test] fn bindgen_test_layout_MemTxAttrs() { @@ -1672,50 +1677,6 @@ impl MemTxAttrs { } } #[inline] - pub fn byte_swap(&self) -> ::std::os::raw::c_uint { - unsafe { ::std::mem::transmute(self._bitfield_1.get(22usize, 1u8) as u32) } - } - #[inline] - pub fn set_byte_swap(&mut self, val: ::std::os::raw::c_uint) { - unsafe { - let val: u32 = ::std::mem::transmute(val); - self._bitfield_1.set(22usize, 1u8, val as u64) - } - } - #[inline] - pub fn target_tlb_bit0(&self) -> ::std::os::raw::c_uint { - unsafe { ::std::mem::transmute(self._bitfield_1.get(23usize, 1u8) as u32) } - } - #[inline] - pub fn set_target_tlb_bit0(&mut self, val: ::std::os::raw::c_uint) { - unsafe { - let val: u32 = ::std::mem::transmute(val); - self._bitfield_1.set(23usize, 1u8, val as u64) - } - } - #[inline] - pub fn target_tlb_bit1(&self) -> ::std::os::raw::c_uint { - unsafe { ::std::mem::transmute(self._bitfield_1.get(24usize, 1u8) as u32) } - } - #[inline] - pub fn set_target_tlb_bit1(&mut self, val: ::std::os::raw::c_uint) { - unsafe { - let val: u32 = ::std::mem::transmute(val); - self._bitfield_1.set(24usize, 1u8, val as u64) - } - } - #[inline] - pub fn target_tlb_bit2(&self) -> ::std::os::raw::c_uint { - unsafe { ::std::mem::transmute(self._bitfield_1.get(25usize, 1u8) as u32) } - } - #[inline] - pub fn set_target_tlb_bit2(&mut self, val: ::std::os::raw::c_uint) { - unsafe { - let val: u32 = ::std::mem::transmute(val); - self._bitfield_1.set(25usize, 1u8, val as u64) - } - } - #[inline] pub fn new_bitfield_1( unspecified: ::std::os::raw::c_uint, secure: ::std::os::raw::c_uint, @@ -1723,12 +1684,8 @@ impl MemTxAttrs { user: ::std::os::raw::c_uint, memory: ::std::os::raw::c_uint, requester_id: ::std::os::raw::c_uint, - byte_swap: ::std::os::raw::c_uint, - target_tlb_bit0: ::std::os::raw::c_uint, - target_tlb_bit1: ::std::os::raw::c_uint, - target_tlb_bit2: ::std::os::raw::c_uint, - ) -> __BindgenBitfieldUnit<[u8; 4usize]> { - let mut __bindgen_bitfield_unit: __BindgenBitfieldUnit<[u8; 4usize]> = Default::default(); + ) -> __BindgenBitfieldUnit<[u8; 3usize]> { + let mut __bindgen_bitfield_unit: __BindgenBitfieldUnit<[u8; 3usize]> = Default::default(); __bindgen_bitfield_unit.set(0usize, 1u8, { let unspecified: u32 = unsafe { ::std::mem::transmute(unspecified) }; unspecified as u64 @@ -1753,22 +1710,6 @@ impl MemTxAttrs { let requester_id: u32 = unsafe { ::std::mem::transmute(requester_id) }; requester_id as u64 }); - __bindgen_bitfield_unit.set(22usize, 1u8, { - let byte_swap: u32 = unsafe { ::std::mem::transmute(byte_swap) }; - byte_swap as u64 - }); - __bindgen_bitfield_unit.set(23usize, 1u8, { - let target_tlb_bit0: u32 = unsafe { ::std::mem::transmute(target_tlb_bit0) }; - target_tlb_bit0 as u64 - }); - __bindgen_bitfield_unit.set(24usize, 1u8, { - let target_tlb_bit1: u32 = unsafe { ::std::mem::transmute(target_tlb_bit1) }; - target_tlb_bit1 as u64 - }); - __bindgen_bitfield_unit.set(25usize, 1u8, { - let target_tlb_bit2: u32 = unsafe { ::std::mem::transmute(target_tlb_bit2) }; - target_tlb_bit2 as u64 - }); __bindgen_bitfield_unit } } @@ -2399,6 +2340,7 @@ pub type DeviceReset = ::std::option::Option(), - 208usize, + 216usize, concat!("Size of: ", stringify!(disassemble_info)) ); assert_eq!( @@ -4312,8 +4255,18 @@ fn bindgen_test_layout_disassemble_info() { ) ); assert_eq!( - unsafe { ::std::ptr::addr_of!((*ptr).target_info) as usize - ptr as usize }, + unsafe { ::std::ptr::addr_of!((*ptr).show_opcodes) as usize - ptr as usize }, 184usize, + concat!( + "Offset of field: ", + stringify!(disassemble_info), + "::", + stringify!(show_opcodes) + ) + ); + assert_eq!( + unsafe { ::std::ptr::addr_of!((*ptr).target_info) as usize - ptr as usize }, + 192usize, concat!( "Offset of field: ", stringify!(disassemble_info), @@ -4323,7 +4276,7 @@ fn bindgen_test_layout_disassemble_info() { ); assert_eq!( unsafe { ::std::ptr::addr_of!((*ptr).cap_arch) as usize - ptr as usize }, - 192usize, + 200usize, concat!( "Offset of field: ", stringify!(disassemble_info), @@ -4333,7 +4286,7 @@ fn bindgen_test_layout_disassemble_info() { ); assert_eq!( unsafe { ::std::ptr::addr_of!((*ptr).cap_mode) as usize - ptr as usize }, - 196usize, + 204usize, concat!( "Offset of field: ", stringify!(disassemble_info), @@ -4343,7 +4296,7 @@ fn bindgen_test_layout_disassemble_info() { ); assert_eq!( unsafe { ::std::ptr::addr_of!((*ptr).cap_insn_unit) as usize - ptr as usize }, - 200usize, + 208usize, concat!( "Offset of field: ", stringify!(disassemble_info), @@ -4353,7 +4306,7 @@ fn bindgen_test_layout_disassemble_info() { ); assert_eq!( unsafe { ::std::ptr::addr_of!((*ptr).cap_insn_split) as usize - ptr as usize }, - 204usize, + 212usize, concat!( "Offset of field: ", stringify!(disassemble_info), @@ -4371,18 +4324,9 @@ impl Default for disassemble_info { } } } +pub type hwaddr = u64; #[doc = " vaddr:\n Type wide enough to contain any #target_ulong virtual address."] pub type vaddr = u64; -extern "C" { - pub fn cpu_memory_rw_debug( - cpu: *mut CPUState, - addr: vaddr, - ptr: *mut ::std::os::raw::c_void, - len: usize, - is_write: bool, - ) -> ::std::os::raw::c_int; -} -pub type hwaddr = u64; #[repr(C)] #[derive(Copy, Clone)] pub union CPUTLBEntry { @@ -4585,15 +4529,10 @@ impl ::std::ops::BitAndAssign for ShutdownCause { pub struct ShutdownCause(pub ::std::os::raw::c_uint); #[repr(C)] #[derive(Debug, Copy, Clone)] -pub struct TCGCPUOps { - _unused: [u8; 0], -} -#[repr(C)] -#[derive(Debug, Copy, Clone)] pub struct SysemuCPUOps { _unused: [u8; 0], } -#[doc = " CPUClass:\n @class_by_name: Callback to map -cpu command line model name to an\n instantiatable CPU type.\n @parse_features: Callback to parse command line arguments.\n @reset_dump_flags: #CPUDumpFlags to use for reset logging.\n @has_work: Callback for checking if there is work to do.\n @memory_rw_debug: Callback for GDB memory access.\n @dump_state: Callback for dumping state.\n @query_cpu_fast:\n Fill in target specific information for the \"query-cpus-fast\"\n QAPI call.\n @get_arch_id: Callback for getting architecture-dependent CPU ID.\n @set_pc: Callback for setting the Program Counter register. This\n should have the semantics used by the target architecture when\n setting the PC from a source such as an ELF file entry point;\n for example on Arm it will also set the Thumb mode bit based\n on the least significant bit of the new PC value.\n If the target behaviour here is anything other than \"set\n the PC register to the value passed in\" then the target must\n also implement the synchronize_from_tb hook.\n @get_pc: Callback for getting the Program Counter register.\n As above, with the semantics of the target architecture.\n @gdb_read_register: Callback for letting GDB read a register.\n @gdb_write_register: Callback for letting GDB write a register.\n @gdb_adjust_breakpoint: Callback for adjusting the address of a\n breakpoint. Used by AVR to handle a gdb mis-feature with\n its Harvard architecture split code and data.\n @gdb_num_core_regs: Number of core registers accessible to GDB.\n @gdb_core_xml_file: File name for core registers GDB XML description.\n @gdb_stop_before_watchpoint: Indicates whether GDB expects the CPU to stop\n before the insn which triggers a watchpoint rather than after it.\n @gdb_arch_name: Optional callback that returns the architecture name known\n to GDB. The caller must free the returned string with g_free.\n @gdb_get_dynamic_xml: Callback to return dynamically generated XML for the\n gdb stub. Returns a pointer to the XML contents for the specified XML file\n or NULL if the CPU doesn't have a dynamically generated content for it.\n @disas_set_info: Setup architecture specific components of disassembly info\n @adjust_watchpoint_address: Perform a target-specific adjustment to an\n address before attempting to match it against watchpoints.\n @deprecation_note: If this CPUClass is deprecated, this field provides\n related information.\n\n Represents a CPU family or model."] +#[doc = " CPUClass:\n @class_by_name: Callback to map -cpu command line model name to an\n instantiatable CPU type.\n @parse_features: Callback to parse command line arguments.\n @reset_dump_flags: #CPUDumpFlags to use for reset logging.\n @has_work: Callback for checking if there is work to do.\n @mmu_index: Callback for choosing softmmu mmu index;\n may be used internally by memory_rw_debug without TCG.\n @memory_rw_debug: Callback for GDB memory access.\n @dump_state: Callback for dumping state.\n @query_cpu_fast:\n Fill in target specific information for the \"query-cpus-fast\"\n QAPI call.\n @get_arch_id: Callback for getting architecture-dependent CPU ID.\n @set_pc: Callback for setting the Program Counter register. This\n should have the semantics used by the target architecture when\n setting the PC from a source such as an ELF file entry point;\n for example on Arm it will also set the Thumb mode bit based\n on the least significant bit of the new PC value.\n If the target behaviour here is anything other than \"set\n the PC register to the value passed in\" then the target must\n also implement the synchronize_from_tb hook.\n @get_pc: Callback for getting the Program Counter register.\n As above, with the semantics of the target architecture.\n @gdb_read_register: Callback for letting GDB read a register.\n @gdb_write_register: Callback for letting GDB write a register.\n @gdb_adjust_breakpoint: Callback for adjusting the address of a\n breakpoint. Used by AVR to handle a gdb mis-feature with\n its Harvard architecture split code and data.\n @gdb_num_core_regs: Number of core registers accessible to GDB or 0 to infer\n from @gdb_core_xml_file.\n @gdb_core_xml_file: File name for core registers GDB XML description.\n @gdb_stop_before_watchpoint: Indicates whether GDB expects the CPU to stop\n before the insn which triggers a watchpoint rather than after it.\n @gdb_arch_name: Optional callback that returns the architecture name known\n to GDB. The caller must free the returned string with g_free.\n @disas_set_info: Setup architecture specific components of disassembly info\n @adjust_watchpoint_address: Perform a target-specific adjustment to an\n address before attempting to match it against watchpoints.\n @deprecation_note: If this CPUClass is deprecated, this field provides\n related information.\n\n Represents a CPU family or model."] #[repr(C)] #[derive(Debug, Copy, Clone)] pub struct CPUClass { @@ -4609,6 +4548,9 @@ pub struct CPUClass { ), >, pub has_work: ::std::option::Option bool>, + pub mmu_index: ::std::option::Option< + unsafe extern "C" fn(cpu: *mut CPUState, ifetch: bool) -> ::std::os::raw::c_int, + >, pub memory_rw_debug: ::std::option::Option< unsafe extern "C" fn( cpu: *mut CPUState, @@ -4645,12 +4587,6 @@ pub struct CPUClass { pub gdb_core_xml_file: *const ::std::os::raw::c_char, pub gdb_arch_name: ::std::option::Option *const gchar>, - pub gdb_get_dynamic_xml: ::std::option::Option< - unsafe extern "C" fn( - cpu: *mut CPUState, - xmlname: *const ::std::os::raw::c_char, - ) -> *const ::std::os::raw::c_char, - >, pub disas_set_info: ::std::option::Option< unsafe extern "C" fn(cpu: *mut CPUState, info: *mut disassemble_info), >, @@ -4720,8 +4656,18 @@ fn bindgen_test_layout_CPUClass() { ) ); assert_eq!( - unsafe { ::std::ptr::addr_of!((*ptr).memory_rw_debug) as usize - ptr as usize }, + unsafe { ::std::ptr::addr_of!((*ptr).mmu_index) as usize - ptr as usize }, 200usize, + concat!( + "Offset of field: ", + stringify!(CPUClass), + "::", + stringify!(mmu_index) + ) + ); + assert_eq!( + unsafe { ::std::ptr::addr_of!((*ptr).memory_rw_debug) as usize - ptr as usize }, + 208usize, concat!( "Offset of field: ", stringify!(CPUClass), @@ -4731,7 +4677,7 @@ fn bindgen_test_layout_CPUClass() { ); assert_eq!( unsafe { ::std::ptr::addr_of!((*ptr).dump_state) as usize - ptr as usize }, - 208usize, + 216usize, concat!( "Offset of field: ", stringify!(CPUClass), @@ -4741,7 +4687,7 @@ fn bindgen_test_layout_CPUClass() { ); assert_eq!( unsafe { ::std::ptr::addr_of!((*ptr).query_cpu_fast) as usize - ptr as usize }, - 216usize, + 224usize, concat!( "Offset of field: ", stringify!(CPUClass), @@ -4751,7 +4697,7 @@ fn bindgen_test_layout_CPUClass() { ); assert_eq!( unsafe { ::std::ptr::addr_of!((*ptr).get_arch_id) as usize - ptr as usize }, - 224usize, + 232usize, concat!( "Offset of field: ", stringify!(CPUClass), @@ -4761,7 +4707,7 @@ fn bindgen_test_layout_CPUClass() { ); assert_eq!( unsafe { ::std::ptr::addr_of!((*ptr).set_pc) as usize - ptr as usize }, - 232usize, + 240usize, concat!( "Offset of field: ", stringify!(CPUClass), @@ -4771,7 +4717,7 @@ fn bindgen_test_layout_CPUClass() { ); assert_eq!( unsafe { ::std::ptr::addr_of!((*ptr).get_pc) as usize - ptr as usize }, - 240usize, + 248usize, concat!( "Offset of field: ", stringify!(CPUClass), @@ -4781,7 +4727,7 @@ fn bindgen_test_layout_CPUClass() { ); assert_eq!( unsafe { ::std::ptr::addr_of!((*ptr).gdb_read_register) as usize - ptr as usize }, - 248usize, + 256usize, concat!( "Offset of field: ", stringify!(CPUClass), @@ -4791,7 +4737,7 @@ fn bindgen_test_layout_CPUClass() { ); assert_eq!( unsafe { ::std::ptr::addr_of!((*ptr).gdb_write_register) as usize - ptr as usize }, - 256usize, + 264usize, concat!( "Offset of field: ", stringify!(CPUClass), @@ -4801,7 +4747,7 @@ fn bindgen_test_layout_CPUClass() { ); assert_eq!( unsafe { ::std::ptr::addr_of!((*ptr).gdb_adjust_breakpoint) as usize - ptr as usize }, - 264usize, + 272usize, concat!( "Offset of field: ", stringify!(CPUClass), @@ -4811,7 +4757,7 @@ fn bindgen_test_layout_CPUClass() { ); assert_eq!( unsafe { ::std::ptr::addr_of!((*ptr).gdb_core_xml_file) as usize - ptr as usize }, - 272usize, + 280usize, concat!( "Offset of field: ", stringify!(CPUClass), @@ -4821,22 +4767,12 @@ fn bindgen_test_layout_CPUClass() { ); assert_eq!( unsafe { ::std::ptr::addr_of!((*ptr).gdb_arch_name) as usize - ptr as usize }, - 280usize, - concat!( - "Offset of field: ", - stringify!(CPUClass), - "::", - stringify!(gdb_arch_name) - ) - ); - assert_eq!( - unsafe { ::std::ptr::addr_of!((*ptr).gdb_get_dynamic_xml) as usize - ptr as usize }, 288usize, concat!( "Offset of field: ", stringify!(CPUClass), "::", - stringify!(gdb_get_dynamic_xml) + stringify!(gdb_arch_name) ) ); assert_eq!( @@ -4947,6 +4883,7 @@ pub struct CPUTLBEntryFull { pub attrs: MemTxAttrs, pub prot: u8, pub lg_page_size: u8, + pub tlb_fill_flags: u8, pub slow_flags: [u8; 3usize], pub extra: CPUTLBEntryFull__bindgen_ty_1, } @@ -5119,8 +5056,18 @@ fn bindgen_test_layout_CPUTLBEntryFull() { ) ); assert_eq!( - unsafe { ::std::ptr::addr_of!((*ptr).slow_flags) as usize - ptr as usize }, + unsafe { ::std::ptr::addr_of!((*ptr).tlb_fill_flags) as usize - ptr as usize }, 22usize, + concat!( + "Offset of field: ", + stringify!(CPUTLBEntryFull), + "::", + stringify!(tlb_fill_flags) + ) + ); + assert_eq!( + unsafe { ::std::ptr::addr_of!((*ptr).slow_flags) as usize - ptr as usize }, + 23usize, concat!( "Offset of field: ", stringify!(CPUTLBEntryFull), @@ -5130,7 +5077,7 @@ fn bindgen_test_layout_CPUTLBEntryFull() { ); assert_eq!( unsafe { ::std::ptr::addr_of!((*ptr).extra) as usize - ptr as usize }, - 25usize, + 26usize, concat!( "Offset of field: ", stringify!(CPUTLBEntryFull), @@ -5892,7 +5839,7 @@ pub struct kvm_run { pub struct qemu_work_item { _unused: [u8; 0], } -#[doc = " CPUState:\n @cpu_index: CPU index (informative).\n @cluster_index: Identifies which cluster this CPU is in.\n For boards which don't define clusters or for \"loose\" CPUs not assigned\n to a cluster this will be UNASSIGNED_CLUSTER_INDEX; otherwise it will\n be the same as the cluster-id property of the CPU object's TYPE_CPU_CLUSTER\n QOM parent.\n Under TCG this value is propagated to @tcg_cflags.\n See TranslationBlock::TCG CF_CLUSTER_MASK.\n @tcg_cflags: Pre-computed cflags for this cpu.\n @nr_cores: Number of cores within this CPU package.\n @nr_threads: Number of threads within this CPU core.\n @running: #true if CPU is currently running (lockless).\n @has_waiter: #true if a CPU is currently waiting for the cpu_exec_end;\n valid under cpu_list_lock.\n @created: Indicates whether the CPU thread has been successfully created.\n @interrupt_request: Indicates a pending interrupt request.\n @halted: Nonzero if the CPU is in suspended state.\n @stop: Indicates a pending stop request.\n @stopped: Indicates the CPU has been artificially stopped.\n @unplug: Indicates a pending CPU unplug request.\n @crash_occurred: Indicates the OS reported a crash (panic) for this CPU\n @singlestep_enabled: Flags for single-stepping.\n @icount_extra: Instructions until next timer event.\n @neg.can_do_io: True if memory-mapped IO is allowed.\n @cpu_ases: Pointer to array of CPUAddressSpaces (which define the\n AddressSpaces this CPU has)\n @num_ases: number of CPUAddressSpaces in @cpu_ases\n @as: Pointer to the first AddressSpace, for the convenience of targets which\n only have a single AddressSpace\n @gdb_regs: Additional GDB registers.\n @gdb_num_regs: Number of total registers accessible to GDB.\n @gdb_num_g_regs: Number of registers in GDB 'g' packets.\n @next_cpu: Next CPU sharing TB cache.\n @opaque: User data.\n @mem_io_pc: Host Program Counter at which the memory was accessed.\n @accel: Pointer to accelerator specific state.\n @kvm_fd: vCPU file descriptor for KVM.\n @work_mutex: Lock to prevent multiple access to @work_list.\n @work_list: List of pending asynchronous work.\n @trace_dstate_delayed: Delayed changes to trace_dstate (includes all changes\n to @trace_dstate).\n @trace_dstate: Dynamic tracing state of events for this vCPU (bitmask).\n @plugin_mask: Plugin event bitmap. Modified only via async work.\n @ignore_memory_transaction_failures: Cached copy of the MachineState\n flag of the same name: allows the board to suppress calling of the\n CPU do_transaction_failed hook function.\n @kvm_dirty_gfns: Points to the KVM dirty ring for this CPU when KVM dirty\n ring is enabled.\n @kvm_fetch_index: Keeps the index that we last fetched from the per-vCPU\n dirty ring structure.\n\n State of one CPU core or thread.\n\n Align, in order to match possible alignment required by CPUArchState,\n and eliminate a hole between CPUState and CPUArchState within ArchCPU."] +#[doc = " CPUState:\n @cpu_index: CPU index (informative).\n @cluster_index: Identifies which cluster this CPU is in.\n For boards which don't define clusters or for \"loose\" CPUs not assigned\n to a cluster this will be UNASSIGNED_CLUSTER_INDEX; otherwise it will\n be the same as the cluster-id property of the CPU object's TYPE_CPU_CLUSTER\n QOM parent.\n Under TCG this value is propagated to @tcg_cflags.\n See TranslationBlock::TCG CF_CLUSTER_MASK.\n @tcg_cflags: Pre-computed cflags for this cpu.\n @nr_cores: Number of cores within this CPU package.\n @nr_threads: Number of threads within this CPU core.\n @running: #true if CPU is currently running (lockless).\n @has_waiter: #true if a CPU is currently waiting for the cpu_exec_end;\n valid under cpu_list_lock.\n @created: Indicates whether the CPU thread has been successfully created.\n @interrupt_request: Indicates a pending interrupt request.\n @halted: Nonzero if the CPU is in suspended state.\n @stop: Indicates a pending stop request.\n @stopped: Indicates the CPU has been artificially stopped.\n @unplug: Indicates a pending CPU unplug request.\n @crash_occurred: Indicates the OS reported a crash (panic) for this CPU\n @singlestep_enabled: Flags for single-stepping.\n @icount_extra: Instructions until next timer event.\n @neg.can_do_io: True if memory-mapped IO is allowed.\n @cpu_ases: Pointer to array of CPUAddressSpaces (which define the\n AddressSpaces this CPU has)\n @num_ases: number of CPUAddressSpaces in @cpu_ases\n @as: Pointer to the first AddressSpace, for the convenience of targets which\n only have a single AddressSpace\n @gdb_regs: Additional GDB registers.\n @gdb_num_regs: Number of total registers accessible to GDB.\n @gdb_num_g_regs: Number of registers in GDB 'g' packets.\n @node: QTAILQ of CPUs sharing TB cache.\n @opaque: User data.\n @mem_io_pc: Host Program Counter at which the memory was accessed.\n @accel: Pointer to accelerator specific state.\n @kvm_fd: vCPU file descriptor for KVM.\n @work_mutex: Lock to prevent multiple access to @work_list.\n @work_list: List of pending asynchronous work.\n @plugin_mem_cbs: active plugin memory callbacks\n @plugin_state: per-CPU plugin state\n @ignore_memory_transaction_failures: Cached copy of the MachineState\n flag of the same name: allows the board to suppress calling of the\n CPU do_transaction_failed hook function.\n @kvm_dirty_gfns: Points to the KVM dirty ring for this CPU when KVM dirty\n ring is enabled.\n @kvm_fetch_index: Keeps the index that we last fetched from the per-vCPU\n dirty ring structure.\n\n State of one CPU core or thread.\n\n Align, in order to match possible alignment required by CPUArchState,\n and eliminate a hole between CPUState and CPUArchState within ArchCPU."] #[repr(C)] #[repr(align(16))] pub struct CPUState { @@ -5945,8 +5892,8 @@ pub struct CPUState { pub dirty_pages: u64, pub kvm_vcpu_stats_fd: ::std::os::raw::c_int, pub in_ioctl_lock: QemuLockCnt, - pub plugin_mask: [::std::os::raw::c_ulong; 1usize], pub plugin_mem_cbs: *mut GArray, + pub plugin_state: *mut CPUPluginState, pub cpu_index: ::std::os::raw::c_int, pub cluster_index: ::std::os::raw::c_int, pub tcg_cflags: u32, @@ -6687,23 +6634,23 @@ fn bindgen_test_layout_CPUState() { ) ); assert_eq!( - unsafe { ::std::ptr::addr_of!((*ptr).plugin_mask) as usize - ptr as usize }, + unsafe { ::std::ptr::addr_of!((*ptr).plugin_mem_cbs) as usize - ptr as usize }, 696usize, concat!( "Offset of field: ", stringify!(CPUState), "::", - stringify!(plugin_mask) + stringify!(plugin_mem_cbs) ) ); assert_eq!( - unsafe { ::std::ptr::addr_of!((*ptr).plugin_mem_cbs) as usize - ptr as usize }, + unsafe { ::std::ptr::addr_of!((*ptr).plugin_state) as usize - ptr as usize }, 704usize, concat!( "Offset of field: ", stringify!(CPUState), "::", - stringify!(plugin_mem_cbs) + stringify!(plugin_state) ) ); assert_eq!( @@ -6860,7 +6807,7 @@ impl Default for CPUState { } impl ::std::fmt::Debug for CPUState { fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result { - write ! (f , "CPUState {{ parent_obj: {:?}, cc: {:?}, nr_cores: {:?}, nr_threads: {:?}, thread: {:?}, thread_id: {:?}, running: {:?}, has_waiter: {:?}, halt_cond: {:?}, thread_kicked: {:?}, created: {:?}, stop: {:?}, stopped: {:?}, start_powered_off: {:?}, unplug: {:?}, crash_occurred: {:?}, exit_request: {:?}, exclusive_context_count: {:?}, singlestep_enabled: {:?}, jmp_env: {:?}, work_mutex: {:?}, work_list: {:?}, cpu_ases: {:?}, num_ases: {:?}, as: {:?}, memory: {:?}, tb_jmp_cache: {:?}, gdb_regs: {:?}, gdb_num_regs: {:?}, gdb_num_g_regs: {:?}, node: {:?}, breakpoints: {:?}, watchpoints: {:?}, watchpoint_hit: {:?}, opaque: {:?}, kvm_fd: {:?}, kvm_state: {:?}, kvm_run: {:?}, kvm_dirty_gfns: {:?}, kvm_vcpu_stats_fd: {:?}, in_ioctl_lock: {:?}, plugin_mask: {:?}, plugin_mem_cbs: {:?}, cpu_index: {:?}, cluster_index: {:?}, accel: {:?}, vcpu_dirty: {:?}, throttle_thread_scheduled: {:?}, ignore_memory_transaction_failures: {:?}, prctl_unalign_sigbus: {:?}, iommu_notifiers: {:?}, neg_align: {:?}, neg: {:?} }}" , self . parent_obj , self . cc , self . nr_cores , self . nr_threads , self . thread , self . thread_id , self . running , self . has_waiter , self . halt_cond , self . thread_kicked , self . created , self . stop , self . stopped , self . start_powered_off , self . unplug , self . crash_occurred , self . exit_request , self . exclusive_context_count , self . singlestep_enabled , self . jmp_env , self . work_mutex , self . work_list , self . cpu_ases , self . num_ases , self . as_ , self . memory , self . tb_jmp_cache , self . gdb_regs , self . gdb_num_regs , self . gdb_num_g_regs , self . node , self . breakpoints , self . watchpoints , self . watchpoint_hit , self . opaque , self . kvm_fd , self . kvm_state , self . kvm_run , self . kvm_dirty_gfns , self . kvm_vcpu_stats_fd , self . in_ioctl_lock , self . plugin_mask , self . plugin_mem_cbs , self . cpu_index , self . cluster_index , self . accel , self . vcpu_dirty , self . throttle_thread_scheduled , self . ignore_memory_transaction_failures , self . prctl_unalign_sigbus , self . iommu_notifiers , self . neg_align , self . neg) + write ! (f , "CPUState {{ parent_obj: {:?}, cc: {:?}, nr_cores: {:?}, nr_threads: {:?}, thread: {:?}, thread_id: {:?}, running: {:?}, has_waiter: {:?}, halt_cond: {:?}, thread_kicked: {:?}, created: {:?}, stop: {:?}, stopped: {:?}, start_powered_off: {:?}, unplug: {:?}, crash_occurred: {:?}, exit_request: {:?}, exclusive_context_count: {:?}, singlestep_enabled: {:?}, jmp_env: {:?}, work_mutex: {:?}, work_list: {:?}, cpu_ases: {:?}, num_ases: {:?}, as: {:?}, memory: {:?}, tb_jmp_cache: {:?}, gdb_regs: {:?}, gdb_num_regs: {:?}, gdb_num_g_regs: {:?}, node: {:?}, breakpoints: {:?}, watchpoints: {:?}, watchpoint_hit: {:?}, opaque: {:?}, kvm_fd: {:?}, kvm_state: {:?}, kvm_run: {:?}, kvm_dirty_gfns: {:?}, kvm_vcpu_stats_fd: {:?}, in_ioctl_lock: {:?}, plugin_mem_cbs: {:?}, plugin_state: {:?}, cpu_index: {:?}, cluster_index: {:?}, accel: {:?}, vcpu_dirty: {:?}, throttle_thread_scheduled: {:?}, ignore_memory_transaction_failures: {:?}, prctl_unalign_sigbus: {:?}, iommu_notifiers: {:?}, neg_align: {:?}, neg: {:?} }}" , self . parent_obj , self . cc , self . nr_cores , self . nr_threads , self . thread , self . thread_id , self . running , self . has_waiter , self . halt_cond , self . thread_kicked , self . created , self . stop , self . stopped , self . start_powered_off , self . unplug , self . crash_occurred , self . exit_request , self . exclusive_context_count , self . singlestep_enabled , self . jmp_env , self . work_mutex , self . work_list , self . cpu_ases , self . num_ases , self . as_ , self . memory , self . tb_jmp_cache , self . gdb_regs , self . gdb_num_regs , self . gdb_num_g_regs , self . node , self . breakpoints , self . watchpoints , self . watchpoint_hit , self . opaque , self . kvm_fd , self . kvm_state , self . kvm_run , self . kvm_dirty_gfns , self . kvm_vcpu_stats_fd , self . in_ioctl_lock , self . plugin_mem_cbs , self . plugin_state , self . cpu_index , self . cluster_index , self . accel , self . vcpu_dirty , self . throttle_thread_scheduled , self . ignore_memory_transaction_failures , self . prctl_unalign_sigbus , self . iommu_notifiers , self . neg_align , self . neg) } } extern "C" { @@ -11485,108 +11432,15 @@ impl ::std::fmt::Debug for ArchCPU { write ! (f , "ArchCPU {{ parent_obj: {:?}, env: {:?}, vmsentry: {:?}, hyperv_vendor: {:?}, hyperv_synic_kvm_only: {:?}, hyperv_passthrough: {:?}, hyperv_no_nonarch_cs: {:?}, hyperv_vendor_id: {:?}, hyperv_interface_id: {:?}, hyperv_limits: {:?}, hyperv_enforce_cpuid: {:?}, check_cpuid: {:?}, enforce_cpuid: {:?}, force_features: {:?}, expose_kvm: {:?}, expose_tcg: {:?}, migratable: {:?}, migrate_smi_count: {:?}, max_features: {:?}, vmware_cpuid_freq: {:?}, cache_info_passthrough: {:?}, mwait: {:?}, filtered_features: {:?}, enable_pmu: {:?}, enable_lmce: {:?}, enable_l3_cache: {:?}, legacy_cache: {:?}, enable_cpuid_0xb: {:?}, full_cpuid_auto_level: {:?}, vendor_cpuid_only: {:?}, intel_pt_auto_level: {:?}, fill_mtrr_mask: {:?}, host_phys_bits: {:?}, kvm_no_smi_migration: {:?}, kvm_pv_enforce_cpuid: {:?}, apic_state: {:?}, cpu_as_root: {:?}, cpu_as_mem: {:?}, smram: {:?}, machine_done: {:?}, kvm_msr_buf: {:?}, xen_vapic: {:?} }}" , self . parent_obj , self . env , self . vmsentry , self . hyperv_vendor , self . hyperv_synic_kvm_only , self . hyperv_passthrough , self . hyperv_no_nonarch_cs , self . hyperv_vendor_id , self . hyperv_interface_id , self . hyperv_limits , self . hyperv_enforce_cpuid , self . check_cpuid , self . enforce_cpuid , self . force_features , self . expose_kvm , self . expose_tcg , self . migratable , self . migrate_smi_count , self . max_features , self . vmware_cpuid_freq , self . cache_info_passthrough , self . mwait , self . filtered_features , self . enable_pmu , self . enable_lmce , self . enable_l3_cache , self . legacy_cache , self . enable_cpuid_0xb , self . full_cpuid_auto_level , self . vendor_cpuid_only , self . intel_pt_auto_level , self . fill_mtrr_mask , self . host_phys_bits , self . kvm_no_smi_migration , self . kvm_pv_enforce_cpuid , self . apic_state , self . cpu_as_root , self . cpu_as_mem , self . smram , self . machine_done , self . kvm_msr_buf , self . xen_vapic) } } -pub type abi_ulong = target_ulong; -pub type abi_long = target_long; extern "C" { - #[doc = " page_check_range\n @start: first byte of range\n @len: length of range\n @flags: flags required for each page\n\n Return true if every page in [@start, @start+@len) has @flags set.\n Return false if any page is unmapped. Thus testing flags == 0 is\n equivalent to testing for flags == PAGE_VALID."] - pub fn page_check_range( - start: target_ulong, - last: target_ulong, - flags: ::std::os::raw::c_int, - ) -> bool; -} -pub const MemOp_MO_8: MemOp = MemOp(0); -pub const MemOp_MO_16: MemOp = MemOp(1); -pub const MemOp_MO_32: MemOp = MemOp(2); -pub const MemOp_MO_64: MemOp = MemOp(3); -pub const MemOp_MO_128: MemOp = MemOp(4); -pub const MemOp_MO_256: MemOp = MemOp(5); -pub const MemOp_MO_512: MemOp = MemOp(6); -pub const MemOp_MO_1024: MemOp = MemOp(7); -pub const MemOp_MO_SIZE: MemOp = MemOp(7); -pub const MemOp_MO_SIGN: MemOp = MemOp(8); -pub const MemOp_MO_BSWAP: MemOp = MemOp(16); -pub const MemOp_MO_LE: MemOp = MemOp(0); -pub const MemOp_MO_BE: MemOp = MemOp(16); -pub const MemOp_MO_TE: MemOp = MemOp(0); -pub const MemOp_MO_ASHIFT: MemOp = MemOp(5); -pub const MemOp_MO_AMASK: MemOp = MemOp(224); -pub const MemOp_MO_UNALN: MemOp = MemOp(0); -pub const MemOp_MO_ALIGN_2: MemOp = MemOp(32); -pub const MemOp_MO_ALIGN_4: MemOp = MemOp(64); -pub const MemOp_MO_ALIGN_8: MemOp = MemOp(96); -pub const MemOp_MO_ALIGN_16: MemOp = MemOp(128); -pub const MemOp_MO_ALIGN_32: MemOp = MemOp(160); -pub const MemOp_MO_ALIGN_64: MemOp = MemOp(192); -pub const MemOp_MO_ALIGN: MemOp = MemOp(224); -pub const MemOp_MO_ATOM_SHIFT: MemOp = MemOp(8); -pub const MemOp_MO_ATOM_IFALIGN: MemOp = MemOp(0); -pub const MemOp_MO_ATOM_IFALIGN_PAIR: MemOp = MemOp(256); -pub const MemOp_MO_ATOM_WITHIN16: MemOp = MemOp(512); -pub const MemOp_MO_ATOM_WITHIN16_PAIR: MemOp = MemOp(768); -pub const MemOp_MO_ATOM_SUBALIGN: MemOp = MemOp(1024); -pub const MemOp_MO_ATOM_NONE: MemOp = MemOp(1280); -pub const MemOp_MO_ATOM_MASK: MemOp = MemOp(1792); -pub const MemOp_MO_UB: MemOp = MemOp(0); -pub const MemOp_MO_UW: MemOp = MemOp(1); -pub const MemOp_MO_UL: MemOp = MemOp(2); -pub const MemOp_MO_UQ: MemOp = MemOp(3); -pub const MemOp_MO_UO: MemOp = MemOp(4); -pub const MemOp_MO_SB: MemOp = MemOp(8); -pub const MemOp_MO_SW: MemOp = MemOp(9); -pub const MemOp_MO_SL: MemOp = MemOp(10); -pub const MemOp_MO_SQ: MemOp = MemOp(11); -pub const MemOp_MO_SO: MemOp = MemOp(12); -pub const MemOp_MO_LEUW: MemOp = MemOp(1); -pub const MemOp_MO_LEUL: MemOp = MemOp(2); -pub const MemOp_MO_LEUQ: MemOp = MemOp(3); -pub const MemOp_MO_LESW: MemOp = MemOp(9); -pub const MemOp_MO_LESL: MemOp = MemOp(10); -pub const MemOp_MO_LESQ: MemOp = MemOp(11); -pub const MemOp_MO_BEUW: MemOp = MemOp(17); -pub const MemOp_MO_BEUL: MemOp = MemOp(18); -pub const MemOp_MO_BEUQ: MemOp = MemOp(19); -pub const MemOp_MO_BESW: MemOp = MemOp(25); -pub const MemOp_MO_BESL: MemOp = MemOp(26); -pub const MemOp_MO_BESQ: MemOp = MemOp(27); -pub const MemOp_MO_TEUW: MemOp = MemOp(1); -pub const MemOp_MO_TEUL: MemOp = MemOp(2); -pub const MemOp_MO_TEUQ: MemOp = MemOp(3); -pub const MemOp_MO_TEUO: MemOp = MemOp(4); -pub const MemOp_MO_TESW: MemOp = MemOp(9); -pub const MemOp_MO_TESL: MemOp = MemOp(10); -pub const MemOp_MO_TESQ: MemOp = MemOp(11); -pub const MemOp_MO_SSIZE: MemOp = MemOp(15); -impl ::std::ops::BitOr for MemOp { - type Output = Self; - #[inline] - fn bitor(self, other: Self) -> Self { - MemOp(self.0 | other.0) - } -} -impl ::std::ops::BitOrAssign for MemOp { - #[inline] - fn bitor_assign(&mut self, rhs: MemOp) { - self.0 |= rhs.0; - } -} -impl ::std::ops::BitAnd for MemOp { - type Output = Self; - #[inline] - fn bitand(self, other: Self) -> Self { - MemOp(self.0 & other.0) - } -} -impl ::std::ops::BitAndAssign for MemOp { - #[inline] - fn bitand_assign(&mut self, rhs: MemOp) { - self.0 &= rhs.0; - } + pub fn cpu_memory_rw_debug( + cpu: *mut CPUState, + addr: vaddr, + ptr: *mut ::std::os::raw::c_void, + len: usize, + is_write: bool, + ) -> ::std::os::raw::c_int; } -#[repr(transparent)] -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct MemOp(pub ::std::os::raw::c_uint); -pub type MemOpIdx = u32; #[repr(C)] #[derive(Debug, Copy, Clone)] pub struct RBNode { @@ -11650,34 +11504,123 @@ impl Default for RBNode { } #[repr(C)] #[derive(Debug, Copy, Clone)] -pub struct IntervalTreeNode { - pub rb: RBNode, - pub start: u64, - pub last: u64, - pub subtree_last: u64, +pub struct RBRoot { + pub rb_node: *mut RBNode, } #[test] -fn bindgen_test_layout_IntervalTreeNode() { - const UNINIT: ::std::mem::MaybeUninit = ::std::mem::MaybeUninit::uninit(); +fn bindgen_test_layout_RBRoot() { + const UNINIT: ::std::mem::MaybeUninit = ::std::mem::MaybeUninit::uninit(); let ptr = UNINIT.as_ptr(); assert_eq!( - ::std::mem::size_of::(), - 48usize, - concat!("Size of: ", stringify!(IntervalTreeNode)) + ::std::mem::size_of::(), + 8usize, + concat!("Size of: ", stringify!(RBRoot)) ); assert_eq!( - ::std::mem::align_of::(), + ::std::mem::align_of::(), 8usize, - concat!("Alignment of ", stringify!(IntervalTreeNode)) + concat!("Alignment of ", stringify!(RBRoot)) ); assert_eq!( - unsafe { ::std::ptr::addr_of!((*ptr).rb) as usize - ptr as usize }, + unsafe { ::std::ptr::addr_of!((*ptr).rb_node) as usize - ptr as usize }, 0usize, concat!( "Offset of field: ", - stringify!(IntervalTreeNode), + stringify!(RBRoot), "::", - stringify!(rb) + stringify!(rb_node) + ) + ); +} +impl Default for RBRoot { + fn default() -> Self { + let mut s = ::std::mem::MaybeUninit::::uninit(); + unsafe { + ::std::ptr::write_bytes(s.as_mut_ptr(), 0, 1); + s.assume_init() + } + } +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct RBRootLeftCached { + pub rb_root: RBRoot, + pub rb_leftmost: *mut RBNode, +} +#[test] +fn bindgen_test_layout_RBRootLeftCached() { + const UNINIT: ::std::mem::MaybeUninit = ::std::mem::MaybeUninit::uninit(); + let ptr = UNINIT.as_ptr(); + assert_eq!( + ::std::mem::size_of::(), + 16usize, + concat!("Size of: ", stringify!(RBRootLeftCached)) + ); + assert_eq!( + ::std::mem::align_of::(), + 8usize, + concat!("Alignment of ", stringify!(RBRootLeftCached)) + ); + assert_eq!( + unsafe { ::std::ptr::addr_of!((*ptr).rb_root) as usize - ptr as usize }, + 0usize, + concat!( + "Offset of field: ", + stringify!(RBRootLeftCached), + "::", + stringify!(rb_root) + ) + ); + assert_eq!( + unsafe { ::std::ptr::addr_of!((*ptr).rb_leftmost) as usize - ptr as usize }, + 8usize, + concat!( + "Offset of field: ", + stringify!(RBRootLeftCached), + "::", + stringify!(rb_leftmost) + ) + ); +} +impl Default for RBRootLeftCached { + fn default() -> Self { + let mut s = ::std::mem::MaybeUninit::::uninit(); + unsafe { + ::std::ptr::write_bytes(s.as_mut_ptr(), 0, 1); + s.assume_init() + } + } +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct IntervalTreeNode { + pub rb: RBNode, + pub start: u64, + pub last: u64, + pub subtree_last: u64, +} +#[test] +fn bindgen_test_layout_IntervalTreeNode() { + const UNINIT: ::std::mem::MaybeUninit = ::std::mem::MaybeUninit::uninit(); + let ptr = UNINIT.as_ptr(); + assert_eq!( + ::std::mem::size_of::(), + 48usize, + concat!("Size of: ", stringify!(IntervalTreeNode)) + ); + assert_eq!( + ::std::mem::align_of::(), + 8usize, + concat!("Alignment of ", stringify!(IntervalTreeNode)) + ); + assert_eq!( + unsafe { ::std::ptr::addr_of!((*ptr).rb) as usize - ptr as usize }, + 0usize, + concat!( + "Offset of field: ", + stringify!(IntervalTreeNode), + "::", + stringify!(rb) ) ); assert_eq!( @@ -11720,6 +11663,113 @@ impl Default for IntervalTreeNode { } } } +pub type IntervalTreeRoot = RBRootLeftCached; +pub type abi_ulong = target_ulong; +pub type abi_long = target_long; +extern "C" { + #[doc = " --- Begin LibAFL code ---"] + pub fn pageflags_get_root() -> *mut IntervalTreeRoot; +} +extern "C" { + #[doc = " page_check_range\n @start: first byte of range\n @len: length of range\n @flags: flags required for each page\n\n Return true if every page in [@start, @start+@len) has @flags set.\n Return false if any page is unmapped. Thus testing flags == 0 is\n equivalent to testing for flags == PAGE_VALID."] + pub fn page_check_range( + start: target_ulong, + last: target_ulong, + flags: ::std::os::raw::c_int, + ) -> bool; +} +pub const MemOp_MO_8: MemOp = MemOp(0); +pub const MemOp_MO_16: MemOp = MemOp(1); +pub const MemOp_MO_32: MemOp = MemOp(2); +pub const MemOp_MO_64: MemOp = MemOp(3); +pub const MemOp_MO_128: MemOp = MemOp(4); +pub const MemOp_MO_256: MemOp = MemOp(5); +pub const MemOp_MO_512: MemOp = MemOp(6); +pub const MemOp_MO_1024: MemOp = MemOp(7); +pub const MemOp_MO_SIZE: MemOp = MemOp(7); +pub const MemOp_MO_SIGN: MemOp = MemOp(8); +pub const MemOp_MO_BSWAP: MemOp = MemOp(16); +pub const MemOp_MO_LE: MemOp = MemOp(0); +pub const MemOp_MO_BE: MemOp = MemOp(16); +pub const MemOp_MO_TE: MemOp = MemOp(0); +pub const MemOp_MO_ASHIFT: MemOp = MemOp(5); +pub const MemOp_MO_AMASK: MemOp = MemOp(224); +pub const MemOp_MO_UNALN: MemOp = MemOp(0); +pub const MemOp_MO_ALIGN_2: MemOp = MemOp(32); +pub const MemOp_MO_ALIGN_4: MemOp = MemOp(64); +pub const MemOp_MO_ALIGN_8: MemOp = MemOp(96); +pub const MemOp_MO_ALIGN_16: MemOp = MemOp(128); +pub const MemOp_MO_ALIGN_32: MemOp = MemOp(160); +pub const MemOp_MO_ALIGN_64: MemOp = MemOp(192); +pub const MemOp_MO_ALIGN: MemOp = MemOp(224); +pub const MemOp_MO_ATOM_SHIFT: MemOp = MemOp(8); +pub const MemOp_MO_ATOM_IFALIGN: MemOp = MemOp(0); +pub const MemOp_MO_ATOM_IFALIGN_PAIR: MemOp = MemOp(256); +pub const MemOp_MO_ATOM_WITHIN16: MemOp = MemOp(512); +pub const MemOp_MO_ATOM_WITHIN16_PAIR: MemOp = MemOp(768); +pub const MemOp_MO_ATOM_SUBALIGN: MemOp = MemOp(1024); +pub const MemOp_MO_ATOM_NONE: MemOp = MemOp(1280); +pub const MemOp_MO_ATOM_MASK: MemOp = MemOp(1792); +pub const MemOp_MO_UB: MemOp = MemOp(0); +pub const MemOp_MO_UW: MemOp = MemOp(1); +pub const MemOp_MO_UL: MemOp = MemOp(2); +pub const MemOp_MO_UQ: MemOp = MemOp(3); +pub const MemOp_MO_UO: MemOp = MemOp(4); +pub const MemOp_MO_SB: MemOp = MemOp(8); +pub const MemOp_MO_SW: MemOp = MemOp(9); +pub const MemOp_MO_SL: MemOp = MemOp(10); +pub const MemOp_MO_SQ: MemOp = MemOp(11); +pub const MemOp_MO_SO: MemOp = MemOp(12); +pub const MemOp_MO_LEUW: MemOp = MemOp(1); +pub const MemOp_MO_LEUL: MemOp = MemOp(2); +pub const MemOp_MO_LEUQ: MemOp = MemOp(3); +pub const MemOp_MO_LESW: MemOp = MemOp(9); +pub const MemOp_MO_LESL: MemOp = MemOp(10); +pub const MemOp_MO_LESQ: MemOp = MemOp(11); +pub const MemOp_MO_BEUW: MemOp = MemOp(17); +pub const MemOp_MO_BEUL: MemOp = MemOp(18); +pub const MemOp_MO_BEUQ: MemOp = MemOp(19); +pub const MemOp_MO_BESW: MemOp = MemOp(25); +pub const MemOp_MO_BESL: MemOp = MemOp(26); +pub const MemOp_MO_BESQ: MemOp = MemOp(27); +pub const MemOp_MO_TEUW: MemOp = MemOp(1); +pub const MemOp_MO_TEUL: MemOp = MemOp(2); +pub const MemOp_MO_TEUQ: MemOp = MemOp(3); +pub const MemOp_MO_TEUO: MemOp = MemOp(4); +pub const MemOp_MO_TESW: MemOp = MemOp(9); +pub const MemOp_MO_TESL: MemOp = MemOp(10); +pub const MemOp_MO_TESQ: MemOp = MemOp(11); +pub const MemOp_MO_SSIZE: MemOp = MemOp(15); +impl ::std::ops::BitOr for MemOp { + type Output = Self; + #[inline] + fn bitor(self, other: Self) -> Self { + MemOp(self.0 | other.0) + } +} +impl ::std::ops::BitOrAssign for MemOp { + #[inline] + fn bitor_assign(&mut self, rhs: MemOp) { + self.0 |= rhs.0; + } +} +impl ::std::ops::BitAnd for MemOp { + type Output = Self; + #[inline] + fn bitand(self, other: Self) -> Self { + MemOp(self.0 & other.0) + } +} +impl ::std::ops::BitAndAssign for MemOp { + #[inline] + fn bitand_assign(&mut self, rhs: MemOp) { + self.0 &= rhs.0; + } +} +#[repr(transparent)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct MemOp(pub ::std::os::raw::c_uint); +pub type MemOpIdx = u32; #[repr(C)] #[derive(Debug, Copy, Clone)] pub struct tb_tc { @@ -11983,6 +12033,14 @@ extern "C" { extern "C" { pub fn target_munmap(start: abi_ulong, len: abi_ulong) -> ::std::os::raw::c_int; } +extern "C" { + #[doc = " read_self_maps:\n\n Read /proc/self/maps and return a tree of MapInfo structures."] + pub fn read_self_maps() -> *mut IntervalTreeRoot; +} +extern "C" { + #[doc = " free_self_maps:\n @info: an interval tree\n\n Free a tree of MapInfo structures."] + pub fn free_self_maps(root: *mut IntervalTreeRoot); +} extern "C" { pub fn libafl_breakpoint_invalidate(cpu: *mut CPUState, pc: target_ulong); } @@ -12311,6 +12369,121 @@ extern "C" { } #[repr(C)] #[derive(Debug, Copy, Clone)] +pub struct libafl_mapinfo { + pub start: target_ulong, + pub end: target_ulong, + pub offset: target_ulong, + pub path: *const ::std::os::raw::c_char, + pub flags: ::std::os::raw::c_int, + pub is_priv: ::std::os::raw::c_int, + pub is_valid: bool, +} +#[test] +fn bindgen_test_layout_libafl_mapinfo() { + const UNINIT: ::std::mem::MaybeUninit = ::std::mem::MaybeUninit::uninit(); + let ptr = UNINIT.as_ptr(); + assert_eq!( + ::std::mem::size_of::(), + 48usize, + concat!("Size of: ", stringify!(libafl_mapinfo)) + ); + assert_eq!( + ::std::mem::align_of::(), + 8usize, + concat!("Alignment of ", stringify!(libafl_mapinfo)) + ); + assert_eq!( + unsafe { ::std::ptr::addr_of!((*ptr).start) as usize - ptr as usize }, + 0usize, + concat!( + "Offset of field: ", + stringify!(libafl_mapinfo), + "::", + stringify!(start) + ) + ); + assert_eq!( + unsafe { ::std::ptr::addr_of!((*ptr).end) as usize - ptr as usize }, + 8usize, + concat!( + "Offset of field: ", + stringify!(libafl_mapinfo), + "::", + stringify!(end) + ) + ); + assert_eq!( + unsafe { ::std::ptr::addr_of!((*ptr).offset) as usize - ptr as usize }, + 16usize, + concat!( + "Offset of field: ", + stringify!(libafl_mapinfo), + "::", + stringify!(offset) + ) + ); + assert_eq!( + unsafe { ::std::ptr::addr_of!((*ptr).path) as usize - ptr as usize }, + 24usize, + concat!( + "Offset of field: ", + stringify!(libafl_mapinfo), + "::", + stringify!(path) + ) + ); + assert_eq!( + unsafe { ::std::ptr::addr_of!((*ptr).flags) as usize - ptr as usize }, + 32usize, + concat!( + "Offset of field: ", + stringify!(libafl_mapinfo), + "::", + stringify!(flags) + ) + ); + assert_eq!( + unsafe { ::std::ptr::addr_of!((*ptr).is_priv) as usize - ptr as usize }, + 36usize, + concat!( + "Offset of field: ", + stringify!(libafl_mapinfo), + "::", + stringify!(is_priv) + ) + ); + assert_eq!( + unsafe { ::std::ptr::addr_of!((*ptr).is_valid) as usize - ptr as usize }, + 40usize, + concat!( + "Offset of field: ", + stringify!(libafl_mapinfo), + "::", + stringify!(is_valid) + ) + ); +} +impl Default for libafl_mapinfo { + fn default() -> Self { + let mut s = ::std::mem::MaybeUninit::::uninit(); + unsafe { + ::std::ptr::write_bytes(s.as_mut_ptr(), 0, 1); + s.assume_init() + } + } +} +extern "C" { + pub fn libafl_maps_first(map_info: *mut IntervalTreeRoot) -> *mut IntervalTreeNode; +} +extern "C" { + pub fn libafl_maps_next( + pageflags_maps_node: *mut IntervalTreeNode, + proc_maps_node: *mut IntervalTreeRoot, + ret: *mut libafl_mapinfo, + ) -> *mut IntervalTreeNode; +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] pub struct AccelCPUClass { pub parent_class: ObjectClass, pub cpu_class_init: ::std::option::Option, @@ -12428,6 +12601,37 @@ extern "C" { #[doc = " qemu_plugin_hwaddr_phys_addr() - query physical address for memory operation\n @haddr: address handle from qemu_plugin_get_hwaddr()\n\n Returns the physical address associated with the memory operation\n\n Note that the returned physical address may not be unique if you are dealing\n with multiple address spaces."] pub fn qemu_plugin_hwaddr_phys_addr(haddr: *const qemu_plugin_hwaddr) -> u64; } +#[doc = " struct CPUPluginState - per-CPU state for plugins\n @event_mask: plugin event bitmap. Modified only via async work."] +#[repr(C)] +#[derive(Debug, Default, Copy, Clone)] +pub struct CPUPluginState { + pub event_mask: [::std::os::raw::c_ulong; 1usize], +} +#[test] +fn bindgen_test_layout_CPUPluginState() { + const UNINIT: ::std::mem::MaybeUninit = ::std::mem::MaybeUninit::uninit(); + let ptr = UNINIT.as_ptr(); + assert_eq!( + ::std::mem::size_of::(), + 8usize, + concat!("Size of: ", stringify!(CPUPluginState)) + ); + assert_eq!( + ::std::mem::align_of::(), + 8usize, + concat!("Alignment of ", stringify!(CPUPluginState)) + ); + assert_eq!( + unsafe { ::std::ptr::addr_of!((*ptr).event_mask) as usize - ptr as usize }, + 0usize, + concat!( + "Offset of field: ", + stringify!(CPUPluginState), + "::", + stringify!(event_mask) + ) + ); +} pub const TCGReg_TCG_REG_EAX: TCGReg = TCGReg(0); pub const TCGReg_TCG_REG_ECX: TCGReg = TCGReg(1); pub const TCGReg_TCG_REG_EDX: TCGReg = TCGReg(2); @@ -12818,13 +13022,13 @@ impl TCGTemp { } #[inline] pub fn temp_subindex(&self) -> ::std::os::raw::c_uint { - unsafe { ::std::mem::transmute(self._bitfield_1.get(40usize, 1u8) as u32) } + unsafe { ::std::mem::transmute(self._bitfield_1.get(40usize, 2u8) as u32) } } #[inline] pub fn set_temp_subindex(&mut self, val: ::std::os::raw::c_uint) { unsafe { let val: u32 = ::std::mem::transmute(val); - self._bitfield_1.set(40usize, 1u8, val as u64) + self._bitfield_1.set(40usize, 2u8, val as u64) } } #[inline] @@ -12882,7 +13086,7 @@ impl TCGTemp { let temp_allocated: u32 = unsafe { ::std::mem::transmute(temp_allocated) }; temp_allocated as u64 }); - __bindgen_bitfield_unit.set(40usize, 1u8, { + __bindgen_bitfield_unit.set(40usize, 2u8, { let temp_subindex: u32 = unsafe { ::std::mem::transmute(temp_subindex) }; temp_subindex as u64 }); @@ -13423,7 +13627,7 @@ impl Default for libafl_hook { extern "C" { pub fn libafl_qemu_set_hook( pc: target_ulong, - callback: ::std::option::Option, + callback: ::std::option::Option< extern "C" fn(data: u64, pc: target_ulong)>, data: u64, invalidate: ::std::os::raw::c_int, ) -> usize; @@ -13445,7 +13649,9 @@ extern "C" { } extern "C" { pub fn libafl_add_backdoor_hook( - exec: ::std::option::Option, + exec: ::std::option::Option< + extern "C" fn(data: u64, cpu: *mut CPUArchState, pc: target_ulong), + >, data: u64, ) -> usize; } @@ -13458,9 +13664,9 @@ extern "C" { extern "C" { pub fn libafl_add_edge_hook( gen: ::std::option::Option< - extern "C" fn(data: u64, src: target_ulong, dst: target_ulong) -> u64, + extern "C" fn(data: u64, src: target_ulong, dst: target_ulong) -> u64, >, - exec: ::std::option::Option, + exec: ::std::option::Option< extern "C" fn(data: u64, id: u64)>, data: u64, ) -> usize; } @@ -13478,11 +13684,11 @@ extern "C" { } extern "C" { pub fn libafl_add_block_hook( - gen: ::std::option::Option u64>, + gen: ::std::option::Option< extern "C" fn(data: u64, pc: target_ulong) -> u64>, post_gen: ::std::option::Option< - extern "C" fn(data: u64, pc: target_ulong, block_length: target_ulong), + extern "C" fn(data: u64, pc: target_ulong, block_length: target_ulong), >, - exec: ::std::option::Option, + exec: ::std::option::Option< extern "C" fn(data: u64, id: u64)>, data: u64, ) -> usize; } @@ -13500,26 +13706,40 @@ extern "C" { } extern "C" { pub fn libafl_add_read_hook( - gen: ::std::option::Option u64>, - exec1: ::std::option::Option, - exec2: ::std::option::Option, - exec4: ::std::option::Option, - exec8: ::std::option::Option, + gen: ::std::option::Option< + unsafe extern "C" fn( + data: u64, + pc: target_ulong, + addr: *mut TCGTemp, + oi: MemOpIdx, + ) -> u64, + >, + exec1: ::std::option::Option< extern "C" fn(data: u64, id: u64, addr: target_ulong)>, + exec2: ::std::option::Option< extern "C" fn(data: u64, id: u64, addr: target_ulong)>, + exec4: ::std::option::Option< extern "C" fn(data: u64, id: u64, addr: target_ulong)>, + exec8: ::std::option::Option< extern "C" fn(data: u64, id: u64, addr: target_ulong)>, execN: ::std::option::Option< - extern "C" fn(data: u64, id: u64, addr: target_ulong, size: usize), + extern "C" fn(data: u64, id: u64, addr: target_ulong, size: usize), >, data: u64, ) -> usize; } extern "C" { pub fn libafl_add_write_hook( - gen: ::std::option::Option u64>, - exec1: ::std::option::Option, - exec2: ::std::option::Option, - exec4: ::std::option::Option, - exec8: ::std::option::Option, + gen: ::std::option::Option< + unsafe extern "C" fn( + data: u64, + pc: target_ulong, + addr: *mut TCGTemp, + oi: MemOpIdx, + ) -> u64, + >, + exec1: ::std::option::Option< extern "C" fn(data: u64, id: u64, addr: target_ulong)>, + exec2: ::std::option::Option< extern "C" fn(data: u64, id: u64, addr: target_ulong)>, + exec4: ::std::option::Option< extern "C" fn(data: u64, id: u64, addr: target_ulong)>, + exec8: ::std::option::Option< extern "C" fn(data: u64, id: u64, addr: target_ulong)>, execN: ::std::option::Option< - extern "C" fn(data: u64, id: u64, addr: target_ulong, size: usize), + extern "C" fn(data: u64, id: u64, addr: target_ulong, size: usize), >, data: u64, ) -> usize; @@ -13544,11 +13764,13 @@ extern "C" { } extern "C" { pub fn libafl_add_cmp_hook( - gen: ::std::option::Option u64>, - exec1: ::std::option::Option, - exec2: ::std::option::Option, - exec4: ::std::option::Option, - exec8: ::std::option::Option, + gen: ::std::option::Option< + extern "C" fn(data: u64, pc: target_ulong, size: usize) -> u64, + >, + exec1: ::std::option::Option< extern "C" fn(data: u64, id: u64, v0: u8, v1: u8)>, + exec2: ::std::option::Option< extern "C" fn(data: u64, id: u64, v0: u16, v1: u16)>, + exec4: ::std::option::Option< extern "C" fn(data: u64, id: u64, v0: u32, v1: u32)>, + exec8: ::std::option::Option< extern "C" fn(data: u64, id: u64, v0: u64, v1: u64)>, data: u64, ) -> usize; } @@ -13646,13 +13868,16 @@ extern "C" { } extern "C" { pub fn libafl_add_new_thread_hook( - callback: ::std::option::Option bool>, + callback: ::std::option::Option< extern "C" fn(data: u64, tid: u32) -> bool>, data: u64, ) -> usize; } extern "C" { pub fn libafl_qemu_remove_new_thread_hook(num: usize) -> ::std::os::raw::c_int; } +extern "C" { + pub fn libafl_tcg_gen_asan(addr: *mut TCGTemp, size: usize); +} extern "C" { pub fn libafl_jit_trace_edge_hitcount(data: u64, id: u64) -> usize; } diff --git a/libafl_qemu/libqasan/Makefile b/libafl_qemu/libqasan/Makefile index 1044994f7c..98b5066b98 100644 --- a/libafl_qemu/libqasan/Makefile +++ b/libafl_qemu/libqasan/Makefile @@ -15,13 +15,16 @@ OUT_DIR ?= . -override CFLAGS += -Wno-int-to-void-pointer-cast -ggdb -O1 -fno-builtin +override CFLAGS += -Wno-int-to-void-pointer-cast -ggdb -O1 -fno-builtin -Wno-unused-result override LDFLAGS += -ldl -pthread -SRC := libqasan.c hooks.c malloc.c string.c uninstrument.c patch.c dlmalloc.c printf/printf.c +SRC := libqasan.c hooks.c malloc.c mmap.c string.c uninstrument.c patch.c dlmalloc.c printf/printf.c HDR := libqasan.h qasan.h map_macro.h printf/printf.h -all: $(OUT_DIR)/libqasan.so +all: $(OUT_DIR)/libgasan.so $(OUT_DIR)/libqasan.so + +$(OUT_DIR)/libgasan.so: $(HDR) $(SRC) + $(CC) $(CFLAGS) -DASAN_GUEST=1 -fPIC -shared $(SRC) -o $@ $(LDFLAGS) $(OUT_DIR)/libqasan.so: $(HDR) $(SRC) $(CC) $(CFLAGS) -fPIC -shared $(SRC) -o $@ $(LDFLAGS) @@ -30,4 +33,5 @@ $(OUT_DIR)/libqasan.so: $(HDR) $(SRC) clean: rm -f *.o *.so *~ a.out core core.[1-9][0-9]* + rm -f $(OUT_DIR)/libgasan.so rm -f $(OUT_DIR)/libqasan.so diff --git a/libafl_qemu/libqasan/hooks.c b/libafl_qemu/libqasan/hooks.c index c0adf97759..42ead87a14 100644 --- a/libafl_qemu/libqasan/hooks.c +++ b/libafl_qemu/libqasan/hooks.c @@ -24,6 +24,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. *******************************************************************************/ #include "libqasan.h" +#include "printf/printf.h" #include "map_macro.h" #include #include @@ -470,6 +471,21 @@ char *strdup(const char *s) { return r; } +char *strndup(const char *s, size_t n) { + void *rtv = __builtin_return_address(0); + + QASAN_DEBUG("%14p: strndup(%p, %zu)\n", rtv, s, n); + size_t l = __libqasan_strnlen(s, n); + if (l > n) { l = n; } + QASAN_LOAD(s, l + 1); + void *r = __libqasan_malloc(l + 1); + __libqasan_memcpy(r, s, l); + ((char *)r)[l] = 0; + QASAN_DEBUG("\t\t = %p\n", r); + + return r; +} + size_t strlen(const char *s) { void *rtv = __builtin_return_address(0); @@ -593,3 +609,46 @@ int wcscmp(const wchar_t *s1, const wchar_t *s2) { return r; } + +int asprintf(char **restrict strp, const char *restrict fmt, ...) { + void *rtv = __builtin_return_address(0); + + QASAN_DEBUG("%14p: asprintf(%p, %p)\n", rtv, strp, fmt); + va_list va; + va_start(va, fmt); + int len = __libqasan_vasprintf(strp, fmt, va); + va_end(va); + QASAN_DEBUG("\t\t = %d [*strp = %p]\n", len, *strp); + + return len; +} + +int vasprintf(char **restrict strp, const char *restrict fmt, va_list ap) { + void *rtv = __builtin_return_address(0); + + QASAN_DEBUG("%14p: vasprintf(%p, %p)\n", rtv, strp, fmt); + int len = __libqasan_vasprintf(strp, fmt, ap); + QASAN_DEBUG("\t\t = %d [*strp = %p]\n", len, *strp); + + return len; +} + +void *mmap(void *addr, size_t length, int prot, int flags, int fd, + off_t offset) { + void *rtv = __builtin_return_address(0); + QASAN_DEBUG("%14p: mmap(%p, %zu, %d, %d, %d, %ld)\n", rtv, addr, length, prot, + flags, fd, offset); + void *r = __libqasan_mmap(addr, length, prot, flags, fd, offset); + QASAN_DEBUG("\t\t = %p\n", r); + + return r; +} + +int munmap(void *addr, size_t length) { + void *rtv = __builtin_return_address(0); + QASAN_DEBUG("%14p: munmap(%p, %zu)\n", rtv, addr, length); + int r = __libqasan_munmap(addr, length); + QASAN_DEBUG("\t\t = %d\n", r); + + return r; +} diff --git a/libafl_qemu/libqasan/libqasan.c b/libafl_qemu/libqasan/libqasan.c index b9927d7697..5d6397a4a7 100644 --- a/libafl_qemu/libqasan/libqasan.c +++ b/libafl_qemu/libqasan/libqasan.c @@ -57,6 +57,47 @@ void __libqasan_print_maps(void) { int __libqasan_is_initialized = 0; +__attribute__((always_inline)) inline size_t qasan_align_down(size_t val, + size_t align) { + return (val & ~(align - 1)); +} + +__attribute__((always_inline)) inline size_t qasan_align_up(size_t val, + size_t align) { + return qasan_align_down(val + align - 1, align); +} + +#ifdef ASAN_GUEST +static void __libqasan_map_shadow(void *addr, void *limit) { + size_t size = (limit - addr) + 1; + void *map = mmap(addr, size, PROT_READ | PROT_WRITE, + MAP_FIXED | MAP_FIXED_NOREPLACE | MAP_PRIVATE | + MAP_ANONYMOUS | MAP_NORESERVE, + -1, 0); + if (map != addr) { + QASAN_LOG("Failed to map shadow: %p-%p, errno: %d", addr, limit + 1, errno); + abort(); + } + + if (madvise(addr, size, MADV_HUGEPAGE) != 0) { + QASAN_LOG("Failed to madvise (MADV_HUGEPAGE) shadow: %p-%p, errno: %d", + addr, limit + 1, errno); + abort(); + } +} +#endif + +#ifdef ASAN_GUEST + +const size_t ALLOC_ALIGN_POW = 3; +const size_t ALLOC_ALIGN_SIZE = (1UL << ALLOC_ALIGN_POW); + #if defined(__x86_64__) || defined(__aarch64__) + #define SHADOW_OFFSET (0x7fff8000) + #else + #define SHADOW_OFFSET (0x20000000) + #endif +#endif + __attribute__((constructor)) void __libqasan_init() { if (__libqasan_is_initialized) { return; } __libqasan_is_initialized = 1; @@ -77,9 +118,252 @@ __attribute__((constructor)) void __libqasan_init() { "Copyright (C) 2019-2021 Andrea Fioraldi \n"); QASAN_LOG("\n"); +#ifdef ASAN_GUEST + QASAN_DEBUG("QASAN - Debugging is enabled!!!\n"); + /* MMap our shadow and madvise to use huge pages */ + #if defined(__x86_64__) || defined(__aarch64__) + // [0x10007fff8000, 0x7fffffffffff] HighMem + // [0x02008fff7000, 0x10007fff7fff] HighShadow + // [0x00008fff7000, 0x02008fff6fff] ShadowGap + // [0x00007fff8000, 0x00008fff6fff] LowShadow + // [0x000000000000, 0x00007fff7fff] LowMem + __libqasan_map_shadow((void *)0x02008fff7000, (void *)0x10007fff7fff); + __libqasan_map_shadow((void *)0x00007fff8000, (void *)0x00008fff6fff); + + #else + // [0x40000000, 0xffffffff] HighMem + // [0x28000000, 0x3fffffff] HighShadow + // [0x24000000, 0x27ffffff] ShadowGap + // [0x20000000, 0x23ffffff] LowShadow + // [0x00000000, 0x1fffffff] LowMem + __libqasan_map_shadow((void *)0x28000000, (void *)0x3fffffff); + __libqasan_map_shadow((void *)0x20000000, (void *)0x23ffffff); + #endif + +#endif + // if (__qasan_log) { __libqasan_print_maps(); } } +#ifdef ASAN_GUEST + +__attribute__((always_inline)) static inline char *qasan_get_shadow( + const char *start) { + size_t shadow_addr = ((size_t)start >> ALLOC_ALIGN_POW) + SHADOW_OFFSET; + return ((char *)shadow_addr); +} + +__attribute__((always_inline)) static inline const char *qasan_align_ptr_down( + const char *start, size_t n) { + return (const char *)qasan_align_down((size_t)start, n); +} + +__attribute__((always_inline)) static inline const char *qasan_align_ptr_up( + const char *start, size_t n) { + return qasan_align_ptr_down(&start[n - 1], n); +} + +static bool qemu_mem_test(const char *k_start, const char *k_end) { + for (const char *cursor = k_start; cursor < k_end; cursor++) { + char k = *cursor; + if (k != 0) { + QASAN_DEBUG("qemu_mem_test - k_start: %p, k_end: %p, cursor: %p, k: %d\n", + k_start, k_end, cursor, k); + return true; + } + } + + return false; +} + +static void qemu_mem_set(char *k_start, char *k_end, char val) { + for (char *cursor = (char *)k_start; cursor < k_end; cursor++) { + *cursor = val; + } +} + +/* Our end point should be 8-byte aligned */ +void qasan_load(const char *start, size_t len) { + QASAN_DEBUG("LOAD: %p-%p\n", start, &start[len]); + if (qasan_is_poison(start, len)) { + QASAN_LOG("Region is poisoned: %p-%p\n", start, &start[len]); + abort(); + } +} + +void qasan_store(const char *start, size_t len) { + QASAN_DEBUG("STORE: %p-%p\n", start, &start[len]); + if (qasan_is_poison(start, len)) { + QASAN_LOG("Region is poisoned: %p-%p\n", start, &start[len]); + abort(); + } +} + +void qasan_poison(const char *start, size_t len, char val) { + const char *end = &start[len]; + QASAN_DEBUG("POISON: %p-%p, (%zu) 0x%02x\n", start, end, len, val); + + const char *start_aligned = qasan_align_ptr_up(start, ALLOC_ALIGN_SIZE); + const char *end_aligned = qasan_align_ptr_down(end, ALLOC_ALIGN_SIZE); + + if (len == 0) return; + + if (end != end_aligned) { + QASAN_LOG("Region end is unaligned: %p-%p, end_aligned: %p\n", start, end, + end_aligned); + abort(); + } + + /* k > 0 (first k bytes are UN-poisoned */ + size_t first_unpoisoned = ALLOC_ALIGN_SIZE - (start_aligned - start); + QASAN_DEBUG("UNPOIS - first_unpoisoned: %zu\n", first_unpoisoned); + + char *k_start = qasan_get_shadow(start); + QASAN_DEBUG("UNPOISON - k_start: %p\n", k_start); + + if (first_unpoisoned == 0) { + *k_start = val; + } else { + *k_start = first_unpoisoned; + } + + /* + * The end is aligned, so we can round up the start and deal with the + * remaining aligned buffer now + */ + char *k_start_aligned = qasan_get_shadow(start_aligned); + char *k_end_aligned = qasan_get_shadow(end_aligned); + + QASAN_DEBUG("POISONk: %p-%p\n", k_start_aligned, k_end_aligned); + + qemu_mem_set(k_start_aligned, k_end_aligned, val); + QASAN_DEBUG("POISONED: %p-%p, 0x%02x\n", start, end, val); +} + +void qasan_unpoison(const char *start, size_t len) { + const char *end = &start[len]; + QASAN_DEBUG("UNPOISON: %p-%p (%zu)\n", start, end, len); + + const char *start_aligned = qasan_align_ptr_up(start, ALLOC_ALIGN_SIZE); + const char *end_aligned = qasan_align_ptr_down(end, ALLOC_ALIGN_SIZE); + + if (len == 0) return; + + if (start_aligned != start) { + QASAN_LOG("Region start is unaligned: %p-%p, start_aligned: %p\n", start, + end, start_aligned); + abort(); + } + + char *k_start_aligned = qasan_get_shadow(k_start_aligned); + char *k_end_aligned = qasan_get_shadow(k_end_aligned); + + QASAN_DEBUG("UNPOISONk: %p-%p\n", k_start_aligned, k_end_aligned); + + qemu_mem_set(k_start_aligned, k_end_aligned, 0); + + size_t last_unpoisoned = end - end_aligned; + QASAN_DEBUG("UNPOISON - last_unpoisoned: %zu\n", last_unpoisoned); + + char *k_end = qasan_get_shadow(end); + QASAN_DEBUG("UNPOISON - k_end: %p\n", k_end); + + *k_end = (char)last_unpoisoned; + + QASAN_DEBUG("UNPOISONED: %p-%p\n", start, end); +} + +bool qasan_is_poison(const char *start, size_t len) { + const char *end = &start[len]; + QASAN_DEBUG("IS POISON: %p-%p (%zu)\n", start, end, len); + + const char *start_aligned = qasan_align_ptr_up(start, ALLOC_ALIGN_SIZE); + const char *end_aligned = qasan_align_ptr_down(end, ALLOC_ALIGN_SIZE); + + if (len == 0) return false; + + /* If our start is unaligned */ + if (start_aligned != start) { + char *k_start = qasan_get_shadow(start); + QASAN_DEBUG("IS POISON - k_start: %p\n", k_start); + + size_t first_k = (size_t)*k_start; + QASAN_DEBUG("IS POISON - first_k: %zu\n", first_k); + + /* If our buffer ends within the first shadow byte */ + if (end < start_aligned) { + size_t first_len = end - end_aligned; + QASAN_DEBUG("IS POISON - first_len: %zu\n", first_len); + + if ((first_k != 0) && (first_len > first_k)) { + QASAN_DEBUG( + "qasan_is_poison #1 - start_aligned: %p, end_aligned: %p, first_k: " + "%d, first_len: %zu\n", + start_aligned, end_aligned, first_k, first_len); + return true; + } + + return false; + } + + /* + * If our buffer extends beyond the first shadow byte, then it must be + * zero + */ + if (first_k != 0) { + QASAN_DEBUG( + "qasan_is_poison #2 - start_aligned: %p, end_aligned: %p, first_k: " + "%d\n", + start_aligned, end_aligned, first_k); + return true; + } + } + + /* If our end is unaligned */ + if (end_aligned != end) { + size_t last_len = end - end_aligned; + QASAN_DEBUG("IS POISON - last_len: %zu\n", last_len); + + char *k_end = qasan_get_shadow(end); + QASAN_DEBUG("IS POISON - k_end: %p\n", k_end); + + char last_k = *k_end; + QASAN_DEBUG("IS POISON - last_k: %zu\n", last_k); + + if ((last_k != 0) && (last_len > last_k)) { + QASAN_DEBUG( + "qasan_is_poison #3 - start_aligned: %p, end_aligned: %p, last_k: " + "%d, last_len: %zu\n", + start_aligned, end_aligned, last_k, last_len); + return true; + } + } + + const char *k_start_aligned = qasan_get_shadow(start_aligned); + QASAN_DEBUG("IS POISON - k_start_aligned: %p\n", k_start_aligned); + + const char *k_end_aligned = qasan_get_shadow(end_aligned); + QASAN_DEBUG("IS POISON - k_end_aligned: %p\n", k_end_aligned); + + return qemu_mem_test(k_start_aligned, k_end_aligned); +} + +void qasan_alloc(const char *start, const char *end) { + QASAN_DEBUG("ALLOC: %p-%p\n", start, end); + /* Do Nothing - We don't track allocations */ +} + +void qasan_dealloc(const char *start) { + QASAN_DEBUG("DEALLOC: %p\n", start); + /* Do Nothing - We don't track allocations */ +} + +int qasan_swap(int state) { + QASAN_DEBUG("SWAP: %d\n", state); + /* Do Nothing */ +} +#endif + int __libc_start_main(int (*main)(int, char **, char **), int argc, char **argv, int (*init)(int, char **, char **), void (*fini)(void), void (*rtld_fini)(void), void *stack_end) { diff --git a/libafl_qemu/libqasan/libqasan.h b/libafl_qemu/libqasan/libqasan.h index e611513000..99af9e3099 100644 --- a/libafl_qemu/libqasan/libqasan.h +++ b/libafl_qemu/libqasan/libqasan.h @@ -42,6 +42,11 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "qasan.h" #include "printf/printf.h" +#ifdef ASAN_GUEST + #include + #include +#endif + #define QASAN_LOG(msg...) \ do { \ if (__qasan_log) { \ @@ -81,6 +86,9 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. extern int __qasan_debug; extern int __qasan_log; +size_t qasan_align_down(size_t val, size_t align); +size_t qasan_align_up(size_t val, size_t align); + void __libqasan_init_hooks(void); void __libqasan_init_malloc(void); @@ -117,5 +125,8 @@ char *__libqasan_strrchr(const char *s, int c); size_t __libqasan_wcslen(const wchar_t *s); wchar_t *__libqasan_wcscpy(wchar_t *d, const wchar_t *s); int __libqasan_wcscmp(const wchar_t *s1, const wchar_t *s2); +void *__libqasan_mmap(void *addr, size_t length, int prot, int flags, int fd, + off_t offset); +int __libqasan_munmap(void *addr, size_t length); #endif diff --git a/libafl_qemu/libqasan/malloc.c b/libafl_qemu/libqasan/malloc.c index afb389bc78..be5f56aff2 100644 --- a/libafl_qemu/libqasan/malloc.c +++ b/libafl_qemu/libqasan/malloc.c @@ -72,9 +72,9 @@ struct chunk_struct { #ifdef USE_LIBC_ALLOC -void *(*__lq_libc_malloc)(size_t); +void *(*__lq_libc_memalign)(size_t, size_t); void (*__lq_libc_free)(void *); - #define backend_malloc __lq_libc_malloc + #define backend_memalign __lq_libc_memalign #define backend_free __lq_libc_free #define TMP_ZONE_SIZE 4096 @@ -84,9 +84,9 @@ static unsigned char __tmp_alloc_zone[TMP_ZONE_SIZE]; #else // From dlmalloc.c -void *dlmalloc(size_t); +void *dlmemalign(size_t, size_t); void dlfree(void *); - #define backend_malloc dlmalloc + #define backend_memalign dlmemalign #define backend_free dlfree #endif @@ -140,7 +140,7 @@ void __libqasan_init_malloc(void) { if (__libqasan_malloc_initialized) return; #ifdef USE_LIBC_ALLOC - __lq_libc_malloc = dlsym(RTLD_NEXT, "malloc"); + __lq_libc_memalign = dlsym(RTLD_NEXT, "memalign"); __lq_libc_free = dlsym(RTLD_NEXT, "free"); #endif @@ -168,26 +168,23 @@ void *__libqasan_malloc(size_t size) { #ifdef USE_LIBC_ALLOC void *r = &__tmp_alloc_zone[__tmp_alloc_zone_idx]; - - if (size & (ALLOC_ALIGN_SIZE - 1)) - __tmp_alloc_zone_idx += - (size & ~(ALLOC_ALIGN_SIZE - 1)) + ALLOC_ALIGN_SIZE; - else - __tmp_alloc_zone_idx += size; - + __tmp_alloc_zone_idx += qasan_align_up(size, ALLOC_ALIGN_SIZE); return r; #endif } int state = QASAN_SWAP(QASAN_DISABLED); // disable qasan for this thread - struct chunk_begin *p = backend_malloc(sizeof(struct chunk_struct) + size); + struct chunk_begin *p = backend_memalign( + ALLOC_ALIGN_SIZE, + sizeof(struct chunk_struct) + qasan_align_up(size, ALLOC_ALIGN_SIZE)); QASAN_SWAP(state); if (!p) return NULL; - QASAN_UNPOISON(p, sizeof(struct chunk_struct) + size); + QASAN_UNPOISON( + p, sizeof(struct chunk_struct) + qasan_align_up(size, ALLOC_ALIGN_SIZE)); p->requested_size = size; p->aligned_orig = NULL; @@ -195,12 +192,9 @@ void *__libqasan_malloc(size_t size) { QASAN_ALLOC(&p[1], (char *)&p[1] + size); QASAN_POISON(p->redzone, REDZONE_SIZE, ASAN_HEAP_LEFT_RZ); - if (size & (ALLOC_ALIGN_SIZE - 1)) - QASAN_POISON((char *)&p[1] + size, - (size & ~(ALLOC_ALIGN_SIZE - 1)) + 8 - size + REDZONE_SIZE, - ASAN_HEAP_RIGHT_RZ); - else - QASAN_POISON((char *)&p[1] + size, REDZONE_SIZE, ASAN_HEAP_RIGHT_RZ); + QASAN_POISON((char *)&p[1] + size, + qasan_align_up(size, ALLOC_ALIGN_SIZE) - size + REDZONE_SIZE, + ASAN_HEAP_RIGHT_RZ); __libqasan_memset(&p[1], 0xff, size); @@ -235,11 +229,7 @@ void __libqasan_free(void *ptr) { } QASAN_SWAP(state); - - if (n & (ALLOC_ALIGN_SIZE - 1)) - n = (n & ~(ALLOC_ALIGN_SIZE - 1)) + ALLOC_ALIGN_SIZE; - - QASAN_POISON(ptr, n, ASAN_HEAP_FREED); + QASAN_POISON(ptr, qasan_align_up(n, ALLOC_ALIGN_SIZE), ASAN_HEAP_FREED); QASAN_DEALLOC(ptr); } @@ -289,13 +279,16 @@ int __libqasan_posix_memalign(void **ptr, size_t align, size_t len) { int state = QASAN_SWAP(QASAN_DISABLED); // disable qasan for this thread - char *orig = backend_malloc(sizeof(struct chunk_struct) + size); + char *orig = backend_memalign( + ALLOC_ALIGN_SIZE, + sizeof(struct chunk_struct) + qasan_align_up(size, ALLOC_ALIGN_SIZE)); QASAN_SWAP(state); if (!orig) return ENOMEM; - QASAN_UNPOISON(orig, sizeof(struct chunk_struct) + size); + QASAN_UNPOISON(orig, sizeof(struct chunk_struct) + + qasan_align_up(size, ALLOC_ALIGN_SIZE)); char *data = orig + sizeof(struct chunk_begin); data += align - ((uintptr_t)data % align); @@ -307,13 +300,9 @@ int __libqasan_posix_memalign(void **ptr, size_t align, size_t len) { QASAN_ALLOC(data, data + len); QASAN_POISON(p->redzone, REDZONE_SIZE, ASAN_HEAP_LEFT_RZ); - if (len & (ALLOC_ALIGN_SIZE - 1)) - QASAN_POISON( - data + len, - (len & ~(ALLOC_ALIGN_SIZE - 1)) + ALLOC_ALIGN_SIZE - len + REDZONE_SIZE, - ASAN_HEAP_RIGHT_RZ); - else - QASAN_POISON(data + len, REDZONE_SIZE, ASAN_HEAP_RIGHT_RZ); + QASAN_POISON(data + len, + qasan_align_up(len, ALLOC_ALIGN_SIZE) - len + REDZONE_SIZE, + ASAN_HEAP_RIGHT_RZ); __libqasan_memset(data, 0xff, len); diff --git a/libafl_qemu/libqasan/mmap.c b/libafl_qemu/libqasan/mmap.c new file mode 100644 index 0000000000..2a6757855f --- /dev/null +++ b/libafl_qemu/libqasan/mmap.c @@ -0,0 +1,118 @@ +/******************************************************************************* +Copyright (c) 2019-2024, Andrea Fioraldi + + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*******************************************************************************/ + +/* +Mmap hooks for libqasan by Alessandro "cube" De Vito + + +*/ + +#include "libqasan.h" +#include +#include +#include +#include +#include +#include + +#ifdef __GLIBC__ + #define USE_LIBC_ALLOC +#endif + +#if __STDC_VERSION__ < 201112L || \ + (defined(__FreeBSD__) && __FreeBSD_version < 1200000) +// use this hack if not C11 +typedef struct { + long long __ll; + long double __ld; + +} max_align_t; + +#endif + +#ifdef USE_LIBC_ALLOC + +void *(*__lq_libc_mmap)(void *, size_t, int, int, int, off_t); +int (*__lq_libc_munmap)(void *, size_t); + +#else + +// TODO: include from mmap.c + +#endif + +int __libqasan_mmap_initialized; + +void __libqasan_init_mmap(void) { + if (__libqasan_mmap_initialized) return; + +#ifdef USE_LIBC_ALLOC + __lq_libc_mmap = dlsym(RTLD_NEXT, "mmap"); + __lq_libc_munmap = dlsym(RTLD_NEXT, "munmap"); +#endif + + __libqasan_mmap_initialized = 1; + QASAN_LOG("\n"); + QASAN_LOG("mmap initialization done.\n"); + QASAN_LOG("\n"); +} + +void *__libqasan_mmap(void *addr, size_t length, int prot, int flags, int fd, + off_t offset) { + __libqasan_init_mmap(); + + int state = QASAN_SWAP(QASAN_DISABLED); // disable qasan for this thread + void *p = __lq_libc_mmap(addr, length, prot, flags, fd, offset); + QASAN_SWAP(state); + + if (!p) return NULL; + + QASAN_UNPOISON(p, length); + + QASAN_ALLOC(p, (uintptr_t)p + length); + + // We don't memset the memory, as it's not guaranteed to be writable. + + return p; +} + +int __libqasan_munmap(void *addr, size_t length) { + __libqasan_init_mmap(); + + int state = QASAN_SWAP(QASAN_DISABLED); // disable qasan for this thread + int ret = __lq_libc_munmap(addr, length); + QASAN_SWAP(state); + + if (ret == -1) return -1; + + // Omitting memory poisoning for unmapped regions as accessing them would + // result in an error anyway. + + // TODO: add a syscall to deallocate addr->addr + length + QASAN_DEALLOC(addr); + + return ret; +} diff --git a/libafl_qemu/libqasan/patch.c b/libafl_qemu/libqasan/patch.c index 9dafd3d786..443c5a9b76 100644 --- a/libafl_qemu/libqasan/patch.c +++ b/libafl_qemu/libqasan/patch.c @@ -200,6 +200,9 @@ void __libqasan_hotpatch(void) { HOTPATCH(bcmp) #endif + HOTPATCH(asprintf) + HOTPATCH(vasprintf) + HOTPATCH(strchr) HOTPATCH(strrchr) HOTPATCH(strcasecmp) @@ -211,6 +214,7 @@ void __libqasan_hotpatch(void) { HOTPATCH(strncpy) HOTPATCH(stpcpy) HOTPATCH(strdup) + HOTPATCH(strndup) HOTPATCH(strlen) HOTPATCH(strnlen) HOTPATCH(strstr) diff --git a/libafl_qemu/libqasan/printf/printf.c b/libafl_qemu/libqasan/printf/printf.c index b4e1a03eb4..e3f0afea4c 100644 --- a/libafl_qemu/libqasan/printf/printf.c +++ b/libafl_qemu/libqasan/printf/printf.c @@ -35,6 +35,7 @@ #include #include "printf.h" +#include "../libqasan.h" // qasan define #define PRINTF_SUPPORT_FLOAT @@ -911,6 +912,17 @@ int __libqasan_vprintf(const char *format, va_list va) { return _vsnprintf(_out_char, buffer, (size_t)-1, format, va); } +int __libqasan_vasprintf(char **restrict strp, const char *restrict format, + va_list va) { + // get the string size + const int len = _vsnprintf(NULL, NULL, (size_t)-1, format, va); + + void *buffer = __libqasan_malloc(len + 1); + *strp = buffer; + const int ret = _vsnprintf(_out_buffer, buffer, len + 1, format, va); + return ret; +} + int __libqasan_vsnprintf(char *buffer, size_t count, const char *format, va_list va) { return _vsnprintf(_out_buffer, buffer, count, format, va); diff --git a/libafl_qemu/libqasan/printf/printf.h b/libafl_qemu/libqasan/printf/printf.h index 8f329fd7ef..dc85409af3 100644 --- a/libafl_qemu/libqasan/printf/printf.h +++ b/libafl_qemu/libqasan/printf/printf.h @@ -93,6 +93,17 @@ int __libqasan_vsnprintf(char *buffer, size_t count, const char *format, */ int __libqasan_vprintf(const char *format, va_list va); +/** + * Tiny vasprintf implementation + * \param strp This function will write the pointer to the allocated string + * \param format A string that specifies the format of the output + * \param va A value identifying a variable arguments list + * \return The number of characters that are WRITTEN into the buffer, not + * counting the terminating null character + */ +int __libqasan_vasprintf(char **restrict strp, const char *restrict format, + va_list va); + /** * printf with output function * You may use this as dynamic alternative to printf() with its fixed _putchar() diff --git a/libafl_qemu/libqasan/qasan.h b/libafl_qemu/libqasan/qasan.h index 05b59ef5c7..cff2fd81b3 100644 --- a/libafl_qemu/libqasan/qasan.h +++ b/libafl_qemu/libqasan/qasan.h @@ -76,27 +76,58 @@ enum { #include -#define QASAN_CALL0(action) syscall(QASAN_FAKESYS_NR, action, NULL, NULL, NULL) -#define QASAN_CALL1(action, arg1) \ - syscall(QASAN_FAKESYS_NR, action, arg1, NULL, NULL) -#define QASAN_CALL2(action, arg1, arg2) \ - syscall(QASAN_FAKESYS_NR, action, arg1, arg2, NULL) -#define QASAN_CALL3(action, arg1, arg2, arg3) \ - syscall(QASAN_FAKESYS_NR, action, arg1, arg2, arg3) - -#define QASAN_LOAD(ptr, len) QASAN_CALL2(QASAN_ACTION_CHECK_LOAD, ptr, len) -#define QASAN_STORE(ptr, len) QASAN_CALL2(QASAN_ACTION_CHECK_STORE, ptr, len) - -#define QASAN_POISON(ptr, len, poison_byte) \ - QASAN_CALL3(QASAN_ACTION_POISON, ptr, len, poison_byte) -#define QASAN_USER_POISON(ptr, len) \ - QASAN_CALL3(QASAN_ACTION_POISON, ptr, len, ASAN_USER) -#define QASAN_UNPOISON(ptr, len) QASAN_CALL2(QASAN_ACTION_UNPOISON, ptr, len) -#define QASAN_IS_POISON(ptr, len) QASAN_CALL2(QASAN_ACTION_IS_POISON, ptr, len) - -#define QASAN_ALLOC(start, end) QASAN_CALL2(QASAN_ACTION_ALLOC, start, end) -#define QASAN_DEALLOC(ptr) QASAN_CALL1(QASAN_ACTION_DEALLOC, ptr) - -#define QASAN_SWAP(state) QASAN_CALL1(QASAN_ACTION_SWAP_STATE, state) +#ifdef ASAN_GUEST + #include + +void qasan_load(const char *start, size_t len); +void qasan_store(const char *start, size_t len); +void qasan_poison(const char *start, size_t len, char val); +void qasan_unpoison(const char *start, size_t len); +bool qasan_is_poison(const char *start, size_t len); + +void qasan_alloc(const char *start, const char *end); +void qasan_dealloc(const char *start); +int qasan_swap(int state); + + #define QASAN_LOAD(ptr, len) qasan_load((const char *)(ptr), (size_t)(len)) + #define QASAN_STORE(ptr, len) qasan_store((const char *)(ptr), (size_t)(len)) + #define QASAN_POISON(ptr, len, poison_byte) \ + qasan_poison((const char *)(ptr), (size_t)(len), (char)(poison_byte)) + #define QASAN_USER_POISON(ptr, len) QASAN_POISON(ptr, len, ASAN_USER) + #define QASAN_UNPOISON(ptr, len) \ + qasan_unpoison((const char *)(ptr), (size_t)(len)) + #define QASAN_IS_POISON(ptr, len) \ + qasan_is_poison((const char *)(ptr), (size_t)(len)) + #define QASAN_ALLOC(start, end) \ + qasan_alloc((const char *)(start), (const char *)(end)) + #define QASAN_DEALLOC(ptr) qasan_dealloc((const char *)(ptr)) + #define QASAN_SWAP(state) qasan_swap((int)(state)) +#else + + #define QASAN_CALL0(action) \ + syscall(QASAN_FAKESYS_NR, action, NULL, NULL, NULL) + #define QASAN_CALL1(action, arg1) \ + syscall(QASAN_FAKESYS_NR, action, arg1, NULL, NULL) + #define QASAN_CALL2(action, arg1, arg2) \ + syscall(QASAN_FAKESYS_NR, action, arg1, arg2, NULL) + #define QASAN_CALL3(action, arg1, arg2, arg3) \ + syscall(QASAN_FAKESYS_NR, action, arg1, arg2, arg3) + + #define QASAN_LOAD(ptr, len) QASAN_CALL2(QASAN_ACTION_CHECK_LOAD, ptr, len) + #define QASAN_STORE(ptr, len) QASAN_CALL2(QASAN_ACTION_CHECK_STORE, ptr, len) + + #define QASAN_POISON(ptr, len, poison_byte) \ + QASAN_CALL3(QASAN_ACTION_POISON, ptr, len, poison_byte) + #define QASAN_USER_POISON(ptr, len) \ + QASAN_CALL3(QASAN_ACTION_POISON, ptr, len, ASAN_USER) + #define QASAN_UNPOISON(ptr, len) QASAN_CALL2(QASAN_ACTION_UNPOISON, ptr, len) + #define QASAN_IS_POISON(ptr, len) \ + QASAN_CALL2(QASAN_ACTION_IS_POISON, ptr, len) + + #define QASAN_ALLOC(start, end) QASAN_CALL2(QASAN_ACTION_ALLOC, start, end) + #define QASAN_DEALLOC(ptr) QASAN_CALL1(QASAN_ACTION_DEALLOC, ptr) + + #define QASAN_SWAP(state) QASAN_CALL1(QASAN_ACTION_SWAP_STATE, state) +#endif #endif diff --git a/libafl_qemu/runtime/libafl_qemu.h b/libafl_qemu/runtime/libafl_qemu.h new file mode 100644 index 0000000000..0337fd9873 --- /dev/null +++ b/libafl_qemu/runtime/libafl_qemu.h @@ -0,0 +1,215 @@ +#ifndef LIBAFL_QEMU_H +#define LIBAFL_QEMU_H + +/** + * LibAFL QEMU header file. + * + * This file is a portable header file used to build target harnesses more + * conveniently. Its main purpose is to generate ready-to-use calls to + * communicate with the fuzzer. The list of commands is available at the bottom + * of this file. The rest mostly consists of macros generating the code used by + * the commands. + */ + +/* === The private part starts here === */ + +/* This part should not be useful for most people. Callable commands are + * available at the end of this file. */ + +#define STRINGIFY(s) #s +#define XSTRINGIFY(s) STRINGIFY(s) + +// Target Specific imports / definitions +#ifdef _WIN32 + #include + #include + +typedef UINT64 libafl_word; + #define LIBAFL_CALLING_CONVENTION __fastcall + +#else + #include + + #ifdef __x86_64__ + typedef uint64_t libafl_word; + #define LIBAFL_CALLING_CONVENTION __attribute__(()) + #endif + + #ifdef __arm__ + typedef uint32_t libafl_word; + #define LIBAFL_CALLING_CONVENTION __attribute__(()) + #endif +#endif + +#define LIBAFL_SYNC_EXIT_OPCODE 0x66f23a0f +#define LIBAFL_BACKDOOR_OPCODE 0x44f23a0f + +#define LIBAFL_QEMU_HDR_VERSION_NUMBER 0111 // TODO: find a nice way to set it. + +typedef enum LibaflQemuCommand { + LIBAFL_QEMU_COMMAND_START_VIRT = 0, + LIBAFL_QEMU_COMMAND_START_PHYS = 1, + LIBAFL_QEMU_COMMAND_INPUT_VIRT = 2, + LIBAFL_QEMU_COMMAND_INPUT_PHYS = 3, + LIBAFL_QEMU_COMMAND_END = 4, + LIBAFL_QEMU_COMMAND_SAVE = 5, + LIBAFL_QEMU_COMMAND_LOAD = 6, + LIBAFL_QEMU_COMMAND_VERSION = 7, + LIBAFL_QEMU_COMMAND_VADDR_FILTER_ALLOW = 8, +} LibaflExit; + +typedef enum LibaflQemuEndStatus { + LIBAFL_QEMU_END_UNKNOWN = 0, + LIBAFL_QEMU_END_OK = 1, + LIBAFL_QEMU_END_CRASH = 2, +} LibaflExitEndParams; + +#ifdef _WIN32 + #define LIBAFL_DEFINE_FUNCTIONS(name, _opcode) \ + #ifdef __cplusplus \ + extern "C" { \ + #endif \ + libafl_word LIBAFL_CALLING_CONVENTION _libafl_##name##_call0(libafl_word action); \ + libafl_word LIBAFL_CALLING_CONVENTION _libafl_##name##_call1(libafl_word action, \ + ##name## libafl_word arg1); \ + libafl_word LIBAFL_CALLING_CONVENTION _libafl_##name##_call2(libafl_word action, \ + libafl_word arg1, \ + libafl_word arg2); \ + #ifdef __cplusplus \ + } \ + #endif +#else + + #ifdef __x86_64__ + #define LIBAFL_DEFINE_FUNCTIONS(name, opcode) \ + libafl_word LIBAFL_CALLING_CONVENTION _libafl_##name##_call0( \ + libafl_word action) { \ + libafl_word ret; \ + __asm__ volatile ( \ + "mov %1, %%rax\n" \ + ".dword " XSTRINGIFY(opcode) "\n" \ + "mov %%rax, %0\n" \ + : "=g"(ret) \ + : "g"(action) \ + : "%rax" \ + ); \ + return ret; \ + } \ + \ + libafl_word LIBAFL_CALLING_CONVENTION _libafl_##name##_call1( \ + libafl_word action, libafl_word arg1) { \ + libafl_word ret; \ + __asm__ volatile ( \ + "mov %1, %%rax\n" \ + "mov %2, %%rdi\n" \ + ".dword " XSTRINGIFY(opcode) "\n" \ + "mov %%rax, %0\n" \ + : "=g"(ret) \ + : "g"(action), "g"(arg1) \ + : "%rax", "%rdi" \ + ); \ + return ret; \ + } \ + \ + libafl_word LIBAFL_CALLING_CONVENTION _libafl_##name##_call2( \ + libafl_word action, libafl_word arg1, libafl_word arg2) { \ + libafl_word ret; \ + __asm__ volatile ( \ + "mov %1, %%rax\n" \ + "mov %2, %%rdi\n" \ + "mov %3, %%rsi\n" \ + ".dword " XSTRINGIFY(opcode) "\n" \ + "mov %%rax, %0\n" \ + : "=g"(ret) \ + : "g"(action), "g"(arg1), "g"(arg2) \ + : "%rax", "%rdi", "%rsi" \ + ); \ + return ret; \ + } + #endif + + #ifdef __arm__ + #define LIBAFL_DEFINE_FUNCTIONS(name, opcode) \ + libafl_word LIBAFL_CALLING_CONVENTION _libafl_##name##_call0( \ + libafl_word action) { \ + libafl_word ret; \ + __asm__ volatile ( \ + "mov r0, %1\n" \ + ".word " XSTRINGIFY(opcode) "\n" \ + "mov %0, r0\n" \ + : "=r"(ret) \ + : "r"(action) \ + : "r0" \ + ); \ + return ret; \ + } \ + \ + libafl_word LIBAFL_CALLING_CONVENTION _libafl_##name##_call1( \ + libafl_word action, libafl_word arg1) { \ + libafl_word ret; \ + __asm__ volatile ( \ + "mov r0, %1\n" \ + "mov r1, %2\n" \ + ".word " XSTRINGIFY(opcode) "\n" \ + "mov %0, r0\n" \ + : "=r"(ret) \ + : "r"(action), "r"(arg1) \ + : "r0", "r1" \ + ); \ + return ret; \ + } \ + \ + libafl_word LIBAFL_CALLING_CONVENTION _libafl_##name##_call2( \ + libafl_word action, libafl_word arg1, libafl_word arg2) { \ + libafl_word ret; \ + __asm__ volatile ( \ + "mov r0, %1\n" \ + "mov r1, %2\n" \ + "mov r2, %3\n" \ + ".word " XSTRINGIFY(opcode) "\n" \ + "mov %0, r0\n" \ + : "=r"(ret) \ + : "r"(action), "r"(arg1), "r"(arg2) \ + : "r0", "r1", "r2" \ + ); \ + return ret; \ + } + #endif + +#endif + +// Generates sync exit functions +LIBAFL_DEFINE_FUNCTIONS(sync_exit, LIBAFL_SYNC_EXIT_OPCODE) + +// Generates backdoor functions +LIBAFL_DEFINE_FUNCTIONS(backdoor, LIBAFL_BACKDOOR_OPCODE) + +/* === The private part ends here === */ + +/* === The public part starts here === */ + +/* LibAFL QEMU Commands */ + +#define LIBAFL_QEMU_START_VIRT(buf_vaddr, max_len) \ + _libafl_sync_exit_call2(LIBAFL_QEMU_COMMAND_START_VIRT, buf_vaddr, max_len) + +#define LIBAFL_QEMU_START_PHYS(buf_paddr, max_len) \ + _libafl_sync_exit_call2(LIBAFL_QEMU_COMMAND_START_PHYS, buf_paddr, max_len) + +#define LIBAFL_QEMU_INPUT_VIRT(buf_vaddr, max_len) \ + _libafl_sync_exit_call2(LIBAFL_QEMU_COMMAND_INPUT_VIRT, buf_vaddr, max_len) + +#define LIBAFL_QEMU_INPUT_PHYS(buf_paddr, max_len) \ + _libafl_exit_call2(LIBAFL_QEMU_COMMAND_INPUT_PHYS, buf_paddr, max_len) + +#define LIBAFL_QEMU_END(status) _libafl_sync_exit_call1(LIBAFL_QEMU_COMMAND_END, status) + +#define LIBAFL_QEMU_SAVE() _libafl_sync_exit_call0(LIBAFL_QEMU_COMMAND_SAVE) + +#define LIBAFL_QEMU_LOAD() _libafl_sync_exit_call0(LIBAFL_QEMU_COMMAND_LOAD) + +#define LIBAFL_QEMU_VERSION() _libafl_sync_exit_call0(LIBAFL_QEMU_COMMAND_VERSION) + +/* === The public part ends here === */ + +#endif diff --git a/libafl_qemu/runtime/libafl_qemu_stub_bindings.rs b/libafl_qemu/runtime/libafl_qemu_stub_bindings.rs new file mode 100644 index 0000000000..b7dbc82441 --- /dev/null +++ b/libafl_qemu/runtime/libafl_qemu_stub_bindings.rs @@ -0,0 +1,314 @@ +/* automatically generated by rust-bindgen 0.69.4 */ + +pub const _STDINT_H: u32 = 1; +pub const _FEATURES_H: u32 = 1; +pub const _DEFAULT_SOURCE: u32 = 1; +pub const __GLIBC_USE_ISOC2X: u32 = 0; +pub const __USE_ISOC11: u32 = 1; +pub const __USE_ISOC99: u32 = 1; +pub const __USE_ISOC95: u32 = 1; +pub const __USE_POSIX_IMPLICITLY: u32 = 1; +pub const _POSIX_SOURCE: u32 = 1; +pub const _POSIX_C_SOURCE: u32 = 200809; +pub const __USE_POSIX: u32 = 1; +pub const __USE_POSIX2: u32 = 1; +pub const __USE_POSIX199309: u32 = 1; +pub const __USE_POSIX199506: u32 = 1; +pub const __USE_XOPEN2K: u32 = 1; +pub const __USE_XOPEN2K8: u32 = 1; +pub const _ATFILE_SOURCE: u32 = 1; +pub const __WORDSIZE: u32 = 64; +pub const __WORDSIZE_TIME64_COMPAT32: u32 = 1; +pub const __SYSCALL_WORDSIZE: u32 = 64; +pub const __TIMESIZE: u32 = 64; +pub const __USE_MISC: u32 = 1; +pub const __USE_ATFILE: u32 = 1; +pub const __USE_FORTIFY_LEVEL: u32 = 0; +pub const __GLIBC_USE_DEPRECATED_GETS: u32 = 0; +pub const __GLIBC_USE_DEPRECATED_SCANF: u32 = 0; +pub const _STDC_PREDEF_H: u32 = 1; +pub const __STDC_IEC_559__: u32 = 1; +pub const __STDC_IEC_60559_BFP__: u32 = 201404; +pub const __STDC_IEC_559_COMPLEX__: u32 = 1; +pub const __STDC_IEC_60559_COMPLEX__: u32 = 201404; +pub const __STDC_ISO_10646__: u32 = 201706; +pub const __GNU_LIBRARY__: u32 = 6; +pub const __GLIBC__: u32 = 2; +pub const __GLIBC_MINOR__: u32 = 35; +pub const _SYS_CDEFS_H: u32 = 1; +pub const __glibc_c99_flexarr_available: u32 = 1; +pub const __LDOUBLE_REDIRECTS_TO_FLOAT128_ABI: u32 = 0; +pub const __HAVE_GENERIC_SELECTION: u32 = 1; +pub const __GLIBC_USE_LIB_EXT2: u32 = 0; +pub const __GLIBC_USE_IEC_60559_BFP_EXT: u32 = 0; +pub const __GLIBC_USE_IEC_60559_BFP_EXT_C2X: u32 = 0; +pub const __GLIBC_USE_IEC_60559_EXT: u32 = 0; +pub const __GLIBC_USE_IEC_60559_FUNCS_EXT: u32 = 0; +pub const __GLIBC_USE_IEC_60559_FUNCS_EXT_C2X: u32 = 0; +pub const __GLIBC_USE_IEC_60559_TYPES_EXT: u32 = 0; +pub const _BITS_TYPES_H: u32 = 1; +pub const _BITS_TYPESIZES_H: u32 = 1; +pub const __OFF_T_MATCHES_OFF64_T: u32 = 1; +pub const __INO_T_MATCHES_INO64_T: u32 = 1; +pub const __RLIM_T_MATCHES_RLIM64_T: u32 = 1; +pub const __STATFS_MATCHES_STATFS64: u32 = 1; +pub const __KERNEL_OLD_TIMEVAL_MATCHES_TIMEVAL64: u32 = 1; +pub const __FD_SETSIZE: u32 = 1024; +pub const _BITS_TIME64_H: u32 = 1; +pub const _BITS_WCHAR_H: u32 = 1; +pub const _BITS_STDINT_INTN_H: u32 = 1; +pub const _BITS_STDINT_UINTN_H: u32 = 1; +pub const INT8_MIN: i32 = -128; +pub const INT16_MIN: i32 = -32768; +pub const INT32_MIN: i32 = -2147483648; +pub const INT8_MAX: u32 = 127; +pub const INT16_MAX: u32 = 32767; +pub const INT32_MAX: u32 = 2147483647; +pub const UINT8_MAX: u32 = 255; +pub const UINT16_MAX: u32 = 65535; +pub const UINT32_MAX: u32 = 4294967295; +pub const INT_LEAST8_MIN: i32 = -128; +pub const INT_LEAST16_MIN: i32 = -32768; +pub const INT_LEAST32_MIN: i32 = -2147483648; +pub const INT_LEAST8_MAX: u32 = 127; +pub const INT_LEAST16_MAX: u32 = 32767; +pub const INT_LEAST32_MAX: u32 = 2147483647; +pub const UINT_LEAST8_MAX: u32 = 255; +pub const UINT_LEAST16_MAX: u32 = 65535; +pub const UINT_LEAST32_MAX: u32 = 4294967295; +pub const INT_FAST8_MIN: i32 = -128; +pub const INT_FAST16_MIN: i64 = -9223372036854775808; +pub const INT_FAST32_MIN: i64 = -9223372036854775808; +pub const INT_FAST8_MAX: u32 = 127; +pub const INT_FAST16_MAX: u64 = 9223372036854775807; +pub const INT_FAST32_MAX: u64 = 9223372036854775807; +pub const UINT_FAST8_MAX: u32 = 255; +pub const UINT_FAST16_MAX: i32 = -1; +pub const UINT_FAST32_MAX: i32 = -1; +pub const INTPTR_MIN: i64 = -9223372036854775808; +pub const INTPTR_MAX: u64 = 9223372036854775807; +pub const UINTPTR_MAX: i32 = -1; +pub const PTRDIFF_MIN: i64 = -9223372036854775808; +pub const PTRDIFF_MAX: u64 = 9223372036854775807; +pub const SIG_ATOMIC_MIN: i32 = -2147483648; +pub const SIG_ATOMIC_MAX: u32 = 2147483647; +pub const SIZE_MAX: i32 = -1; +pub const WINT_MIN: u32 = 0; +pub const WINT_MAX: u32 = 4294967295; +pub const LIBAFL_SYNC_EXIT_OPCODE: u32 = 1727150607; +pub const LIBAFL_BACKDOOR_OPCODE: u32 = 1156725263; +pub const LIBAFL_QEMU_HDR_VERSION_NUMBER: u32 = 73; +pub type __u_char = ::std::os::raw::c_uchar; +pub type __u_short = ::std::os::raw::c_ushort; +pub type __u_int = ::std::os::raw::c_uint; +pub type __u_long = ::std::os::raw::c_ulong; +pub type __int8_t = ::std::os::raw::c_schar; +pub type __uint8_t = ::std::os::raw::c_uchar; +pub type __int16_t = ::std::os::raw::c_short; +pub type __uint16_t = ::std::os::raw::c_ushort; +pub type __int32_t = ::std::os::raw::c_int; +pub type __uint32_t = ::std::os::raw::c_uint; +pub type __int64_t = ::std::os::raw::c_long; +pub type __uint64_t = ::std::os::raw::c_ulong; +pub type __int_least8_t = __int8_t; +pub type __uint_least8_t = __uint8_t; +pub type __int_least16_t = __int16_t; +pub type __uint_least16_t = __uint16_t; +pub type __int_least32_t = __int32_t; +pub type __uint_least32_t = __uint32_t; +pub type __int_least64_t = __int64_t; +pub type __uint_least64_t = __uint64_t; +pub type __quad_t = ::std::os::raw::c_long; +pub type __u_quad_t = ::std::os::raw::c_ulong; +pub type __intmax_t = ::std::os::raw::c_long; +pub type __uintmax_t = ::std::os::raw::c_ulong; +pub type __dev_t = ::std::os::raw::c_ulong; +pub type __uid_t = ::std::os::raw::c_uint; +pub type __gid_t = ::std::os::raw::c_uint; +pub type __ino_t = ::std::os::raw::c_ulong; +pub type __ino64_t = ::std::os::raw::c_ulong; +pub type __mode_t = ::std::os::raw::c_uint; +pub type __nlink_t = ::std::os::raw::c_ulong; +pub type __off_t = ::std::os::raw::c_long; +pub type __off64_t = ::std::os::raw::c_long; +pub type __pid_t = ::std::os::raw::c_int; +#[repr(C)] +#[derive(Debug, Default, Copy, Clone)] +pub struct __fsid_t { + pub __val: [::std::os::raw::c_int; 2usize], +} +#[test] +fn bindgen_test_layout___fsid_t() { + const UNINIT: ::std::mem::MaybeUninit<__fsid_t> = ::std::mem::MaybeUninit::uninit(); + let ptr = UNINIT.as_ptr(); + assert_eq!( + ::std::mem::size_of::<__fsid_t>(), + 8usize, + concat!("Size of: ", stringify!(__fsid_t)) + ); + assert_eq!( + ::std::mem::align_of::<__fsid_t>(), + 4usize, + concat!("Alignment of ", stringify!(__fsid_t)) + ); + assert_eq!( + unsafe { ::std::ptr::addr_of!((*ptr).__val) as usize - ptr as usize }, + 0usize, + concat!( + "Offset of field: ", + stringify!(__fsid_t), + "::", + stringify!(__val) + ) + ); +} +pub type __clock_t = ::std::os::raw::c_long; +pub type __rlim_t = ::std::os::raw::c_ulong; +pub type __rlim64_t = ::std::os::raw::c_ulong; +pub type __id_t = ::std::os::raw::c_uint; +pub type __time_t = ::std::os::raw::c_long; +pub type __useconds_t = ::std::os::raw::c_uint; +pub type __suseconds_t = ::std::os::raw::c_long; +pub type __suseconds64_t = ::std::os::raw::c_long; +pub type __daddr_t = ::std::os::raw::c_int; +pub type __key_t = ::std::os::raw::c_int; +pub type __clockid_t = ::std::os::raw::c_int; +pub type __timer_t = *mut ::std::os::raw::c_void; +pub type __blksize_t = ::std::os::raw::c_long; +pub type __blkcnt_t = ::std::os::raw::c_long; +pub type __blkcnt64_t = ::std::os::raw::c_long; +pub type __fsblkcnt_t = ::std::os::raw::c_ulong; +pub type __fsblkcnt64_t = ::std::os::raw::c_ulong; +pub type __fsfilcnt_t = ::std::os::raw::c_ulong; +pub type __fsfilcnt64_t = ::std::os::raw::c_ulong; +pub type __fsword_t = ::std::os::raw::c_long; +pub type __ssize_t = ::std::os::raw::c_long; +pub type __syscall_slong_t = ::std::os::raw::c_long; +pub type __syscall_ulong_t = ::std::os::raw::c_ulong; +pub type __loff_t = __off64_t; +pub type __caddr_t = *mut ::std::os::raw::c_char; +pub type __intptr_t = ::std::os::raw::c_long; +pub type __socklen_t = ::std::os::raw::c_uint; +pub type __sig_atomic_t = ::std::os::raw::c_int; +pub type int_least8_t = __int_least8_t; +pub type int_least16_t = __int_least16_t; +pub type int_least32_t = __int_least32_t; +pub type int_least64_t = __int_least64_t; +pub type uint_least8_t = __uint_least8_t; +pub type uint_least16_t = __uint_least16_t; +pub type uint_least32_t = __uint_least32_t; +pub type uint_least64_t = __uint_least64_t; +pub type int_fast8_t = ::std::os::raw::c_schar; +pub type int_fast16_t = ::std::os::raw::c_long; +pub type int_fast32_t = ::std::os::raw::c_long; +pub type int_fast64_t = ::std::os::raw::c_long; +pub type uint_fast8_t = ::std::os::raw::c_uchar; +pub type uint_fast16_t = ::std::os::raw::c_ulong; +pub type uint_fast32_t = ::std::os::raw::c_ulong; +pub type uint_fast64_t = ::std::os::raw::c_ulong; +pub type intmax_t = __intmax_t; +pub type uintmax_t = __uintmax_t; +pub type libafl_word = u64; +pub const LibaflQemuCommand_LIBAFL_QEMU_COMMAND_START_VIRT: LibaflQemuCommand = + LibaflQemuCommand(0); +pub const LibaflQemuCommand_LIBAFL_QEMU_COMMAND_START_PHYS: LibaflQemuCommand = + LibaflQemuCommand(1); +pub const LibaflQemuCommand_LIBAFL_QEMU_COMMAND_INPUT_VIRT: LibaflQemuCommand = + LibaflQemuCommand(2); +pub const LibaflQemuCommand_LIBAFL_QEMU_COMMAND_INPUT_PHYS: LibaflQemuCommand = + LibaflQemuCommand(3); +pub const LibaflQemuCommand_LIBAFL_QEMU_COMMAND_END: LibaflQemuCommand = LibaflQemuCommand(4); +pub const LibaflQemuCommand_LIBAFL_QEMU_COMMAND_SAVE: LibaflQemuCommand = LibaflQemuCommand(5); +pub const LibaflQemuCommand_LIBAFL_QEMU_COMMAND_LOAD: LibaflQemuCommand = LibaflQemuCommand(6); +pub const LibaflQemuCommand_LIBAFL_QEMU_COMMAND_VERSION: LibaflQemuCommand = LibaflQemuCommand(7); +pub const LibaflQemuCommand_LIBAFL_QEMU_COMMAND_VADDR_FILTER_ALLOW: LibaflQemuCommand = + LibaflQemuCommand(8); +impl ::std::ops::BitOr for LibaflQemuCommand { + type Output = Self; + #[inline] + fn bitor(self, other: Self) -> Self { + LibaflQemuCommand(self.0 | other.0) + } +} +impl ::std::ops::BitOrAssign for LibaflQemuCommand { + #[inline] + fn bitor_assign(&mut self, rhs: LibaflQemuCommand) { + self.0 |= rhs.0; + } +} +impl ::std::ops::BitAnd for LibaflQemuCommand { + type Output = Self; + #[inline] + fn bitand(self, other: Self) -> Self { + LibaflQemuCommand(self.0 & other.0) + } +} +impl ::std::ops::BitAndAssign for LibaflQemuCommand { + #[inline] + fn bitand_assign(&mut self, rhs: LibaflQemuCommand) { + self.0 &= rhs.0; + } +} +#[repr(transparent)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct LibaflQemuCommand(pub ::std::os::raw::c_uint); +pub use self::LibaflQemuCommand as LibaflExit; +pub const LibaflQemuEndStatus_LIBAFL_QEMU_END_UNKNOWN: LibaflQemuEndStatus = LibaflQemuEndStatus(0); +pub const LibaflQemuEndStatus_LIBAFL_QEMU_END_OK: LibaflQemuEndStatus = LibaflQemuEndStatus(1); +pub const LibaflQemuEndStatus_LIBAFL_QEMU_END_CRASH: LibaflQemuEndStatus = LibaflQemuEndStatus(2); +impl ::std::ops::BitOr for LibaflQemuEndStatus { + type Output = Self; + #[inline] + fn bitor(self, other: Self) -> Self { + LibaflQemuEndStatus(self.0 | other.0) + } +} +impl ::std::ops::BitOrAssign for LibaflQemuEndStatus { + #[inline] + fn bitor_assign(&mut self, rhs: LibaflQemuEndStatus) { + self.0 |= rhs.0; + } +} +impl ::std::ops::BitAnd for LibaflQemuEndStatus { + type Output = Self; + #[inline] + fn bitand(self, other: Self) -> Self { + LibaflQemuEndStatus(self.0 & other.0) + } +} +impl ::std::ops::BitAndAssign for LibaflQemuEndStatus { + #[inline] + fn bitand_assign(&mut self, rhs: LibaflQemuEndStatus) { + self.0 &= rhs.0; + } +} +#[repr(transparent)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct LibaflQemuEndStatus(pub ::std::os::raw::c_uint); +pub use self::LibaflQemuEndStatus as LibaflExitEndParams; +extern "C" { + pub fn _libafl_sync_exit_call0(action: libafl_word) -> libafl_word; +} +extern "C" { + pub fn _libafl_sync_exit_call1(action: libafl_word, arg1: libafl_word) -> libafl_word; +} +extern "C" { + pub fn _libafl_sync_exit_call2( + action: libafl_word, + arg1: libafl_word, + arg2: libafl_word, + ) -> libafl_word; +} +extern "C" { + pub fn _libafl_backdoor_call0(action: libafl_word) -> libafl_word; +} +extern "C" { + pub fn _libafl_backdoor_call1(action: libafl_word, arg1: libafl_word) -> libafl_word; +} +extern "C" { + pub fn _libafl_backdoor_call2( + action: libafl_word, + arg1: libafl_word, + arg2: libafl_word, + ) -> libafl_word; +} diff --git a/libafl_qemu/runtime/libafl_qemu_windows.asm b/libafl_qemu/runtime/libafl_qemu_windows.asm new file mode 100755 index 0000000000..1b8163e887 --- /dev/null +++ b/libafl_qemu/runtime/libafl_qemu_windows.asm @@ -0,0 +1,115 @@ +; LibAFL QEMU Windows ASM companion file. It should be used together with libafl_qemu.h +; Since Windows does not support extended inline assembly, it is more convenient to use asm files directly. + +PUBLIC _libafl_sync_exit_call0, _libafl_sync_exit_call1, _libafl_sync_exit_call2 +PUBLIC _libafl_backdoor_call0, _libafl_backdoor_call1, _libafl_backdoor_call2 + +LIBAFL_SYNC_EXIT_OPCODE MACRO + dd 66f23a0fh +ENDM + +LIBAFL_BACKDOOR_OPCODE MACRO + dd 44f23a0fh +ENDM + +.code + +; Execute LibAFL sync exit (no argument) +; Parameters: +; [RAX, OUT] Hook return value +; [RCX, IN] LibAFL QEMU Command +_libafl_sync_exit_call0: + mov rax, rcx + + LIBAFL_SYNC_EXIT_OPCODE + + ret + +; Execute LibAFL sync exit (one argument) +; Parameters: +; [RAX, OUT] Hook return value +; [RCX, IN] LibAFL QEMU Command +; [RDX, IN] Arg1 +_libafl_sync_exit_call1: + push rdi + + mov rax, rcx + mov rdi, rdx + + LIBAFL_SYNC_EXIT_OPCODE + + pop rdi + + ret + +; Execute LibAFL sync exit (two arguments) +; Parameters: +; [RAX, OUT] Hook return value +; [RCX, IN] LibAFL QEMU Command +; [RDX, IN] Arg1 +; [R8, IN] Arg2 +_libafl_sync_exit_call2: + push rdi + push rsi + + mov rax, rcx + mov rdi, rdx + mov rsi, r8 + + LIBAFL_SYNC_EXIT_OPCODE + + pop rsi + pop rdi + + ret + +; Execute LibAFL backdoor (no argument) +; Parameters: +; [RAX, OUT] Hook return value +; [RCX, IN] LibAFL QEMU Command +_libafl_backdoor_call0: + mov rax, rcx + + LIBAFL_BACKDOOR_OPCODE + + ret + +; Execute LibAFL backdoor (one argument) +; Parameters: +; [RAX, OUT] Hook return value +; [RCX, IN] LibAFL QEMU Command +; [RDX, IN] Arg1 +_libafl_backdoor_call1: + push rdi + + mov rax, rcx + mov rdi, rdx + + LIBAFL_BACKDOOR_OPCODE + + pop rdi + + ret + +; Execute LibAFL backdoor (two arguments) +; Parameters: +; [RAX, OUT] Hook return value +; [RCX, IN] LibAFL QEMU Command +; [RDX, IN] Arg1 +; [R8, IN] Arg2 +_libafl_backdoor_call2: + push rdi + push rsi + + mov rax, rcx + mov rdi, rdx + mov rsi, r8 + + LIBAFL_BACKDOOR_OPCODE + + pop rsi + pop rdi + + ret + +END diff --git a/libafl_qemu/src/aarch64.rs b/libafl_qemu/src/arch/aarch64.rs similarity index 81% rename from libafl_qemu/src/aarch64.rs rename to libafl_qemu/src/arch/aarch64.rs index 4489b57171..023cb34da5 100644 --- a/libafl_qemu/src/aarch64.rs +++ b/libafl_qemu/src/arch/aarch64.rs @@ -8,7 +8,7 @@ use pyo3::prelude::*; pub use strum_macros::EnumIter; pub use syscall_numbers::aarch64::*; -use crate::{sync_backdoor::SyncBackdoorArgs, CallingConvention}; +use crate::{sync_exit::BackdoorArgs, CallingConvention}; #[derive(IntoPrimitive, TryFromPrimitive, Debug, Clone, Copy, EnumIter)] #[repr(i32)] @@ -49,19 +49,19 @@ pub enum Regs { Pstate = 33, } -static SYNC_BACKDOOR_ARCH_REGS: OnceLock> = OnceLock::new(); +static BACKDOOR_ARCH_REGS: OnceLock> = OnceLock::new(); -pub fn get_sync_backdoor_arch_regs() -> &'static EnumMap { - SYNC_BACKDOOR_ARCH_REGS.get_or_init(|| { +pub fn get_backdoor_arch_regs() -> &'static EnumMap { + BACKDOOR_ARCH_REGS.get_or_init(|| { enum_map! { - SyncBackdoorArgs::Ret => Regs::X0, - SyncBackdoorArgs::Cmd => Regs::X0, - SyncBackdoorArgs::Arg1 => Regs::X1, - SyncBackdoorArgs::Arg2 => Regs::X2, - SyncBackdoorArgs::Arg3 => Regs::X3, - SyncBackdoorArgs::Arg4 => Regs::X4, - SyncBackdoorArgs::Arg5 => Regs::X5, - SyncBackdoorArgs::Arg6 => Regs::X6, + BackdoorArgs::Ret => Regs::X0, + BackdoorArgs::Cmd => Regs::X0, + BackdoorArgs::Arg1 => Regs::X1, + BackdoorArgs::Arg2 => Regs::X2, + BackdoorArgs::Arg3 => Regs::X3, + BackdoorArgs::Arg4 => Regs::X4, + BackdoorArgs::Arg5 => Regs::X5, + BackdoorArgs::Arg6 => Regs::X6, } }) } diff --git a/libafl_qemu/src/arm.rs b/libafl_qemu/src/arch/arm.rs similarity index 82% rename from libafl_qemu/src/arm.rs rename to libafl_qemu/src/arch/arm.rs index 926f8bef89..42b6d8d24f 100644 --- a/libafl_qemu/src/arm.rs +++ b/libafl_qemu/src/arch/arm.rs @@ -8,7 +8,7 @@ use pyo3::prelude::*; pub use strum_macros::EnumIter; pub use syscall_numbers::arm::*; -use crate::{sync_backdoor::SyncBackdoorArgs, CallingConvention}; +use crate::{sync_exit::BackdoorArgs, CallingConvention}; /// Registers for the ARM instruction set. #[derive(IntoPrimitive, TryFromPrimitive, Debug, Clone, Copy, EnumIter)] @@ -33,19 +33,19 @@ pub enum Regs { R25 = 25, } -static SYNC_BACKDOOR_ARCH_REGS: OnceLock> = OnceLock::new(); +static BACKDOOR_ARCH_REGS: OnceLock> = OnceLock::new(); -pub fn get_sync_backdoor_arch_regs() -> &'static EnumMap { - SYNC_BACKDOOR_ARCH_REGS.get_or_init(|| { +pub fn get_backdoor_arch_regs() -> &'static EnumMap { + BACKDOOR_ARCH_REGS.get_or_init(|| { enum_map! { - SyncBackdoorArgs::Ret => Regs::R0, - SyncBackdoorArgs::Cmd => Regs::R0, - SyncBackdoorArgs::Arg1 => Regs::R1, - SyncBackdoorArgs::Arg2 => Regs::R2, - SyncBackdoorArgs::Arg3 => Regs::R3, - SyncBackdoorArgs::Arg4 => Regs::R4, - SyncBackdoorArgs::Arg5 => Regs::R5, - SyncBackdoorArgs::Arg6 => Regs::R6, + BackdoorArgs::Ret => Regs::R0, + BackdoorArgs::Cmd => Regs::R0, + BackdoorArgs::Arg1 => Regs::R1, + BackdoorArgs::Arg2 => Regs::R2, + BackdoorArgs::Arg3 => Regs::R3, + BackdoorArgs::Arg4 => Regs::R4, + BackdoorArgs::Arg5 => Regs::R5, + BackdoorArgs::Arg6 => Regs::R6, } }) } diff --git a/libafl_qemu/src/hexagon.rs b/libafl_qemu/src/arch/hexagon.rs similarity index 80% rename from libafl_qemu/src/hexagon.rs rename to libafl_qemu/src/arch/hexagon.rs index 726c3b0a54..bd4ff32cd9 100644 --- a/libafl_qemu/src/hexagon.rs +++ b/libafl_qemu/src/arch/hexagon.rs @@ -6,7 +6,7 @@ use num_enum::{IntoPrimitive, TryFromPrimitive}; use pyo3::prelude::*; pub use strum_macros::EnumIter; -use crate::{sync_backdoor::SyncBackdoorArgs, CallingConvention}; +use crate::{sync_exit::BackdoorArgs, CallingConvention}; #[derive(IntoPrimitive, TryFromPrimitive, Debug, Clone, Copy, EnumIter)] #[repr(i32)] @@ -64,19 +64,19 @@ pub enum Regs { Pktcnthi = 51, } -static SYNC_BACKDOOR_ARCH_REGS: OnceLock> = OnceLock::new(); +static BACKDOOR_ARCH_REGS: OnceLock> = OnceLock::new(); -pub fn get_sync_backdoor_arch_regs() -> &'static EnumMap { - SYNC_BACKDOOR_ARCH_REGS.get_or_init(|| { +pub fn get_backdoor_arch_regs() -> &'static EnumMap { + BACKDOOR_ARCH_REGS.get_or_init(|| { enum_map! { - SyncBackdoorArgs::Ret => Regs::R0, - SyncBackdoorArgs::Cmd => Regs::R0, - SyncBackdoorArgs::Arg1 => Regs::R1, - SyncBackdoorArgs::Arg2 => Regs::R2, - SyncBackdoorArgs::Arg3 => Regs::R3, - SyncBackdoorArgs::Arg4 => Regs::R4, - SyncBackdoorArgs::Arg5 => Regs::R5, - SyncBackdoorArgs::Arg6 => Regs::R6, + BackdoorArgs::Ret => Regs::R0, + BackdoorArgs::Cmd => Regs::R0, + BackdoorArgs::Arg1 => Regs::R1, + BackdoorArgs::Arg2 => Regs::R2, + BackdoorArgs::Arg3 => Regs::R3, + BackdoorArgs::Arg4 => Regs::R4, + BackdoorArgs::Arg5 => Regs::R5, + BackdoorArgs::Arg6 => Regs::R6, } }) } diff --git a/libafl_qemu/src/i386.rs b/libafl_qemu/src/arch/i386.rs similarity index 84% rename from libafl_qemu/src/i386.rs rename to libafl_qemu/src/arch/i386.rs index 44bbd0ca92..ca91b8ddf6 100644 --- a/libafl_qemu/src/i386.rs +++ b/libafl_qemu/src/arch/i386.rs @@ -8,7 +8,7 @@ use pyo3::prelude::*; pub use strum_macros::EnumIter; pub use syscall_numbers::x86::*; -use crate::{sync_backdoor::SyncBackdoorArgs, CallingConvention, GuestAddr}; +use crate::{sync_exit::BackdoorArgs, CallingConvention, GuestAddr}; #[derive(IntoPrimitive, TryFromPrimitive, Debug, Clone, Copy, EnumIter)] #[repr(i32)] @@ -25,19 +25,19 @@ pub enum Regs { Eflags = 9, } -static SYNC_BACKDOOR_ARCH_REGS: OnceLock> = OnceLock::new(); +static BACKDOOR_ARCH_REGS: OnceLock> = OnceLock::new(); -pub fn get_sync_backdoor_arch_regs() -> &'static EnumMap { - SYNC_BACKDOOR_ARCH_REGS.get_or_init(|| { +pub fn get_backdoor_arch_regs() -> &'static EnumMap { + BACKDOOR_ARCH_REGS.get_or_init(|| { enum_map! { - SyncBackdoorArgs::Ret => Regs::Eax, - SyncBackdoorArgs::Cmd => Regs::Eax, - SyncBackdoorArgs::Arg1 => Regs::Edi, - SyncBackdoorArgs::Arg2 => Regs::Esi, - SyncBackdoorArgs::Arg3 => Regs::Edx, - SyncBackdoorArgs::Arg4 => Regs::Ebx, - SyncBackdoorArgs::Arg5 => Regs::Ecx, - SyncBackdoorArgs::Arg6 => Regs::Ebp, + BackdoorArgs::Ret => Regs::Eax, + BackdoorArgs::Cmd => Regs::Eax, + BackdoorArgs::Arg1 => Regs::Edi, + BackdoorArgs::Arg2 => Regs::Esi, + BackdoorArgs::Arg3 => Regs::Edx, + BackdoorArgs::Arg4 => Regs::Ebx, + BackdoorArgs::Arg5 => Regs::Ecx, + BackdoorArgs::Arg6 => Regs::Ebp, } }) } diff --git a/libafl_qemu/src/mips.rs b/libafl_qemu/src/arch/mips.rs similarity index 80% rename from libafl_qemu/src/mips.rs rename to libafl_qemu/src/arch/mips.rs index bd39e4da21..4810b33508 100644 --- a/libafl_qemu/src/mips.rs +++ b/libafl_qemu/src/arch/mips.rs @@ -7,7 +7,7 @@ use pyo3::prelude::*; pub use strum_macros::EnumIter; pub use syscall_numbers::mips::*; -use crate::{sync_backdoor::SyncBackdoorArgs, CallingConvention}; +use crate::{sync_exit::BackdoorArgs, CallingConvention}; /// Registers for the MIPS instruction set. #[derive(IntoPrimitive, TryFromPrimitive, Debug, Clone, Copy, EnumIter)] @@ -49,19 +49,19 @@ pub enum Regs { Pc = 37, } -static SYNC_BACKDOOR_ARCH_REGS: OnceLock> = OnceLock::new(); +static BACKDOOR_ARCH_REGS: OnceLock> = OnceLock::new(); -pub fn get_sync_backdoor_arch_regs() -> &'static EnumMap { - SYNC_BACKDOOR_ARCH_REGS.get_or_init(|| { +pub fn get_backdoor_arch_regs() -> &'static EnumMap { + BACKDOOR_ARCH_REGS.get_or_init(|| { enum_map! { - SyncBackdoorArgs::Ret => Regs::V0, - SyncBackdoorArgs::Cmd => Regs::V0, - SyncBackdoorArgs::Arg1 => Regs::A0, - SyncBackdoorArgs::Arg2 => Regs::A1, - SyncBackdoorArgs::Arg3 => Regs::A2, - SyncBackdoorArgs::Arg4 => Regs::A3, - SyncBackdoorArgs::Arg5 => Regs::T0, - SyncBackdoorArgs::Arg6 => Regs::T1, + BackdoorArgs::Ret => Regs::V0, + BackdoorArgs::Cmd => Regs::V0, + BackdoorArgs::Arg1 => Regs::A0, + BackdoorArgs::Arg2 => Regs::A1, + BackdoorArgs::Arg3 => Regs::A2, + BackdoorArgs::Arg4 => Regs::A3, + BackdoorArgs::Arg5 => Regs::T0, + BackdoorArgs::Arg6 => Regs::T1, } }) } diff --git a/libafl_qemu/src/arch/mod.rs b/libafl_qemu/src/arch/mod.rs new file mode 100644 index 0000000000..ff95a150be --- /dev/null +++ b/libafl_qemu/src/arch/mod.rs @@ -0,0 +1,34 @@ +#[cfg(cpu_target = "aarch64")] +pub mod aarch64; +#[cfg(all(cpu_target = "aarch64", not(feature = "clippy")))] +pub use aarch64::*; + +#[cfg(cpu_target = "arm")] +pub mod arm; +#[cfg(all(cpu_target = "arm", not(feature = "clippy")))] +pub use arm::*; + +#[cfg(cpu_target = "i386")] +pub mod i386; +#[cfg(all(cpu_target = "i386", not(feature = "clippy")))] +pub use i386::*; + +#[cfg(cpu_target = "x86_64")] +pub mod x86_64; +#[cfg(cpu_target = "x86_64")] +pub use x86_64::*; + +#[cfg(cpu_target = "mips")] +pub mod mips; +#[cfg(cpu_target = "mips")] +pub use mips::*; + +#[cfg(cpu_target = "ppc")] +pub mod ppc; +#[cfg(cpu_target = "ppc")] +pub use ppc::*; + +#[cfg(cpu_target = "hexagon")] +pub mod hexagon; +#[cfg(cpu_target = "hexagon")] +pub use hexagon::*; diff --git a/libafl_qemu/src/ppc.rs b/libafl_qemu/src/arch/ppc.rs similarity index 83% rename from libafl_qemu/src/ppc.rs rename to libafl_qemu/src/arch/ppc.rs index 3f4bbcdccc..53dd4c1a91 100644 --- a/libafl_qemu/src/ppc.rs +++ b/libafl_qemu/src/arch/ppc.rs @@ -7,7 +7,7 @@ use pyo3::prelude::*; pub use strum_macros::EnumIter; pub use syscall_numbers::powerpc::*; -use crate::{sync_backdoor::SyncBackdoorArgs, CallingConvention}; +use crate::{sync_exit::BackdoorArgs, CallingConvention}; /// Registers for the MIPS instruction set. #[derive(IntoPrimitive, TryFromPrimitive, Debug, Clone, Copy, EnumIter)] @@ -88,19 +88,19 @@ pub enum Regs { Fpscr = 70, } -static SYNC_BACKDOOR_ARCH_REGS: OnceLock> = OnceLock::new(); +static BACKDOOR_ARCH_REGS: OnceLock> = OnceLock::new(); -pub fn get_sync_backdoor_arch_regs() -> &'static EnumMap { - SYNC_BACKDOOR_ARCH_REGS.get_or_init(|| { +pub fn get_backdoor_arch_regs() -> &'static EnumMap { + BACKDOOR_ARCH_REGS.get_or_init(|| { enum_map! { - SyncBackdoorArgs::Ret => Regs::R3, - SyncBackdoorArgs::Cmd => Regs::R0, - SyncBackdoorArgs::Arg1 => Regs::R3, - SyncBackdoorArgs::Arg2 => Regs::R4, - SyncBackdoorArgs::Arg3 => Regs::R5, - SyncBackdoorArgs::Arg4 => Regs::R6, - SyncBackdoorArgs::Arg5 => Regs::R7, - SyncBackdoorArgs::Arg6 => Regs::R8, + BackdoorArgs::Ret => Regs::R3, + BackdoorArgs::Cmd => Regs::R0, + BackdoorArgs::Arg1 => Regs::R3, + BackdoorArgs::Arg2 => Regs::R4, + BackdoorArgs::Arg3 => Regs::R5, + BackdoorArgs::Arg4 => Regs::R6, + BackdoorArgs::Arg5 => Regs::R7, + BackdoorArgs::Arg6 => Regs::R8, } }) } diff --git a/libafl_qemu/src/x86_64.rs b/libafl_qemu/src/arch/x86_64.rs similarity index 82% rename from libafl_qemu/src/x86_64.rs rename to libafl_qemu/src/arch/x86_64.rs index 3003a922f1..bdeddc0f75 100644 --- a/libafl_qemu/src/x86_64.rs +++ b/libafl_qemu/src/arch/x86_64.rs @@ -8,7 +8,7 @@ use pyo3::prelude::*; pub use strum_macros::EnumIter; pub use syscall_numbers::x86_64::*; -use crate::{sync_backdoor::SyncBackdoorArgs, CallingConvention}; +use crate::{sync_exit::BackdoorArgs, CallingConvention}; #[derive(IntoPrimitive, TryFromPrimitive, Debug, Clone, Copy, EnumIter)] #[repr(i32)] @@ -33,19 +33,19 @@ pub enum Regs { Rflags = 17, } -static SYNC_BACKDOOR_ARCH_REGS: OnceLock> = OnceLock::new(); +static BACKDOOR_ARCH_REGS: OnceLock> = OnceLock::new(); -pub fn get_sync_backdoor_arch_regs() -> &'static EnumMap { - SYNC_BACKDOOR_ARCH_REGS.get_or_init(|| { +pub fn get_backdoor_arch_regs() -> &'static EnumMap { + BACKDOOR_ARCH_REGS.get_or_init(|| { enum_map! { - SyncBackdoorArgs::Ret => Regs::Rax, - SyncBackdoorArgs::Cmd => Regs::Rax, - SyncBackdoorArgs::Arg1 => Regs::Rdi, - SyncBackdoorArgs::Arg2 => Regs::Rsi, - SyncBackdoorArgs::Arg3 => Regs::Rdx, - SyncBackdoorArgs::Arg4 => Regs::R10, - SyncBackdoorArgs::Arg5 => Regs::R8, - SyncBackdoorArgs::Arg6 => Regs::R9, + BackdoorArgs::Ret => Regs::Rax, + BackdoorArgs::Cmd => Regs::Rax, + BackdoorArgs::Arg1 => Regs::Rdi, + BackdoorArgs::Arg2 => Regs::Rsi, + BackdoorArgs::Arg3 => Regs::Rdx, + BackdoorArgs::Arg4 => Regs::R10, + BackdoorArgs::Arg5 => Regs::R8, + BackdoorArgs::Arg6 => Regs::R9, } }) } diff --git a/libafl_qemu/src/breakpoint.rs b/libafl_qemu/src/breakpoint.rs new file mode 100644 index 0000000000..ec9ed7ff94 --- /dev/null +++ b/libafl_qemu/src/breakpoint.rs @@ -0,0 +1,95 @@ +use std::{ + borrow::Borrow, + fmt::{Display, Formatter}, + hash::{Hash, Hasher}, +}; + +use libafl_qemu_sys::GuestAddr; + +use crate::{command::Command, Qemu}; + +// TODO: distinguish breakpoints with IDs instead of addresses to avoid collisions. +#[derive(Debug, Clone)] +pub struct Breakpoint { + addr: GuestAddr, + cmd: Option, + disable_on_trigger: bool, + enabled: bool, +} + +impl Hash for Breakpoint { + fn hash(&self, state: &mut H) { + self.addr.hash(state); + } +} + +impl PartialEq for Breakpoint { + fn eq(&self, other: &Self) -> bool { + self.addr == other.addr + } +} + +impl Eq for Breakpoint {} + +impl Display for Breakpoint { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "Breakpoint @vaddr 0x{:x}", self.addr) + } +} + +impl Borrow for Breakpoint { + fn borrow(&self) -> &GuestAddr { + &self.addr + } +} + +impl Breakpoint { + // Emu will return with the breakpoint as exit reason. + #[must_use] + pub fn without_command(addr: GuestAddr, disable_on_trigger: bool) -> Self { + Self { + addr, + cmd: None, + disable_on_trigger, + enabled: false, + } + } + + // Emu will execute the command when it meets the breakpoint. + #[must_use] + pub fn with_command(addr: GuestAddr, cmd: Command, disable_on_trigger: bool) -> Self { + Self { + addr, + cmd: Some(cmd), + disable_on_trigger, + enabled: false, + } + } + + #[must_use] + pub fn addr(&self) -> GuestAddr { + self.addr + } + + pub fn enable(&mut self, qemu: &Qemu) { + if !self.enabled { + qemu.set_breakpoint(self.addr); + self.enabled = true; + } + } + + pub fn disable(&mut self, qemu: &Qemu) { + if self.enabled { + qemu.remove_breakpoint(self.addr.into()); + self.enabled = false; + } + } + + pub fn trigger(&mut self, qemu: &Qemu) -> Option<&Command> { + if self.disable_on_trigger { + self.disable(qemu); + } + + self.cmd.as_ref() + } +} diff --git a/libafl_qemu/src/command.rs b/libafl_qemu/src/command.rs new file mode 100644 index 0000000000..3535b76b52 --- /dev/null +++ b/libafl_qemu/src/command.rs @@ -0,0 +1,678 @@ +#[cfg(emulation_mode = "systemmode")] +use std::collections::HashSet; +use std::fmt::{Debug, Display, Formatter}; + +use enum_map::Enum; +use libafl::{ + executors::ExitKind, + inputs::HasTargetBytes, + state::{HasExecutions, State}, +}; +use libafl_bolts::AsSlice; +use libafl_qemu_sys::{GuestPhysAddr, GuestVirtAddr}; +use num_enum::TryFromPrimitive; + +#[cfg(emulation_mode = "systemmode")] +use crate::QemuInstrumentationPagingFilter; +use crate::{ + executor::QemuExecutorState, sync_exit::SyncBackdoorError, EmuExitHandler, Emulator, + GuestAddrKind, GuestReg, HandlerError, HasInstrumentationFilter, InnerHandlerResult, + InputLocation, IsFilter, IsSnapshotManager, Qemu, QemuHelperTuple, + QemuInstrumentationAddressRangeFilter, Regs, StdEmuExitHandler, StdInstrumentationFilter, CPU, +}; + +pub const VERSION: u64 = bindings::LIBAFL_QEMU_HDR_VERSION_NUMBER as u64; + +mod bindings { + #![allow(non_upper_case_globals)] + #![allow(non_camel_case_types)] + #![allow(non_snake_case)] + #![allow(improper_ctypes)] + #![allow(unused_mut)] + #![allow(unused)] + #![allow(unused_variables)] + #![allow(clippy::all)] + #![allow(clippy::pedantic)] + + include!(concat!(env!("OUT_DIR"), "/libafl_qemu_bindings.rs")); +} + +#[derive(Debug, Clone, TryFromPrimitive)] +#[repr(u64)] +pub enum NativeBackdoorCommand { + StartVirt = bindings::LibaflQemuCommand_LIBAFL_QEMU_COMMAND_START_VIRT.0 as u64, // Shortcut for Save + InputVirt + StartPhys = bindings::LibaflQemuCommand_LIBAFL_QEMU_COMMAND_START_PHYS.0 as u64, // Shortcut for Save + InputPhys + InputVirt = bindings::LibaflQemuCommand_LIBAFL_QEMU_COMMAND_INPUT_VIRT.0 as u64, // The address is a virtual address using the paging currently running in the VM. + InputPhys = bindings::LibaflQemuCommand_LIBAFL_QEMU_COMMAND_INPUT_PHYS.0 as u64, // The address is a physical address + End = bindings::LibaflQemuCommand_LIBAFL_QEMU_COMMAND_END.0 as u64, // Implies reloading of the target. The first argument gives the exit status. + Save = bindings::LibaflQemuCommand_LIBAFL_QEMU_COMMAND_SAVE.0 as u64, // Save the VM + Load = bindings::LibaflQemuCommand_LIBAFL_QEMU_COMMAND_LOAD.0 as u64, // Reload the target without ending the run? + Version = bindings::LibaflQemuCommand_LIBAFL_QEMU_COMMAND_VERSION.0 as u64, // Version of the bindings used in the target + VaddrFilterAllowRange = + bindings::LibaflQemuCommand_LIBAFL_QEMU_COMMAND_VADDR_FILTER_ALLOW.0 as u64, // Allow given address range +} + +#[derive(Debug, Clone, Enum, TryFromPrimitive)] +#[repr(u64)] +pub enum NativeExitKind { + Unknown = bindings::LibaflQemuEndStatus_LIBAFL_QEMU_END_UNKNOWN.0 as u64, // Should not be used + Ok = bindings::LibaflQemuEndStatus_LIBAFL_QEMU_END_OK.0 as u64, // Normal exit + Crash = bindings::LibaflQemuEndStatus_LIBAFL_QEMU_END_CRASH.0 as u64, // Crash reported in the VM +} + +pub trait IsCommand +where + QT: QemuHelperTuple, + S: State + HasExecutions, + E: EmuExitHandler, +{ + /// Used to know whether the command can be run during a backdoor, or if it is necessary to go out of + /// the QEMU VM to run the command. + fn usable_at_runtime(&self) -> bool; + + /// Command handler. + /// - `input`: The input for the current emulator run. + /// - `ret_reg`: The register in which the guest return value should be written, if any. + /// Returns + /// - `InnerHandlerResult`: How the high-level handler should behave + fn run( + &self, + emu: &Emulator, + qemu_executor_state: &mut QemuExecutorState, + input: &S::Input, + ret_reg: Option, + ) -> Result; +} + +#[cfg(emulation_mode = "systemmode")] +pub type PagingFilterCommand = FilterCommand; + +pub type AddressRangeFilterCommand = FilterCommand; + +#[derive(Debug, Clone)] +pub enum Command { + SaveCommand(SaveCommand), + LoadCommand(LoadCommand), + InputCommand(InputCommand), + StartCommand(StartCommand), + EndCommand(EndCommand), + VersionCommand(VersionCommand), + #[cfg(emulation_mode = "systemmode")] + PagingFilterCommand(PagingFilterCommand), + AddressRangeFilterCommand(AddressRangeFilterCommand), +} + +// TODO: Replace with enum_dispatch implementation +impl IsCommand> for Command +where + SM: IsSnapshotManager, + QT: QemuHelperTuple + StdInstrumentationFilter + Debug, + S: State + HasExecutions, + S::Input: HasTargetBytes, +{ + fn usable_at_runtime(&self) -> bool { + match self { + Command::SaveCommand(cmd) => { + >>::usable_at_runtime(cmd) + } + Command::LoadCommand(cmd) => { + >>::usable_at_runtime(cmd) + } + Command::InputCommand(cmd) => { + >>::usable_at_runtime(cmd) + } + Command::StartCommand(cmd) => { + >>::usable_at_runtime(cmd) + } + Command::EndCommand(cmd) => { + >>::usable_at_runtime(cmd) + } + Command::VersionCommand(cmd) => { + >>::usable_at_runtime(cmd) + } + #[cfg(emulation_mode = "systemmode")] + Command::PagingFilterCommand(cmd) => { + >>::usable_at_runtime( + cmd, + ) + } + Command::AddressRangeFilterCommand(cmd) => , + >>::usable_at_runtime(cmd), + } + } + + fn run( + &self, + emu: &Emulator>, + qemu_executor_state: &mut QemuExecutorState, + input: &S::Input, + ret_reg: Option, + ) -> Result { + match self { + Command::SaveCommand(cmd) => { + >>::run( + cmd, + emu, + qemu_executor_state, + input, + ret_reg, + ) + } + Command::LoadCommand(cmd) => { + >>::run( + cmd, + emu, + qemu_executor_state, + input, + ret_reg, + ) + } + Command::InputCommand(cmd) => , + >>::run( + cmd, emu, qemu_executor_state, input, ret_reg + ), + Command::StartCommand(cmd) => , + >>::run( + cmd, emu, qemu_executor_state, input, ret_reg + ), + Command::EndCommand(cmd) => { + >>::run( + cmd, + emu, + qemu_executor_state, + input, + ret_reg, + ) + } + Command::VersionCommand(cmd) => , + >>::run( + cmd, emu, qemu_executor_state, input, ret_reg + ), + #[cfg(emulation_mode = "systemmode")] + Command::PagingFilterCommand(cmd) => , + >>::run( + cmd, emu, qemu_executor_state, input, ret_reg + ), + Command::AddressRangeFilterCommand(cmd) => { + >>::run( + cmd, + emu, + qemu_executor_state, + input, + ret_reg, + ) + } + } + } +} + +#[derive(Debug, Clone)] +pub struct EmulatorMemoryChunk { + addr: GuestAddrKind, + size: GuestReg, + cpu: Option, +} + +#[derive(Debug, Clone)] +pub struct SaveCommand; + +impl IsCommand> for SaveCommand +where + SM: IsSnapshotManager, + QT: QemuHelperTuple + StdInstrumentationFilter + Debug, + S: State + HasExecutions, + S::Input: HasTargetBytes, +{ + fn usable_at_runtime(&self) -> bool { + false + } + + fn run( + &self, + emu: &Emulator>, + #[cfg(emulation_mode = "systemmode")] qemu_executor_state: &mut QemuExecutorState, + #[cfg(not(emulation_mode = "systemmode"))] _qemu_executor_state: &mut QemuExecutorState< + QT, + S, + >, + _input: &S::Input, + _ret_reg: Option, + ) -> Result { + let qemu = emu.qemu(); + let emu_exit_handler = emu.exit_handler().borrow_mut(); + + let snapshot_id = emu_exit_handler.snapshot_manager_borrow_mut().save(qemu); + emu_exit_handler + .set_snapshot_id(snapshot_id) + .map_err(|_| HandlerError::MultipleSnapshotDefinition)?; + + #[cfg(emulation_mode = "systemmode")] + { + let qemu_helpers = qemu_executor_state.hooks_mut().helpers_mut(); + + let mut allowed_paging_ids = HashSet::new(); + + let current_paging_id = qemu.current_cpu().unwrap().current_paging_id().unwrap(); + allowed_paging_ids.insert(current_paging_id); + + let paging_filter = + HasInstrumentationFilter::::filter_mut( + qemu_helpers, + ); + + *paging_filter = QemuInstrumentationPagingFilter::AllowList(allowed_paging_ids); + } + + Ok(InnerHandlerResult::Continue) + } +} + +#[derive(Debug, Clone)] +pub struct LoadCommand; + +impl IsCommand> for LoadCommand +where + SM: IsSnapshotManager, + QT: QemuHelperTuple + StdInstrumentationFilter + Debug, + S: State + HasExecutions, + S::Input: HasTargetBytes, +{ + fn usable_at_runtime(&self) -> bool { + false + } + + fn run( + &self, + emu: &Emulator>, + _qemu_executor_state: &mut QemuExecutorState, + _input: &S::Input, + _ret_reg: Option, + ) -> Result { + let qemu = emu.qemu(); + let emu_exit_handler = emu.exit_handler().borrow_mut(); + + let snapshot_id = emu_exit_handler + .snapshot_id() + .ok_or(HandlerError::SnapshotNotFound)?; + + emu_exit_handler + .snapshot_manager_borrow_mut() + .restore(&snapshot_id, qemu)?; + + Ok(InnerHandlerResult::Continue) + } +} + +#[derive(Debug, Clone)] +pub struct InputCommand { + location: EmulatorMemoryChunk, + cpu: CPU, +} + +impl IsCommand> for InputCommand +where + SM: IsSnapshotManager, + QT: QemuHelperTuple + StdInstrumentationFilter + Debug, + S: State + HasExecutions, + S::Input: HasTargetBytes, +{ + fn usable_at_runtime(&self) -> bool { + true + } + + fn run( + &self, + emu: &Emulator>, + _qemu_executor_state: &mut QemuExecutorState, + input: &S::Input, + ret_reg: Option, + ) -> Result { + let qemu = emu.qemu(); + + let ret_value = self.location.write(qemu, input.target_bytes().as_slice()); + + if let Some(reg) = ret_reg { + self.cpu.write_reg(reg, ret_value).unwrap(); + } + + Ok(InnerHandlerResult::Continue) + } +} + +#[derive(Debug, Clone)] +pub struct StartCommand { + input_location: EmulatorMemoryChunk, +} + +impl IsCommand> for StartCommand +where + SM: IsSnapshotManager, + QT: QemuHelperTuple + StdInstrumentationFilter + Debug, + S: State + HasExecutions, + S::Input: HasTargetBytes, +{ + fn usable_at_runtime(&self) -> bool { + false + } + + fn run( + &self, + emu: &Emulator>, + _qemu_executor_state: &mut QemuExecutorState, + input: &S::Input, + ret_reg: Option, + ) -> Result { + let emu_exit_handler = emu.exit_handler().borrow_mut(); + let qemu = emu.qemu(); + let snapshot_id = emu_exit_handler.snapshot_manager_borrow_mut().save(qemu); + + emu_exit_handler + .set_snapshot_id(snapshot_id) + .map_err(|_| HandlerError::MultipleSnapshotDefinition)?; + + emu_exit_handler + .set_input_location(InputLocation::new( + self.input_location.clone(), + qemu.current_cpu().unwrap(), + ret_reg, + )) + .unwrap(); + + let ret_value = self + .input_location + .write(qemu, input.target_bytes().as_slice()); + + if let Some(reg) = ret_reg { + qemu.write_reg(reg, ret_value).unwrap(); + } + + Ok(InnerHandlerResult::Continue) + } +} + +#[derive(Debug, Clone)] +pub struct EndCommand(Option); + +impl IsCommand> for EndCommand +where + SM: IsSnapshotManager, + QT: QemuHelperTuple + StdInstrumentationFilter + Debug, + S: State + HasExecutions, + S::Input: HasTargetBytes, +{ + fn usable_at_runtime(&self) -> bool { + false + } + + fn run( + &self, + emu: &Emulator>, + _qemu_executor_state: &mut QemuExecutorState, + _input: &S::Input, + _ret_reg: Option, + ) -> Result { + let emu_exit_handler = emu.exit_handler().borrow_mut(); + + let snapshot_id = emu_exit_handler + .snapshot_id() + .ok_or(HandlerError::SnapshotNotFound)?; + + emu_exit_handler + .snapshot_manager_borrow_mut() + .restore(&snapshot_id, emu.qemu())?; + + Ok(InnerHandlerResult::EndOfRun(self.0.unwrap())) + } +} + +#[derive(Debug, Clone)] +pub struct VersionCommand(u64); + +impl IsCommand> for VersionCommand +where + SM: IsSnapshotManager, + QT: QemuHelperTuple + StdInstrumentationFilter + Debug, + S: State + HasExecutions, + S::Input: HasTargetBytes, +{ + fn usable_at_runtime(&self) -> bool { + true + } + + fn run( + &self, + _emu: &Emulator>, + _qemu_executor_state: &mut QemuExecutorState, + _input: &S::Input, + _ret_reg: Option, + ) -> Result { + let guest_version = self.0; + + if VERSION == guest_version { + Ok(InnerHandlerResult::Continue) + } else { + Err(HandlerError::SyncBackdoorError( + SyncBackdoorError::VersionDifference(guest_version), + )) + } + } +} + +#[derive(Debug, Clone)] +pub struct FilterCommand +where + T: IsFilter + Debug, +{ + filter: T, +} + +#[cfg(emulation_mode = "systemmode")] +impl IsCommand> for PagingFilterCommand +where + SM: IsSnapshotManager, + QT: QemuHelperTuple + StdInstrumentationFilter + Debug, + S: State + HasExecutions, + S::Input: HasTargetBytes, +{ + fn usable_at_runtime(&self) -> bool { + true + } + + fn run( + &self, + _emu: &Emulator>, + qemu_executor_state: &mut QemuExecutorState, + _input: &S::Input, + _ret_reg: Option, + ) -> Result { + let qemu_helpers = qemu_executor_state.hooks_mut().helpers_mut(); + + let paging_filter = + HasInstrumentationFilter::::filter_mut( + qemu_helpers, + ); + + *paging_filter = self.filter.clone(); + + Ok(InnerHandlerResult::Continue) + } +} + +impl IsCommand> for AddressRangeFilterCommand +where + SM: IsSnapshotManager, + QT: QemuHelperTuple + StdInstrumentationFilter + Debug, + S: State + HasExecutions, + S::Input: HasTargetBytes, +{ + fn usable_at_runtime(&self) -> bool { + true + } + + #[allow(clippy::type_complexity)] // TODO: refactor with correct type. + fn run( + &self, + _emu: &Emulator>, + qemu_executor_state: &mut QemuExecutorState, + _input: &S::Input, + _ret_reg: Option, + ) -> Result { + let qemu_helpers = qemu_executor_state.hooks_mut().helpers_mut(); + + let addr_range_filter = + HasInstrumentationFilter::::filter_mut( + qemu_helpers, + ); + + *addr_range_filter = self.filter.clone(); + + Ok(InnerHandlerResult::Continue) + } +} + +impl VersionCommand { + #[must_use] + pub fn new(version: u64) -> Self { + Self(version) + } +} + +impl FilterCommand +where + T: IsFilter + Debug, +{ + pub fn new(filter: T) -> Self { + Self { filter } + } +} + +// TODO: rewrite with display implementation for each command. +impl Display for Command { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Command::SaveCommand(_) => write!(f, "Save VM"), + Command::LoadCommand(_) => write!(f, "Reload VM"), + Command::InputCommand(input_command) => { + write!(f, "Set fuzzing input @{}", input_command.location.addr) + } + Command::StartCommand(start_command) => { + write!( + f, + "Start fuzzing with input @{}", + start_command.input_location.addr + ) + } + Command::EndCommand(end_command) => write!(f, "Exit of kind {:?}", end_command.0), + Command::VersionCommand(version_command) => { + write!(f, "Client version: {}", version_command.0) + } + Command::AddressRangeFilterCommand(addr_range_filter) => { + write!(f, "Addr range filter: {:?}", addr_range_filter.filter,) + } + #[cfg(emulation_mode = "systemmode")] + Command::PagingFilterCommand(paging_filter) => { + write!(f, "Addr range filter: {:?}", paging_filter.filter,) + } + } + } +} + +impl StartCommand { + #[must_use] + pub fn new(input_location: EmulatorMemoryChunk) -> Self { + Self { input_location } + } +} + +impl EndCommand { + #[must_use] + pub fn new(exit_kind: Option) -> Self { + Self(exit_kind) + } +} + +impl InputCommand { + #[must_use] + pub fn new(location: EmulatorMemoryChunk, cpu: CPU) -> Self { + Self { location, cpu } + } +} + +impl EmulatorMemoryChunk { + #[must_use] + pub fn phys(addr: GuestPhysAddr, size: GuestReg, cpu: Option) -> Self { + Self { + addr: GuestAddrKind::Physical(addr), + size, + cpu, + } + } + + #[must_use] + pub fn virt(addr: GuestVirtAddr, size: GuestReg, cpu: CPU) -> Self { + Self { + addr: GuestAddrKind::Virtual(addr), + size, + cpu: Some(cpu), + } + } + + /// Returns the number of bytes effectively written. + #[must_use] + pub fn write(&self, qemu: &Qemu, input: &[u8]) -> GuestReg { + let max_len: usize = self.size.try_into().unwrap(); + + let input_sliced = if input.len() > max_len { + &input[0..max_len] + } else { + input + }; + + match self.addr { + GuestAddrKind::Physical(hwaddr) => unsafe { + #[cfg(emulation_mode = "usermode")] + { + // For now the default behaviour is to fall back to virtual addresses + qemu.write_mem(hwaddr.try_into().unwrap(), input_sliced); + } + #[cfg(emulation_mode = "systemmode")] + { + qemu.write_phys_mem(hwaddr, input_sliced); + } + }, + GuestAddrKind::Virtual(vaddr) => unsafe { + self.cpu + .as_ref() + .unwrap() + .write_mem(vaddr.try_into().unwrap(), input_sliced); + }, + }; + + input_sliced.len().try_into().unwrap() + } +} + +impl Display for InputCommand { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{} (0x{:x} max nb bytes)", + self.location.addr, self.location.size + ) + } +} diff --git a/libafl_qemu/src/elf.rs b/libafl_qemu/src/elf.rs index a92a89d5d4..40f0726941 100644 --- a/libafl_qemu/src/elf.rs +++ b/libafl_qemu/src/elf.rs @@ -1,11 +1,10 @@ //! Utilities to parse and process ELFs -use std::{convert::AsRef, fs::File, io::Read, ops::Range, path::Path, str}; +use std::{fs::File, io::Read, ops::Range, path::Path, str}; use goblin::elf::{header::ET_DYN, Elf}; use libafl::Error; - -use crate::GuestAddr; +use libafl_qemu_sys::GuestAddr; pub struct EasyElf<'a> { elf: Elf<'a>, diff --git a/libafl_qemu/src/emu.rs b/libafl_qemu/src/emu.rs deleted file mode 100644 index e870dbeeca..0000000000 --- a/libafl_qemu/src/emu.rs +++ /dev/null @@ -1,1955 +0,0 @@ -//! Expose QEMU user `LibAFL` C api to Rust - -use core::{ - convert::Into, - ffi::c_void, - fmt, - mem::{transmute, MaybeUninit}, - ptr::{addr_of, copy_nonoverlapping, null}, -}; -#[cfg(emulation_mode = "usermode")] -use std::cell::OnceCell; -#[cfg(emulation_mode = "systemmode")] -use std::{ffi::CStr, ptr::null_mut}; -use std::{ffi::CString, ptr, slice::from_raw_parts, str::from_utf8_unchecked}; - -#[cfg(emulation_mode = "usermode")] -use libc::c_int; -use num_enum::{IntoPrimitive, TryFromPrimitive}; -use num_traits::Num; -use paste::paste; -use strum::IntoEnumIterator; -use strum_macros::EnumIter; - -use crate::{GuestReg, Regs}; - -/// Safe linking with of extern "C" functions. -/// This macro makes sure the declared symbol is defined *at link time*, avoiding declaring non-existant symbols -/// that could be silently ignored during linking if unused. -/// -/// This macro relies on a nightly feature, and can only be used in this mode -/// It is (nearly) a drop-in replacement for extern "C" { } blocks containing function and static declarations, and will have the same effect in practice. -macro_rules! extern_c_checked { - () => {}; - - ($visibility:vis fn $c_fn:ident($($param_ident:ident : $param_ty:ty),*) $( -> $ret_ty:ty )?; $($tail:tt)*) => { - paste! { - #[cfg_attr(nightly, used(linker))] - static [<__ $c_fn:upper __>]: unsafe extern "C" fn($($param_ty),*) $( -> $ret_ty )? = $c_fn; - } - - extern "C" { - $visibility fn $c_fn($($param_ident : $param_ty),*) $( -> $ret_ty )?; - } - - extern_c_checked!($($tail)*); - }; - - ($visibility:vis static $c_var:ident : $c_var_ty:ty; $($tail:tt)*) => { - paste! { - #[allow(non_camel_case_types)] - #[allow(unused)] - struct [<__ $c_var:upper _STRUCT__>] { member: *const $c_var_ty } - - unsafe impl Sync for [<__ $c_var:upper _STRUCT__>] {} - - #[cfg_attr(nightly, used(linker))] - static [<__ $c_var:upper __>]: [<__ $c_var:upper _STRUCT__>] = unsafe { [<__ $c_var:upper _STRUCT__>] { member: core::ptr::addr_of!($c_var) } }; - } - - extern "C" { - $visibility static $c_var: $c_var_ty; - } - - extern_c_checked!($($tail)*); - }; - - ($visibility:vis static mut $c_var:ident : $c_var_ty:ty; $($tail:tt)*) => { - paste! { - #[allow(non_camel_case_types)] - #[allow(unused)] - struct [<__ $c_var:upper _STRUCT__>] { member: *const $c_var_ty } - - unsafe impl Sync for [<__ $c_var:upper _STRUCT__>] {} - - #[cfg_attr(nightly, used(linker))] - static mut [<__ $c_var:upper __>]: [<__ $c_var:upper _STRUCT__>] = unsafe { [<__ $c_var:upper _STRUCT__>] { member: core::ptr::addr_of!($c_var) } }; - } - - extern "C" { - $visibility static mut $c_var: $c_var_ty; - } - - extern_c_checked!($($tail)*); - }; -} - -pub type GuestAddr = libafl_qemu_sys::target_ulong; -pub type GuestUsize = libafl_qemu_sys::target_ulong; -pub type GuestIsize = libafl_qemu_sys::target_long; -pub type GuestVirtAddr = libafl_qemu_sys::vaddr; -pub type GuestPhysAddr = libafl_qemu_sys::hwaddr; - -pub type GuestHwAddrInfo = libafl_qemu_sys::qemu_plugin_hwaddr; - -#[derive(Debug, Clone)] -pub enum GuestAddrKind { - Physical(GuestPhysAddr), - Virtual(GuestVirtAddr), -} - -impl fmt::Display for GuestAddrKind { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - GuestAddrKind::Physical(phys_addr) => write!(f, "hwaddr 0x{phys_addr:x}"), - GuestAddrKind::Virtual(virt_addr) => write!(f, "vaddr 0x{virt_addr:x}"), - } - } -} - -#[cfg(emulation_mode = "systemmode")] -pub type FastSnapshot = *mut libafl_qemu_sys::SyxSnapshot; - -#[cfg(emulation_mode = "systemmode")] -pub enum DeviceSnapshotFilter { - All, - AllowList(Vec), - DenyList(Vec), -} - -#[cfg(emulation_mode = "systemmode")] -impl DeviceSnapshotFilter { - fn enum_id(&self) -> libafl_qemu_sys::DeviceSnapshotKind { - match self { - DeviceSnapshotFilter::All => libafl_qemu_sys::DeviceSnapshotKind_DEVICE_SNAPSHOT_ALL, - DeviceSnapshotFilter::AllowList(_) => { - libafl_qemu_sys::DeviceSnapshotKind_DEVICE_SNAPSHOT_ALLOWLIST - } - DeviceSnapshotFilter::DenyList(_) => { - libafl_qemu_sys::DeviceSnapshotKind_DEVICE_SNAPSHOT_DENYLIST - } - } - } - - fn devices(&self, v: &mut Vec<*mut i8>) -> *mut *mut i8 { - v.clear(); - match self { - DeviceSnapshotFilter::All => null_mut(), - DeviceSnapshotFilter::AllowList(l) | DeviceSnapshotFilter::DenyList(l) => { - for name in l { - v.push(name.as_bytes().as_ptr() as *mut i8); - } - v.as_mut_ptr() - } - } - } -} - -#[repr(transparent)] -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct MemAccessInfo { - oi: libafl_qemu_sys::MemOpIdx, -} - -impl MemAccessInfo { - #[must_use] - pub fn memop(&self) -> libafl_qemu_sys::MemOp { - libafl_qemu_sys::MemOp(self.oi >> 4) - } - - #[must_use] - pub fn memopidx(&self) -> libafl_qemu_sys::MemOpIdx { - self.oi - } - - #[must_use] - pub fn mmu_index(&self) -> u32 { - self.oi & 15 - } - - #[must_use] - pub fn size(&self) -> usize { - libafl_qemu_sys::memop_size(self.memop()) as usize - } - - #[must_use] - pub fn is_big_endian(&self) -> bool { - libafl_qemu_sys::memop_big_endian(self.memop()) - } - - #[must_use] - pub fn encode_with(&self, other: u32) -> u64 { - (u64::from(self.oi) << 32) | u64::from(other) - } - - #[must_use] - pub fn decode_from(encoded: u64) -> (Self, u32) { - let low = (encoded & 0xFFFFFFFF) as u32; - let high = (encoded >> 32) as u32; - (Self { oi: high }, low) - } - - #[must_use] - pub fn new(oi: libafl_qemu_sys::MemOpIdx) -> Self { - Self { oi } - } -} - -impl From for MemAccessInfo { - fn from(oi: libafl_qemu_sys::MemOpIdx) -> Self { - Self { oi } - } -} - -#[cfg(feature = "python")] -use pyo3::prelude::*; - -pub const SKIP_EXEC_HOOK: u64 = u64::MAX; - -pub use libafl_qemu_sys::{CPUArchState, CPUState}; - -use crate::sync_backdoor::{SyncBackdoor, SyncBackdoorError}; - -pub type CPUStatePtr = *mut libafl_qemu_sys::CPUState; -pub type CPUArchStatePtr = *mut libafl_qemu_sys::CPUArchState; - -pub type ExitReasonPtr = *mut libafl_qemu_sys::libafl_exit_reason; - -#[derive(IntoPrimitive, TryFromPrimitive, Debug, Clone, Copy, EnumIter, PartialEq, Eq)] -#[repr(i32)] -pub enum MmapPerms { - None = 0, - Read = libc::PROT_READ, - Write = libc::PROT_WRITE, - Execute = libc::PROT_EXEC, - ReadWrite = libc::PROT_READ | libc::PROT_WRITE, - ReadExecute = libc::PROT_READ | libc::PROT_EXEC, - WriteExecute = libc::PROT_WRITE | libc::PROT_EXEC, - ReadWriteExecute = libc::PROT_READ | libc::PROT_WRITE | libc::PROT_EXEC, -} - -impl MmapPerms { - #[must_use] - pub fn is_r(&self) -> bool { - matches!( - self, - MmapPerms::Read - | MmapPerms::ReadWrite - | MmapPerms::ReadExecute - | MmapPerms::ReadWriteExecute - ) - } - - #[must_use] - pub fn is_w(&self) -> bool { - matches!( - self, - MmapPerms::Write - | MmapPerms::ReadWrite - | MmapPerms::WriteExecute - | MmapPerms::ReadWriteExecute - ) - } - - #[must_use] - pub fn is_x(&self) -> bool { - matches!( - self, - MmapPerms::Execute - | MmapPerms::ReadExecute - | MmapPerms::WriteExecute - | MmapPerms::ReadWriteExecute - ) - } -} - -#[cfg(feature = "python")] -impl IntoPy for MmapPerms { - fn into_py(self, py: Python) -> PyObject { - let n: i32 = self.into(); - n.into_py(py) - } -} - -#[cfg(emulation_mode = "usermode")] -#[derive(IntoPrimitive, TryFromPrimitive, Debug, Clone, Copy, EnumIter, PartialEq, Eq)] -#[repr(i32)] -pub enum VerifyAccess { - Read = libc::PROT_READ, - Write = libc::PROT_READ | libc::PROT_WRITE, -} - -// syshook_ret -#[repr(C)] -#[cfg_attr(feature = "python", pyclass)] -#[cfg_attr(feature = "python", derive(FromPyObject))] -pub struct SyscallHookResult { - pub retval: GuestAddr, - pub skip_syscall: bool, -} - -#[cfg(feature = "python")] -#[pymethods] -impl SyscallHookResult { - #[new] - #[must_use] - pub fn new(value: Option) -> Self { - value.map_or( - Self { - retval: 0, - skip_syscall: false, - }, - |v| Self { - retval: v, - skip_syscall: true, - }, - ) - } -} - -#[cfg(not(feature = "python"))] -impl SyscallHookResult { - #[must_use] - pub fn new(value: Option) -> Self { - value.map_or( - Self { - retval: 0, - skip_syscall: false, - }, - |v| Self { - retval: v, - skip_syscall: true, - }, - ) - } -} - -#[repr(C)] -#[cfg_attr(feature = "python", pyclass(unsendable))] -pub struct MapInfo { - start: GuestAddr, - end: GuestAddr, - offset: GuestAddr, - path: *const u8, - flags: i32, - is_priv: i32, -} - -#[cfg_attr(feature = "python", pymethods)] -impl MapInfo { - #[must_use] - pub fn start(&self) -> GuestAddr { - self.start - } - - #[must_use] - pub fn end(&self) -> GuestAddr { - self.end - } - - #[must_use] - pub fn offset(&self) -> GuestAddr { - self.offset - } - - #[must_use] - pub fn path(&self) -> Option<&str> { - if self.path.is_null() { - None - } else { - unsafe { - Some(from_utf8_unchecked(from_raw_parts( - self.path, - strlen(self.path), - ))) - } - } - } - - #[must_use] - pub fn flags(&self) -> MmapPerms { - MmapPerms::try_from(self.flags).unwrap() - } - - #[must_use] - pub fn is_priv(&self) -> bool { - self.is_priv != 0 - } -} - -#[cfg(emulation_mode = "usermode")] -extern_c_checked! { - fn qemu_user_init(argc: i32, argv: *const *const u8, envp: *const *const u8) -> i32; - - fn libafl_qemu_run() -> i32; - - fn libafl_load_addr() -> u64; - fn libafl_get_brk() -> u64; - fn libafl_set_brk(brk: u64) -> u64; - - fn read_self_maps() -> *const c_void; - fn free_self_maps(map_info: *const c_void); - - fn libafl_maps_next(map_info: *const c_void, ret: *mut MapInfo) -> *const c_void; - - static exec_path: *const u8; - static guest_base: usize; - static mut mmap_next_start: GuestAddr; - - static mut libafl_dump_core_hook: unsafe extern "C" fn(i32); - static mut libafl_force_dfl: i32; -} - -#[cfg(emulation_mode = "systemmode")] -extern_c_checked! { - fn qemu_init(argc: i32, argv: *const *const u8, envp: *const *const u8); - - fn vm_start(); - fn qemu_main_loop(); - fn qemu_cleanup(); - - fn libafl_save_qemu_snapshot(name: *const u8, sync: bool); - fn libafl_load_qemu_snapshot(name: *const u8, sync: bool); - - fn libafl_qemu_current_paging_id(cpu: CPUStatePtr) -> GuestPhysAddr; -} - -#[cfg(emulation_mode = "systemmode")] -extern "C" fn qemu_cleanup_atexit() { - unsafe { - qemu_cleanup(); - } -} - -// TODO rely completely on libafl_qemu_sys -extern_c_checked! { - //static libafl_page_size: GuestUsize; - fn libafl_page_from_addr(addr: GuestAddr) -> GuestAddr; - - // CPUState* libafl_qemu_get_cpu(int cpu_index); - fn libafl_qemu_get_cpu(cpu_index: i32) -> CPUStatePtr; - // int libafl_qemu_num_cpus(void); - fn libafl_qemu_num_cpus() -> i32; - // CPUState* libafl_qemu_current_cpu(void); - fn libafl_qemu_current_cpu() -> CPUStatePtr; - - // struct libafl_exit_reason* libafl_get_exit_reason(void); - fn libafl_get_exit_reason() -> ExitReasonPtr; - - fn libafl_qemu_cpu_index(cpu: CPUStatePtr) -> i32; - - fn libafl_qemu_write_reg(cpu: CPUStatePtr, reg: i32, val: *const u8) -> i32; - fn libafl_qemu_read_reg(cpu: CPUStatePtr, reg: i32, val: *mut u8) -> i32; - fn libafl_qemu_num_regs(cpu: CPUStatePtr) -> i32; - - fn libafl_qemu_set_breakpoint(addr: u64) -> i32; - fn libafl_qemu_remove_breakpoint(addr: u64) -> i32; - fn libafl_flush_jit(); - fn libafl_qemu_trigger_breakpoint(cpu: CPUStatePtr); - - fn strlen(s: *const u8) -> usize; - - fn libafl_qemu_add_gdb_cmd( - callback: extern "C" fn(*const (), *const u8, usize) -> i32, - data: *const () - ); - fn libafl_qemu_gdb_reply(buf: *const u8, len: usize); -} - -#[cfg(emulation_mode = "usermode")] -#[cfg_attr(feature = "python", pyclass(unsendable))] -pub struct GuestMaps { - orig_c_iter: *const c_void, - c_iter: *const c_void, -} - -// Consider a private new only for Emulator -#[cfg(emulation_mode = "usermode")] -impl GuestMaps { - #[must_use] - pub(crate) fn new() -> Self { - unsafe { - let maps = read_self_maps(); - Self { - orig_c_iter: maps, - c_iter: maps, - } - } - } -} - -#[cfg(emulation_mode = "usermode")] -impl Iterator for GuestMaps { - type Item = MapInfo; - - #[allow(clippy::uninit_assumed_init)] - fn next(&mut self) -> Option { - if self.c_iter.is_null() { - return None; - } - unsafe { - let mut ret = MaybeUninit::uninit(); - self.c_iter = libafl_maps_next(self.c_iter, ret.as_mut_ptr()); - if self.c_iter.is_null() { - None - } else { - Some(ret.assume_init()) - } - } - } -} - -#[cfg(all(emulation_mode = "usermode", feature = "python"))] -#[pymethods] -impl GuestMaps { - fn __iter__(slf: PyRef) -> PyRef { - slf - } - fn __next__(mut slf: PyRefMut) -> Option { - Python::with_gil(|py| slf.next().map(|x| x.into_py(py))) - } -} - -#[cfg(emulation_mode = "usermode")] -impl Drop for GuestMaps { - fn drop(&mut self) { - unsafe { - free_self_maps(self.orig_c_iter); - } - } -} - -#[repr(C)] -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) struct FatPtr(pub *const c_void, pub *const c_void); - -#[allow(clippy::vec_box)] -static mut GDB_COMMANDS: Vec> = vec![]; - -extern "C" fn gdb_cmd(data: *const (), buf: *const u8, len: usize) -> i32 { - unsafe { - let closure = &mut *(data as *mut Box FnMut(&Emulator, &'r str) -> bool>); - let cmd = std::str::from_utf8_unchecked(std::slice::from_raw_parts(buf, len)); - let emu = Emulator::new_empty(); - i32::from(closure(&emu, cmd)) - } -} - -#[derive(Debug)] -#[repr(transparent)] -pub struct CPU { - ptr: CPUStatePtr, -} - -#[derive(Debug, PartialEq)] -pub enum CallingConvention { - Cdecl, -} - -pub trait ArchExtras { - fn read_return_address(&self) -> Result - where - T: From; - fn write_return_address(&self, val: T) -> Result<(), String> - where - T: Into; - fn read_function_argument(&self, conv: CallingConvention, idx: u8) -> Result - where - T: From; - fn write_function_argument( - &self, - conv: CallingConvention, - idx: i32, - val: T, - ) -> Result<(), String> - where - T: Into; -} - -#[allow(clippy::unused_self)] -impl CPU { - #[must_use] - pub fn emulator(&self) -> Emulator { - unsafe { Emulator::new_empty() } - } - - #[must_use] - #[allow(clippy::cast_sign_loss)] - pub fn index(&self) -> usize { - unsafe { libafl_qemu_cpu_index(self.ptr) as usize } - } - - pub fn trigger_breakpoint(&self) { - unsafe { - libafl_qemu_trigger_breakpoint(self.ptr); - } - } - - #[cfg(emulation_mode = "usermode")] - #[must_use] - pub fn g2h(&self, addr: GuestAddr) -> *mut T { - unsafe { (addr as usize + guest_base) as *mut T } - } - - #[cfg(emulation_mode = "usermode")] - #[must_use] - pub fn h2g(&self, addr: *const T) -> GuestAddr { - unsafe { (addr as usize - guest_base) as GuestAddr } - } - - #[cfg(emulation_mode = "usermode")] - #[must_use] - pub fn access_ok(&self, kind: VerifyAccess, addr: GuestAddr, size: usize) -> bool { - unsafe { - // TODO add support for tagged GuestAddr - libafl_qemu_sys::page_check_range(addr, size as GuestAddr, kind.into()) - } - } - - #[cfg(emulation_mode = "systemmode")] - #[must_use] - pub fn get_phys_addr(&self, vaddr: GuestAddr) -> Option { - unsafe { - let page = libafl_page_from_addr(vaddr); - let mut attrs = MaybeUninit::::uninit(); - let paddr = libafl_qemu_sys::cpu_get_phys_page_attrs_debug( - self.ptr, - page as GuestVirtAddr, - attrs.as_mut_ptr(), - ); - if paddr == (-1i64 as GuestPhysAddr) { - None - } else { - Some(paddr) - } - } - } - - #[cfg(emulation_mode = "systemmode")] - #[must_use] - pub fn get_phys_addr_tlb( - &self, - vaddr: GuestAddr, - info: MemAccessInfo, - is_store: bool, - ) -> Option { - unsafe { - let pminfo = libafl_qemu_sys::make_plugin_meminfo( - info.oi, - if is_store { - libafl_qemu_sys::qemu_plugin_mem_rw_QEMU_PLUGIN_MEM_W - } else { - libafl_qemu_sys::qemu_plugin_mem_rw_QEMU_PLUGIN_MEM_R - }, - ); - let phwaddr = libafl_qemu_sys::qemu_plugin_get_hwaddr(pminfo, vaddr as GuestVirtAddr); - if phwaddr.is_null() { - None - } else { - Some(libafl_qemu_sys::qemu_plugin_hwaddr_phys_addr(phwaddr) as GuestPhysAddr) - } - } - } - - #[cfg(emulation_mode = "systemmode")] - #[must_use] - pub fn get_current_paging_id(&self) -> Option { - let paging_id = unsafe { libafl_qemu_current_paging_id(self.ptr) }; - - if paging_id == 0 { - None - } else { - Some(paging_id) - } - } - - // TODO expose tlb_set_dirty and tlb_reset_dirty - - /// Write a value to a guest address. - /// - /// # Safety - /// This will write to a translated guest address (using `g2h`). - /// It just adds `guest_base` and writes to that location, without checking the bounds. - /// This may only be safely used for valid guest addresses! - pub unsafe fn write_mem(&self, addr: GuestAddr, buf: &[u8]) { - #[cfg(emulation_mode = "usermode")] - { - let host_addr = Emulator::new_empty().g2h(addr); - copy_nonoverlapping(buf.as_ptr(), host_addr, buf.len()); - } - // TODO use gdbstub's target_cpu_memory_rw_debug - #[cfg(emulation_mode = "systemmode")] - libafl_qemu_sys::cpu_memory_rw_debug( - self.ptr, - addr as GuestVirtAddr, - buf.as_ptr() as *mut _, - buf.len(), - true, - ); - } - - /// Read a value from a guest address. - /// - /// # Safety - /// This will read from a translated guest address (using `g2h`). - /// It just adds `guest_base` and writes to that location, without checking the bounds. - /// This may only be safely used for valid guest addresses! - pub unsafe fn read_mem(&self, addr: GuestAddr, buf: &mut [u8]) { - #[cfg(emulation_mode = "usermode")] - { - let host_addr = Emulator::new_empty().g2h(addr); - copy_nonoverlapping(host_addr, buf.as_mut_ptr(), buf.len()); - } - // TODO use gdbstub's target_cpu_memory_rw_debug - #[cfg(emulation_mode = "systemmode")] - libafl_qemu_sys::cpu_memory_rw_debug( - self.ptr, - addr as GuestVirtAddr, - buf.as_mut_ptr() as *mut _, - buf.len(), - false, - ); - } - - #[must_use] - pub fn num_regs(&self) -> i32 { - unsafe { libafl_qemu_num_regs(self.ptr) } - } - - pub fn write_reg(&self, reg: R, val: T) -> Result<(), String> - where - R: Into, - T: Into, - { - let reg = reg.into(); - #[cfg(feature = "be")] - let val = GuestReg::to_be(val.into()); - - #[cfg(not(feature = "be"))] - let val = GuestReg::to_le(val.into()); - - let success = unsafe { libafl_qemu_write_reg(self.ptr, reg, addr_of!(val) as *const u8) }; - if success == 0 { - Err(format!("Failed to write to register {reg}")) - } else { - Ok(()) - } - } - - pub fn read_reg(&self, reg: R) -> Result - where - R: Into, - T: From, - { - unsafe { - let reg = reg.into(); - let mut val = MaybeUninit::uninit(); - let success = libafl_qemu_read_reg(self.ptr, reg, val.as_mut_ptr() as *mut u8); - if success == 0 { - Err(format!("Failed to read register {reg}")) - } else { - #[cfg(feature = "be")] - return Ok(GuestReg::from_be(val.assume_init()).into()); - - #[cfg(not(feature = "be"))] - return Ok(GuestReg::from_le(val.assume_init()).into()); - } - } - } - - pub fn reset(&self) { - unsafe { libafl_qemu_sys::cpu_reset(self.ptr) }; - } - - #[must_use] - pub fn save_state(&self) -> CPUArchState { - unsafe { - let mut saved = MaybeUninit::::uninit(); - copy_nonoverlapping( - libafl_qemu_sys::cpu_env(self.ptr.as_mut().unwrap()), - saved.as_mut_ptr(), - 1, - ); - saved.assume_init() - } - } - - pub fn restore_state(&self, saved: &CPUArchState) { - unsafe { - copy_nonoverlapping( - saved, - libafl_qemu_sys::cpu_env(self.ptr.as_mut().unwrap()), - 1, - ); - } - } - - #[must_use] - pub fn raw_ptr(&self) -> CPUStatePtr { - self.ptr - } - - #[must_use] - pub fn page_size(&self) -> usize { - #[cfg(emulation_mode = "usermode")] - { - thread_local! { - static PAGE_SIZE: OnceCell = const { OnceCell::new() }; - } - - PAGE_SIZE.with(|s| { - *s.get_or_init(|| { - unsafe { libc::sysconf(libc::_SC_PAGE_SIZE) } - .try_into() - .expect("Invalid page size") - }) - }) - } - #[cfg(emulation_mode = "systemmode")] - { - unsafe { libafl_qemu_sys::qemu_target_page_size() } - } - } - - #[must_use] - pub fn display_context(&self) -> String { - let mut display = String::new(); - let mut maxl = 0; - for r in Regs::iter() { - maxl = std::cmp::max(format!("{r:#?}").len(), maxl); - } - for (i, r) in Regs::iter().enumerate() { - let v: GuestAddr = self.read_reg(r).unwrap(); - let sr = format!("{r:#?}"); - display += &format!("{sr:>maxl$}: {v:#016x} "); - if (i + 1) % 4 == 0 { - display += "\n"; - } - } - if !display.ends_with('\n') { - display += "\n"; - } - display - } -} - -pub trait HookId { - fn remove(&self, invalidate_block: bool) -> bool; -} - -macro_rules! create_hook_id { - ($name:ident, $sys:ident, true) => { - paste::paste! { - #[derive(Clone, Copy, PartialEq, Debug)] - pub struct [<$name HookId>](pub(crate) usize); - impl HookId for [<$name HookId>] { - fn remove(&self, invalidate_block: bool) -> bool { - unsafe { libafl_qemu_sys::$sys(self.0, invalidate_block.into()) != 0 } - } - } - } - }; - ($name:ident, $sys:ident, false) => { - paste::paste! { - #[derive(Clone, Copy, PartialEq, Debug)] - pub struct [<$name HookId>](pub(crate) usize); - impl HookId for [<$name HookId>] { - fn remove(&self, _invalidate_block: bool) -> bool { - unsafe { libafl_qemu_sys::$sys(self.0) != 0 } - } - } - } - }; -} - -create_hook_id!(Instruction, libafl_qemu_remove_hook, true); -create_hook_id!(Backdoor, libafl_qemu_remove_backdoor_hook, true); -create_hook_id!(Edge, libafl_qemu_remove_edge_hook, true); -create_hook_id!(Block, libafl_qemu_remove_block_hook, true); -create_hook_id!(Read, libafl_qemu_remove_read_hook, true); -create_hook_id!(Write, libafl_qemu_remove_write_hook, true); -create_hook_id!(Cmp, libafl_qemu_remove_cmp_hook, true); -create_hook_id!(PreSyscall, libafl_qemu_remove_pre_syscall_hook, false); -create_hook_id!(PostSyscall, libafl_qemu_remove_post_syscall_hook, false); -create_hook_id!(NewThread, libafl_qemu_remove_new_thread_hook, false); - -use std::pin::Pin; - -#[derive(Debug)] -pub struct HookData(u64); - -impl From> for HookData { - fn from(value: Pin<&mut T>) -> Self { - unsafe { HookData(core::mem::transmute(value)) } - } -} - -impl From> for HookData { - fn from(value: Pin<&T>) -> Self { - unsafe { HookData(core::mem::transmute(value)) } - } -} - -impl From<&'static mut T> for HookData { - fn from(value: &'static mut T) -> Self { - unsafe { HookData(core::mem::transmute(value)) } - } -} - -impl From<&'static T> for HookData { - fn from(value: &'static T) -> Self { - unsafe { HookData(core::mem::transmute(value)) } - } -} - -impl From<*mut T> for HookData { - fn from(value: *mut T) -> Self { - HookData(value as u64) - } -} - -impl From<*const T> for HookData { - fn from(value: *const T) -> Self { - HookData(value as u64) - } -} - -impl From for HookData { - fn from(value: u64) -> Self { - HookData(value) - } -} - -impl From for HookData { - fn from(value: u32) -> Self { - HookData(u64::from(value)) - } -} - -impl From for HookData { - fn from(value: u16) -> Self { - HookData(u64::from(value)) - } -} - -impl From for HookData { - fn from(value: u8) -> Self { - HookData(u64::from(value)) - } -} - -#[derive(Debug)] -pub enum EmuError { - MultipleInstances, - EmptyArgs, - TooManyArgs(usize), -} - -#[derive(Debug, Clone)] -pub enum EmuExitReason { - End, // QEMU ended for some reason. - Breakpoint(GuestVirtAddr), // Breakpoint triggered. Contains the virtual address of the trigger. - SyncBackdoor(SyncBackdoor), // Synchronous backdoor: The guest triggered a backdoor and should return to LibAFL. -} - -impl fmt::Display for EmuExitReason { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - EmuExitReason::End => write!(f, "End"), - EmuExitReason::Breakpoint(vaddr) => write!(f, "Breakpoint @vaddr 0x{vaddr:x}"), - EmuExitReason::SyncBackdoor(sync_backdoor) => { - write!(f, "Sync backdoor exit: {sync_backdoor}") - } - } - } -} - -#[derive(Debug, Clone)] -pub enum EmuExitReasonError { - UnknownKind(), - UnexpectedExit, - SyncBackdoorError(SyncBackdoorError), -} - -impl From for EmuExitReasonError { - fn from(sync_backdoor_error: SyncBackdoorError) -> Self { - EmuExitReasonError::SyncBackdoorError(sync_backdoor_error) - } -} - -impl TryFrom<&Emulator> for EmuExitReason { - type Error = EmuExitReasonError; - fn try_from(emu: &Emulator) -> Result { - let exit_reason = unsafe { libafl_get_exit_reason() }; - if exit_reason.is_null() { - Err(EmuExitReasonError::UnexpectedExit) - } else { - let exit_reason: &mut libafl_qemu_sys::libafl_exit_reason = - unsafe { transmute(&mut *exit_reason) }; - Ok(match exit_reason.kind { - libafl_qemu_sys::libafl_exit_reason_kind_BREAKPOINT => unsafe { - EmuExitReason::Breakpoint(exit_reason.data.breakpoint.addr.into()) - }, - libafl_qemu_sys::libafl_exit_reason_kind_SYNC_BACKDOOR => { - EmuExitReason::SyncBackdoor(emu.try_into()?) - } - _ => return Err(EmuExitReasonError::UnknownKind()), - }) - } - } -} - -impl std::error::Error for EmuError {} - -impl fmt::Display for EmuError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - EmuError::MultipleInstances => { - write!(f, "Only one instance of the QEMU Emulator is permitted") - } - EmuError::EmptyArgs => { - write!(f, "QEMU emulator args cannot be empty") - } - EmuError::TooManyArgs(n) => { - write!( - f, - "Too many arguments passed to QEMU emulator ({n} > i32::MAX)" - ) - } - } - } -} - -impl From for libafl::Error { - fn from(err: EmuError) -> Self { - libafl::Error::unknown(format!("{err}")) - } -} - -static mut EMULATOR_IS_INITIALIZED: bool = false; - -#[derive(Clone, Debug)] -pub struct Emulator { - _private: (), -} - -#[allow(clippy::unused_self)] -impl Emulator { - #[allow(clippy::must_use_candidate, clippy::similar_names)] - pub fn new(args: &[String], env: &[(String, String)]) -> Result { - if args.is_empty() { - return Err(EmuError::EmptyArgs); - } - - let argc = args.len(); - if i32::try_from(argc).is_err() { - return Err(EmuError::TooManyArgs(argc)); - } - - unsafe { - if EMULATOR_IS_INITIALIZED { - return Err(EmuError::MultipleInstances); - } - EMULATOR_IS_INITIALIZED = true; - } - - #[allow(clippy::cast_possible_wrap)] - let argc = argc as i32; - - let args: Vec = args - .iter() - .map(|x| CString::new(x.clone()).unwrap()) - .collect(); - let mut argv: Vec<*const u8> = args.iter().map(|x| x.as_ptr() as *const u8).collect(); - argv.push(ptr::null()); // argv is always null terminated. - let env_strs: Vec = env - .iter() - .map(|(k, v)| format!("{}={}\0", &k, &v)) - .collect(); - let mut envp: Vec<*const u8> = env_strs.iter().map(|x| x.as_bytes().as_ptr()).collect(); - envp.push(null()); - unsafe { - #[cfg(emulation_mode = "usermode")] - qemu_user_init(argc, argv.as_ptr(), envp.as_ptr()); - #[cfg(emulation_mode = "systemmode")] - { - qemu_init( - argc, - argv.as_ptr() as *const *const u8, - envp.as_ptr() as *const *const u8, - ); - libc::atexit(qemu_cleanup_atexit); - libafl_qemu_sys::syx_snapshot_init(true); - } - } - Ok(Emulator { _private: () }) - } - - #[must_use] - pub fn get() -> Option { - unsafe { - if EMULATOR_IS_INITIALIZED { - Some(Self::new_empty()) - } else { - None - } - } - } - - /// Get an empty emulator. - /// - /// # Safety - /// - /// Should not be used if `Emulator::new` has never been used before (otherwise QEMU will not be initialized). - /// Prefer `Emulator::get` for a safe version of this method. - #[must_use] - pub unsafe fn new_empty() -> Emulator { - Emulator { _private: () } - } - - /// This function gets the memory mappings from the emulator. - #[cfg(emulation_mode = "usermode")] - #[must_use] - pub fn mappings(&self) -> GuestMaps { - GuestMaps::new() - } - - #[must_use] - #[allow(clippy::cast_possible_wrap)] - #[allow(clippy::cast_sign_loss)] - pub fn num_cpus(&self) -> usize { - unsafe { libafl_qemu_num_cpus() as usize } - } - - #[must_use] - pub fn current_cpu(&self) -> Option { - let ptr = unsafe { libafl_qemu_current_cpu() }; - if ptr.is_null() { - None - } else { - Some(CPU { ptr }) - } - } - - #[must_use] - #[allow(clippy::cast_possible_wrap)] - pub fn cpu_from_index(&self, index: usize) -> CPU { - unsafe { - CPU { - ptr: libafl_qemu_get_cpu(index as i32), - } - } - } - - #[must_use] - pub fn page_from_addr(addr: GuestAddr) -> GuestAddr { - unsafe { libafl_page_from_addr(addr) } - } - - //#[must_use] - /*pub fn page_size() -> GuestUsize { - unsafe { libafl_page_size } - }*/ - - #[cfg(emulation_mode = "usermode")] - #[must_use] - pub fn g2h(&self, addr: GuestAddr) -> *mut T { - unsafe { (addr as usize + guest_base) as *mut T } - } - - #[cfg(emulation_mode = "usermode")] - #[must_use] - pub fn h2g(&self, addr: *const T) -> GuestAddr { - unsafe { (addr as usize - guest_base) as GuestAddr } - } - - #[cfg(emulation_mode = "usermode")] - #[must_use] - pub fn access_ok(&self, kind: VerifyAccess, addr: GuestAddr, size: usize) -> bool { - self.current_cpu() - .unwrap_or_else(|| self.cpu_from_index(0)) - .access_ok(kind, addr, size) - } - - pub unsafe fn write_mem(&self, addr: GuestAddr, buf: &[u8]) { - self.current_cpu() - .unwrap_or_else(|| self.cpu_from_index(0)) - .write_mem(addr, buf); - } - - pub unsafe fn read_mem(&self, addr: GuestAddr, buf: &mut [u8]) { - self.current_cpu() - .unwrap_or_else(|| self.cpu_from_index(0)) - .read_mem(addr, buf); - } - - /// Write a value to a phsical guest address, including ROM areas. - #[cfg(emulation_mode = "systemmode")] - pub unsafe fn write_phys_mem(&self, paddr: GuestPhysAddr, buf: &[u8]) { - libafl_qemu_sys::cpu_physical_memory_rw( - paddr, - buf.as_ptr() as *mut _, - buf.len() as u64, - true, - ); - } - - /// Read a value from a physical guest address. - #[cfg(emulation_mode = "systemmode")] - pub unsafe fn read_phys_mem(&self, paddr: GuestPhysAddr, buf: &mut [u8]) { - libafl_qemu_sys::cpu_physical_memory_rw( - paddr, - buf.as_mut_ptr() as *mut _, - buf.len() as u64, - false, - ); - } - - #[must_use] - pub fn num_regs(&self) -> i32 { - self.current_cpu().unwrap().num_regs() - } - - pub fn write_reg(&self, reg: R, val: T) -> Result<(), String> - where - T: Num + PartialOrd + Copy + Into, - R: Into, - { - self.current_cpu().unwrap().write_reg(reg, val) - } - - pub fn read_reg(&self, reg: R) -> Result - where - T: Num + PartialOrd + Copy + From, - R: Into, - { - self.current_cpu().unwrap().read_reg(reg) - } - - pub fn set_breakpoint(&self, addr: GuestAddr) { - unsafe { - libafl_qemu_set_breakpoint(addr.into()); - } - } - - pub fn remove_breakpoint(&self, addr: GuestAddr) { - unsafe { - libafl_qemu_remove_breakpoint(addr.into()); - } - } - - pub fn entry_break(&self, addr: GuestAddr) { - self.set_breakpoint(addr); - unsafe { - // TODO: decide what to do with sync exit here: ignore or check for bp exit? - let _ = self.run(); - } - self.remove_breakpoint(addr); - } - - #[cfg(emulation_mode = "usermode")] - pub fn force_dfl(&self) { - unsafe { - libafl_force_dfl = 1; - } - } - /// This function will run the emulator until the next breakpoint, or until finish. - /// # Safety - /// - /// Should, in general, be safe to call. - /// Of course, the emulated target is not contained securely and can corrupt state or interact with the operating system. - pub unsafe fn run(&self) -> Result { - #[cfg(emulation_mode = "usermode")] - libafl_qemu_run(); - #[cfg(emulation_mode = "systemmode")] - { - vm_start(); - qemu_main_loop(); - } - EmuExitReason::try_from(self) - } - - #[cfg(emulation_mode = "usermode")] - #[must_use] - pub fn binary_path<'a>(&self) -> &'a str { - unsafe { from_utf8_unchecked(from_raw_parts(exec_path, strlen(exec_path))) } - } - - #[cfg(emulation_mode = "usermode")] - #[must_use] - pub fn load_addr(&self) -> GuestAddr { - unsafe { libafl_load_addr() as GuestAddr } - } - - #[cfg(emulation_mode = "usermode")] - #[must_use] - pub fn get_brk(&self) -> GuestAddr { - unsafe { libafl_get_brk() as GuestAddr } - } - - #[cfg(emulation_mode = "usermode")] - pub fn set_brk(&self, brk: GuestAddr) { - unsafe { libafl_set_brk(brk.into()) }; - } - - #[cfg(emulation_mode = "usermode")] - #[must_use] - pub fn get_mmap_start(&self) -> GuestAddr { - unsafe { mmap_next_start } - } - - #[cfg(emulation_mode = "usermode")] - pub fn set_mmap_start(&self, start: GuestAddr) { - unsafe { mmap_next_start = start }; - } - - #[cfg(emulation_mode = "usermode")] - #[allow(clippy::cast_sign_loss)] - fn mmap( - &self, - addr: GuestAddr, - size: usize, - perms: MmapPerms, - flags: c_int, - ) -> Result { - let res = unsafe { - libafl_qemu_sys::target_mmap(addr, size as GuestUsize, perms.into(), flags, -1, 0) - }; - if res <= 0 { - Err(()) - } else { - Ok(res as GuestAddr) - } - } - - #[cfg(emulation_mode = "usermode")] - pub fn map_private( - &self, - addr: GuestAddr, - size: usize, - perms: MmapPerms, - ) -> Result { - self.mmap(addr, size, perms, libc::MAP_PRIVATE | libc::MAP_ANONYMOUS) - .map_err(|()| format!("Failed to map {addr}")) - .map(|addr| addr as GuestAddr) - } - - #[cfg(emulation_mode = "usermode")] - pub fn map_fixed( - &self, - addr: GuestAddr, - size: usize, - perms: MmapPerms, - ) -> Result { - self.mmap( - addr, - size, - perms, - libc::MAP_FIXED | libc::MAP_PRIVATE | libc::MAP_ANONYMOUS, - ) - .map_err(|()| format!("Failed to map {addr}")) - .map(|addr| addr as GuestAddr) - } - - #[cfg(emulation_mode = "usermode")] - pub fn mprotect(&self, addr: GuestAddr, size: usize, perms: MmapPerms) -> Result<(), String> { - let res = unsafe { - libafl_qemu_sys::target_mprotect(addr.into(), size as GuestUsize, perms.into()) - }; - if res == 0 { - Ok(()) - } else { - Err(format!("Failed to mprotect {addr}")) - } - } - - #[cfg(emulation_mode = "usermode")] - pub fn unmap(&self, addr: GuestAddr, size: usize) -> Result<(), String> { - if unsafe { libafl_qemu_sys::target_munmap(addr.into(), size as GuestUsize) } == 0 { - Ok(()) - } else { - Err(format!("Failed to unmap {addr}")) - } - } - - pub fn flush_jit(&self) { - unsafe { - libafl_flush_jit(); - } - } - - // TODO set T lifetime to be like Emulator - pub fn set_hook>( - &self, - data: T, - addr: GuestAddr, - callback: extern "C" fn(T, GuestAddr), - invalidate_block: bool, - ) -> InstructionHookId { - unsafe { - let data: u64 = data.into().0; - let callback: extern "C" fn(u64, GuestAddr) = core::mem::transmute(callback); - let num = libafl_qemu_sys::libafl_qemu_set_hook( - addr.into(), - Some(callback), - data, - i32::from(invalidate_block), - ); - InstructionHookId(num) - } - } - - #[must_use] - pub fn remove_hook(&self, id: impl HookId, invalidate_block: bool) -> bool { - id.remove(invalidate_block) - } - - #[must_use] - pub fn remove_hooks_at(&self, addr: GuestAddr, invalidate_block: bool) -> usize { - unsafe { - libafl_qemu_sys::libafl_qemu_remove_hooks_at(addr.into(), i32::from(invalidate_block)) - } - } - - pub fn add_edge_hooks>( - &self, - data: T, - gen: Option u64>, - exec: Option, - ) -> EdgeHookId { - unsafe { - let data: u64 = data.into().0; - let gen: Option u64> = - core::mem::transmute(gen); - let exec: Option = core::mem::transmute(exec); - let num = libafl_qemu_sys::libafl_add_edge_hook(gen, exec, data); - EdgeHookId(num) - } - } - - pub fn add_block_hooks>( - &self, - data: T, - gen: Option u64>, - post_gen: Option, - exec: Option, - ) -> BlockHookId { - unsafe { - let data: u64 = data.into().0; - let gen: Option u64> = core::mem::transmute(gen); - let post_gen: Option = - core::mem::transmute(post_gen); - let exec: Option = core::mem::transmute(exec); - let num = libafl_qemu_sys::libafl_add_block_hook(gen, post_gen, exec, data); - BlockHookId(num) - } - } - - pub fn add_read_hooks>( - &self, - data: T, - gen: Option u64>, - exec1: Option, - exec2: Option, - exec4: Option, - exec8: Option, - exec_n: Option, - ) -> ReadHookId { - unsafe { - let data: u64 = data.into().0; - let gen: Option u64> = - core::mem::transmute(gen); - let exec1: Option = core::mem::transmute(exec1); - let exec2: Option = core::mem::transmute(exec2); - let exec4: Option = core::mem::transmute(exec4); - let exec8: Option = core::mem::transmute(exec8); - let exec_n: Option = - core::mem::transmute(exec_n); - let num = libafl_qemu_sys::libafl_add_read_hook( - gen, exec1, exec2, exec4, exec8, exec_n, data, - ); - ReadHookId(num) - } - } - - // TODO add MemOp info - pub fn add_write_hooks>( - &self, - data: T, - gen: Option u64>, - exec1: Option, - exec2: Option, - exec4: Option, - exec8: Option, - exec_n: Option, - ) -> WriteHookId { - unsafe { - let data: u64 = data.into().0; - let gen: Option u64> = - core::mem::transmute(gen); - let exec1: Option = core::mem::transmute(exec1); - let exec2: Option = core::mem::transmute(exec2); - let exec4: Option = core::mem::transmute(exec4); - let exec8: Option = core::mem::transmute(exec8); - let exec_n: Option = - core::mem::transmute(exec_n); - let num = libafl_qemu_sys::libafl_add_write_hook( - gen, exec1, exec2, exec4, exec8, exec_n, data, - ); - WriteHookId(num) - } - } - - pub fn add_cmp_hooks>( - &self, - data: T, - gen: Option u64>, - exec1: Option, - exec2: Option, - exec4: Option, - exec8: Option, - ) -> CmpHookId { - unsafe { - let data: u64 = data.into().0; - let gen: Option u64> = - core::mem::transmute(gen); - let exec1: Option = core::mem::transmute(exec1); - let exec2: Option = core::mem::transmute(exec2); - let exec4: Option = core::mem::transmute(exec4); - let exec8: Option = core::mem::transmute(exec8); - let num = libafl_qemu_sys::libafl_add_cmp_hook(gen, exec1, exec2, exec4, exec8, data); - CmpHookId(num) - } - } - - pub fn add_backdoor_hook>( - &self, - data: T, - callback: extern "C" fn(T, GuestAddr), - ) -> BackdoorHookId { - unsafe { - let data: u64 = data.into().0; - let callback: extern "C" fn(u64, GuestAddr) = core::mem::transmute(callback); - let num = libafl_qemu_sys::libafl_add_backdoor_hook(Some(callback), data); - BackdoorHookId(num) - } - } - - #[cfg(emulation_mode = "usermode")] - #[allow(clippy::type_complexity)] - pub fn add_pre_syscall_hook>( - &self, - data: T, - callback: extern "C" fn( - T, - i32, - GuestAddr, - GuestAddr, - GuestAddr, - GuestAddr, - GuestAddr, - GuestAddr, - GuestAddr, - GuestAddr, - ) -> SyscallHookResult, - ) -> PreSyscallHookId { - unsafe { - let data: u64 = data.into().0; - let callback: extern "C" fn( - u64, - i32, - GuestAddr, - GuestAddr, - GuestAddr, - GuestAddr, - GuestAddr, - GuestAddr, - GuestAddr, - GuestAddr, - ) -> libafl_qemu_sys::syshook_ret = core::mem::transmute(callback); - let num = libafl_qemu_sys::libafl_add_pre_syscall_hook(Some(callback), data); - PreSyscallHookId(num) - } - } - - #[cfg(emulation_mode = "usermode")] - #[allow(clippy::type_complexity)] - pub fn add_post_syscall_hook>( - &self, - data: T, - callback: extern "C" fn( - T, - GuestAddr, - i32, - GuestAddr, - GuestAddr, - GuestAddr, - GuestAddr, - GuestAddr, - GuestAddr, - GuestAddr, - GuestAddr, - ) -> GuestAddr, - ) -> PostSyscallHookId { - unsafe { - let data: u64 = data.into().0; - let callback: extern "C" fn( - u64, - GuestAddr, - i32, - GuestAddr, - GuestAddr, - GuestAddr, - GuestAddr, - GuestAddr, - GuestAddr, - GuestAddr, - GuestAddr, - ) -> GuestAddr = core::mem::transmute(callback); - let num = libafl_qemu_sys::libafl_add_post_syscall_hook(Some(callback), data); - PostSyscallHookId(num) - } - } - - #[cfg(emulation_mode = "usermode")] - pub fn add_new_thread_hook>( - &self, - data: T, - callback: extern "C" fn(T, tid: u32) -> bool, - ) -> NewThreadHookId { - unsafe { - let data: u64 = data.into().0; - let callback: extern "C" fn(u64, u32) -> bool = core::mem::transmute(callback); - let num = libafl_qemu_sys::libafl_add_new_thread_hook(Some(callback), data); - NewThreadHookId(num) - } - } - - #[cfg(emulation_mode = "systemmode")] - pub fn save_snapshot(&self, name: &str, sync: bool) { - let s = CString::new(name).expect("Invalid snapshot name"); - unsafe { libafl_save_qemu_snapshot(s.as_ptr() as *const _, sync) }; - } - - #[cfg(emulation_mode = "systemmode")] - pub fn load_snapshot(&self, name: &str, sync: bool) { - let s = CString::new(name).expect("Invalid snapshot name"); - unsafe { libafl_load_qemu_snapshot(s.as_ptr() as *const _, sync) }; - } - - #[cfg(emulation_mode = "systemmode")] - #[must_use] - pub fn create_fast_snapshot(&self, track: bool) -> FastSnapshot { - unsafe { - libafl_qemu_sys::syx_snapshot_new( - track, - true, - libafl_qemu_sys::DeviceSnapshotKind_DEVICE_SNAPSHOT_ALL, - null_mut(), - ) - } - } - - #[cfg(emulation_mode = "systemmode")] - #[must_use] - pub fn create_fast_snapshot_filter( - &self, - track: bool, - device_filter: &DeviceSnapshotFilter, - ) -> FastSnapshot { - let mut v = vec![]; - unsafe { - libafl_qemu_sys::syx_snapshot_new( - track, - true, - device_filter.enum_id(), - device_filter.devices(&mut v), - ) - } - } - - #[cfg(emulation_mode = "systemmode")] - pub fn restore_fast_snapshot(&self, snapshot: FastSnapshot) { - unsafe { libafl_qemu_sys::syx_snapshot_root_restore(snapshot) } - } - - #[cfg(emulation_mode = "systemmode")] - pub fn list_devices(&self) -> Vec { - let mut r = vec![]; - unsafe { - let devices = libafl_qemu_sys::device_list_all(); - if devices.is_null() { - return r; - } - - let mut ptr = devices; - while !(*ptr).is_null() { - let c_str: &CStr = CStr::from_ptr(*ptr); - let name = c_str.to_str().unwrap().to_string(); - r.push(name); - - ptr = ptr.add(1); - } - - libc::free(devices as *mut c_void); - r - } - } - - #[allow(clippy::type_complexity)] - pub fn add_gdb_cmd(&self, callback: Box bool>) { - unsafe { - let fat: Box = Box::new(transmute(callback)); - libafl_qemu_add_gdb_cmd(gdb_cmd, core::ptr::from_ref(&*fat) as *const ()); - GDB_COMMANDS.push(fat); - } - } - - pub fn gdb_reply(&self, output: &str) { - unsafe { libafl_qemu_gdb_reply(output.as_bytes().as_ptr(), output.len()) }; - } - - #[cfg(emulation_mode = "usermode")] - #[allow(clippy::type_complexity)] - pub fn set_crash_hook(&self, callback: extern "C" fn(i32)) { - unsafe { - libafl_dump_core_hook = callback; - } - } -} - -impl ArchExtras for Emulator { - fn read_return_address(&self) -> Result - where - T: From, - { - self.current_cpu() - .ok_or("Failed to get current CPU")? - .read_return_address::() - } - - fn write_return_address(&self, val: T) -> Result<(), String> - where - T: Into, - { - self.current_cpu() - .ok_or("Failed to get current CPU")? - .write_return_address::(val) - } - - fn read_function_argument(&self, conv: CallingConvention, idx: u8) -> Result - where - T: From, - { - self.current_cpu() - .ok_or("Failed to get current CPU")? - .read_function_argument::(conv, idx) - } - - fn write_function_argument( - &self, - conv: CallingConvention, - idx: i32, - val: T, - ) -> Result<(), String> - where - T: Into, - { - self.current_cpu() - .ok_or("Failed to get current CPU")? - .write_function_argument::(conv, idx, val) - } -} - -#[cfg(feature = "python")] -pub mod pybind { - use std::convert::TryFrom; - - use pyo3::{exceptions::PyValueError, prelude::*, types::PyInt}; - - use super::{GuestAddr, GuestUsize, MmapPerms, SyscallHookResult}; - - static mut PY_SYSCALL_HOOK: Option = None; - static mut PY_GENERIC_HOOKS: Vec<(GuestAddr, PyObject)> = vec![]; - - extern "C" fn py_syscall_hook_wrapper( - _data: u64, - sys_num: i32, - a0: u64, - a1: u64, - a2: u64, - a3: u64, - a4: u64, - a5: u64, - a6: u64, - a7: u64, - ) -> SyscallHookResult { - unsafe { PY_SYSCALL_HOOK.as_ref() }.map_or_else( - || SyscallHookResult::new(None), - |obj| { - let args = (sys_num, a0, a1, a2, a3, a4, a5, a6, a7); - Python::with_gil(|py| { - let ret = obj.call1(py, args).expect("Error in the syscall hook"); - let any = ret.as_ref(py); - if any.is_none() { - SyscallHookResult::new(None) - } else { - let a: Result<&PyInt, _> = any.downcast(); - if let Ok(i) = a { - SyscallHookResult::new(Some( - i.extract().expect("Invalid syscall hook return value"), - )) - } else { - SyscallHookResult::extract(any) - .expect("The syscall hook must return a SyscallHookResult") - } - } - }) - }, - ) - } - - extern "C" fn py_generic_hook_wrapper(idx: u64, _pc: GuestAddr) { - let obj = unsafe { &PY_GENERIC_HOOKS[idx as usize].1 }; - Python::with_gil(|py| { - obj.call0(py).expect("Error in the hook"); - }); - } - - #[pyclass(unsendable)] - pub struct Emulator { - pub emu: super::Emulator, - } - - #[pymethods] - impl Emulator { - #[allow(clippy::needless_pass_by_value)] - #[new] - fn new(args: Vec, env: Vec<(String, String)>) -> PyResult { - let emu = super::Emulator::new(&args, &env) - .map_err(|e| PyValueError::new_err(format!("{e}")))?; - Ok(Emulator { emu }) - } - - fn write_mem(&self, addr: GuestAddr, buf: &[u8]) { - unsafe { - self.emu.write_mem(addr, buf); - } - } - - fn read_mem(&self, addr: GuestAddr, size: usize) -> Vec { - let mut buf = vec![0; size]; - unsafe { - self.emu.read_mem(addr, &mut buf); - } - buf - } - - fn num_regs(&self) -> i32 { - self.emu.num_regs() - } - - fn write_reg(&self, reg: i32, val: GuestUsize) -> PyResult<()> { - self.emu.write_reg(reg, val).map_err(PyValueError::new_err) - } - - fn read_reg(&self, reg: i32) -> PyResult { - self.emu.read_reg(reg).map_err(PyValueError::new_err) - } - - fn set_breakpoint(&self, addr: GuestAddr) { - self.emu.set_breakpoint(addr); - } - - fn entry_break(&self, addr: GuestAddr) { - self.emu.entry_break(addr); - } - - fn remove_breakpoint(&self, addr: GuestAddr) { - self.emu.remove_breakpoint(addr); - } - - fn run(&self) { - unsafe { - self.emu.run().unwrap(); - } - } - - fn g2h(&self, addr: GuestAddr) -> u64 { - self.emu.g2h::<*const u8>(addr) as u64 - } - - fn h2g(&self, addr: u64) -> GuestAddr { - self.emu.h2g(addr as *const u8) - } - - fn binary_path(&self) -> String { - self.emu.binary_path().to_owned() - } - - fn load_addr(&self) -> GuestAddr { - self.emu.load_addr() - } - - fn flush_jit(&self) { - self.emu.flush_jit(); - } - - fn map_private(&self, addr: GuestAddr, size: usize, perms: i32) -> PyResult { - if let Ok(p) = MmapPerms::try_from(perms) { - self.emu - .map_private(addr, size, p) - .map_err(PyValueError::new_err) - } else { - Err(PyValueError::new_err("Invalid perms")) - } - } - - fn map_fixed(&self, addr: GuestAddr, size: usize, perms: i32) -> PyResult { - if let Ok(p) = MmapPerms::try_from(perms) { - self.emu - .map_fixed(addr, size, p) - .map_err(PyValueError::new_err) - } else { - Err(PyValueError::new_err("Invalid perms")) - } - } - - fn mprotect(&self, addr: GuestAddr, size: usize, perms: i32) -> PyResult<()> { - if let Ok(p) = MmapPerms::try_from(perms) { - self.emu - .mprotect(addr, size, p) - .map_err(PyValueError::new_err) - } else { - Err(PyValueError::new_err("Invalid perms")) - } - } - - fn unmap(&self, addr: GuestAddr, size: usize) -> PyResult<()> { - self.emu.unmap(addr, size).map_err(PyValueError::new_err) - } - - fn set_syscall_hook(&self, hook: PyObject) { - unsafe { - PY_SYSCALL_HOOK = Some(hook); - } - self.emu.add_pre_syscall_hook(0u64, py_syscall_hook_wrapper); - } - - fn set_hook(&self, addr: GuestAddr, hook: PyObject) { - unsafe { - let idx = PY_GENERIC_HOOKS.len(); - PY_GENERIC_HOOKS.push((addr, hook)); - self.emu - .set_hook(idx as u64, addr, py_generic_hook_wrapper, true); - } - } - - fn remove_hooks_at(&self, addr: GuestAddr) -> usize { - unsafe { - PY_GENERIC_HOOKS.retain(|(a, _)| *a != addr); - } - self.emu.remove_hooks_at(addr, true) - } - } -} diff --git a/libafl_qemu/src/emu/mod.rs b/libafl_qemu/src/emu/mod.rs new file mode 100644 index 0000000000..91d9a71cb4 --- /dev/null +++ b/libafl_qemu/src/emu/mod.rs @@ -0,0 +1,1978 @@ +//! Expose QEMU user `LibAFL` C api to Rust + +use core::{ + fmt::{self, Debug, Display, Formatter}, + marker::PhantomData, + mem::{transmute, MaybeUninit}, + ptr::{addr_of, copy_nonoverlapping, null}, +}; +use std::{ + cell::{OnceCell, Ref, RefCell, RefMut}, + collections::HashSet, + ffi::CString, + ptr, +}; + +use libafl::{events::CTRL_C_EXIT, executors::ExitKind}; +#[cfg(emulation_mode = "systemmode")] +use libafl_qemu_sys::qemu_init; +#[cfg(emulation_mode = "usermode")] +use libafl_qemu_sys::{guest_base, qemu_user_init, VerifyAccess}; +use libafl_qemu_sys::{ + libafl_flush_jit, libafl_get_exit_reason, libafl_page_from_addr, libafl_qemu_add_gdb_cmd, + libafl_qemu_cpu_index, libafl_qemu_current_cpu, libafl_qemu_gdb_reply, libafl_qemu_get_cpu, + libafl_qemu_num_cpus, libafl_qemu_num_regs, libafl_qemu_read_reg, + libafl_qemu_remove_breakpoint, libafl_qemu_set_breakpoint, libafl_qemu_trigger_breakpoint, + libafl_qemu_write_reg, CPUArchStatePtr, CPUStatePtr, FatPtr, GuestUsize, +}; +pub use libafl_qemu_sys::{GuestAddr, GuestPhysAddr, GuestVirtAddr}; +#[cfg(emulation_mode = "usermode")] +pub use libafl_qemu_sys::{MapInfo, MmapPerms, MmapPermsIter}; +use num_traits::Num; +use strum::IntoEnumIterator; + +use crate::{ + command::IsCommand, sys::TCGTemp, GuestReg, QemuHelperTuple, Regs, StdInstrumentationFilter, +}; + +#[cfg(emulation_mode = "systemmode")] +pub mod systemmode; +#[cfg(emulation_mode = "systemmode")] +pub use systemmode::*; + +#[cfg(emulation_mode = "usermode")] +pub mod usermode; +#[cfg(emulation_mode = "usermode")] +pub use usermode::*; + +#[derive(Clone)] +pub enum GuestAddrKind { + Physical(GuestPhysAddr), + Virtual(GuestVirtAddr), +} + +impl Debug for GuestAddrKind { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + GuestAddrKind::Physical(paddr) => write!(f, "vaddr {paddr:x}"), + GuestAddrKind::Virtual(vaddr) => write!(f, "paddr {vaddr:x}"), + } + } +} + +#[derive(Debug, Clone)] +pub enum QemuShutdownCause { + None, + HostError, + HostQmpQuit, + HostQmpSystemReset, + HostSignal(Signal), + HostUi, + GuestShutdown, + GuestReset, + GuestPanic, + SubsystemReset, + SnapshotLoad, +} + +impl Display for GuestAddrKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + GuestAddrKind::Physical(phys_addr) => write!(f, "hwaddr 0x{phys_addr:x}"), + GuestAddrKind::Virtual(virt_addr) => write!(f, "vaddr 0x{virt_addr:x}"), + } + } +} + +#[derive(Debug, Clone)] +pub enum HandlerError { + QemuExitReasonError(EmuExitReasonError), + SMError(SnapshotManagerError), + SyncBackdoorError(SyncBackdoorError), + MultipleSnapshotDefinition, + MultipleInputDefinition, + SnapshotNotFound, +} + +impl From for HandlerError { + fn from(sm_error: SnapshotManagerError) -> Self { + HandlerError::SMError(sm_error) + } +} + +#[derive(Debug, Clone)] +pub enum SnapshotManagerError { + SnapshotIdNotFound(SnapshotId), + MemoryInconsistencies(u64), +} + +#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] +pub struct SnapshotId { + id: u64, +} + +pub trait IsSnapshotManager: Debug + Clone { + fn save(&mut self, qemu: &Qemu) -> SnapshotId; + fn restore( + &mut self, + snapshot_id: &SnapshotId, + qemu: &Qemu, + ) -> Result<(), SnapshotManagerError>; +} + +// TODO: Rework with generics for command handlers? +pub trait EmuExitHandler: Sized + Debug + Clone +where + QT: QemuHelperTuple, + S: State + HasExecutions, +{ + fn try_put_input( + emu: &Emulator, + qemu_executor_state: &mut QemuExecutorState, + input: &S::Input, + ); + + fn handle( + emu: &Emulator, + exit_reason: Result, + qemu_executor_state: &mut QemuExecutorState, + input: &S::Input, + ) -> Result; +} + +pub enum InnerHandlerResult { + EndOfRun(ExitKind), // The run is over and the emulator is ready for the next iteration. + ReturnToHarness(EmuExitReason), // Return to the harness immediately. Can happen at any point of the run when the handler is not supposed to handle a request. + Continue, // Resume QEMU and continue to run the handler. + Interrupt, // QEMU has been interrupted by user. +} + +/// Special kind of Exit handler with no data embedded. +/// As a result, it is safe to transmute from any `Emulator` implementing `EmuExitHandler` to this one, +/// since it won't use any data which could cause type confusion. +#[derive(Clone, Debug)] +pub struct NopEmuExitHandler; + +impl EmuExitHandler for NopEmuExitHandler +where + QT: QemuHelperTuple, + S: State + HasExecutions, +{ + fn try_put_input(_: &Emulator, _: &mut QemuExecutorState, _: &S::Input) {} + + fn handle( + _: &Emulator, + exit_reason: Result, + _: &mut QemuExecutorState, + _: &S::Input, + ) -> Result { + match exit_reason { + Ok(reason) => Ok(InnerHandlerResult::ReturnToHarness(reason)), + Err(error) => Err(error)?, + } + } +} + +#[derive(Debug, Clone)] +pub struct InputLocation { + mem_chunk: EmulatorMemoryChunk, + cpu: CPU, + ret_register: Option, +} + +impl InputLocation { + #[must_use] + pub fn new(mem_chunk: EmulatorMemoryChunk, cpu: CPU, ret_register: Option) -> Self { + Self { + mem_chunk, + cpu, + ret_register, + } + } +} + +/// Synchronous Exit handler maintaining only one snapshot. +#[derive(Debug, Clone)] +pub struct StdEmuExitHandler +where + SM: IsSnapshotManager + Clone, +{ + snapshot_manager: RefCell, + snapshot_id: OnceCell, + input_location: OnceCell, +} + +impl StdEmuExitHandler +where + SM: IsSnapshotManager, +{ + pub fn new(snapshot_manager: SM) -> Self { + Self { + snapshot_manager: RefCell::new(snapshot_manager), + snapshot_id: OnceCell::new(), + input_location: OnceCell::new(), + } + } + + pub fn set_input_location(&self, input_location: InputLocation) -> Result<(), InputLocation> { + self.input_location.set(input_location) + } + + pub fn set_snapshot_id(&self, snapshot_id: SnapshotId) -> Result<(), SnapshotId> { + self.snapshot_id.set(snapshot_id) + } + + pub fn snapshot_id(&self) -> Option { + Some(*self.snapshot_id.get()?) + } + + pub fn snapshot_manager_borrow(&self) -> Ref { + self.snapshot_manager.borrow() + } + + pub fn snapshot_manager_borrow_mut(&self) -> RefMut { + self.snapshot_manager.borrow_mut() + } +} + +// TODO: replace handlers with generics to permit compile-time customization of handlers +impl EmuExitHandler for StdEmuExitHandler +where + SM: IsSnapshotManager, + QT: QemuHelperTuple + StdInstrumentationFilter + Debug, + S: State + HasExecutions, + S::Input: HasTargetBytes, +{ + fn try_put_input( + emu: &Emulator, + qemu_executor_state: &mut QemuExecutorState, + input: &S::Input, + ) { + let exit_handler = emu.state().exit_handler.borrow(); + + if let Some(input_location) = exit_handler.input_location.get() { + let input_command = + InputCommand::new(input_location.mem_chunk.clone(), input_location.cpu.clone()); + input_command + .run(emu, qemu_executor_state, input, input_location.ret_register) + .unwrap(); + } + } + + fn handle( + emu: &Emulator, + exit_reason: Result, + qemu_executor_state: &mut QemuExecutorState, + input: &S::Input, + ) -> Result { + let exit_handler = emu.exit_handler().borrow_mut(); + let qemu = emu.qemu(); + + let mut exit_reason = match exit_reason { + Ok(exit_reason) => exit_reason, + Err(exit_error) => match exit_error { + EmuExitReasonError::UnexpectedExit => { + if let Some(snapshot_id) = exit_handler.snapshot_id.get() { + exit_handler + .snapshot_manager + .borrow_mut() + .restore(snapshot_id, qemu)?; + } + return Ok(InnerHandlerResult::EndOfRun(ExitKind::Crash)); + } + _ => Err(exit_error)?, + }, + }; + + let (command, ret_reg): (Option, Option) = match &mut exit_reason { + EmuExitReason::End(shutdown_cause) => match shutdown_cause { + QemuShutdownCause::HostSignal(Signal::SigInterrupt) => { + std::process::exit(CTRL_C_EXIT); + } + QemuShutdownCause::GuestPanic => { + return Ok(InnerHandlerResult::EndOfRun(ExitKind::Crash)) + } + _ => panic!("Unhandled QEMU shutdown cause: {shutdown_cause:?}."), + }, + EmuExitReason::Breakpoint(bp) => (bp.trigger(qemu).cloned(), None), + EmuExitReason::SyncBackdoor(sync_backdoor) => { + let command = sync_backdoor.command().clone(); + (Some(command), Some(sync_backdoor.ret_reg())) + } + }; + + // manually drop ref cell here to avoid keeping it alive in cmd. + drop(exit_handler); + + if let Some(cmd) = command { + cmd.run(emu, qemu_executor_state, input, ret_reg) + } else { + Ok(InnerHandlerResult::ReturnToHarness(exit_reason)) + } + } +} + +#[repr(transparent)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct MemAccessInfo { + oi: libafl_qemu_sys::MemOpIdx, +} + +impl MemAccessInfo { + #[must_use] + pub fn memop(&self) -> libafl_qemu_sys::MemOp { + libafl_qemu_sys::MemOp(self.oi >> 4) + } + + #[must_use] + pub fn memopidx(&self) -> libafl_qemu_sys::MemOpIdx { + self.oi + } + + #[must_use] + pub fn mmu_index(&self) -> u32 { + self.oi & 15 + } + + #[must_use] + pub fn size(&self) -> usize { + libafl_qemu_sys::memop_size(self.memop()) as usize + } + + #[must_use] + pub fn is_big_endian(&self) -> bool { + libafl_qemu_sys::memop_big_endian(self.memop()) + } + + #[must_use] + pub fn encode_with(&self, other: u32) -> u64 { + (u64::from(self.oi) << 32) | u64::from(other) + } + + #[must_use] + pub fn decode_from(encoded: u64) -> (Self, u32) { + let low = (encoded & 0xFFFFFFFF) as u32; + let high = (encoded >> 32) as u32; + (Self { oi: high }, low) + } + + #[must_use] + pub fn new(oi: libafl_qemu_sys::MemOpIdx) -> Self { + Self { oi } + } +} + +impl From for MemAccessInfo { + fn from(oi: libafl_qemu_sys::MemOpIdx) -> Self { + Self { oi } + } +} + +#[cfg(feature = "python")] +use pyo3::prelude::*; + +pub const SKIP_EXEC_HOOK: u64 = u64::MAX; + +pub use libafl_qemu_sys::{CPUArchState, CPUState}; + +use crate::sync_exit::{SyncBackdoor, SyncBackdoorError}; + +// syshook_ret +#[repr(C)] +#[cfg_attr(feature = "python", pyclass)] +#[cfg_attr(feature = "python", derive(FromPyObject))] +pub struct SyscallHookResult { + pub retval: GuestAddr, + pub skip_syscall: bool, +} + +#[cfg(feature = "python")] +#[pymethods] +impl SyscallHookResult { + #[new] + #[must_use] + pub fn new(value: Option) -> Self { + value.map_or( + Self { + retval: 0, + skip_syscall: false, + }, + |v| Self { + retval: v, + skip_syscall: true, + }, + ) + } +} + +#[cfg(not(feature = "python"))] +impl SyscallHookResult { + #[must_use] + pub fn new(value: Option) -> Self { + value.map_or( + Self { + retval: 0, + skip_syscall: false, + }, + |v| Self { + retval: v, + skip_syscall: true, + }, + ) + } +} + +#[allow(clippy::vec_box)] +static mut GDB_COMMANDS: Vec> = vec![]; + +extern "C" fn gdb_cmd(data: *const (), buf: *const u8, len: usize) -> i32 { + unsafe { + let closure = &mut *(data as *mut Box FnMut(&Qemu, &'r str) -> bool>); + let cmd = std::str::from_utf8_unchecked(std::slice::from_raw_parts(buf, len)); + let qemu = Qemu::get_unchecked(); + i32::from(closure(&qemu, cmd)) + } +} + +#[derive(Debug, Clone)] +#[repr(transparent)] +pub struct CPU { + ptr: CPUStatePtr, +} + +#[derive(Debug, PartialEq)] +pub enum CallingConvention { + Cdecl, +} + +pub trait ArchExtras { + fn read_return_address(&self) -> Result + where + T: From; + fn write_return_address(&self, val: T) -> Result<(), String> + where + T: Into; + fn read_function_argument(&self, conv: CallingConvention, idx: u8) -> Result + where + T: From; + fn write_function_argument( + &self, + conv: CallingConvention, + idx: i32, + val: T, + ) -> Result<(), String> + where + T: Into; +} + +#[allow(clippy::unused_self)] +impl CPU { + #[must_use] + pub fn qemu(&self) -> Qemu { + unsafe { Qemu::get_unchecked() } + } + + #[must_use] + #[allow(clippy::cast_sign_loss)] + pub fn index(&self) -> usize { + unsafe { libafl_qemu_cpu_index(self.ptr) as usize } + } + + pub fn trigger_breakpoint(&self) { + unsafe { + libafl_qemu_trigger_breakpoint(self.ptr); + } + } + + #[cfg(emulation_mode = "usermode")] + #[must_use] + pub fn g2h(&self, addr: GuestAddr) -> *mut T { + unsafe { (addr as usize + guest_base) as *mut T } + } + + #[cfg(emulation_mode = "usermode")] + #[must_use] + pub fn h2g(&self, addr: *const T) -> GuestAddr { + unsafe { (addr as usize - guest_base) as GuestAddr } + } + + #[cfg(emulation_mode = "usermode")] + #[must_use] + pub fn access_ok(&self, kind: VerifyAccess, addr: GuestAddr, size: usize) -> bool { + unsafe { + // TODO add support for tagged GuestAddr + libafl_qemu_sys::page_check_range(addr, size as GuestAddr, kind.into()) + } + } + + // TODO expose tlb_set_dirty and tlb_reset_dirty + + #[must_use] + pub fn num_regs(&self) -> i32 { + unsafe { libafl_qemu_num_regs(self.ptr) } + } + + pub fn write_reg(&self, reg: R, val: T) -> Result<(), String> + where + R: Into, + T: Into, + { + let reg = reg.into(); + #[cfg(feature = "be")] + let val = GuestReg::to_be(val.into()); + + #[cfg(not(feature = "be"))] + let val = GuestReg::to_le(val.into()); + + let success = unsafe { libafl_qemu_write_reg(self.ptr, reg, addr_of!(val) as *const u8) }; + if success == 0 { + Err(format!("Failed to write to register {reg}")) + } else { + Ok(()) + } + } + + pub fn read_reg(&self, reg: R) -> Result + where + R: Into, + T: From, + { + unsafe { + let reg = reg.into(); + let mut val = MaybeUninit::uninit(); + let success = libafl_qemu_read_reg(self.ptr, reg, val.as_mut_ptr() as *mut u8); + if success == 0 { + Err(format!("Failed to read register {reg}")) + } else { + #[cfg(feature = "be")] + return Ok(GuestReg::from_be(val.assume_init()).into()); + + #[cfg(not(feature = "be"))] + return Ok(GuestReg::from_le(val.assume_init()).into()); + } + } + } + + pub fn reset(&self) { + unsafe { libafl_qemu_sys::cpu_reset(self.ptr) }; + } + + #[must_use] + pub fn save_state(&self) -> CPUArchState { + unsafe { + let mut saved = MaybeUninit::::uninit(); + copy_nonoverlapping( + libafl_qemu_sys::cpu_env(self.ptr.as_mut().unwrap()), + saved.as_mut_ptr(), + 1, + ); + saved.assume_init() + } + } + + pub fn restore_state(&self, saved: &CPUArchState) { + unsafe { + copy_nonoverlapping( + saved, + libafl_qemu_sys::cpu_env(self.ptr.as_mut().unwrap()), + 1, + ); + } + } + + #[must_use] + pub fn raw_ptr(&self) -> CPUStatePtr { + self.ptr + } + + #[must_use] + pub fn display_context(&self) -> String { + let mut display = String::new(); + let mut maxl = 0; + for r in Regs::iter() { + maxl = std::cmp::max(format!("{r:#?}").len(), maxl); + } + for (i, r) in Regs::iter().enumerate() { + let v: GuestAddr = self.read_reg(r).unwrap(); + let sr = format!("{r:#?}"); + display += &format!("{sr:>maxl$}: {v:#016x} "); + if (i + 1) % 4 == 0 { + display += "\n"; + } + } + if !display.ends_with('\n') { + display += "\n"; + } + display + } +} + +pub trait HookId { + fn remove(&self, invalidate_block: bool) -> bool; +} + +macro_rules! create_hook_id { + ($name:ident, $sys:ident, true) => { + paste::paste! { + #[derive(Clone, Copy, PartialEq, Debug)] + pub struct [<$name HookId>](pub(crate) usize); + impl HookId for [<$name HookId>] { + fn remove(&self, invalidate_block: bool) -> bool { + unsafe { libafl_qemu_sys::$sys(self.0, invalidate_block.into()) != 0 } + } + } + } + }; + ($name:ident, $sys:ident, false) => { + paste::paste! { + #[derive(Clone, Copy, PartialEq, Debug)] + pub struct [<$name HookId>](pub(crate) usize); + impl HookId for [<$name HookId>] { + fn remove(&self, _invalidate_block: bool) -> bool { + unsafe { libafl_qemu_sys::$sys(self.0) != 0 } + } + } + } + }; +} + +create_hook_id!(Instruction, libafl_qemu_remove_hook, true); +create_hook_id!(Backdoor, libafl_qemu_remove_backdoor_hook, true); +create_hook_id!(Edge, libafl_qemu_remove_edge_hook, true); +create_hook_id!(Block, libafl_qemu_remove_block_hook, true); +create_hook_id!(Read, libafl_qemu_remove_read_hook, true); +create_hook_id!(Write, libafl_qemu_remove_write_hook, true); +create_hook_id!(Cmp, libafl_qemu_remove_cmp_hook, true); +create_hook_id!(PreSyscall, libafl_qemu_remove_pre_syscall_hook, false); +create_hook_id!(PostSyscall, libafl_qemu_remove_post_syscall_hook, false); +create_hook_id!(NewThread, libafl_qemu_remove_new_thread_hook, false); + +use std::{pin::Pin, ptr::NonNull}; + +use libafl::{ + inputs::HasTargetBytes, + state::{HasExecutions, State}, +}; +use libafl_bolts::os::unix_signals::Signal; + +use crate::{ + breakpoint::Breakpoint, + command::{Command, EmulatorMemoryChunk, InputCommand}, + executor::QemuExecutorState, +}; + +#[derive(Debug)] +pub struct HookData(u64); + +impl From> for HookData { + fn from(value: Pin<&mut T>) -> Self { + unsafe { HookData(transmute::, u64>(value)) } + } +} + +impl From> for HookData { + fn from(value: Pin<&T>) -> Self { + unsafe { HookData(transmute::, u64>(value)) } + } +} + +impl From<&'static mut T> for HookData { + fn from(value: &'static mut T) -> Self { + unsafe { HookData(transmute::<&mut T, u64>(value)) } + } +} + +impl From<&'static T> for HookData { + fn from(value: &'static T) -> Self { + unsafe { HookData(transmute::<&T, u64>(value)) } + } +} + +impl From<*mut T> for HookData { + fn from(value: *mut T) -> Self { + HookData(value as u64) + } +} + +impl From<*const T> for HookData { + fn from(value: *const T) -> Self { + HookData(value as u64) + } +} + +impl From for HookData { + fn from(value: u64) -> Self { + HookData(value) + } +} + +impl From for HookData { + fn from(value: u32) -> Self { + HookData(u64::from(value)) + } +} + +impl From for HookData { + fn from(value: u16) -> Self { + HookData(u64::from(value)) + } +} + +impl From for HookData { + fn from(value: u8) -> Self { + HookData(u64::from(value)) + } +} + +#[derive(Debug)] +pub enum EmuError { + MultipleInstances, + EmptyArgs, + TooManyArgs(usize), +} + +#[derive(Debug, Clone)] +pub enum EmuExitReason { + End(QemuShutdownCause), // QEMU ended for some reason. + Breakpoint(Breakpoint), // Breakpoint triggered. Contains the address of the trigger. + SyncBackdoor(SyncBackdoor), // Synchronous backdoor: The guest triggered a backdoor and should return to LibAFL. +} + +#[derive(Debug, Clone)] +pub enum QemuExitReason { + End(QemuShutdownCause), // QEMU ended for some reason. + Breakpoint(GuestAddr), // Breakpoint triggered. Contains the address of the trigger. + SyncBackdoor, // Synchronous backdoor: The guest triggered a backdoor and should return to LibAFL. +} + +/// High level result when finishing to handle requests +#[derive(Debug, Clone)] +pub enum HandlerResult { + UnhandledExit(EmuExitReason), // QEMU exit not handled by the current exit handler. + EndOfRun(ExitKind), // QEMU ended the current run and should pass some exit kind. + Interrupted, // User sent an interrupt signal +} + +impl From for HandlerError { + fn from(error: EmuExitReasonError) -> Self { + HandlerError::QemuExitReasonError(error) + } +} + +impl From for HandlerError { + fn from(error: SyncBackdoorError) -> Self { + HandlerError::SyncBackdoorError(error) + } +} + +impl Display for QemuExitReason { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + match self { + QemuExitReason::End(shutdown_cause) => write!(f, "End: {shutdown_cause:?}"), + QemuExitReason::Breakpoint(bp) => write!(f, "Breakpoint: {bp}"), + QemuExitReason::SyncBackdoor => write!(f, "Sync Backdoor"), // QemuExitReason::SyncBackdoor(sync_backdoor) => { + // write!(f, "Sync backdoor exit: {sync_backdoor}") + // } + } + } +} + +impl Display for EmuExitReason { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + match self { + EmuExitReason::End(shutdown_cause) => write!(f, "End: {shutdown_cause:?}"), + EmuExitReason::Breakpoint(bp) => write!(f, "{bp}"), + EmuExitReason::SyncBackdoor(sync_backdoor) => { + write!(f, "Sync backdoor exit: {sync_backdoor}") + } + } + } +} + +#[derive(Debug, Clone)] +pub enum QemuExitReasonError { + UnknownKind, // Exit reason was not NULL, but exit kind is unknown. Should never happen. + UnexpectedExit, // Qemu exited without going through an expected exit point. Can be caused by a crash for example. +} + +#[derive(Debug, Clone)] +pub enum EmuExitReasonError { + UnknownKind, + UnexpectedExit, + SyncBackdoorError(SyncBackdoorError), + BreakpointNotFound(GuestAddr), +} + +impl From for EmuExitReasonError { + fn from(sync_backdoor_error: SyncBackdoorError) -> Self { + EmuExitReasonError::SyncBackdoorError(sync_backdoor_error) + } +} + +impl std::error::Error for EmuError {} + +impl Display for EmuError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + EmuError::MultipleInstances => { + write!(f, "Only one instance of the QEMU Emulator is permitted") + } + EmuError::EmptyArgs => { + write!(f, "QEMU emulator args cannot be empty") + } + EmuError::TooManyArgs(n) => { + write!( + f, + "Too many arguments passed to QEMU emulator ({n} > i32::MAX)" + ) + } + } + } +} + +impl From for libafl::Error { + fn from(err: EmuError) -> Self { + libafl::Error::unknown(format!("{err}")) + } +} + +static mut EMULATOR_STATE: *mut () = ptr::null_mut(); +static mut QEMU_IS_INITIALIZED: bool = false; + +/// The thin wrapper around QEMU. +/// It is considered unsafe to use it directly. +/// Prefer using `Emulator` instead in case of doubt. +#[derive(Clone, Copy, Debug)] +pub struct Qemu { + _private: (), +} + +pub struct EmulatorState +where + QT: QemuHelperTuple, + S: State + HasExecutions, + E: EmuExitHandler, +{ + exit_handler: RefCell, + breakpoints: RefCell>, + _phantom: PhantomData<(QT, S)>, +} + +#[derive(Clone, Debug)] +pub struct Emulator +where + QT: QemuHelperTuple, + S: State + HasExecutions, + E: EmuExitHandler, +{ + state: ptr::NonNull>, + qemu: Qemu, +} + +#[allow(clippy::unused_self)] +impl Qemu { + #[allow(clippy::must_use_candidate, clippy::similar_names)] + pub fn init(args: &[String], env: &[(String, String)]) -> Result { + if args.is_empty() { + return Err(EmuError::EmptyArgs); + } + + let argc = args.len(); + if i32::try_from(argc).is_err() { + return Err(EmuError::TooManyArgs(argc)); + } + + unsafe { + if QEMU_IS_INITIALIZED { + return Err(EmuError::MultipleInstances); + } + QEMU_IS_INITIALIZED = true; + } + + #[allow(clippy::cast_possible_wrap)] + let argc = argc as i32; + + let args: Vec = args + .iter() + .map(|x| CString::new(x.clone()).unwrap()) + .collect(); + let mut argv: Vec<*const u8> = args.iter().map(|x| x.as_ptr() as *const u8).collect(); + argv.push(ptr::null()); // argv is always null terminated. + let env_strs: Vec = env + .iter() + .map(|(k, v)| format!("{}={}\0", &k, &v)) + .collect(); + let mut envp: Vec<*const u8> = env_strs.iter().map(|x| x.as_bytes().as_ptr()).collect(); + envp.push(null()); + unsafe { + #[cfg(emulation_mode = "usermode")] + qemu_user_init(argc, argv.as_ptr(), envp.as_ptr()); + #[cfg(emulation_mode = "systemmode")] + { + qemu_init( + argc, + argv.as_ptr() as *const *const u8, + envp.as_ptr() as *const *const u8, + ); + libc::atexit(qemu_cleanup_atexit); + libafl_qemu_sys::syx_snapshot_init(true); + } + } + + Ok(Qemu { _private: () }) + } + + /// Get a QEMU object. + /// Same as `Qemu::get`, but without checking whether QEMU has been correctly initialized. + /// + /// # Safety + /// + /// Should not be used if `Qemu::init` has never been used before (otherwise QEMU will not be initialized, and a crash will occur). + /// Prefer `Qemu::get` for a safe version of this method. + #[must_use] + pub unsafe fn get_unchecked() -> Self { + Qemu { _private: () } + } + + #[must_use] + pub fn get() -> Option { + unsafe { + if QEMU_IS_INITIALIZED { + Some(Qemu { _private: () }) + } else { + None + } + } + } + + fn post_run(&self) -> Result { + let exit_reason = unsafe { libafl_get_exit_reason() }; + if exit_reason.is_null() { + Err(QemuExitReasonError::UnexpectedExit) + } else { + let exit_reason: &mut libafl_qemu_sys::libafl_exit_reason = + unsafe { transmute(&mut *exit_reason) }; + Ok(match exit_reason.kind { + libafl_qemu_sys::libafl_exit_reason_kind_INTERNAL => unsafe { + let qemu_shutdown_cause: QemuShutdownCause = + match exit_reason.data.internal.cause { + libafl_qemu_sys::ShutdownCause_SHUTDOWN_CAUSE_NONE => { + QemuShutdownCause::None + } + libafl_qemu_sys::ShutdownCause_SHUTDOWN_CAUSE_HOST_ERROR => { + QemuShutdownCause::HostError + } + libafl_qemu_sys::ShutdownCause_SHUTDOWN_CAUSE_HOST_QMP_QUIT => { + QemuShutdownCause::HostQmpQuit + } + libafl_qemu_sys::ShutdownCause_SHUTDOWN_CAUSE_HOST_QMP_SYSTEM_RESET => { + QemuShutdownCause::HostQmpSystemReset + } + libafl_qemu_sys::ShutdownCause_SHUTDOWN_CAUSE_HOST_SIGNAL => { + QemuShutdownCause::HostSignal( + Signal::try_from(exit_reason.data.internal.signal).unwrap(), + ) + } + libafl_qemu_sys::ShutdownCause_SHUTDOWN_CAUSE_HOST_UI => { + QemuShutdownCause::HostUi + } + libafl_qemu_sys::ShutdownCause_SHUTDOWN_CAUSE_GUEST_SHUTDOWN => { + QemuShutdownCause::GuestShutdown + } + libafl_qemu_sys::ShutdownCause_SHUTDOWN_CAUSE_GUEST_RESET => { + QemuShutdownCause::GuestReset + } + libafl_qemu_sys::ShutdownCause_SHUTDOWN_CAUSE_GUEST_PANIC => { + QemuShutdownCause::GuestPanic + } + libafl_qemu_sys::ShutdownCause_SHUTDOWN_CAUSE_SUBSYSTEM_RESET => { + QemuShutdownCause::SubsystemReset + } + libafl_qemu_sys::ShutdownCause_SHUTDOWN_CAUSE_SNAPSHOT_LOAD => { + QemuShutdownCause::SnapshotLoad + } + + _ => panic!("shutdown cause not handled."), + }; + + QemuExitReason::End(qemu_shutdown_cause) + }, + libafl_qemu_sys::libafl_exit_reason_kind_BREAKPOINT => unsafe { + let bp_addr = exit_reason.data.breakpoint.addr; + QemuExitReason::Breakpoint(bp_addr) + }, + libafl_qemu_sys::libafl_exit_reason_kind_SYNC_BACKDOOR => { + QemuExitReason::SyncBackdoor + } + _ => return Err(QemuExitReasonError::UnknownKind), + }) + } + } + + #[must_use] + #[allow(clippy::cast_possible_wrap)] + #[allow(clippy::cast_sign_loss)] + pub fn num_cpus(&self) -> usize { + unsafe { libafl_qemu_num_cpus() as usize } + } + + #[must_use] + pub fn current_cpu(&self) -> Option { + let ptr = unsafe { libafl_qemu_current_cpu() }; + if ptr.is_null() { + None + } else { + Some(CPU { ptr }) + } + } + + #[must_use] + #[allow(clippy::cast_possible_wrap)] + pub fn cpu_from_index(&self, index: usize) -> CPU { + unsafe { + CPU { + ptr: libafl_qemu_get_cpu(index as i32), + } + } + } + + #[must_use] + pub fn page_from_addr(&self, addr: GuestAddr) -> GuestAddr { + unsafe { libafl_page_from_addr(addr) } + } + + //#[must_use] + /*pub fn page_size() -> GuestUsize { + unsafe { libafl_page_size } + }*/ + + pub unsafe fn write_mem(&self, addr: GuestAddr, buf: &[u8]) { + self.current_cpu() + .unwrap_or_else(|| self.cpu_from_index(0)) + .write_mem(addr, buf); + } + + pub unsafe fn read_mem(&self, addr: GuestAddr, buf: &mut [u8]) { + self.current_cpu() + .unwrap_or_else(|| self.cpu_from_index(0)) + .read_mem(addr, buf); + } + + #[must_use] + pub fn num_regs(&self) -> i32 { + self.current_cpu().unwrap().num_regs() + } + + pub fn write_reg(&self, reg: R, val: T) -> Result<(), String> + where + T: Num + PartialOrd + Copy + Into, + R: Into, + { + self.current_cpu().unwrap().write_reg(reg, val) + } + + pub fn read_reg(&self, reg: R) -> Result + where + T: Num + PartialOrd + Copy + From, + R: Into, + { + self.current_cpu().unwrap().read_reg(reg) + } + + pub fn set_breakpoint(&self, addr: GuestAddr) { + unsafe { + libafl_qemu_set_breakpoint(addr.into()); + } + } + + pub fn remove_breakpoint(&self, addr: GuestAddr) { + unsafe { + libafl_qemu_remove_breakpoint(addr.into()); + } + } + + pub fn entry_break(&self, addr: GuestAddr) { + self.set_breakpoint(addr); + unsafe { + match self.run() { + Ok(QemuExitReason::Breakpoint(_)) => {} + _ => panic!("Unexpected QEMU exit."), + } + } + self.remove_breakpoint(addr); + } + + pub fn flush_jit(&self) { + unsafe { + libafl_flush_jit(); + } + } + + // TODO set T lifetime to be like Emulator + #[allow(clippy::missing_transmute_annotations)] + pub fn set_hook>( + &self, + data: T, + addr: GuestAddr, + callback: extern "C" fn(T, GuestAddr), + invalidate_block: bool, + ) -> InstructionHookId { + unsafe { + let data: u64 = data.into().0; + let callback: extern "C" fn(u64, GuestAddr) = transmute(callback); + let num = libafl_qemu_sys::libafl_qemu_set_hook( + addr.into(), + Some(callback), + data, + i32::from(invalidate_block), + ); + InstructionHookId(num) + } + } + + #[must_use] + pub fn remove_hook(&self, id: impl HookId, invalidate_block: bool) -> bool { + id.remove(invalidate_block) + } + + #[must_use] + pub fn remove_hooks_at(&self, addr: GuestAddr, invalidate_block: bool) -> usize { + unsafe { + libafl_qemu_sys::libafl_qemu_remove_hooks_at(addr.into(), i32::from(invalidate_block)) + } + } + + #[allow(clippy::missing_transmute_annotations)] + pub fn add_edge_hooks>( + &self, + data: T, + gen: Option u64>, + exec: Option, + ) -> EdgeHookId { + unsafe { + let data: u64 = data.into().0; + let gen: Option u64> = transmute(gen); + let exec: Option = transmute(exec); + let num = libafl_qemu_sys::libafl_add_edge_hook(gen, exec, data); + EdgeHookId(num) + } + } + + #[allow(clippy::missing_transmute_annotations)] + pub fn add_block_hooks>( + &self, + data: T, + gen: Option u64>, + post_gen: Option, + exec: Option, + ) -> BlockHookId { + unsafe { + let data: u64 = data.into().0; + let gen: Option u64> = transmute(gen); + let post_gen: Option = transmute(post_gen); + let exec: Option = transmute(exec); + let num = libafl_qemu_sys::libafl_add_block_hook(gen, post_gen, exec, data); + BlockHookId(num) + } + } + + /// `data` can be used to pass data that can be accessed as the first argument in the `gen` and the `exec` functions + /// + /// `gen` gets passed the current programm counter, mutable access to a `TCGTemp` and information about the memory + /// access being performed. + /// The `u64` return value is an id that gets passed to the `exec` functions as their second argument. + /// + /// `exec` hooks get invoked on every read performed by the guest + /// + /// `exec1`-`exec8` special case accesses of width 1-8 + /// + /// If there is no specialized hook for a given read width, the `exec_n` will be + /// called and its last argument will specify the access width + #[allow(clippy::missing_transmute_annotations)] + pub fn add_read_hooks>( + &self, + data: T, + gen: Option u64>, + exec1: Option, + exec2: Option, + exec4: Option, + exec8: Option, + exec_n: Option, + ) -> ReadHookId { + unsafe { + let data: u64 = data.into().0; + let gen: Option< + unsafe extern "C" fn( + u64, + GuestAddr, + *mut TCGTemp, + libafl_qemu_sys::MemOpIdx, + ) -> u64, + > = transmute(gen); + let exec1: Option = transmute(exec1); + let exec2: Option = transmute(exec2); + let exec4: Option = transmute(exec4); + let exec8: Option = transmute(exec8); + let exec_n: Option = transmute(exec_n); + let num = libafl_qemu_sys::libafl_add_read_hook( + gen, exec1, exec2, exec4, exec8, exec_n, data, + ); + ReadHookId(num) + } + } + + // TODO add MemOp info + #[allow(clippy::missing_transmute_annotations)] + pub fn add_write_hooks>( + &self, + data: T, + gen: Option u64>, + exec1: Option, + exec2: Option, + exec4: Option, + exec8: Option, + exec_n: Option, + ) -> WriteHookId { + unsafe { + let data: u64 = data.into().0; + let gen: Option< + unsafe extern "C" fn( + u64, + GuestAddr, + *mut TCGTemp, + libafl_qemu_sys::MemOpIdx, + ) -> u64, + > = transmute(gen); + let exec1: Option = transmute(exec1); + let exec2: Option = transmute(exec2); + let exec4: Option = transmute(exec4); + let exec8: Option = transmute(exec8); + let exec_n: Option = transmute(exec_n); + let num = libafl_qemu_sys::libafl_add_write_hook( + gen, exec1, exec2, exec4, exec8, exec_n, data, + ); + WriteHookId(num) + } + } + + #[allow(clippy::missing_transmute_annotations)] + pub fn add_cmp_hooks>( + &self, + data: T, + gen: Option u64>, + exec1: Option, + exec2: Option, + exec4: Option, + exec8: Option, + ) -> CmpHookId { + unsafe { + let data: u64 = data.into().0; + let gen: Option u64> = transmute(gen); + let exec1: Option = transmute(exec1); + let exec2: Option = transmute(exec2); + let exec4: Option = transmute(exec4); + let exec8: Option = transmute(exec8); + let num = libafl_qemu_sys::libafl_add_cmp_hook(gen, exec1, exec2, exec4, exec8, data); + CmpHookId(num) + } + } + + #[allow(clippy::missing_transmute_annotations)] + pub fn add_backdoor_hook>( + &self, + data: T, + callback: extern "C" fn(T, CPUArchStatePtr, GuestAddr), + ) -> BackdoorHookId { + unsafe { + let data: u64 = data.into().0; + let callback: extern "C" fn(u64, CPUArchStatePtr, GuestAddr) = transmute(callback); + let num = libafl_qemu_sys::libafl_add_backdoor_hook(Some(callback), data); + BackdoorHookId(num) + } + } + + #[allow(clippy::type_complexity)] + pub fn add_gdb_cmd(&self, callback: Box bool>) { + unsafe { + let fat: Box = Box::new(transmute::< + Box FnMut(&'a Qemu, &'b str) -> bool>, + FatPtr, + >(callback)); + libafl_qemu_add_gdb_cmd(gdb_cmd, core::ptr::from_ref(&*fat) as *const ()); + GDB_COMMANDS.push(fat); + } + } + + pub fn gdb_reply(&self, output: &str) { + unsafe { libafl_qemu_gdb_reply(output.as_bytes().as_ptr(), output.len()) }; + } +} + +impl ArchExtras for Qemu { + fn read_return_address(&self) -> Result + where + T: From, + { + self.current_cpu() + .ok_or("Failed to get current CPU")? + .read_return_address::() + } + + fn write_return_address(&self, val: T) -> Result<(), String> + where + T: Into, + { + self.current_cpu() + .ok_or("Failed to get current CPU")? + .write_return_address::(val) + } + + fn read_function_argument(&self, conv: CallingConvention, idx: u8) -> Result + where + T: From, + { + self.current_cpu() + .ok_or("Failed to get current CPU")? + .read_function_argument::(conv, idx) + } + + fn write_function_argument( + &self, + conv: CallingConvention, + idx: i32, + val: T, + ) -> Result<(), String> + where + T: Into, + { + self.current_cpu() + .ok_or("Failed to get current CPU")? + .write_function_argument::(conv, idx, val) + } +} + +#[allow(clippy::unused_self)] +impl Emulator +where + QT: QemuHelperTuple, + S: State + HasExecutions, + E: EmuExitHandler, +{ + #[allow(clippy::must_use_candidate, clippy::similar_names)] + pub fn new( + args: &[String], + env: &[(String, String)], + exit_handler: E, + ) -> Result { + let qemu = Qemu::init(args, env)?; + + Self::new_with_qemu(qemu, exit_handler) + } + + pub fn new_with_qemu(qemu: Qemu, exit_handler: E) -> Result { + let emu_state = Box::new(EmulatorState { + exit_handler: RefCell::new(exit_handler), + breakpoints: RefCell::new(HashSet::new()), + _phantom: PhantomData, + }); + + let emu_state_ptr = unsafe { + let emu_ptr = NonNull::from(Box::leak(emu_state)); + EMULATOR_STATE = emu_ptr.as_ptr() as *mut (); + emu_ptr + }; + + Ok(Emulator { + state: emu_state_ptr, + qemu, + }) + } + + #[must_use] + pub fn qemu(&self) -> &Qemu { + &self.qemu + } + + #[must_use] + pub fn state(&self) -> &EmulatorState { + unsafe { self.state.as_ref() } + } + + #[must_use] + pub fn state_mut(&mut self) -> &mut EmulatorState { + unsafe { self.state.as_mut() } + } + + #[must_use] + pub fn exit_handler(&self) -> &RefCell { + &self.state().exit_handler + } + + #[must_use] + pub fn get() -> Option> { + unsafe { + if QEMU_IS_INITIALIZED { + Some(Emulator::::get_unchecked()) + } else { + None + } + } + } + + /// Get an empty emulator. + /// Same as `Emulator::get`, but without checking whether QEMU has been correctly initialized. + /// + /// # Safety + /// + /// Should not be used if `Qemu::init` or `Emulator::new` has never been used before (otherwise QEMU will not be initialized, and a crash will occur). + /// Prefer `Emulator::get` for a safe version of this method. + #[must_use] + pub unsafe fn get_unchecked() -> Emulator { + Emulator { + state: NonNull::dangling(), + qemu: Qemu::get_unchecked(), + } + } + + #[must_use] + #[allow(clippy::cast_possible_wrap)] + #[allow(clippy::cast_sign_loss)] + #[deprecated( + note = "This function has been moved to the `Qemu` low-level structure. Please access it through `emu.qemu()`." + )] + pub fn num_cpus(&self) -> usize { + self.qemu.num_cpus() + } + + #[must_use] + #[deprecated( + note = "This function has been moved to the `Qemu` low-level structure. Please access it through `emu.qemu()`." + )] + pub fn current_cpu(&self) -> Option { + self.qemu.current_cpu() + } + + #[must_use] + #[allow(clippy::cast_possible_wrap)] + #[deprecated( + note = "This function has been moved to the `Qemu` low-level structure. Please access it through `emu.qemu()`." + )] + pub fn cpu_from_index(&self, index: usize) -> CPU { + self.qemu.cpu_from_index(index) + } + + #[must_use] + #[deprecated( + note = "This function has been moved to the `Qemu` low-level structure. Please access it through `emu.qemu()`." + )] + pub fn page_from_addr(&self, addr: GuestAddr) -> GuestAddr { + self.qemu.page_from_addr(addr) + } + + //#[must_use] + /*pub fn page_size() -> GuestUsize { + unsafe { libafl_page_size } + }*/ + + #[deprecated( + note = "This function has been moved to the `Qemu` low-level structure. Please access it through `emu.qemu()`." + )] + pub unsafe fn write_mem(&self, addr: GuestAddr, buf: &[u8]) { + self.qemu.write_mem(addr, buf); + } + + #[deprecated( + note = "This function has been moved to the `Qemu` low-level structure. Please access it through `emu.qemu()`." + )] + pub unsafe fn read_mem(&self, addr: GuestAddr, buf: &mut [u8]) { + self.qemu.read_mem(addr, buf); + } + + #[must_use] + #[deprecated( + note = "This function has been moved to the `Qemu` low-level structure. Please access it through `emu.qemu()`." + )] + pub fn num_regs(&self) -> i32 { + self.qemu.num_regs() + } + + #[deprecated( + note = "This function has been moved to the `Qemu` low-level structure. Please access it through `emu.qemu()`." + )] + pub fn write_reg(&self, reg: R, val: T) -> Result<(), String> + where + T: Num + PartialOrd + Copy + Into, + R: Into, + { + self.qemu.write_reg(reg, val) + } + + #[deprecated( + note = "This function has been moved to the `Qemu` low-level structure. Please access it through `emu.qemu()`." + )] + pub fn read_reg(&self, reg: R) -> Result + where + T: Num + PartialOrd + Copy + From, + R: Into, + { + self.qemu.read_reg(reg) + } + + pub fn add_breakpoint(&self, mut bp: Breakpoint, enable: bool) { + if enable { + bp.enable(&self.qemu); + } + + self.state().breakpoints.borrow_mut().insert(bp); + } + + pub fn remove_breakpoint(&self, bp: &mut Breakpoint) { + bp.disable(&self.qemu); + + self.state().breakpoints.borrow_mut().remove(bp); + } + + #[deprecated( + note = "This function has been moved to the `Qemu` low-level structure. Please access it through `emu.qemu()`." + )] + pub fn entry_break(&self, addr: GuestAddr) { + self.qemu.entry_break(addr); + } + + /// This function will run the emulator until the next breakpoint, or until finish. + /// # Safety + /// + /// Should, in general, be safe to call. + /// Of course, the emulated target is not contained securely and can corrupt state or interact with the operating system. + unsafe fn run_qemu(&self) -> Result { + match self.qemu.run() { + Ok(qemu_exit_reason) => Ok(match qemu_exit_reason { + QemuExitReason::End(qemu_shutdown_cause) => EmuExitReason::End(qemu_shutdown_cause), + QemuExitReason::Breakpoint(bp_addr) => { + let bp = self + .state() + .breakpoints + .borrow() + .get(&bp_addr) + .ok_or(EmuExitReasonError::BreakpointNotFound(bp_addr))? + .clone(); + EmuExitReason::Breakpoint(bp) + } + QemuExitReason::SyncBackdoor => EmuExitReason::SyncBackdoor(self.try_into()?), + }), + Err(qemu_exit_reason_error) => Err(match qemu_exit_reason_error { + QemuExitReasonError::UnexpectedExit => EmuExitReasonError::UnexpectedExit, + QemuExitReasonError::UnknownKind => EmuExitReasonError::UnknownKind, + }), + } + } + + /// This function will run the emulator until the exit handler decides to stop the execution for + /// whatever reason, depending on the choosen handler. + /// It is a higher-level abstraction of [`Emulator::run`] that will take care of some part of the runtime logic, + /// returning only when something interesting happen. + /// + /// # Safety + /// Should, in general, be safe to call. + /// Of course, the emulated target is not contained securely and can corrupt state or interact with the operating system. + pub unsafe fn run( + &self, + input: &S::Input, + qemu_executor_state: &mut QemuExecutorState, + ) -> Result { + loop { + // Insert input if the location is already known + E::try_put_input(self, qemu_executor_state, input); + + // Run QEMU + let exit_reason = self.run_qemu(); + + // Handle QEMU exit + let handler_res = E::handle(self, exit_reason, qemu_executor_state, input)?; + + // Return to harness + match handler_res { + InnerHandlerResult::ReturnToHarness(exit_reason) => { + return Ok(HandlerResult::UnhandledExit(exit_reason)) + } + InnerHandlerResult::EndOfRun(exit_kind) => { + return Ok(HandlerResult::EndOfRun(exit_kind)) + } + InnerHandlerResult::Interrupt => return Ok(HandlerResult::Interrupted), + InnerHandlerResult::Continue => {} + } + } + } + + #[deprecated( + note = "This function has been moved to the `Qemu` low-level structure. Please access it through `emu.qemu()`." + )] + pub fn flush_jit(&self) { + self.qemu.flush_jit(); + } + + // TODO set T lifetime to be like Emulator + #[deprecated( + note = "This function has been moved to the `Qemu` low-level structure. Please access it through `emu.qemu()`." + )] + pub fn set_hook>( + &self, + data: T, + addr: GuestAddr, + callback: extern "C" fn(T, GuestAddr), + invalidate_block: bool, + ) -> InstructionHookId { + self.qemu.set_hook(data, addr, callback, invalidate_block) + } + + #[must_use] + #[deprecated( + note = "This function has been moved to the `Qemu` low-level structure. Please access it through `emu.qemu()`." + )] + pub fn remove_hook(&self, id: impl HookId, invalidate_block: bool) -> bool { + self.qemu.remove_hook(id, invalidate_block) + } + + #[must_use] + #[deprecated( + note = "This function has been moved to the `Qemu` low-level structure. Please access it through `emu.qemu()`." + )] + pub fn remove_hooks_at(&self, addr: GuestAddr, invalidate_block: bool) -> usize { + self.qemu.remove_hooks_at(addr, invalidate_block) + } + + #[deprecated( + note = "This function has been moved to the `Qemu` low-level structure. Please access it through `emu.qemu()`." + )] + pub fn add_edge_hooks>( + &self, + data: T, + gen: Option u64>, + exec: Option, + ) -> EdgeHookId { + self.qemu.add_edge_hooks(data, gen, exec) + } + + #[deprecated( + note = "This function has been moved to the `Qemu` low-level structure. Please access it through `emu.qemu()`." + )] + pub fn add_block_hooks>( + &self, + data: T, + gen: Option u64>, + post_gen: Option, + exec: Option, + ) -> BlockHookId { + self.qemu.add_block_hooks(data, gen, post_gen, exec) + } + + #[deprecated( + note = "This function has been moved to the `Qemu` low-level structure. Please access it through `emu.qemu()`." + )] + pub fn add_read_hooks>( + &self, + data: T, + gen: Option u64>, + exec1: Option, + exec2: Option, + exec4: Option, + exec8: Option, + exec_n: Option, + ) -> ReadHookId { + self.qemu + .add_read_hooks(data, gen, exec1, exec2, exec4, exec8, exec_n) + } + + // TODO add MemOp info + #[deprecated( + note = "This function has been moved to the `Qemu` low-level structure. Please access it through `emu.qemu()`." + )] + pub fn add_write_hooks>( + &self, + data: T, + gen: Option u64>, + exec1: Option, + exec2: Option, + exec4: Option, + exec8: Option, + exec_n: Option, + ) -> WriteHookId { + self.qemu + .add_write_hooks(data, gen, exec1, exec2, exec4, exec8, exec_n) + } + + #[deprecated( + note = "This function has been moved to the `Qemu` low-level structure. Please access it through `emu.qemu()`." + )] + pub fn add_cmp_hooks>( + &self, + data: T, + gen: Option u64>, + exec1: Option, + exec2: Option, + exec4: Option, + exec8: Option, + ) -> CmpHookId { + self.qemu + .add_cmp_hooks(data, gen, exec1, exec2, exec4, exec8) + } + + #[deprecated( + note = "This function has been moved to the `Qemu` low-level structure. Please access it through `emu.qemu()`." + )] + pub fn add_backdoor_hook>( + &self, + data: T, + callback: extern "C" fn(T, CPUArchStatePtr, GuestAddr), + ) -> BackdoorHookId { + self.qemu.add_backdoor_hook(data, callback) + } + + #[allow(clippy::type_complexity)] + #[deprecated( + note = "This function has been moved to the `Qemu` low-level structure. Please access it through `emu.qemu()`." + )] + pub fn add_gdb_cmd(&self, callback: Box bool>) { + self.qemu.add_gdb_cmd(callback); + } + + #[deprecated( + note = "This function has been moved to the `Qemu` low-level structure. Please access it through `emu.qemu()`." + )] + pub fn gdb_reply(&self, output: &str) { + self.qemu.gdb_reply(output); + } +} + +// impl ArchExtras for Emulator +// where +// QT: QemuHelperTuple, +// S: State + HasExecutions, +// E: EmuExitHandler, +// { +// fn read_return_address(&self) -> Result +// where +// T: From, +// { +// self.qemu.read_return_address() +// } +// +// fn write_return_address(&self, val: T) -> Result<(), String> +// where +// T: Into, +// { +// self.qemu.write_return_address(val) +// } +// +// fn read_function_argument(&self, conv: CallingConvention, idx: u8) -> Result +// where +// T: From, +// { +// self.qemu.read_function_argument(conv, idx) +// } +// +// fn write_function_argument( +// &self, +// conv: CallingConvention, +// idx: i32, +// val: T, +// ) -> Result<(), String> +// where +// T: Into, +// { +// self.qemu.write_function_argument(conv, idx, val) +// } +// } + +#[cfg(feature = "python")] +pub mod pybind { + use pyo3::{exceptions::PyValueError, prelude::*, types::PyInt}; + + use super::{GuestAddr, GuestUsize, MmapPerms, SyscallHookResult}; + + static mut PY_SYSCALL_HOOK: Option = None; + static mut PY_GENERIC_HOOKS: Vec<(GuestAddr, PyObject)> = vec![]; + + extern "C" fn py_syscall_hook_wrapper( + _data: u64, + sys_num: i32, + a0: u64, + a1: u64, + a2: u64, + a3: u64, + a4: u64, + a5: u64, + a6: u64, + a7: u64, + ) -> SyscallHookResult { + unsafe { PY_SYSCALL_HOOK.as_ref() }.map_or_else( + || SyscallHookResult::new(None), + |obj| { + let args = (sys_num, a0, a1, a2, a3, a4, a5, a6, a7); + Python::with_gil(|py| { + let ret = obj.call1(py, args).expect("Error in the syscall hook"); + let any = ret.as_ref(py); + if any.is_none() { + SyscallHookResult::new(None) + } else { + let a: Result<&PyInt, _> = any.downcast(); + if let Ok(i) = a { + SyscallHookResult::new(Some( + i.extract().expect("Invalid syscall hook return value"), + )) + } else { + SyscallHookResult::extract(any) + .expect("The syscall hook must return a SyscallHookResult") + } + } + }) + }, + ) + } + + extern "C" fn py_generic_hook_wrapper(idx: u64, _pc: GuestAddr) { + let obj = unsafe { &PY_GENERIC_HOOKS[idx as usize].1 }; + Python::with_gil(|py| { + obj.call0(py).expect("Error in the hook"); + }); + } + + #[pyclass(unsendable)] + pub struct Qemu { + pub qemu: super::Qemu, + } + + #[pymethods] + impl Qemu { + #[allow(clippy::needless_pass_by_value)] + #[new] + fn new(args: Vec, env: Vec<(String, String)>) -> PyResult { + let qemu = super::Qemu::init(&args, &env) + .map_err(|e| PyValueError::new_err(format!("{e}")))?; + + Ok(Qemu { qemu }) + } + + fn run(&self) { + unsafe { + self.qemu.run().unwrap(); + } + } + + fn write_mem(&self, addr: GuestAddr, buf: &[u8]) { + unsafe { + self.qemu.write_mem(addr, buf); + } + } + + fn read_mem(&self, addr: GuestAddr, size: usize) -> Vec { + let mut buf = vec![0; size]; + unsafe { + self.qemu.read_mem(addr, &mut buf); + } + buf + } + + fn num_regs(&self) -> i32 { + self.qemu.num_regs() + } + + fn write_reg(&self, reg: i32, val: GuestUsize) -> PyResult<()> { + self.qemu.write_reg(reg, val).map_err(PyValueError::new_err) + } + + fn read_reg(&self, reg: i32) -> PyResult { + self.qemu.read_reg(reg).map_err(PyValueError::new_err) + } + + fn set_breakpoint(&self, addr: GuestAddr) { + self.qemu.set_breakpoint(addr); + } + + fn entry_break(&self, addr: GuestAddr) { + self.qemu.entry_break(addr); + } + + fn remove_breakpoint(&self, addr: GuestAddr) { + self.qemu.remove_breakpoint(addr); + } + + fn g2h(&self, addr: GuestAddr) -> u64 { + self.qemu.g2h::<*const u8>(addr) as u64 + } + + fn h2g(&self, addr: u64) -> GuestAddr { + self.qemu.h2g(addr as *const u8) + } + + fn binary_path(&self) -> String { + self.qemu.binary_path().to_owned() + } + + fn load_addr(&self) -> GuestAddr { + self.qemu.load_addr() + } + + fn flush_jit(&self) { + self.qemu.flush_jit(); + } + + fn map_private(&self, addr: GuestAddr, size: usize, perms: i32) -> PyResult { + if let Ok(p) = MmapPerms::try_from(perms) { + self.qemu + .map_private(addr, size, p) + .map_err(PyValueError::new_err) + } else { + Err(PyValueError::new_err("Invalid perms")) + } + } + + fn map_fixed(&self, addr: GuestAddr, size: usize, perms: i32) -> PyResult { + if let Ok(p) = MmapPerms::try_from(perms) { + self.qemu + .map_fixed(addr, size, p) + .map_err(PyValueError::new_err) + } else { + Err(PyValueError::new_err("Invalid perms")) + } + } + + fn mprotect(&self, addr: GuestAddr, size: usize, perms: i32) -> PyResult<()> { + if let Ok(p) = MmapPerms::try_from(perms) { + self.qemu + .mprotect(addr, size, p) + .map_err(PyValueError::new_err) + } else { + Err(PyValueError::new_err("Invalid perms")) + } + } + + fn unmap(&self, addr: GuestAddr, size: usize) -> PyResult<()> { + self.qemu.unmap(addr, size).map_err(PyValueError::new_err) + } + + fn set_syscall_hook(&self, hook: PyObject) { + unsafe { + PY_SYSCALL_HOOK = Some(hook); + } + self.qemu + .add_pre_syscall_hook(0u64, py_syscall_hook_wrapper); + } + + fn set_hook(&self, addr: GuestAddr, hook: PyObject) { + unsafe { + let idx = PY_GENERIC_HOOKS.len(); + PY_GENERIC_HOOKS.push((addr, hook)); + self.qemu + .set_hook(idx as u64, addr, py_generic_hook_wrapper, true); + } + } + + fn remove_hooks_at(&self, addr: GuestAddr) -> usize { + unsafe { + PY_GENERIC_HOOKS.retain(|(a, _)| *a != addr); + } + self.qemu.remove_hooks_at(addr, true) + } + } +} diff --git a/libafl_qemu/src/emu/systemmode.rs b/libafl_qemu/src/emu/systemmode.rs new file mode 100644 index 0000000000..4db07028dc --- /dev/null +++ b/libafl_qemu/src/emu/systemmode.rs @@ -0,0 +1,451 @@ +use std::{ + collections::HashMap, + ffi::{c_void, CStr, CString}, + fmt::Debug, + mem::MaybeUninit, + ptr::null_mut, + sync::atomic::{AtomicU64, Ordering}, +}; + +use libafl::state::{HasExecutions, State}; +use libafl_qemu_sys::{ + libafl_load_qemu_snapshot, libafl_qemu_current_paging_id, libafl_save_qemu_snapshot, + qemu_cleanup, qemu_main_loop, vm_start, GuestAddr, GuestPhysAddr, GuestVirtAddr, +}; + +use crate::{ + emu::{libafl_page_from_addr, IsSnapshotManager}, + EmuExitHandler, Emulator, MemAccessInfo, Qemu, QemuExitReason, QemuExitReasonError, + QemuHelperTuple, SnapshotId, SnapshotManagerError, CPU, +}; + +impl SnapshotId { + fn gen_unique_id() -> SnapshotId { + static UNIQUE_ID: AtomicU64 = AtomicU64::new(0); + + let unique_id = UNIQUE_ID.fetch_add(1, Ordering::SeqCst); + + SnapshotId { + id: unique_id.clone(), + } + } + + fn inner(&self) -> u64 { + self.id + } +} + +#[derive(Debug, Clone)] +pub enum SnapshotManager { + Qemu(QemuSnapshotManager), + Fast(FastSnapshotManager), +} + +impl IsSnapshotManager for SnapshotManager { + fn save(&mut self, qemu: &Qemu) -> SnapshotId { + match self { + SnapshotManager::Qemu(qemu_sm) => qemu_sm.save(qemu), + SnapshotManager::Fast(fast_sm) => fast_sm.save(qemu), + } + } + + fn restore( + &mut self, + snapshot_id: &SnapshotId, + qemu: &Qemu, + ) -> Result<(), SnapshotManagerError> { + match self { + SnapshotManager::Qemu(qemu_sm) => qemu_sm.restore(snapshot_id, qemu), + SnapshotManager::Fast(fast_sm) => fast_sm.restore(snapshot_id, qemu), + } + } +} + +pub type FastSnapshotPtr = *mut libafl_qemu_sys::SyxSnapshot; + +#[derive(Debug, Clone)] +pub struct FastSnapshotManager { + snapshots: HashMap, + check_memory_consistency: bool, +} + +impl Default for FastSnapshotManager { + fn default() -> Self { + Self::new(false) + } +} + +impl FastSnapshotManager { + pub fn new(check_memory_consistency: bool) -> Self { + Self { + snapshots: HashMap::new(), + check_memory_consistency, + } + } + + pub unsafe fn get(&self, id: &SnapshotId) -> FastSnapshotPtr { + self.snapshots.get(id).unwrap().clone() + } +} + +#[derive(Debug, Clone)] +pub struct QemuSnapshotManager { + is_sync: bool, +} + +impl QemuSnapshotManager { + pub fn new(is_sync: bool) -> Self { + Self { is_sync } + } + + pub fn snapshot_id_to_name(&self, snapshot_id: &SnapshotId) -> String { + format!("__libafl_qemu_snapshot_{}", snapshot_id.inner()) + } +} + +impl IsSnapshotManager for QemuSnapshotManager { + fn save(&mut self, qemu: &Qemu) -> SnapshotId { + let snapshot_id = SnapshotId::gen_unique_id(); + qemu.save_snapshot( + self.snapshot_id_to_name(&snapshot_id).as_str(), + self.is_sync, + ); + snapshot_id + } + + fn restore( + &mut self, + snapshot_id: &SnapshotId, + qemu: &Qemu, + ) -> Result<(), SnapshotManagerError> { + qemu.load_snapshot(self.snapshot_id_to_name(snapshot_id).as_str(), self.is_sync); + Ok(()) + } +} + +impl IsSnapshotManager for FastSnapshotManager { + fn save(&mut self, qemu: &Qemu) -> SnapshotId { + let snapshot_id = SnapshotId::gen_unique_id(); + self.snapshots + .insert(snapshot_id, qemu.create_fast_snapshot(true)); + snapshot_id + } + + fn restore( + &mut self, + snapshot_id: &SnapshotId, + qemu: &Qemu, + ) -> Result<(), SnapshotManagerError> { + let fast_snapshot_ptr = self + .snapshots + .get(snapshot_id) + .ok_or(SnapshotManagerError::SnapshotIdNotFound( + snapshot_id.clone(), + ))? + .clone(); + + qemu.restore_fast_snapshot(fast_snapshot_ptr); + + if self.check_memory_consistency { + let nb_inconsistencies = qemu.check_fast_snapshot_memory_consistency(fast_snapshot_ptr); + + if nb_inconsistencies > 0 { + return Err(SnapshotManagerError::MemoryInconsistencies( + nb_inconsistencies, + )); + } + } + + Ok(()) + } +} + +pub enum DeviceSnapshotFilter { + All, + AllowList(Vec), + DenyList(Vec), +} + +impl DeviceSnapshotFilter { + fn enum_id(&self) -> libafl_qemu_sys::DeviceSnapshotKind { + match self { + DeviceSnapshotFilter::All => libafl_qemu_sys::DeviceSnapshotKind_DEVICE_SNAPSHOT_ALL, + DeviceSnapshotFilter::AllowList(_) => { + libafl_qemu_sys::DeviceSnapshotKind_DEVICE_SNAPSHOT_ALLOWLIST + } + DeviceSnapshotFilter::DenyList(_) => { + libafl_qemu_sys::DeviceSnapshotKind_DEVICE_SNAPSHOT_DENYLIST + } + } + } + + fn devices(&self, v: &mut Vec<*mut i8>) -> *mut *mut i8 { + v.clear(); + match self { + DeviceSnapshotFilter::All => null_mut(), + DeviceSnapshotFilter::AllowList(l) | DeviceSnapshotFilter::DenyList(l) => { + for name in l { + v.push(name.as_bytes().as_ptr() as *mut i8); + } + v.as_mut_ptr() + } + } + } +} + +pub(super) extern "C" fn qemu_cleanup_atexit() { + unsafe { + qemu_cleanup(); + } +} + +impl CPU { + #[must_use] + pub fn get_phys_addr(&self, vaddr: GuestAddr) -> Option { + unsafe { + let page = libafl_page_from_addr(vaddr); + let mut attrs = MaybeUninit::::uninit(); + let paddr = libafl_qemu_sys::cpu_get_phys_page_attrs_debug( + self.ptr, + page as GuestVirtAddr, + attrs.as_mut_ptr(), + ); + if paddr == (-1i64 as GuestPhysAddr) { + None + } else { + Some(paddr) + } + } + } + + #[must_use] + pub fn get_phys_addr_tlb( + &self, + vaddr: GuestAddr, + info: MemAccessInfo, + is_store: bool, + ) -> Option { + unsafe { + let pminfo = libafl_qemu_sys::make_plugin_meminfo( + info.oi, + if is_store { + libafl_qemu_sys::qemu_plugin_mem_rw_QEMU_PLUGIN_MEM_W + } else { + libafl_qemu_sys::qemu_plugin_mem_rw_QEMU_PLUGIN_MEM_R + }, + ); + let phwaddr = libafl_qemu_sys::qemu_plugin_get_hwaddr(pminfo, vaddr as GuestVirtAddr); + if phwaddr.is_null() { + None + } else { + Some(libafl_qemu_sys::qemu_plugin_hwaddr_phys_addr(phwaddr) as GuestPhysAddr) + } + } + } + + #[must_use] + pub fn current_paging_id(&self) -> Option { + let paging_id = unsafe { libafl_qemu_current_paging_id(self.ptr) }; + + if paging_id == 0 { + None + } else { + Some(paging_id) + } + } + + /// Write a value to a guest address. + /// + /// # Safety + /// This will write to a translated guest address (using `g2h`). + /// It just adds `guest_base` and writes to that location, without checking the bounds. + /// This may only be safely used for valid guest addresses! + pub unsafe fn write_mem(&self, addr: GuestAddr, buf: &[u8]) { + // TODO use gdbstub's target_cpu_memory_rw_debug + libafl_qemu_sys::cpu_memory_rw_debug( + self.ptr, + addr as GuestVirtAddr, + buf.as_ptr() as *mut _, + buf.len(), + true, + ); + } + + /// Read a value from a guest address. + /// + /// # Safety + /// This will read from a translated guest address (using `g2h`). + /// It just adds `guest_base` and writes to that location, without checking the bounds. + /// This may only be safely used for valid guest addresses! + pub unsafe fn read_mem(&self, addr: GuestAddr, buf: &mut [u8]) { + // TODO use gdbstub's target_cpu_memory_rw_debug + libafl_qemu_sys::cpu_memory_rw_debug( + self.ptr, + addr as GuestVirtAddr, + buf.as_mut_ptr() as *mut _, + buf.len(), + false, + ); + } + + #[must_use] + pub fn page_size(&self) -> usize { + unsafe { libafl_qemu_sys::qemu_target_page_size() } + } +} + +#[allow(clippy::unused_self)] +impl Qemu { + /// Write a value to a phsical guest address, including ROM areas. + pub unsafe fn write_phys_mem(&self, paddr: GuestPhysAddr, buf: &[u8]) { + libafl_qemu_sys::cpu_physical_memory_rw( + paddr, + buf.as_ptr() as *mut _, + buf.len() as u64, + true, + ); + } + + /// Read a value from a physical guest address. + pub unsafe fn read_phys_mem(&self, paddr: GuestPhysAddr, buf: &mut [u8]) { + libafl_qemu_sys::cpu_physical_memory_rw( + paddr, + buf.as_mut_ptr() as *mut _, + buf.len() as u64, + false, + ); + } + + /// This function will run the emulator until the next breakpoint / sync exit, or until finish. + /// It is a low-level function and simply kicks QEMU. + /// # Safety + /// + /// Should, in general, be safe to call. + /// Of course, the emulated target is not contained securely and can corrupt state or interact with the operating system. + pub unsafe fn run(&self) -> Result { + vm_start(); + qemu_main_loop(); + + self.post_run() + } + + pub fn save_snapshot(&self, name: &str, sync: bool) { + let s = CString::new(name).expect("Invalid snapshot name"); + unsafe { libafl_save_qemu_snapshot(s.as_ptr() as *const _, sync) }; + } + + pub fn load_snapshot(&self, name: &str, sync: bool) { + let s = CString::new(name).expect("Invalid snapshot name"); + unsafe { libafl_load_qemu_snapshot(s.as_ptr() as *const _, sync) }; + } + + #[must_use] + pub fn create_fast_snapshot(&self, track: bool) -> FastSnapshotPtr { + unsafe { + libafl_qemu_sys::syx_snapshot_new( + track, + true, + libafl_qemu_sys::DeviceSnapshotKind_DEVICE_SNAPSHOT_ALL, + null_mut(), + ) + } + } + + #[must_use] + pub fn create_fast_snapshot_filter( + &self, + track: bool, + device_filter: &DeviceSnapshotFilter, + ) -> FastSnapshotPtr { + let mut v = vec![]; + unsafe { + libafl_qemu_sys::syx_snapshot_new( + track, + true, + device_filter.enum_id(), + device_filter.devices(&mut v), + ) + } + } + + pub fn restore_fast_snapshot(&self, snapshot: FastSnapshotPtr) { + unsafe { libafl_qemu_sys::syx_snapshot_root_restore(snapshot) } + } + + pub fn check_fast_snapshot_memory_consistency(&self, snapshot: FastSnapshotPtr) -> u64 { + unsafe { libafl_qemu_sys::syx_snapshot_check_memory_consistency(snapshot) } + } + + pub fn list_devices(&self) -> Vec { + let mut r = vec![]; + unsafe { + let devices = libafl_qemu_sys::device_list_all(); + if devices.is_null() { + return r; + } + + let mut ptr = devices; + while !(*ptr).is_null() { + let c_str: &CStr = CStr::from_ptr(*ptr); + let name = c_str.to_str().unwrap().to_string(); + r.push(name); + + ptr = ptr.add(1); + } + + libc::free(devices as *mut c_void); + r + } + } +} + +impl Emulator +where + QT: QemuHelperTuple, + S: State + HasExecutions, + E: EmuExitHandler, +{ + /// Write a value to a phsical guest address, including ROM areas. + pub unsafe fn write_phys_mem(&self, paddr: GuestPhysAddr, buf: &[u8]) { + self.qemu.write_phys_mem(paddr, buf) + } + + /// Read a value from a physical guest address. + pub unsafe fn read_phys_mem(&self, paddr: GuestPhysAddr, buf: &mut [u8]) { + self.qemu.read_phys_mem(paddr, buf) + } + + pub fn save_snapshot(&self, name: &str, sync: bool) { + self.qemu.save_snapshot(name, sync) + } + + pub fn load_snapshot(&self, name: &str, sync: bool) { + self.qemu.load_snapshot(name, sync) + } + + #[must_use] + pub fn create_fast_snapshot(&self, track: bool) -> FastSnapshotPtr { + self.qemu.create_fast_snapshot(track) + } + + #[must_use] + pub fn create_fast_snapshot_filter( + &self, + track: bool, + device_filter: &DeviceSnapshotFilter, + ) -> FastSnapshotPtr { + self.qemu.create_fast_snapshot_filter(track, device_filter) + } + + pub fn restore_fast_snapshot(&self, snapshot: FastSnapshotPtr) { + self.qemu.restore_fast_snapshot(snapshot) + } + + pub fn check_fast_snapshot_memory_consistency(&self, snapshot: FastSnapshotPtr) -> u64 { + self.qemu.check_fast_snapshot_memory_consistency(snapshot) + } + + pub fn list_devices(&self) -> Vec { + self.qemu.list_devices() + } +} diff --git a/libafl_qemu/src/emu/usermode.rs b/libafl_qemu/src/emu/usermode.rs new file mode 100644 index 0000000000..427192e9ed --- /dev/null +++ b/libafl_qemu/src/emu/usermode.rs @@ -0,0 +1,496 @@ +use core::{mem::MaybeUninit, ptr::copy_nonoverlapping}; +use std::{cell::OnceCell, slice::from_raw_parts, str::from_utf8_unchecked}; + +use libafl_qemu_sys::{ + exec_path, free_self_maps, guest_base, libafl_dump_core_hook, libafl_force_dfl, libafl_get_brk, + libafl_load_addr, libafl_maps_first, libafl_maps_next, libafl_qemu_run, libafl_set_brk, + mmap_next_start, pageflags_get_root, read_self_maps, strlen, GuestAddr, GuestUsize, + IntervalTreeNode, IntervalTreeRoot, MapInfo, MmapPerms, VerifyAccess, +}; +use libc::c_int; +#[cfg(feature = "python")] +use pyo3::prelude::*; + +use crate::{ + emu::{HasExecutions, State}, + sync_exit::SyncBackdoorError, + EmuExitHandler, Emulator, HookData, NewThreadHookId, PostSyscallHookId, PreSyscallHookId, Qemu, + QemuExitReason, QemuExitReasonError, QemuHelperTuple, SyscallHookResult, CPU, +}; + +#[derive(Debug, Clone)] +pub enum HandlerError { + EmuExitReasonError(QemuExitReasonError), + SyncBackdoorError(SyncBackdoorError), + MultipleInputDefinition, +} + +#[cfg_attr(feature = "python", pyclass(unsendable))] +pub struct GuestMaps { + self_maps_root: *mut IntervalTreeRoot, + pageflags_node: *mut IntervalTreeNode, +} + +// Consider a private new only for Emulator +impl GuestMaps { + #[must_use] + pub(crate) fn new() -> Self { + unsafe { + let pageflags_root = pageflags_get_root(); + let self_maps_root = read_self_maps(); + let pageflags_first = libafl_maps_first(pageflags_root); + Self { + self_maps_root, + pageflags_node: pageflags_first, + } + } + } +} + +impl Iterator for GuestMaps { + type Item = MapInfo; + + #[allow(clippy::uninit_assumed_init)] + fn next(&mut self) -> Option { + unsafe { + let mut ret = MaybeUninit::uninit(); + + self.pageflags_node = + libafl_maps_next(self.pageflags_node, self.self_maps_root, ret.as_mut_ptr()); + + let ret = ret.assume_init(); + + if ret.is_valid { + Some(ret.into()) + } else { + None + } + } + } +} + +#[cfg(feature = "python")] +#[pymethods] +impl GuestMaps { + fn __iter__(slf: PyRef) -> PyRef { + slf + } + fn __next__(mut slf: PyRefMut) -> Option { + Python::with_gil(|py| slf.next().map(|x| x.into_py(py))) + } +} + +impl Drop for GuestMaps { + fn drop(&mut self) { + unsafe { + free_self_maps(self.self_maps_root); + } + } +} + +impl CPU { + /// Write a value to a guest address. + /// + /// # Safety + /// This will write to a translated guest address (using `g2h`). + /// It just adds `guest_base` and writes to that location, without checking the bounds. + /// This may only be safely used for valid guest addresses! + pub unsafe fn write_mem(&self, addr: GuestAddr, buf: &[u8]) { + let host_addr = Qemu::get().unwrap().g2h(addr); + copy_nonoverlapping(buf.as_ptr(), host_addr, buf.len()); + } + + /// Read a value from a guest address. + /// + /// # Safety + /// This will read from a translated guest address (using `g2h`). + /// It just adds `guest_base` and writes to that location, without checking the bounds. + /// This may only be safely used for valid guest addresses! + pub unsafe fn read_mem(&self, addr: GuestAddr, buf: &mut [u8]) { + let host_addr = Qemu::get().unwrap().g2h(addr); + copy_nonoverlapping(host_addr, buf.as_mut_ptr(), buf.len()); + } + + #[must_use] + pub fn page_size(&self) -> usize { + thread_local! { + static PAGE_SIZE: OnceCell = const { OnceCell::new() }; + } + + PAGE_SIZE.with(|s| { + *s.get_or_init(|| { + unsafe { libc::sysconf(libc::_SC_PAGE_SIZE) } + .try_into() + .expect("Invalid page size") + }) + }) + } +} + +#[allow(clippy::unused_self)] +impl Qemu { + #[must_use] + pub fn mappings(&self) -> GuestMaps { + GuestMaps::new() + } + + #[must_use] + pub fn g2h(&self, addr: GuestAddr) -> *mut T { + unsafe { (addr as usize + guest_base) as *mut T } + } + + #[must_use] + pub fn h2g(&self, addr: *const T) -> GuestAddr { + unsafe { (addr as usize - guest_base) as GuestAddr } + } + + #[must_use] + pub fn access_ok(&self, kind: VerifyAccess, addr: GuestAddr, size: usize) -> bool { + self.current_cpu() + .unwrap_or_else(|| self.cpu_from_index(0)) + .access_ok(kind, addr, size) + } + + pub fn force_dfl(&self) { + unsafe { + libafl_force_dfl = 1; + } + } + + /// This function will run the emulator until the next breakpoint, or until finish. + /// # Safety + /// + /// Should, in general, be safe to call. + /// Of course, the emulated target is not contained securely and can corrupt state or interact with the operating system. + pub unsafe fn run(&self) -> Result { + libafl_qemu_run(); + + self.post_run() + } + + #[must_use] + pub fn binary_path<'a>(&self) -> &'a str { + unsafe { from_utf8_unchecked(from_raw_parts(exec_path, strlen(exec_path))) } + } + + #[must_use] + pub fn load_addr(&self) -> GuestAddr { + unsafe { libafl_load_addr() as GuestAddr } + } + + #[must_use] + pub fn get_brk(&self) -> GuestAddr { + unsafe { libafl_get_brk() as GuestAddr } + } + + pub fn set_brk(&self, brk: GuestAddr) { + unsafe { libafl_set_brk(brk.into()) }; + } + + #[must_use] + pub fn get_mmap_start(&self) -> GuestAddr { + unsafe { mmap_next_start } + } + + pub fn set_mmap_start(&self, start: GuestAddr) { + unsafe { mmap_next_start = start }; + } + + #[allow(clippy::cast_sign_loss)] + fn mmap( + self, + addr: GuestAddr, + size: usize, + perms: MmapPerms, + flags: c_int, + ) -> Result { + let res = unsafe { + libafl_qemu_sys::target_mmap(addr, size as GuestUsize, perms.into(), flags, -1, 0) + }; + if res <= 0 { + Err(()) + } else { + Ok(res as GuestAddr) + } + } + + pub fn map_private( + &self, + addr: GuestAddr, + size: usize, + perms: MmapPerms, + ) -> Result { + self.mmap(addr, size, perms, libc::MAP_PRIVATE | libc::MAP_ANONYMOUS) + .map_err(|()| format!("Failed to map {addr}")) + .map(|addr| addr as GuestAddr) + } + + pub fn map_fixed( + &self, + addr: GuestAddr, + size: usize, + perms: MmapPerms, + ) -> Result { + self.mmap( + addr, + size, + perms, + libc::MAP_FIXED | libc::MAP_PRIVATE | libc::MAP_ANONYMOUS, + ) + .map_err(|()| format!("Failed to map {addr}")) + .map(|addr| addr as GuestAddr) + } + + pub fn mprotect(&self, addr: GuestAddr, size: usize, perms: MmapPerms) -> Result<(), String> { + let res = unsafe { + libafl_qemu_sys::target_mprotect(addr.into(), size as GuestUsize, perms.into()) + }; + if res == 0 { + Ok(()) + } else { + Err(format!("Failed to mprotect {addr}")) + } + } + + pub fn unmap(&self, addr: GuestAddr, size: usize) -> Result<(), String> { + if unsafe { libafl_qemu_sys::target_munmap(addr.into(), size as GuestUsize) } == 0 { + Ok(()) + } else { + Err(format!("Failed to unmap {addr}")) + } + } + + #[allow(clippy::type_complexity)] + pub fn add_pre_syscall_hook>( + &self, + data: T, + callback: extern "C" fn( + T, + i32, + GuestAddr, + GuestAddr, + GuestAddr, + GuestAddr, + GuestAddr, + GuestAddr, + GuestAddr, + GuestAddr, + ) -> SyscallHookResult, + ) -> PreSyscallHookId { + unsafe { + let data: u64 = data.into().0; + let callback: extern "C" fn( + u64, + i32, + GuestAddr, + GuestAddr, + GuestAddr, + GuestAddr, + GuestAddr, + GuestAddr, + GuestAddr, + GuestAddr, + ) -> libafl_qemu_sys::syshook_ret = core::mem::transmute(callback); + let num = libafl_qemu_sys::libafl_add_pre_syscall_hook(Some(callback), data); + PreSyscallHookId(num) + } + } + + #[allow(clippy::type_complexity)] + pub fn add_post_syscall_hook>( + &self, + data: T, + callback: extern "C" fn( + T, + GuestAddr, + i32, + GuestAddr, + GuestAddr, + GuestAddr, + GuestAddr, + GuestAddr, + GuestAddr, + GuestAddr, + GuestAddr, + ) -> GuestAddr, + ) -> PostSyscallHookId { + unsafe { + let data: u64 = data.into().0; + let callback: extern "C" fn( + u64, + GuestAddr, + i32, + GuestAddr, + GuestAddr, + GuestAddr, + GuestAddr, + GuestAddr, + GuestAddr, + GuestAddr, + GuestAddr, + ) -> GuestAddr = core::mem::transmute(callback); + let num = libafl_qemu_sys::libafl_add_post_syscall_hook(Some(callback), data); + PostSyscallHookId(num) + } + } + + pub fn add_new_thread_hook>( + &self, + data: T, + callback: extern "C" fn(T, tid: u32) -> bool, + ) -> NewThreadHookId { + unsafe { + let data: u64 = data.into().0; + let callback: extern "C" fn(u64, u32) -> bool = core::mem::transmute(callback); + let num = libafl_qemu_sys::libafl_add_new_thread_hook(Some(callback), data); + NewThreadHookId(num) + } + } + + #[allow(clippy::type_complexity)] + pub fn set_crash_hook(&self, callback: extern "C" fn(i32)) { + unsafe { + libafl_dump_core_hook = callback; + } + } +} + +impl Emulator +where + QT: QemuHelperTuple, + S: State + HasExecutions, + E: EmuExitHandler, +{ + /// This function gets the memory mappings from the emulator. + #[must_use] + pub fn mappings(&self) -> GuestMaps { + self.qemu.mappings() + } + + #[must_use] + pub fn g2h(&self, addr: GuestAddr) -> *mut T { + self.qemu.g2h(addr) + } + + #[must_use] + pub fn h2g(&self, addr: *const T) -> GuestAddr { + self.qemu.h2g(addr) + } + + #[must_use] + pub fn access_ok(&self, kind: VerifyAccess, addr: GuestAddr, size: usize) -> bool { + self.qemu.access_ok(kind, addr, size) + } + + pub fn force_dfl(&self) { + self.qemu.force_dfl(); + } + + #[must_use] + pub fn binary_path<'a>(&self) -> &'a str { + self.qemu.binary_path() + } + + #[must_use] + pub fn load_addr(&self) -> GuestAddr { + self.qemu.load_addr() + } + + #[must_use] + pub fn get_brk(&self) -> GuestAddr { + self.qemu.get_brk() + } + + pub fn set_brk(&self, brk: GuestAddr) { + self.qemu.set_brk(brk); + } + + #[must_use] + pub fn get_mmap_start(&self) -> GuestAddr { + self.qemu.get_mmap_start() + } + + pub fn set_mmap_start(&self, start: GuestAddr) { + self.qemu.set_mmap_start(start); + } + + pub fn map_private( + &self, + addr: GuestAddr, + size: usize, + perms: MmapPerms, + ) -> Result { + self.qemu.map_private(addr, size, perms) + } + + pub fn map_fixed( + &self, + addr: GuestAddr, + size: usize, + perms: MmapPerms, + ) -> Result { + self.qemu.map_fixed(addr, size, perms) + } + + pub fn mprotect(&self, addr: GuestAddr, size: usize, perms: MmapPerms) -> Result<(), String> { + self.qemu.mprotect(addr, size, perms) + } + + pub fn unmap(&self, addr: GuestAddr, size: usize) -> Result<(), String> { + self.qemu.unmap(addr, size) + } + + #[allow(clippy::type_complexity)] + pub fn add_pre_syscall_hook>( + &self, + data: T, + callback: extern "C" fn( + T, + i32, + GuestAddr, + GuestAddr, + GuestAddr, + GuestAddr, + GuestAddr, + GuestAddr, + GuestAddr, + GuestAddr, + ) -> SyscallHookResult, + ) -> PreSyscallHookId { + self.qemu.add_pre_syscall_hook(data, callback) + } + + #[allow(clippy::type_complexity)] + pub fn add_post_syscall_hook>( + &self, + data: T, + callback: extern "C" fn( + T, + GuestAddr, + i32, + GuestAddr, + GuestAddr, + GuestAddr, + GuestAddr, + GuestAddr, + GuestAddr, + GuestAddr, + GuestAddr, + ) -> GuestAddr, + ) -> PostSyscallHookId { + self.qemu.add_post_syscall_hook(data, callback) + } + + pub fn add_new_thread_hook>( + &self, + data: T, + callback: extern "C" fn(T, tid: u32) -> bool, + ) -> NewThreadHookId { + self.qemu.add_new_thread_hook(data, callback) + } + + #[allow(clippy::type_complexity)] + pub fn set_crash_hook(&self, callback: extern "C" fn(i32)) { + self.qemu.set_crash_hook(callback); + } +} diff --git a/libafl_qemu/src/executor.rs b/libafl_qemu/src/executor/mod.rs similarity index 56% rename from libafl_qemu/src/executor.rs rename to libafl_qemu/src/executor/mod.rs index f1eff488b6..d6cd36f76e 100644 --- a/libafl_qemu/src/executor.rs +++ b/libafl_qemu/src/executor/mod.rs @@ -1,4 +1,6 @@ //! A `QEMU`-based executor for binary-only instrumentation in `LibAFL` +#[cfg(emulation_mode = "usermode")] +use core::ptr; use core::{ ffi::c_void, fmt::{self, Debug, Formatter}, @@ -6,13 +8,7 @@ use core::{ }; #[cfg(feature = "fork")] -use libafl::inputs::UsesInput; -#[cfg(feature = "fork")] -use libafl::{ - events::EventManager, - executors::InProcessForkExecutor, - state::{HasLastReportTime, HasMetadata}, -}; +use libafl::{events::EventManager, executors::InProcessForkExecutor, state::HasLastReportTime}; use libafl::{ events::{EventFirer, EventRestarter}, executors::{ @@ -24,36 +20,50 @@ use libafl::{ fuzzer::HasObjective, observers::{ObserversTuple, UsesObservers}, state::{HasCorpus, HasExecutions, HasSolutions, State, UsesState}, - Error, + Error, HasMetadata, }; -use libafl_bolts::os::unix_signals::{siginfo_t, ucontext_t, Signal}; #[cfg(feature = "fork")] use libafl_bolts::shmem::ShMemProvider; +use libafl_bolts::{ + os::unix_signals::{siginfo_t, ucontext_t, Signal}, + tuples::RefIndexable, +}; -use crate::{emu::Emulator, helper::QemuHelperTuple, hooks::QemuHooks}; +use crate::{helpers::QemuHelperTuple, hooks::QemuHooks, Qemu}; + +/// A version of `QemuExecutor` with a state accessible from the harness. +pub mod stateful; + +pub struct QemuExecutorState<'a, QT, S> +where + QT: QemuHelperTuple, + S: State + HasExecutions, +{ + hooks: &'a mut QemuHooks, + first_exec: bool, +} pub struct QemuExecutor<'a, H, OT, QT, S> where H: FnMut(&S::Input) -> ExitKind, - S: State, + S: State + HasExecutions, OT: ObserversTuple, QT: QemuHelperTuple, { inner: InProcessExecutor<'a, H, OT, S>, - hooks: &'a mut QemuHooks, - first_exec: bool, + state: QemuExecutorState<'a, QT, S>, } impl<'a, H, OT, QT, S> Debug for QemuExecutor<'a, H, OT, QT, S> where H: FnMut(&S::Input) -> ExitKind, - S: State, + S: State + HasExecutions, OT: ObserversTuple + Debug, QT: QemuHelperTuple + Debug, { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { f.debug_struct("QemuExecutor") - .field("hooks", &self.hooks) + .field("hooks", &self.state.hooks) .field("inner", &self.inner) .finish() } @@ -66,27 +76,29 @@ extern "C" { } #[cfg(emulation_mode = "usermode")] -pub unsafe fn inproc_qemu_crash_handler( +pub unsafe fn inproc_qemu_crash_handler<'a, E, EM, OF, Z, QT, S>( signal: Signal, - info: &mut siginfo_t, - mut context: Option<&mut ucontext_t>, - _data: &mut InProcessExecutorHandlerData, + info: &'a mut siginfo_t, + mut context: Option<&'a mut ucontext_t>, + _data: &'a mut InProcessExecutorHandlerData, ) where E: Executor + HasObservers, EM: EventFirer + EventRestarter, OF: Feedback, E::State: HasExecutions + HasSolutions + HasCorpus, Z: HasObjective, + QT: QemuHelperTuple + Debug + 'a, + S: State + HasExecutions + 'a, { let puc = match &mut context { - Some(v) => core::ptr::from_mut::(*v) as *mut c_void, - None => core::ptr::null_mut(), + Some(v) => ptr::from_mut::(*v) as *mut c_void, + None => ptr::null_mut(), }; libafl_qemu_handle_crash(signal as i32, info, puc); } #[cfg(emulation_mode = "systemmode")] -static mut BREAK_ON_TMOUT: bool = false; +pub(crate) static mut BREAK_ON_TMOUT: bool = false; #[cfg(emulation_mode = "systemmode")] extern "C" { @@ -94,13 +106,13 @@ extern "C" { } #[cfg(emulation_mode = "systemmode")] -pub unsafe fn inproc_qemu_timeout_handler( +pub unsafe fn inproc_qemu_timeout_handler<'a, E, EM, OF, Z>( signal: Signal, - info: &mut siginfo_t, - context: Option<&mut ucontext_t>, - data: &mut InProcessExecutorHandlerData, + info: &'a mut siginfo_t, + context: Option<&'a mut ucontext_t>, + data: &'a mut InProcessExecutorHandlerData, ) where - E: Executor + HasObservers + HasInProcessHooks, + E: Executor + HasObservers + HasInProcessHooks, EM: EventFirer + EventRestarter, OF: Feedback, E::State: HasSolutions + HasCorpus + HasExecutions, @@ -115,12 +127,61 @@ pub unsafe fn inproc_qemu_timeout_handler( } } +impl<'a, QT, S> QemuExecutorState<'a, QT, S> +where + S: State + HasExecutions, + QT: QemuHelperTuple + Debug, +{ + pub fn new(hooks: &'a mut QemuHooks) -> Result + where + E: Executor + HasInProcessHooks + HasObservers, + EM: EventFirer + EventRestarter, + OF: Feedback, + OT: ObserversTuple, + S: State + HasExecutions + HasCorpus + HasSolutions, + Z: HasObjective, + { + #[cfg(emulation_mode = "usermode")] + { + let handler = |hooks: &mut QemuHooks, host_sig| { + eprintln!("Crashed with signal {host_sig}"); + unsafe { + libafl::executors::inprocess::generic_inproc_crash_handler::(); + } + if let Some(cpu) = hooks.qemu().current_cpu() { + eprint!("Context:\n{}", cpu.display_context()); + } + }; + + hooks.crash_closure(Box::new(handler)); + } + Ok(QemuExecutorState { + first_exec: true, + hooks, + }) + } + + #[must_use] + pub fn hooks(&self) -> &QemuHooks { + self.hooks + } + + pub fn hooks_mut(&mut self) -> &mut QemuHooks { + self.hooks + } + + #[must_use] + pub fn qemu(&self) -> &Qemu { + self.hooks.qemu() + } +} + impl<'a, H, OT, QT, S> QemuExecutor<'a, H, OT, QT, S> where H: FnMut(&S::Input) -> ExitKind, - S: State, + S: State + HasExecutions, OT: ObserversTuple, - QT: QemuHelperTuple, + QT: QemuHelperTuple + Debug, { pub fn new( hooks: &'a mut QemuHooks, @@ -140,40 +201,25 @@ where let mut inner = InProcessExecutor::with_timeout( harness_fn, observers, fuzzer, state, event_mgr, timeout, )?; + #[cfg(emulation_mode = "usermode")] { inner.inprocess_hooks_mut().crash_handler = - inproc_qemu_crash_handler::, EM, OF, Z> + inproc_qemu_crash_handler::, EM, OF, Z, QT, S> as *const c_void; - - let handler = |hooks: &mut QemuHooks, host_sig| { - eprintln!("Crashed with signal {host_sig}"); - unsafe { - libafl::executors::inprocess::generic_inproc_crash_handler::< - InProcessExecutor<'a, H, OT, S>, - EM, - OF, - Z, - >(); - } - if let Some(cpu) = hooks.emulator().current_cpu() { - eprint!("Context:\n{}", cpu.display_context()); - } - }; - - hooks.crash_closure(Box::new(handler)); } + #[cfg(emulation_mode = "systemmode")] { inner.inprocess_hooks_mut().timeout_handler = inproc_qemu_timeout_handler::, EM, OF, Z> as *const c_void; } - Ok(Self { - first_exec: true, - hooks, - inner, - }) + + let state = + QemuExecutorState::new::, EM, OF, OT, Z>(hooks)?; + + Ok(Self { inner, state }) } pub fn inner(&self) -> &InProcessExecutor<'a, H, OT, S> { @@ -192,26 +238,65 @@ where } pub fn hooks(&self) -> &QemuHooks { - self.hooks + self.state.hooks() } pub fn hooks_mut(&mut self) -> &mut QemuHooks { - self.hooks + self.state.hooks_mut() } - pub fn emulator(&self) -> &Emulator { - self.hooks.emulator() + pub fn emulator(&self) -> &Qemu { + self.state.qemu() } } -impl<'a, EM, H, OT, QT, S, Z> Executor for QemuExecutor<'a, H, OT, QT, S> +impl<'a, QT, S> QemuExecutorState<'a, QT, S> where - EM: UsesState, + S: State + HasExecutions + HasCorpus + HasSolutions, + QT: QemuHelperTuple + Debug, +{ + fn pre_exec(&mut self, input: &E::Input, qemu: Qemu) + where + E: Executor, + EM: EventFirer + EventRestarter, + OF: Feedback, + Z: HasObjective, + { + if self.first_exec { + self.hooks.helpers().first_exec_all(self.hooks); + self.first_exec = false; + } + self.hooks.helpers_mut().pre_exec_all(qemu, input); + } + + fn post_exec( + &mut self, + input: &E::Input, + qemu: Qemu, + observers: &mut OT, + exit_kind: &mut ExitKind, + ) where + E: Executor + HasObservers, + EM: EventFirer + EventRestarter, + OT: ObserversTuple, + OF: Feedback, + Z: HasObjective, + { + self.hooks + .helpers_mut() + .post_exec_all(qemu, input, observers, exit_kind); + } +} + +impl<'a, EM, H, OT, OF, QT, S, Z> Executor for QemuExecutor<'a, H, OT, QT, S> +where + EM: EventFirer + EventRestarter, H: FnMut(&S::Input) -> ExitKind, - S: State + HasExecutions, + S: State + HasExecutions + HasCorpus + HasSolutions, OT: ObserversTuple, - QT: QemuHelperTuple, - Z: UsesState, + OF: Feedback, + QT: QemuHelperTuple + Debug, + Z: HasObjective, { fn run_target( &mut self, @@ -220,17 +305,13 @@ where mgr: &mut EM, input: &Self::Input, ) -> Result { - let emu = Emulator::get().unwrap(); - if self.first_exec { - self.hooks.helpers().first_exec_all(self.hooks); - self.first_exec = false; - } - self.hooks.helpers_mut().pre_exec_all(&emu, input); + let qemu = Qemu::get().unwrap(); + self.state.pre_exec::(input, qemu); let mut exit_kind = self.inner.run_target(fuzzer, state, mgr, input)?; - self.hooks.helpers_mut().post_exec_all( - &emu, + self.state.post_exec::( input, - self.inner.observers_mut(), + qemu, + &mut *self.inner.observers_mut(), &mut exit_kind, ); Ok(exit_kind) @@ -242,7 +323,7 @@ where H: FnMut(&S::Input) -> ExitKind, OT: ObserversTuple, QT: QemuHelperTuple, - S: State, + S: State + HasExecutions, { type State = S; } @@ -252,7 +333,7 @@ where H: FnMut(&S::Input) -> ExitKind, OT: ObserversTuple, QT: QemuHelperTuple, - S: State, + S: State + HasExecutions, { type Observers = OT; } @@ -260,62 +341,69 @@ where impl<'a, H, OT, QT, S> HasObservers for QemuExecutor<'a, H, OT, QT, S> where H: FnMut(&S::Input) -> ExitKind, - S: State, + S: State + HasExecutions, OT: ObserversTuple, QT: QemuHelperTuple, { #[inline] - fn observers(&self) -> &OT { + fn observers(&self) -> RefIndexable<&Self::Observers, Self::Observers> { self.inner.observers() } #[inline] - fn observers_mut(&mut self) -> &mut OT { + fn observers_mut(&mut self) -> RefIndexable<&mut Self::Observers, Self::Observers> { self.inner.observers_mut() } } #[cfg(feature = "fork")] -pub struct QemuForkExecutor<'a, H, OT, QT, S, SP> +pub struct QemuForkExecutor<'a, H, OT, QT, S, SP, EM, Z> where H: FnMut(&S::Input) -> ExitKind, - S: UsesInput, + S: State + HasExecutions, OT: ObserversTuple, QT: QemuHelperTuple, SP: ShMemProvider, + EM: UsesState, + Z: UsesState, { - first_exec: bool, - hooks: &'a mut QemuHooks, - inner: InProcessForkExecutor<'a, H, OT, S, SP>, + inner: InProcessForkExecutor<'a, H, OT, S, SP, EM, Z>, + state: QemuExecutorState<'a, QT, S>, } #[cfg(feature = "fork")] -impl<'a, H, OT, QT, S, SP> Debug for QemuForkExecutor<'a, H, OT, QT, S, SP> +impl<'a, H, OT, QT, S, SP, EM, Z> Debug for QemuForkExecutor<'a, H, OT, QT, S, SP, EM, Z> where H: FnMut(&S::Input) -> ExitKind, - S: UsesInput, + S: State + HasExecutions, OT: ObserversTuple + Debug, QT: QemuHelperTuple + Debug, SP: ShMemProvider, + EM: UsesState, + Z: UsesState, { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { f.debug_struct("QemuForkExecutor") - .field("hooks", &self.hooks) + .field("hooks", &self.state.hooks) .field("inner", &self.inner) .finish() } } #[cfg(feature = "fork")] -impl<'a, H, OT, QT, S, SP> QemuForkExecutor<'a, H, OT, QT, S, SP> +impl<'a, H, OT, QT, S, SP, EM, Z, OF> QemuForkExecutor<'a, H, OT, QT, S, SP, EM, Z> where H: FnMut(&S::Input) -> ExitKind, - S: State, + S: State + HasExecutions, OT: ObserversTuple, QT: QemuHelperTuple, SP: ShMemProvider, + EM: EventFirer + EventRestarter, + OF: Feedback, + S: HasSolutions, + Z: HasObjective, { - pub fn new( + pub fn new( hooks: &'a mut QemuHooks, harness_fn: &'a mut H, observers: OT, @@ -324,18 +412,10 @@ where event_mgr: &mut EM, shmem_provider: SP, timeout: core::time::Duration, - ) -> Result - where - EM: EventFirer + EventRestarter, - OF: Feedback, - S: HasSolutions, - Z: HasObjective, - { + ) -> Result { assert!(!QT::HOOKS_DO_SIDE_EFFECTS, "When using QemuForkExecutor, the hooks must not do any side effect as they will happen in the child process and then discarded"); Ok(Self { - first_exec: true, - hooks, inner: InProcessForkExecutor::new( harness_fn, observers, @@ -345,40 +425,46 @@ where timeout, shmem_provider, )?, + state: QemuExecutorState { + first_exec: true, + hooks, + }, }) } - pub fn inner(&self) -> &InProcessForkExecutor<'a, H, OT, S, SP> { + pub fn inner(&self) -> &InProcessForkExecutor<'a, H, OT, S, SP, EM, Z> { &self.inner } - pub fn inner_mut(&mut self) -> &mut InProcessForkExecutor<'a, H, OT, S, SP> { + pub fn inner_mut(&mut self) -> &mut InProcessForkExecutor<'a, H, OT, S, SP, EM, Z> { &mut self.inner } pub fn hooks(&self) -> &QemuHooks { - self.hooks + self.state.hooks } pub fn hooks_mut(&mut self) -> &mut QemuHooks { - self.hooks + self.state.hooks } - pub fn emulator(&self) -> &Emulator { - self.hooks.emulator() + pub fn qemu(&self) -> &Qemu { + self.state.hooks.qemu() } } #[cfg(feature = "fork")] -impl<'a, EM, H, OT, QT, S, Z, SP> Executor for QemuForkExecutor<'a, H, OT, QT, S, SP> +impl<'a, EM, H, OT, QT, S, Z, SP, OF> Executor + for QemuForkExecutor<'a, H, OT, QT, S, SP, EM, Z> where - EM: EventManager, Z, State = S>, + EM: EventManager, Z, State = S>, H: FnMut(&S::Input) -> ExitKind, - S: State + HasMetadata + HasExecutions + HasLastReportTime, - OT: ObserversTuple, + S: State + HasMetadata + HasExecutions + HasLastReportTime + HasCorpus + HasSolutions, + OT: ObserversTuple + Debug, QT: QemuHelperTuple, SP: ShMemProvider, - Z: UsesState, + OF: Feedback, + Z: HasObjective, { fn run_target( &mut self, @@ -387,17 +473,17 @@ where mgr: &mut EM, input: &Self::Input, ) -> Result { - let emu = Emulator::get().unwrap(); - if self.first_exec { - self.hooks.helpers().first_exec_all(self.hooks); - self.first_exec = false; + let qemu = *self.state.hooks.qemu(); + if self.state.first_exec { + self.state.hooks.helpers().first_exec_all(self.state.hooks); + self.state.first_exec = false; } - self.hooks.helpers_mut().pre_exec_all(&emu, input); + self.state.hooks.helpers_mut().pre_exec_all(qemu, input); let mut exit_kind = self.inner.run_target(fuzzer, state, mgr, input)?; - self.hooks.helpers_mut().post_exec_all( - &emu, + self.state.hooks.helpers_mut().post_exec_all( + qemu, input, - self.inner.observers_mut(), + &mut *self.inner.observers_mut(), &mut exit_kind, ); Ok(exit_kind) @@ -405,45 +491,51 @@ where } #[cfg(feature = "fork")] -impl<'a, H, OT, QT, S, SP> UsesObservers for QemuForkExecutor<'a, H, OT, QT, S, SP> +impl<'a, H, OT, QT, S, SP, EM, Z> UsesObservers for QemuForkExecutor<'a, H, OT, QT, S, SP, EM, Z> where H: FnMut(&S::Input) -> ExitKind, OT: ObserversTuple, QT: QemuHelperTuple, - S: State, + S: State + HasExecutions, SP: ShMemProvider, + EM: UsesState, + Z: UsesState, { type Observers = OT; } #[cfg(feature = "fork")] -impl<'a, H, OT, QT, S, SP> UsesState for QemuForkExecutor<'a, H, OT, QT, S, SP> +impl<'a, H, OT, QT, S, SP, EM, Z> UsesState for QemuForkExecutor<'a, H, OT, QT, S, SP, EM, Z> where H: FnMut(&S::Input) -> ExitKind, OT: ObserversTuple, QT: QemuHelperTuple, - S: State, + S: State + HasExecutions, SP: ShMemProvider, + EM: UsesState, + Z: UsesState, { type State = S; } #[cfg(feature = "fork")] -impl<'a, H, OT, QT, S, SP> HasObservers for QemuForkExecutor<'a, H, OT, QT, S, SP> +impl<'a, H, OT, QT, S, SP, EM, Z> HasObservers for QemuForkExecutor<'a, H, OT, QT, S, SP, EM, Z> where H: FnMut(&S::Input) -> ExitKind, - S: State, + S: State + HasExecutions, OT: ObserversTuple, QT: QemuHelperTuple, SP: ShMemProvider, + EM: UsesState, + Z: UsesState, { #[inline] - fn observers(&self) -> &OT { + fn observers(&self) -> RefIndexable<&Self::Observers, Self::Observers> { self.inner.observers() } #[inline] - fn observers_mut(&mut self) -> &mut OT { + fn observers_mut(&mut self) -> RefIndexable<&mut Self::Observers, Self::Observers> { self.inner.observers_mut() } } diff --git a/libafl_qemu/src/executor/stateful.rs b/libafl_qemu/src/executor/stateful.rs new file mode 100644 index 0000000000..145bb38576 --- /dev/null +++ b/libafl_qemu/src/executor/stateful.rs @@ -0,0 +1,211 @@ +//! A `QEMU`-based executor for binary-only instrumentation in `LibAFL` +use core::{ + ffi::c_void, + fmt::{self, Debug, Formatter}, + time::Duration, +}; + +use libafl::{ + events::{EventFirer, EventRestarter}, + executors::{ + inprocess::{stateful::StatefulInProcessExecutor, HasInProcessHooks}, + Executor, ExitKind, HasObservers, + }, + feedbacks::Feedback, + fuzzer::HasObjective, + observers::{ObserversTuple, UsesObservers}, + state::{HasCorpus, HasExecutions, HasSolutions, State, UsesState}, + Error, +}; +use libafl_bolts::tuples::RefIndexable; + +#[cfg(emulation_mode = "usermode")] +use crate::executor::inproc_qemu_crash_handler; +#[cfg(emulation_mode = "systemmode")] +use crate::executor::{inproc_qemu_timeout_handler, BREAK_ON_TMOUT}; +use crate::{executor::QemuExecutorState, helpers::QemuHelperTuple, hooks::QemuHooks, Qemu}; + +pub struct StatefulQemuExecutor<'a, H, OT, QT, S> +where + H: FnMut(&S::Input, &mut QemuExecutorState<'a, QT, S>) -> ExitKind, + S: State + HasExecutions, + OT: ObserversTuple, + QT: QemuHelperTuple, +{ + inner: StatefulInProcessExecutor<'a, H, OT, S, QemuExecutorState<'a, QT, S>>, +} + +impl<'a, H, OT, QT, S> Debug for StatefulQemuExecutor<'a, H, OT, QT, S> +where + H: FnMut(&S::Input, &mut QemuExecutorState<'a, QT, S>) -> ExitKind, + S: State + HasExecutions, + OT: ObserversTuple + Debug, + QT: QemuHelperTuple + Debug, +{ + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct("QemuExecutor") + .field("inner", &self.inner) + .finish() + } +} + +impl<'a, H, OT, QT, S> StatefulQemuExecutor<'a, H, OT, QT, S> +where + H: FnMut(&S::Input, &mut QemuExecutorState<'a, QT, S>) -> ExitKind, + S: State + HasExecutions, + OT: ObserversTuple, + QT: QemuHelperTuple + Debug, +{ + pub fn new( + hooks: &'a mut QemuHooks, + harness_fn: &'a mut H, + observers: OT, + fuzzer: &mut Z, + state: &mut S, + event_mgr: &mut EM, + timeout: Duration, + ) -> Result + where + EM: EventFirer + EventRestarter, + OF: Feedback, + S: State + HasExecutions + HasCorpus + HasSolutions, + Z: HasObjective, + { + let qemu_state = QemuExecutorState::new::< + StatefulInProcessExecutor<'a, H, OT, S, QemuExecutorState<'a, QT, S>>, + EM, + OF, + OT, + Z, + >(hooks)?; + + let mut inner = StatefulInProcessExecutor::with_timeout( + harness_fn, qemu_state, observers, fuzzer, state, event_mgr, timeout, + )?; + + #[cfg(emulation_mode = "usermode")] + { + inner.inprocess_hooks_mut().crash_handler = inproc_qemu_crash_handler::< + StatefulInProcessExecutor<'a, H, OT, S, QemuExecutorState<'a, QT, S>>, + EM, + OF, + Z, + QT, + S, + > as *const c_void; + } + + #[cfg(emulation_mode = "systemmode")] + { + inner.inprocess_hooks_mut().timeout_handler = inproc_qemu_timeout_handler::< + StatefulInProcessExecutor<'a, H, OT, S, QemuExecutorState<'a, QT, S>>, + EM, + OF, + Z, + > as *const c_void; + } + + Ok(Self { inner }) + } + + pub fn inner(&self) -> &StatefulInProcessExecutor<'a, H, OT, S, QemuExecutorState<'a, QT, S>> { + &self.inner + } + + #[cfg(emulation_mode = "systemmode")] + pub fn break_on_timeout(&mut self) { + unsafe { + BREAK_ON_TMOUT = true; + } + } + + pub fn inner_mut( + &mut self, + ) -> &mut StatefulInProcessExecutor<'a, H, OT, S, QemuExecutorState<'a, QT, S>> { + &mut self.inner + } + + pub fn hooks(&self) -> &QemuHooks { + self.inner.exposed_executor_state().hooks() + } + + pub fn hooks_mut(&mut self) -> &mut QemuHooks { + self.inner.exposed_executor_state_mut().hooks_mut() + } + + pub fn emulator(&self) -> &Qemu { + self.inner.exposed_executor_state().qemu() + } +} + +impl<'a, EM, H, OT, OF, QT, S, Z> Executor for StatefulQemuExecutor<'a, H, OT, QT, S> +where + EM: EventFirer + EventRestarter, + H: FnMut(&S::Input, &mut QemuExecutorState<'a, QT, S>) -> ExitKind, + S: State + HasExecutions + HasCorpus + HasSolutions, + OT: ObserversTuple, + OF: Feedback, + QT: QemuHelperTuple + Debug, + Z: HasObjective, +{ + fn run_target( + &mut self, + fuzzer: &mut Z, + state: &mut Self::State, + mgr: &mut EM, + input: &Self::Input, + ) -> Result { + let qemu = Qemu::get().unwrap(); + self.inner + .exposed_executor_state_mut() + .pre_exec::(input, qemu); + let mut exit_kind = self.inner.run_target(fuzzer, state, mgr, input)?; + self.inner + .exposed_executor_state + .post_exec::( + input, + qemu, + &mut *self.inner.inner.observers_mut(), + &mut exit_kind, + ); + Ok(exit_kind) + } +} + +impl<'a, H, OT, QT, S> UsesState for StatefulQemuExecutor<'a, H, OT, QT, S> +where + H: FnMut(&S::Input, &mut QemuExecutorState<'a, QT, S>) -> ExitKind, + OT: ObserversTuple, + QT: QemuHelperTuple, + S: State + HasExecutions, +{ + type State = S; +} + +impl<'a, H, OT, QT, S> UsesObservers for StatefulQemuExecutor<'a, H, OT, QT, S> +where + H: FnMut(&S::Input, &mut QemuExecutorState<'a, QT, S>) -> ExitKind, + OT: ObserversTuple, + QT: QemuHelperTuple, + S: State + HasExecutions, +{ + type Observers = OT; +} + +impl<'a, H, OT, QT, S> HasObservers for StatefulQemuExecutor<'a, H, OT, QT, S> +where + H: FnMut(&S::Input, &mut QemuExecutorState<'a, QT, S>) -> ExitKind, + S: State + HasExecutions, + OT: ObserversTuple, + QT: QemuHelperTuple, +{ + #[inline] + fn observers(&self) -> RefIndexable<&Self::Observers, Self::Observers> { + self.inner.observers() + } + + #[inline] + fn observers_mut(&mut self) -> RefIndexable<&mut Self::Observers, Self::Observers> { + self.inner.observers_mut() + } +} diff --git a/libafl_qemu/src/asan.rs b/libafl_qemu/src/helpers/asan.rs similarity index 80% rename from libafl_qemu/src/asan.rs rename to libafl_qemu/src/helpers/asan.rs index 3da6ff4f13..c738e9bf53 100644 --- a/libafl_qemu/src/asan.rs +++ b/libafl_qemu/src/helpers/asan.rs @@ -8,9 +8,7 @@ use std::{ }; use addr2line::object::{Object, ObjectSection}; -use libafl::{ - executors::ExitKind, inputs::UsesInput, observers::ObserversTuple, state::HasMetadata, -}; +use libafl::{executors::ExitKind, inputs::UsesInput, observers::ObserversTuple, HasMetadata}; use libc::{ c_void, MAP_ANON, MAP_FAILED, MAP_FIXED, MAP_NORESERVE, MAP_PRIVATE, PROT_READ, PROT_WRITE, }; @@ -19,15 +17,15 @@ use num_enum::{IntoPrimitive, TryFromPrimitive}; use rangemap::RangeMap; use crate::{ - calls::FullBacktraceCollector, - emu::{EmuError, Emulator, MemAccessInfo, SyscallHookResult}, - helper::{ - HasInstrumentationFilter, IsFilter, QemuHelper, QemuHelperTuple, - QemuInstrumentationAddressRangeFilter, + emu::{EmuError, MemAccessInfo, SyscallHookResult}, + helpers::{ + calls::FullBacktraceCollector, HasInstrumentationFilter, IsFilter, QemuHelper, + QemuHelperTuple, QemuInstrumentationAddressRangeFilter, }, hooks::{Hook, QemuHooks}, snapshot::QemuSnapshotHelper, - GuestAddr, Regs, + sys::TCGTemp, + GuestAddr, Qemu, Regs, }; // TODO at some point, merge parts with libafl_frida @@ -131,7 +129,7 @@ impl core::fmt::Display for AsanError { } } -pub type AsanErrorCallback = Box; +pub type AsanErrorCallback = Box; #[derive(Debug, Clone)] pub struct AllocTreeItem { @@ -209,7 +207,7 @@ impl AsanGiovese { } #[must_use] - fn new(emu: &Emulator) -> Pin> { + fn new(emu: Qemu) -> Pin> { let res = Self { alloc_tree: Mutex::new(IntervalTree::new()), saved_tree: IntervalTree::new(), @@ -237,34 +235,34 @@ impl AsanGiovese { ) -> SyscallHookResult { if sys_num == QASAN_FAKESYS_NR { let mut r = 0; - let emulator = Emulator::get().unwrap(); + let qemu = Qemu::get().unwrap(); match QasanAction::try_from(a0).expect("Invalid QASan action number") { QasanAction::Poison => { self.poison( - &emulator, + qemu, a1, a2 as usize, PoisonKind::try_from(a3 as i8).unwrap().into(), ); } QasanAction::UserPoison => { - self.poison(&emulator, a1, a2 as usize, PoisonKind::User.into()); + self.poison(qemu, a1, a2 as usize, PoisonKind::User.into()); } QasanAction::UnPoison => { - Self::unpoison(&emulator, a1, a2 as usize); + Self::unpoison(qemu, a1, a2 as usize); } QasanAction::IsPoison => { - if Self::is_invalid_access(&emulator, a1, a2 as usize) { + if Self::is_invalid_access(qemu, a1, a2 as usize) { r = 1; } } QasanAction::Alloc => { - let pc: GuestAddr = emulator.read_reg(Regs::Pc).unwrap(); + let pc: GuestAddr = qemu.read_reg(Regs::Pc).unwrap(); self.allocation(pc, a1, a2); } QasanAction::Dealloc => { - let pc: GuestAddr = emulator.read_reg(Regs::Pc).unwrap(); - self.deallocation(&emulator, pc, a1); + let pc: GuestAddr = qemu.read_reg(Regs::Pc).unwrap(); + self.deallocation(qemu, pc, a1); } _ => (), } @@ -284,9 +282,9 @@ impl AsanGiovese { #[inline] #[must_use] - pub fn is_invalid_access_1(emu: &Emulator, addr: GuestAddr) -> bool { + pub fn is_invalid_access_1(qemu: Qemu, addr: GuestAddr) -> bool { unsafe { - let h = emu.g2h::<*const c_void>(addr) as isize; + let h = qemu.g2h::<*const c_void>(addr) as isize; let shadow_addr = ((h >> 3) as *mut i8).offset(SHADOW_OFFSET); let k = *shadow_addr as isize; k != 0 && (h & 7).wrapping_add(1) > k @@ -295,9 +293,9 @@ impl AsanGiovese { #[inline] #[must_use] - pub fn is_invalid_access_2(emu: &Emulator, addr: GuestAddr) -> bool { + pub fn is_invalid_access_2(qemu: Qemu, addr: GuestAddr) -> bool { unsafe { - let h = emu.g2h::<*const c_void>(addr) as isize; + let h = qemu.g2h::<*const c_void>(addr) as isize; let shadow_addr = ((h >> 3) as *mut i8).offset(SHADOW_OFFSET); let k = *shadow_addr as isize; k != 0 && (h & 7).wrapping_add(2) > k @@ -306,9 +304,9 @@ impl AsanGiovese { #[inline] #[must_use] - pub fn is_invalid_access_4(emu: &Emulator, addr: GuestAddr) -> bool { + pub fn is_invalid_access_4(qemu: Qemu, addr: GuestAddr) -> bool { unsafe { - let h = emu.g2h::<*const c_void>(addr) as isize; + let h = qemu.g2h::<*const c_void>(addr) as isize; let shadow_addr = ((h >> 3) as *mut i8).offset(SHADOW_OFFSET); let k = *shadow_addr as isize; k != 0 && (h & 7).wrapping_add(4) > k @@ -317,9 +315,9 @@ impl AsanGiovese { #[inline] #[must_use] - pub fn is_invalid_access_8(emu: &Emulator, addr: GuestAddr) -> bool { + pub fn is_invalid_access_8(qemu: Qemu, addr: GuestAddr) -> bool { unsafe { - let h = emu.g2h::<*const c_void>(addr) as isize; + let h = qemu.g2h::<*const c_void>(addr) as isize; let shadow_addr = ((h >> 3) as *mut i8).offset(SHADOW_OFFSET); *shadow_addr != 0 } @@ -328,7 +326,7 @@ impl AsanGiovese { #[inline] #[must_use] #[allow(clippy::cast_sign_loss)] - pub fn is_invalid_access(emu: &Emulator, addr: GuestAddr, n: usize) -> bool { + pub fn is_invalid_access(qemu: Qemu, addr: GuestAddr, n: usize) -> bool { unsafe { if n == 0 { return false; @@ -343,12 +341,12 @@ impl AsanGiovese { let next_8 = (start & !7).wrapping_add(8); let first_size = next_8.wrapping_sub(start) as isize; if n <= first_size { - let h = emu.g2h::<*const c_void>(start) as isize; + let h = qemu.g2h::<*const c_void>(start) as isize; let shadow_addr = ((h >> 3) as *mut i8).offset(SHADOW_OFFSET); let k = *shadow_addr as isize; return k != 0 && (h & 7).wrapping_add(n) > k; } - let h = emu.g2h::<*const c_void>(start) as isize; + let h = qemu.g2h::<*const c_void>(start) as isize; let shadow_addr = ((h >> 3) as *mut i8).offset(SHADOW_OFFSET); let k = *shadow_addr as isize; if k != 0 && (h & 7).wrapping_add(first_size) > k { @@ -358,7 +356,7 @@ impl AsanGiovese { } while start < last_8 { - let h = emu.g2h::<*const c_void>(start) as isize; + let h = qemu.g2h::<*const c_void>(start) as isize; let shadow_addr = ((h >> 3) as *mut i8).offset(SHADOW_OFFSET); if *shadow_addr != 0 { return true; @@ -367,7 +365,7 @@ impl AsanGiovese { } if last_8 != end { - let h = emu.g2h::<*const c_void>(start) as isize; + let h = qemu.g2h::<*const c_void>(start) as isize; let last_size = end.wrapping_sub(last_8) as isize; let shadow_addr = ((h >> 3) as *mut i8).offset(SHADOW_OFFSET); let k = *shadow_addr as isize; @@ -380,7 +378,7 @@ impl AsanGiovese { #[inline] #[allow(clippy::cast_sign_loss)] - pub fn poison(&mut self, emu: &Emulator, addr: GuestAddr, n: usize, poison_byte: i8) -> bool { + pub fn poison(&mut self, qemu: Qemu, addr: GuestAddr, n: usize, poison_byte: i8) -> bool { unsafe { if n == 0 { return false; @@ -406,14 +404,14 @@ impl AsanGiovese { if n < first_size { return false; } - let h = emu.g2h::<*const c_void>(start) as isize; + let h = qemu.g2h::<*const c_void>(start) as isize; let shadow_addr = ((h >> 3) as *mut i8).offset(SHADOW_OFFSET); *shadow_addr = (8isize).wrapping_sub(first_size) as i8; start = next_8; } while start < last_8 { - let h = emu.g2h::<*const c_void>(start) as isize; + let h = qemu.g2h::<*const c_void>(start) as isize; let shadow_addr = ((h >> 3) as *mut i8).offset(SHADOW_OFFSET); *shadow_addr = poison_byte; start = (start).wrapping_add(8); @@ -426,14 +424,14 @@ impl AsanGiovese { #[inline] #[allow(clippy::must_use_candidate)] #[allow(clippy::cast_sign_loss)] - pub fn unpoison(emu: &Emulator, addr: GuestAddr, n: usize) -> bool { + pub fn unpoison(qemu: Qemu, addr: GuestAddr, n: usize) -> bool { unsafe { let n = n as isize; let mut start = addr; let end = start.wrapping_add(n as GuestAddr); while start < end { - let h = emu.g2h::<*const c_void>(start) as isize; + let h = qemu.g2h::<*const c_void>(start) as isize; let shadow_addr = ((h >> 3) as *mut i8).offset(SHADOW_OFFSET); *shadow_addr = 0; start = (start).wrapping_add(8); @@ -443,9 +441,9 @@ impl AsanGiovese { } #[inline] - pub fn unpoison_page(emu: &Emulator, page: GuestAddr) { + pub fn unpoison_page(qemu: Qemu, page: GuestAddr) { unsafe { - let h = emu.g2h::<*const c_void>(page) as isize; + let h = qemu.g2h::<*const c_void>(page) as isize; let shadow_addr = ((h >> 3) as *mut i8).offset(SHADOW_OFFSET); shadow_addr.write_bytes(0, SHADOW_PAGE_SIZE); } @@ -453,26 +451,26 @@ impl AsanGiovese { #[inline] #[allow(clippy::mut_from_ref)] - fn get_shadow_page(emu: &Emulator, page: GuestAddr) -> &mut [i8] { + fn get_shadow_page(qemu: &Qemu, page: GuestAddr) -> &mut [i8] { unsafe { - let h = emu.g2h::<*const c_void>(page) as isize; + let h = qemu.g2h::<*const c_void>(page) as isize; let shadow_addr = ((h >> 3) as *mut i8).offset(SHADOW_OFFSET); std::slice::from_raw_parts_mut(shadow_addr, SHADOW_PAGE_SIZE) } } - pub fn report_or_crash(&mut self, emu: &Emulator, pc: GuestAddr, error: AsanError) { + pub fn report_or_crash(&mut self, qemu: Qemu, pc: GuestAddr, error: AsanError) { if let Some(mut cb) = self.error_callback.take() { - (cb)(self, emu, pc, error); + cb(self, qemu, pc, error); self.error_callback = Some(cb); } else { std::process::abort(); } } - pub fn report(&mut self, emu: &Emulator, pc: GuestAddr, error: AsanError) { + pub fn report(&mut self, qemu: Qemu, pc: GuestAddr, error: AsanError) { if let Some(mut cb) = self.error_callback.take() { - (cb)(self, emu, pc, error); + cb(self, qemu, pc, error); self.error_callback = Some(cb); } } @@ -502,7 +500,7 @@ impl AsanGiovese { } } - pub fn alloc_free(&mut self, emulator: &Emulator, pc: GuestAddr, addr: GuestAddr) { + pub fn alloc_free(&mut self, qemu: Qemu, pc: GuestAddr, addr: GuestAddr) { let mut chunk = None; self.alloc_map_mut(addr, |interval, item| { chunk = Some(*interval); @@ -518,11 +516,11 @@ impl AsanGiovese { if let Some(ck) = chunk { if ck.start != addr { // Free not the start of the chunk - self.report_or_crash(emulator, pc, AsanError::BadFree(addr, Some(ck))); + self.report_or_crash(qemu, pc, AsanError::BadFree(addr, Some(ck))); } } else { // Free of wild ptr - self.report_or_crash(emulator, pc, AsanError::BadFree(addr, None)); + self.report_or_crash(qemu, pc, AsanError::BadFree(addr, None)); } } @@ -596,16 +594,16 @@ impl AsanGiovese { self.alloc_insert(pc, start, end); } - pub fn deallocation(&mut self, emulator: &Emulator, pc: GuestAddr, addr: GuestAddr) { - self.alloc_free(emulator, pc, addr); + pub fn deallocation(&mut self, qemu: Qemu, pc: GuestAddr, addr: GuestAddr) { + self.alloc_free(qemu, pc, addr); } - pub fn snapshot(&mut self, emu: &Emulator) { + pub fn snapshot(&mut self, qemu: Qemu) { if self.snapshot_shadow { let set = self.dirty_shadow.lock().unwrap(); for &page in &*set { - let data = Self::get_shadow_page(emu, page).to_vec(); + let data = Self::get_shadow_page(&qemu, page).to_vec(); self.saved_shadow.insert(page, data); } @@ -614,7 +612,7 @@ impl AsanGiovese { } } - pub fn rollback(&mut self, emu: &Emulator, detect_leaks: bool) -> AsanRollback { + pub fn rollback(&mut self, qemu: Qemu, detect_leaks: bool) -> AsanRollback { let mut leaks = vec![]; { @@ -637,10 +635,10 @@ impl AsanGiovese { for &page in &*set { let original = self.saved_shadow.get(&page); if let Some(data) = original { - let cur = Self::get_shadow_page(emu, page); + let cur = Self::get_shadow_page(&qemu, page); cur.copy_from_slice(data); } else { - Self::unpoison_page(emu, page); + Self::unpoison_page(qemu, page); } } @@ -655,8 +653,8 @@ impl AsanGiovese { for interval in leaks { self.report( - emu, - emu.read_reg(Regs::Pc).unwrap(), + qemu, + qemu.read_reg(Regs::Pc).unwrap(), AsanError::MemLeak(interval), ); } @@ -667,10 +665,10 @@ impl AsanGiovese { static mut ASAN_INITED: bool = false; -pub fn init_with_asan( +pub fn init_qemu_with_asan( args: &mut Vec, env: &mut [(String, String)], -) -> Result<(Emulator, Pin>), EmuError> { +) -> Result<(Qemu, Pin>), EmuError> { let current = env::current_exe().unwrap(); let asan_lib = fs::canonicalize(current) .unwrap() @@ -716,10 +714,10 @@ pub fn init_with_asan( ASAN_INITED = true; } - let emu = Emulator::new(args, env)?; - let rt = AsanGiovese::new(&emu); + let qemu = Qemu::init(args, env)?; + let rt = AsanGiovese::new(qemu); - Ok((emu, rt)) + Ok((qemu, rt)) } pub enum QemuAsanOptions { @@ -756,7 +754,7 @@ impl QemuAsanHelper { filter: QemuInstrumentationAddressRangeFilter, options: QemuAsanOptions, ) -> Self { - assert!(unsafe { ASAN_INITED }, "The ASan runtime is not initialized, use init_with_asan(...) instead of just Emulator::new(...)"); + assert!(unsafe { ASAN_INITED }, "The ASan runtime is not initialized, use init_qemu_with_asan(...) instead of just Qemu::init(...)"); let (snapshot, detect_leaks) = match options { QemuAsanOptions::None => (false, false), QemuAsanOptions::Snapshot => (true, false), @@ -780,7 +778,7 @@ impl QemuAsanHelper { error_callback: AsanErrorCallback, options: QemuAsanOptions, ) -> Self { - assert!(unsafe { ASAN_INITED }, "The ASan runtime is not initialized, use init_with_asan(...) instead of just Emulator::new(...)"); + assert!(unsafe { ASAN_INITED }, "The ASan runtime is not initialized, use init_qemu_with_asan(...) instead of just Qemu::init(...)"); let (snapshot, detect_leaks) = match options { QemuAsanOptions::None => (false, false), QemuAsanOptions::Snapshot => (true, false), @@ -825,107 +823,95 @@ impl QemuAsanHelper { self.rt.allocation(pc, start, end); } - pub fn dealloc(&mut self, emulator: &Emulator, pc: GuestAddr, addr: GuestAddr) { - self.rt.deallocation(emulator, pc, addr); + pub fn dealloc(&mut self, qemu: Qemu, pc: GuestAddr, addr: GuestAddr) { + self.rt.deallocation(qemu, pc, addr); } #[allow(clippy::unused_self)] #[must_use] - pub fn is_poisoned(&self, emulator: &Emulator, addr: GuestAddr, size: usize) -> bool { - AsanGiovese::is_invalid_access(emulator, addr, size) + pub fn is_poisoned(&self, qemu: Qemu, addr: GuestAddr, size: usize) -> bool { + AsanGiovese::is_invalid_access(qemu, addr, size) } - pub fn read_1(&mut self, emulator: &Emulator, pc: GuestAddr, addr: GuestAddr) { - if self.enabled() && AsanGiovese::is_invalid_access_1(emulator, addr) { - self.rt - .report_or_crash(emulator, pc, AsanError::Read(addr, 1)); + pub fn read_1(&mut self, qemu: Qemu, pc: GuestAddr, addr: GuestAddr) { + if self.enabled() && AsanGiovese::is_invalid_access_1(qemu, addr) { + self.rt.report_or_crash(qemu, pc, AsanError::Read(addr, 1)); } } - pub fn read_2(&mut self, emulator: &Emulator, pc: GuestAddr, addr: GuestAddr) { - if self.enabled() && AsanGiovese::is_invalid_access_2(emulator, addr) { - self.rt - .report_or_crash(emulator, pc, AsanError::Read(addr, 2)); + pub fn read_2(&mut self, qemu: Qemu, pc: GuestAddr, addr: GuestAddr) { + if self.enabled() && AsanGiovese::is_invalid_access_2(qemu, addr) { + self.rt.report_or_crash(qemu, pc, AsanError::Read(addr, 2)); } } - pub fn read_4(&mut self, emulator: &Emulator, pc: GuestAddr, addr: GuestAddr) { - if self.enabled() && AsanGiovese::is_invalid_access_4(emulator, addr) { - self.rt - .report_or_crash(emulator, pc, AsanError::Read(addr, 4)); + pub fn read_4(&mut self, qemu: Qemu, pc: GuestAddr, addr: GuestAddr) { + if self.enabled() && AsanGiovese::is_invalid_access_4(qemu, addr) { + self.rt.report_or_crash(qemu, pc, AsanError::Read(addr, 4)); } } - pub fn read_8(&mut self, emulator: &Emulator, pc: GuestAddr, addr: GuestAddr) { - if self.enabled() && AsanGiovese::is_invalid_access_8(emulator, addr) { - self.rt - .report_or_crash(emulator, pc, AsanError::Read(addr, 8)); + pub fn read_8(&mut self, qemu: Qemu, pc: GuestAddr, addr: GuestAddr) { + if self.enabled() && AsanGiovese::is_invalid_access_8(qemu, addr) { + self.rt.report_or_crash(qemu, pc, AsanError::Read(addr, 8)); } } - pub fn read_n(&mut self, emulator: &Emulator, pc: GuestAddr, addr: GuestAddr, size: usize) { - if self.enabled() && AsanGiovese::is_invalid_access(emulator, addr, size) { + pub fn read_n(&mut self, qemu: Qemu, pc: GuestAddr, addr: GuestAddr, size: usize) { + if self.enabled() && AsanGiovese::is_invalid_access(qemu, addr, size) { self.rt - .report_or_crash(emulator, pc, AsanError::Read(addr, size)); + .report_or_crash(qemu, pc, AsanError::Read(addr, size)); } } - pub fn write_1(&mut self, emulator: &Emulator, pc: GuestAddr, addr: GuestAddr) { - if self.enabled() && AsanGiovese::is_invalid_access_1(emulator, addr) { - self.rt - .report_or_crash(emulator, pc, AsanError::Write(addr, 1)); + pub fn write_1(&mut self, qemu: Qemu, pc: GuestAddr, addr: GuestAddr) { + if self.enabled() && AsanGiovese::is_invalid_access_1(qemu, addr) { + self.rt.report_or_crash(qemu, pc, AsanError::Write(addr, 1)); } } - pub fn write_2(&mut self, emulator: &Emulator, pc: GuestAddr, addr: GuestAddr) { - if self.enabled() && AsanGiovese::is_invalid_access_2(emulator, addr) { - self.rt - .report_or_crash(emulator, pc, AsanError::Write(addr, 2)); + pub fn write_2(&mut self, qemu: Qemu, pc: GuestAddr, addr: GuestAddr) { + if self.enabled() && AsanGiovese::is_invalid_access_2(qemu, addr) { + self.rt.report_or_crash(qemu, pc, AsanError::Write(addr, 2)); } } - pub fn write_4(&mut self, emulator: &Emulator, pc: GuestAddr, addr: GuestAddr) { - if self.enabled() && AsanGiovese::is_invalid_access_4(emulator, addr) { - self.rt - .report_or_crash(emulator, pc, AsanError::Write(addr, 4)); + pub fn write_4(&mut self, qemu: Qemu, pc: GuestAddr, addr: GuestAddr) { + if self.enabled() && AsanGiovese::is_invalid_access_4(qemu, addr) { + self.rt.report_or_crash(qemu, pc, AsanError::Write(addr, 4)); } } - pub fn write_8(&mut self, emulator: &Emulator, pc: GuestAddr, addr: GuestAddr) { - if self.enabled() && AsanGiovese::is_invalid_access_8(emulator, addr) { - self.rt - .report_or_crash(emulator, pc, AsanError::Write(addr, 8)); + pub fn write_8(&mut self, qemu: Qemu, pc: GuestAddr, addr: GuestAddr) { + if self.enabled() && AsanGiovese::is_invalid_access_8(qemu, addr) { + self.rt.report_or_crash(qemu, pc, AsanError::Write(addr, 8)); } } - pub fn write_n(&mut self, emulator: &Emulator, pc: GuestAddr, addr: GuestAddr, size: usize) { - if self.enabled() && AsanGiovese::is_invalid_access(emulator, addr, size) { + pub fn write_n(&mut self, qemu: Qemu, pc: GuestAddr, addr: GuestAddr, size: usize) { + if self.enabled() && AsanGiovese::is_invalid_access(qemu, addr, size) { self.rt - .report_or_crash(emulator, pc, AsanError::Write(addr, size)); + .report_or_crash(qemu, pc, AsanError::Write(addr, size)); } } - pub fn poison( - &mut self, - emulator: &Emulator, - addr: GuestAddr, - size: usize, - poison: PoisonKind, - ) { - self.rt.poison(emulator, addr, size, poison.into()); + pub fn poison(&mut self, qemu: Qemu, addr: GuestAddr, size: usize, poison: PoisonKind) { + self.rt.poison(qemu, addr, size, poison.into()); } #[allow(clippy::unused_self)] - pub fn unpoison(&mut self, emulator: &Emulator, addr: GuestAddr, size: usize) { - AsanGiovese::unpoison(emulator, addr, size); + pub fn unpoison(&mut self, qemu: Qemu, addr: GuestAddr, size: usize) { + AsanGiovese::unpoison(qemu, addr, size); } - pub fn reset(&mut self, emulator: &Emulator) -> AsanRollback { - self.rt.rollback(emulator, self.detect_leaks) + pub fn reset(&mut self, qemu: Qemu) -> AsanRollback { + self.rt.rollback(qemu, self.detect_leaks) } } -impl HasInstrumentationFilter for QemuAsanHelper { +impl HasInstrumentationFilter + for QemuAsanHelper +{ fn filter(&self) -> &QemuInstrumentationAddressRangeFilter { &self.filter } @@ -987,23 +973,23 @@ where } } - fn pre_exec(&mut self, emulator: &Emulator, _input: &S::Input) { + fn pre_exec(&mut self, qemu: Qemu, _input: &S::Input) { if self.empty { - self.rt.snapshot(emulator); + self.rt.snapshot(qemu); self.empty = false; } } fn post_exec( &mut self, - emulator: &Emulator, + qemu: Qemu, _input: &S::Input, _observers: &mut OT, exit_kind: &mut ExitKind, ) where OT: ObserversTuple, { - if self.reset(emulator) == AsanRollback::HasLeaks { + if self.reset(qemu) == AsanRollback::HasLeaks { *exit_kind = ExitKind::Crash; } } @@ -1014,16 +1000,17 @@ where S: UsesInput, QT: QemuHelperTuple, { - let emu = hooks.emulator().clone(); + let qemu = *hooks.qemu(); let h = hooks.match_helper_mut::().unwrap(); - let pc: GuestAddr = emu.read_reg(Regs::Pc).unwrap(); - h.rt.report(&emu, pc, AsanError::Signal(target_sig)); + let pc: GuestAddr = qemu.read_reg(Regs::Pc).unwrap(); + h.rt.report(qemu, pc, AsanError::Signal(target_sig)); } pub fn gen_readwrite_asan( hooks: &mut QemuHooks, _state: Option<&mut S>, pc: GuestAddr, + _addr: *mut TCGTemp, _info: MemAccessInfo, ) -> Option where @@ -1047,9 +1034,9 @@ pub fn trace_read1_asan( S: UsesInput, QT: QemuHelperTuple, { - let emulator = hooks.emulator().clone(); + let qemu = *hooks.qemu(); let h = hooks.match_helper_mut::().unwrap(); - h.read_1(&emulator, id as GuestAddr, addr); + h.read_1(qemu, id as GuestAddr, addr); } pub fn trace_read2_asan( @@ -1061,9 +1048,9 @@ pub fn trace_read2_asan( S: UsesInput, QT: QemuHelperTuple, { - let emulator = hooks.emulator().clone(); + let qemu = *hooks.qemu(); let h = hooks.match_helper_mut::().unwrap(); - h.read_2(&emulator, id as GuestAddr, addr); + h.read_2(qemu, id as GuestAddr, addr); } pub fn trace_read4_asan( @@ -1075,9 +1062,9 @@ pub fn trace_read4_asan( S: UsesInput, QT: QemuHelperTuple, { - let emulator = hooks.emulator().clone(); + let qemu = *hooks.qemu(); let h = hooks.match_helper_mut::().unwrap(); - h.read_4(&emulator, id as GuestAddr, addr); + h.read_4(qemu, id as GuestAddr, addr); } pub fn trace_read8_asan( @@ -1089,9 +1076,9 @@ pub fn trace_read8_asan( S: UsesInput, QT: QemuHelperTuple, { - let emulator = hooks.emulator().clone(); + let qemu = *hooks.qemu(); let h = hooks.match_helper_mut::().unwrap(); - h.read_8(&emulator, id as GuestAddr, addr); + h.read_8(qemu, id as GuestAddr, addr); } pub fn trace_read_n_asan( @@ -1104,9 +1091,9 @@ pub fn trace_read_n_asan( S: UsesInput, QT: QemuHelperTuple, { - let emulator = hooks.emulator().clone(); + let qemu = *hooks.qemu(); let h = hooks.match_helper_mut::().unwrap(); - h.read_n(&emulator, id as GuestAddr, addr, size); + h.read_n(qemu, id as GuestAddr, addr, size); } pub fn trace_write1_asan( @@ -1118,9 +1105,9 @@ pub fn trace_write1_asan( S: UsesInput, QT: QemuHelperTuple, { - let emulator = hooks.emulator().clone(); + let qemu = *hooks.qemu(); let h = hooks.match_helper_mut::().unwrap(); - h.write_1(&emulator, id as GuestAddr, addr); + h.write_1(qemu, id as GuestAddr, addr); } pub fn trace_write2_asan( @@ -1132,9 +1119,9 @@ pub fn trace_write2_asan( S: UsesInput, QT: QemuHelperTuple, { - let emulator = hooks.emulator().clone(); + let qemu = *hooks.qemu(); let h = hooks.match_helper_mut::().unwrap(); - h.write_2(&emulator, id as GuestAddr, addr); + h.write_2(qemu, id as GuestAddr, addr); } pub fn trace_write4_asan( @@ -1146,9 +1133,9 @@ pub fn trace_write4_asan( S: UsesInput, QT: QemuHelperTuple, { - let emulator = hooks.emulator().clone(); + let qemu = *hooks.qemu(); let h = hooks.match_helper_mut::().unwrap(); - h.write_4(&emulator, id as GuestAddr, addr); + h.write_4(qemu, id as GuestAddr, addr); } pub fn trace_write8_asan( @@ -1160,9 +1147,9 @@ pub fn trace_write8_asan( S: UsesInput, QT: QemuHelperTuple, { - let emulator = hooks.emulator().clone(); + let qemu = *hooks.qemu(); let h = hooks.match_helper_mut::().unwrap(); - h.write_8(&emulator, id as GuestAddr, addr); + h.write_8(qemu, id as GuestAddr, addr); } pub fn trace_write_n_asan( @@ -1175,15 +1162,16 @@ pub fn trace_write_n_asan( S: UsesInput, QT: QemuHelperTuple, { - let emulator = hooks.emulator().clone(); + let qemu = *hooks.qemu(); let h = hooks.match_helper_mut::().unwrap(); - h.read_n(&emulator, id as GuestAddr, addr, size); + h.read_n(qemu, id as GuestAddr, addr, size); } pub fn gen_write_asan_snapshot( hooks: &mut QemuHooks, _state: Option<&mut S>, pc: GuestAddr, + _addr: *mut TCGTemp, _info: MemAccessInfo, ) -> Option where @@ -1208,9 +1196,9 @@ pub fn trace_write1_asan_snapshot( QT: QemuHelperTuple, { if id != 0 { - let emulator = hooks.emulator().clone(); + let qemu = *hooks.qemu(); let h = hooks.match_helper_mut::().unwrap(); - h.write_1(&emulator, id as GuestAddr, addr); + h.write_1(qemu, id as GuestAddr, addr); } let h = hooks.match_helper_mut::().unwrap(); h.access(addr, 1); @@ -1226,9 +1214,9 @@ pub fn trace_write2_asan_snapshot( QT: QemuHelperTuple, { if id != 0 { - let emulator = hooks.emulator().clone(); + let qemu = *hooks.qemu(); let h = hooks.match_helper_mut::().unwrap(); - h.write_2(&emulator, id as GuestAddr, addr); + h.write_2(qemu, id as GuestAddr, addr); } let h = hooks.match_helper_mut::().unwrap(); h.access(addr, 2); @@ -1244,9 +1232,9 @@ pub fn trace_write4_asan_snapshot( QT: QemuHelperTuple, { if id != 0 { - let emulator = hooks.emulator().clone(); + let qemu = *hooks.qemu(); let h = hooks.match_helper_mut::().unwrap(); - h.write_4(&emulator, id as GuestAddr, addr); + h.write_4(qemu, id as GuestAddr, addr); } let h = hooks.match_helper_mut::().unwrap(); h.access(addr, 4); @@ -1262,9 +1250,9 @@ pub fn trace_write8_asan_snapshot( QT: QemuHelperTuple, { if id != 0 { - let emulator = hooks.emulator().clone(); + let qemu = *hooks.qemu(); let h = hooks.match_helper_mut::().unwrap(); - h.write_8(&emulator, id as GuestAddr, addr); + h.write_8(qemu, id as GuestAddr, addr); } let h = hooks.match_helper_mut::().unwrap(); h.access(addr, 8); @@ -1281,9 +1269,9 @@ pub fn trace_write_n_asan_snapshot( QT: QemuHelperTuple, { if id != 0 { - let emulator = hooks.emulator().clone(); + let qemu = *hooks.qemu(); let h = hooks.match_helper_mut::().unwrap(); - h.read_n(&emulator, id as GuestAddr, addr, size); + h.read_n(qemu, id as GuestAddr, addr, size); } let h = hooks.match_helper_mut::().unwrap(); h.access(addr, size); @@ -1308,16 +1296,16 @@ where QT: QemuHelperTuple, { if sys_num == QASAN_FAKESYS_NR { - let emulator = hooks.emulator().clone(); + let qemu = *hooks.qemu(); let h = hooks.match_helper_mut::().unwrap(); match QasanAction::try_from(a0).expect("Invalid QASan action number") { QasanAction::CheckLoad => { - let pc: GuestAddr = emulator.read_reg(Regs::Pc).unwrap(); - h.read_n(&emulator, pc, a1, a2 as usize); + let pc: GuestAddr = qemu.read_reg(Regs::Pc).unwrap(); + h.read_n(qemu, pc, a1, a2 as usize); } QasanAction::CheckStore => { - let pc: GuestAddr = emulator.read_reg(Regs::Pc).unwrap(); - h.write_n(&emulator, pc, a1, a2 as usize); + let pc: GuestAddr = qemu.read_reg(Regs::Pc).unwrap(); + h.write_n(qemu, pc, a1, a2 as usize); } QasanAction::Enable => { h.set_enabled(true); @@ -1358,9 +1346,9 @@ fn load_file_section<'input, 'arena, Endian: addr2line::gimli::Endianity>( #[allow(clippy::unnecessary_cast)] #[allow(clippy::too_many_lines)] -pub fn asan_report(rt: &AsanGiovese, emu: &Emulator, pc: GuestAddr, err: AsanError) { +pub fn asan_report(rt: &AsanGiovese, qemu: Qemu, pc: GuestAddr, err: AsanError) { let mut regions = std::collections::HashMap::new(); - for region in emu.mappings() { + for region in qemu.mappings() { if let Some(path) = region.path() { let start = region.start(); let end = region.end(); @@ -1562,6 +1550,9 @@ pub fn asan_report(rt: &AsanGiovese, emu: &Emulator, pc: GuestAddr, err: AsanErr } // fix pc in case it is not synced (in hooks) - emu.write_reg(Regs::Pc, pc).unwrap(); - eprint!("Context:\n{}", emu.current_cpu().unwrap().display_context()); + qemu.write_reg(Regs::Pc, pc).unwrap(); + eprint!( + "Context:\n{}", + qemu.current_cpu().unwrap().display_context() + ); } diff --git a/libafl_qemu/src/helpers/asan_guest.rs b/libafl_qemu/src/helpers/asan_guest.rs new file mode 100644 index 0000000000..df08b75f3e --- /dev/null +++ b/libafl_qemu/src/helpers/asan_guest.rs @@ -0,0 +1,300 @@ +#![allow(clippy::cast_possible_wrap)] + +use std::{ + env, + fmt::{self, Debug, Formatter}, + fs, + path::PathBuf, +}; + +use libafl::{inputs::UsesInput, HasMetadata}; + +#[cfg(not(feature = "clippy"))] +use crate::sys::libafl_tcg_gen_asan; +use crate::{ + emu::{EmuError, MemAccessInfo, Qemu}, + helpers::{ + HasInstrumentationFilter, IsFilter, QemuHelper, QemuHelperTuple, + QemuInstrumentationAddressRangeFilter, + }, + hooks::{Hook, QemuHooks}, + sys::TCGTemp, + GuestAddr, MapInfo, +}; + +static mut ASAN_GUEST_INITED: bool = false; + +pub fn init_qemu_with_asan_guest( + args: &mut Vec, + env: &mut [(String, String)], +) -> Result<(Qemu, String), EmuError> { + let current = env::current_exe().unwrap(); + let asan_lib = fs::canonicalize(current) + .unwrap() + .parent() + .unwrap() + .join("libgasan.so"); + + let asan_lib = env::var_os("CUSTOM_ASAN_PATH") + .map_or(asan_lib, |x| PathBuf::from(x.to_string_lossy().to_string())); + + assert!( + asan_lib.as_path().exists(), + "The ASAN library doesn't exist: {asan_lib:#?}" + ); + + let asan_lib = asan_lib + .to_str() + .expect("The path to the asan lib is invalid") + .to_string(); + + println!("Loading ASAN: {asan_lib:}"); + + let add_asan = + |e: &str| "LD_PRELOAD=".to_string() + &asan_lib + " " + &e["LD_PRELOAD=".len()..]; + + let mut added = false; + for (k, v) in &mut *env { + if k == "QEMU_SET_ENV" { + let mut new_v = vec![]; + for e in v.split(',') { + if e.starts_with("LD_PRELOAD=") { + added = true; + new_v.push(add_asan(e)); + } else { + new_v.push(e.to_string()); + } + } + *v = new_v.join(","); + } + } + for i in 0..args.len() { + if args[i] == "-E" && i + 1 < args.len() && args[i + 1].starts_with("LD_PRELOAD=") { + added = true; + args[i + 1] = add_asan(&args[i + 1]); + } + } + + if !added { + args.insert(1, "LD_PRELOAD=".to_string() + &asan_lib); + args.insert(1, "-E".into()); + } + + if env::var("QASAN_DEBUG").is_ok() { + args.push("-E".into()); + args.push("QASAN_DEBUG=1".into()); + } + + if env::var("QASAN_LOG").is_ok() { + args.push("-E".into()); + args.push("QASAN_LOG=1".into()); + } + + unsafe { + ASAN_GUEST_INITED = true; + } + + let emu = Qemu::init(args, env)?; + Ok((emu, asan_lib)) +} + +#[derive(Clone)] +struct QemuAsanGuestMapping { + start: GuestAddr, + end: GuestAddr, + path: String, +} + +impl Debug for QemuAsanGuestMapping { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "0x{:016x}-0x{:016x} {}", self.start, self.end, self.path) + } +} + +impl From<&MapInfo> for QemuAsanGuestMapping { + fn from(map: &MapInfo) -> QemuAsanGuestMapping { + let path = map.path().map(ToString::to_string).unwrap_or_default(); + let start = map.start(); + let end = map.end(); + QemuAsanGuestMapping { start, end, path } + } +} + +#[derive(Debug)] +pub struct QemuAsanGuestHelper { + filter: QemuInstrumentationAddressRangeFilter, + mappings: Vec, +} + +#[cfg(any(cpu_target = "aarch64", cpu_target = "x86_64", feature = "clippy"))] +impl QemuAsanGuestHelper { + const HIGH_SHADOW_START: GuestAddr = 0x02008fff7000; + const HIGH_SHADOW_END: GuestAddr = 0x10007fff7fff; + const LOW_SHADOW_START: GuestAddr = 0x00007fff8000; + const LOW_SHADOW_END: GuestAddr = 0x00008fff6fff; +} + +#[cfg(any( + cpu_target = "arm", + cpu_target = "i386", + cpu_target = "mips", + cpu_target = "ppc" +))] +impl QemuAsanGuestHelper { + const HIGH_SHADOW_START: GuestAddr = 0x28000000; + const HIGH_SHADOW_END: GuestAddr = 0x3fffffff; + const LOW_SHADOW_START: GuestAddr = 0x20000000; + const LOW_SHADOW_END: GuestAddr = 0x23ffffff; +} + +impl QemuAsanGuestHelper { + #[must_use] + pub fn default(emu: &Qemu, asan: String) -> Self { + Self::new(emu, asan, QemuInstrumentationAddressRangeFilter::None) + } + + #[must_use] + pub fn new(emu: &Qemu, asan: String, filter: QemuInstrumentationAddressRangeFilter) -> Self { + for mapping in emu.mappings() { + println!("mapping: {mapping:#?}"); + } + + let mappings = emu + .mappings() + .map(|m| QemuAsanGuestMapping::from(&m)) + .collect::>(); + + for mapping in &mappings { + println!("guest mapping: {mapping:#?}"); + } + + mappings + .iter() + .find(|m| m.start <= Self::HIGH_SHADOW_START && m.end > Self::HIGH_SHADOW_END) + .expect("HighShadow not found, confirm ASAN DSO is loaded in the guest"); + + mappings + .iter() + .find(|m| m.start <= Self::LOW_SHADOW_START && m.end > Self::LOW_SHADOW_END) + .expect("LowShadow not found, confirm ASAN DSO is loaded in the guest"); + + let mappings = mappings + .iter() + .filter(|m| m.path == asan) + .cloned() + .collect::>(); + + for mapping in &mappings { + println!("asan mapping: {mapping:#?}"); + } + + Self { filter, mappings } + } + + #[must_use] + pub fn must_instrument(&self, addr: GuestAddr) -> bool { + self.filter.allowed(addr) + } +} + +impl HasInstrumentationFilter + for QemuAsanGuestHelper +{ + fn filter(&self) -> &QemuInstrumentationAddressRangeFilter { + &self.filter + } + + fn filter_mut(&mut self) -> &mut QemuInstrumentationAddressRangeFilter { + &mut self.filter + } +} + +fn gen_readwrite_guest_asan( + hooks: &mut QemuHooks, + _state: Option<&mut S>, + pc: GuestAddr, + addr: *mut TCGTemp, + info: MemAccessInfo, +) -> Option +where + S: UsesInput, + QT: QemuHelperTuple, +{ + let h = hooks.match_helper_mut::().unwrap(); + if !h.must_instrument(pc) { + return None; + } + + /* Don't sanitize the sanitizer! */ + if h.mappings.iter().any(|m| m.start <= pc && pc < m.end) { + return None; + } + + let size = info.size(); + + /* TODO - If our size is > 8 then do things via a runtime callback */ + assert!(size <= 8, "I shouldn't be here!"); + + unsafe { + libafl_tcg_gen_asan(addr, size); + } + + None +} + +#[cfg(feature = "clippy")] +#[allow(unused_variables)] +unsafe fn libafl_tcg_gen_asan(addr: *mut TCGTemp, size: usize) {} + +fn guest_trace_error_asan( + _hooks: &mut QemuHooks, + _state: Option<&mut S>, + _id: u64, + _addr: GuestAddr, +) where + S: UsesInput, + QT: QemuHelperTuple, +{ + panic!("I really shouldn't be here"); +} + +fn guest_trace_error_n_asan( + _hooks: &mut QemuHooks, + _state: Option<&mut S>, + _id: u64, + _addr: GuestAddr, + _n: usize, +) where + S: UsesInput, + QT: QemuHelperTuple, +{ + panic!("I really shouldn't be here either"); +} + +impl QemuHelper for QemuAsanGuestHelper +where + S: UsesInput + HasMetadata, +{ + fn first_exec(&self, hooks: &QemuHooks) + where + QT: QemuHelperTuple, + { + hooks.reads( + Hook::Function(gen_readwrite_guest_asan::), + Hook::Function(guest_trace_error_asan::), + Hook::Function(guest_trace_error_asan::), + Hook::Function(guest_trace_error_asan::), + Hook::Function(guest_trace_error_asan::), + Hook::Function(guest_trace_error_n_asan::), + ); + + hooks.writes( + Hook::Function(gen_readwrite_guest_asan::), + Hook::Function(guest_trace_error_asan::), + Hook::Function(guest_trace_error_asan::), + Hook::Function(guest_trace_error_asan::), + Hook::Function(guest_trace_error_asan::), + Hook::Function(guest_trace_error_n_asan::), + ); + } +} diff --git a/libafl_qemu/src/calls.rs b/libafl_qemu/src/helpers/calls.rs similarity index 87% rename from libafl_qemu/src/calls.rs rename to libafl_qemu/src/helpers/calls.rs index f024260c6c..d40ed697da 100644 --- a/libafl_qemu/src/calls.rs +++ b/libafl_qemu/src/helpers/calls.rs @@ -6,21 +6,22 @@ use libafl::{ inputs::{Input, UsesInput}, observers::{stacktrace::BacktraceObserver, ObserversTuple}, }; -use libafl_bolts::{tuples::MatchFirstType, Named}; +use libafl_bolts::tuples::{Handle, Handler, MatchFirstType, MatchNameRef}; +use libafl_qemu_sys::GuestAddr; use thread_local::ThreadLocal; use crate::{ capstone, - emu::{ArchExtras, Emulator}, - helper::{ + emu::ArchExtras, + helpers::{ HasInstrumentationFilter, IsFilter, QemuHelper, QemuHelperTuple, QemuInstrumentationAddressRangeFilter, }, hooks::{Hook, QemuHooks}, - GuestAddr, + Qemu, }; -pub trait CallTraceCollector: 'static + Debug { +pub trait CallTraceCollector: 'static { fn on_call( &mut self, hooks: &mut QemuHooks, @@ -42,7 +43,7 @@ pub trait CallTraceCollector: 'static + Debug { QT: QemuHelperTuple; // Frowarded from the `QemuCallTracerHelper` - fn pre_exec(&mut self, _emulator: &Emulator, _input: &I) + fn pre_exec(&mut self, _qemu: Qemu, _input: &I) where I: Input, { @@ -50,7 +51,7 @@ pub trait CallTraceCollector: 'static + Debug { fn post_exec( &mut self, - _emulator: &Emulator, + _qemu: Qemu, _input: &S::Input, _observers: &mut OT, _exit_kind: &mut ExitKind, @@ -61,7 +62,7 @@ pub trait CallTraceCollector: 'static + Debug { } } -pub trait CallTraceCollectorTuple: 'static + MatchFirstType + Debug { +pub trait CallTraceCollectorTuple: 'static + MatchFirstType { fn on_call_all( &mut self, hooks: &mut QemuHooks, @@ -82,13 +83,13 @@ pub trait CallTraceCollectorTuple: 'static + MatchFirstType + Debug { S: UsesInput, QT: QemuHelperTuple; - fn pre_exec_all(&mut self, _emulator: &Emulator, input: &I) + fn pre_exec_all(&mut self, _qemu: Qemu, input: &I) where I: Input; fn post_exec_all( &mut self, - _emulator: &Emulator, + _qemu: Qemu, input: &S::Input, _observers: &mut OT, _exit_kind: &mut ExitKind, @@ -122,7 +123,7 @@ impl CallTraceCollectorTuple for () { { } - fn pre_exec_all(&mut self, _emulator: &Emulator, _input: &I) + fn pre_exec_all(&mut self, _qemu: Qemu, _input: &I) where I: Input, { @@ -130,7 +131,7 @@ impl CallTraceCollectorTuple for () { fn post_exec_all( &mut self, - _emulator: &Emulator, + _emulator: Qemu, _input: &S::Input, _observers: &mut OT, _exit_kind: &mut ExitKind, @@ -190,17 +191,17 @@ where self.1.on_ret_all(hooks, state, pc, ret_addr); } - fn pre_exec_all(&mut self, emulator: &Emulator, input: &I) + fn pre_exec_all(&mut self, qemu: Qemu, input: &I) where I: Input, { - self.0.pre_exec(emulator, input); - self.1.pre_exec_all(emulator, input); + self.0.pre_exec(qemu, input); + self.1.pre_exec_all(qemu, input); } fn post_exec_all( &mut self, - emulator: &Emulator, + qemu: Qemu, input: &S::Input, observers: &mut OT, exit_kind: &mut ExitKind, @@ -208,8 +209,8 @@ where OT: ObserversTuple, S: UsesInput, { - self.0.post_exec(emulator, input, observers, exit_kind); - self.1.post_exec_all(emulator, input, observers, exit_kind); + self.0.post_exec(qemu, input, observers, exit_kind); + self.1.post_exec_all(qemu, input, observers, exit_kind); } } @@ -246,7 +247,7 @@ where S: UsesInput, QT: QemuHelperTuple, { - let ret_addr: GuestAddr = hooks.emulator().read_return_address().unwrap(); + let ret_addr: GuestAddr = hooks.qemu().read_return_address().unwrap(); // log::info!("RET @ 0x{:#x}", ret_addr); @@ -292,7 +293,7 @@ where .unwrap(); } - let emu = hooks.emulator(); + let emu = hooks.qemu(); if let Some(h) = hooks.helpers().match_first_type::() { #[allow(unused_mut)] @@ -383,9 +384,11 @@ where } } -impl HasInstrumentationFilter for QemuCallTracerHelper +impl HasInstrumentationFilter + for QemuCallTracerHelper where T: CallTraceCollectorTuple, + S: UsesInput, { fn filter(&self) -> &QemuInstrumentationAddressRangeFilter { &self.filter @@ -399,7 +402,7 @@ where impl QemuHelper for QemuCallTracerHelper where S: UsesInput, - T: CallTraceCollectorTuple, + T: CallTraceCollectorTuple + Debug, { fn init_hooks(&self, hooks: &QemuHooks) where @@ -412,16 +415,13 @@ where ); } - fn pre_exec(&mut self, emulator: &Emulator, input: &S::Input) { - self.collectors - .as_mut() - .unwrap() - .pre_exec_all(emulator, input); + fn pre_exec(&mut self, qemu: Qemu, input: &S::Input) { + self.collectors.as_mut().unwrap().pre_exec_all(qemu, input); } fn post_exec( &mut self, - emulator: &Emulator, + qemu: Qemu, input: &S::Input, observers: &mut OT, exit_kind: &mut ExitKind, @@ -431,31 +431,23 @@ where self.collectors .as_mut() .unwrap() - .post_exec_all(emulator, input, observers, exit_kind); + .post_exec_all(qemu, input, observers, exit_kind); } } // TODO support multiple threads with thread local callstack #[derive(Debug)] -pub struct OnCrashBacktraceCollector { +pub struct OnCrashBacktraceCollector<'a> { callstack_hash: u64, - observer_name: String, + obs_ref: Handle>, } -impl OnCrashBacktraceCollector { +impl<'a> OnCrashBacktraceCollector<'a> { #[must_use] - pub fn new(observer: &BacktraceObserver<'_>) -> Self { + pub fn new(observer: &BacktraceObserver<'a>) -> Self { Self { callstack_hash: 0, - observer_name: observer.name().to_string(), - } - } - - #[must_use] - pub fn with_name(observer_name: String) -> Self { - Self { - callstack_hash: 0, - observer_name, + obs_ref: observer.handle(), } } @@ -469,7 +461,10 @@ impl OnCrashBacktraceCollector { } } -impl CallTraceCollector for OnCrashBacktraceCollector { +impl<'a> CallTraceCollector for OnCrashBacktraceCollector<'a> +where + 'a: 'static, +{ #[allow(clippy::unnecessary_cast)] fn on_call( &mut self, @@ -498,7 +493,7 @@ impl CallTraceCollector for OnCrashBacktraceCollector { self.callstack_hash ^= ret_addr as u64; } - fn pre_exec(&mut self, _emulator: &Emulator, _input: &I) + fn pre_exec(&mut self, _qemu: Qemu, _input: &I) where I: Input, { @@ -507,7 +502,7 @@ impl CallTraceCollector for OnCrashBacktraceCollector { fn post_exec( &mut self, - _emulator: &Emulator, + _qemu: Qemu, _input: &S::Input, observers: &mut OT, exit_kind: &mut ExitKind, @@ -516,7 +511,7 @@ impl CallTraceCollector for OnCrashBacktraceCollector { S: UsesInput, { let observer = observers - .match_name_mut::>(&self.observer_name) + .get_mut(&self.obs_ref) .expect("A OnCrashBacktraceCollector needs a BacktraceObserver"); observer.fill_external(self.callstack_hash, exit_kind); } @@ -602,7 +597,7 @@ impl CallTraceCollector for FullBacktraceCollector { } } - fn pre_exec(&mut self, _emulator: &Emulator, _input: &I) + fn pre_exec(&mut self, _qemu: Qemu, _input: &I) where I: Input, { diff --git a/libafl_qemu/src/cmplog.rs b/libafl_qemu/src/helpers/cmplog.rs similarity index 91% rename from libafl_qemu/src/cmplog.rs rename to libafl_qemu/src/helpers/cmplog.rs index 7651356ed1..10290dea95 100644 --- a/libafl_qemu/src/cmplog.rs +++ b/libafl_qemu/src/helpers/cmplog.rs @@ -1,7 +1,8 @@ #[cfg(emulation_mode = "usermode")] use capstone::{arch::BuildsCapstone, Capstone, InsnDetail}; use hashbrown::HashMap; -use libafl::{inputs::UsesInput, state::HasMetadata}; +use libafl::{inputs::UsesInput, HasMetadata}; +use libafl_qemu_sys::GuestAddr; pub use libafl_targets::{ cmps::{ __libafl_targets_cmplog_instructions, __libafl_targets_cmplog_routines, CMPLOG_ENABLED, @@ -11,18 +12,13 @@ pub use libafl_targets::{ use serde::{Deserialize, Serialize}; #[cfg(emulation_mode = "usermode")] +use crate::{capstone, emu::ArchExtras, CallingConvention, Qemu}; use crate::{ - capstone, - emu::{ArchExtras, Emulator}, - CallingConvention, -}; -use crate::{ - helper::{ + helpers::{ hash_me, HasInstrumentationFilter, IsFilter, QemuHelper, QemuHelperTuple, QemuInstrumentationAddressRangeFilter, }, hooks::{Hook, QemuHooks}, - GuestAddr, }; #[cfg_attr( @@ -70,7 +66,9 @@ impl Default for QemuCmpLogHelper { } } -impl HasInstrumentationFilter for QemuCmpLogHelper { +impl HasInstrumentationFilter + for QemuCmpLogHelper +{ fn filter(&self) -> &QemuInstrumentationAddressRangeFilter { &self.filter } @@ -246,12 +244,12 @@ impl QemuCmpLogRoutinesHelper { } } - let emu = Emulator::get().unwrap(); + let qemu = Qemu::get().unwrap(); - let a0: GuestAddr = emu + let a0: GuestAddr = qemu .read_function_argument(CallingConvention::Cdecl, 0) .unwrap_or(0); - let a1: GuestAddr = emu + let a1: GuestAddr = qemu .read_function_argument(CallingConvention::Cdecl, 1) .unwrap_or(0); @@ -262,7 +260,7 @@ impl QemuCmpLogRoutinesHelper { // if !emu.access_ok(VerifyAccess::Read, a0, 0x20) || !emu.access_ok(VerifyAccess::Read, a1, 0x20) { return; } unsafe { - __libafl_targets_cmplog_routines(k as usize, emu.g2h(a0), emu.g2h(a1)); + __libafl_targets_cmplog_routines(k as usize, qemu.g2h(a0), qemu.g2h(a1)); } } @@ -289,21 +287,21 @@ impl QemuCmpLogRoutinesHelper { .unwrap(); } - let emu = hooks.emulator(); + let qemu = hooks.qemu(); if let Some(h) = hooks.helpers().match_first_type::() { #[allow(unused_mut)] let mut code = { #[cfg(emulation_mode = "usermode")] unsafe { - std::slice::from_raw_parts(emu.g2h(pc), 512) + std::slice::from_raw_parts(qemu.g2h(pc), 512) } #[cfg(emulation_mode = "systemmode")] &mut [0; 512] }; #[cfg(emulation_mode = "systemmode")] unsafe { - emu.read_mem(pc, code) + qemu.read_mem(pc, code) }; // TODO handle faults let mut iaddr = pc; @@ -318,7 +316,7 @@ impl QemuCmpLogRoutinesHelper { match u32::from(detail.0) { capstone::InsnGroupType::CS_GRP_CALL => { let k = (hash_me(pc.into())) & (CMPLOG_MAP_W as u64 - 1); - emu.set_hook(k, insn.address() as GuestAddr, Self::on_call, false); + qemu.set_hook(k, insn.address() as GuestAddr, Self::on_call, false); } capstone::InsnGroupType::CS_GRP_RET | capstone::InsnGroupType::CS_GRP_INVALID @@ -335,11 +333,11 @@ impl QemuCmpLogRoutinesHelper { #[cfg(emulation_mode = "usermode")] unsafe { - code = std::slice::from_raw_parts(emu.g2h(iaddr), 512); + code = std::slice::from_raw_parts(qemu.g2h(iaddr), 512); } #[cfg(emulation_mode = "systemmode")] unsafe { - emu.read_mem(pc, code); + qemu.read_mem(pc, code); } // TODO handle faults } } @@ -349,7 +347,9 @@ impl QemuCmpLogRoutinesHelper { } #[cfg(emulation_mode = "usermode")] -impl HasInstrumentationFilter for QemuCmpLogRoutinesHelper { +impl HasInstrumentationFilter + for QemuCmpLogRoutinesHelper +{ fn filter(&self) -> &QemuInstrumentationAddressRangeFilter { &self.filter } diff --git a/libafl_qemu/src/drcov.rs b/libafl_qemu/src/helpers/drcov.rs similarity index 94% rename from libafl_qemu/src/drcov.rs rename to libafl_qemu/src/helpers/drcov.rs index 10f0fd1200..c59cd4eb60 100644 --- a/libafl_qemu/src/drcov.rs +++ b/libafl_qemu/src/helpers/drcov.rs @@ -1,21 +1,19 @@ use std::{path::PathBuf, sync::Mutex}; use hashbrown::{hash_map::Entry, HashMap}; -use libafl::{ - executors::ExitKind, inputs::UsesInput, observers::ObserversTuple, state::HasMetadata, -}; +use libafl::{executors::ExitKind, inputs::UsesInput, observers::ObserversTuple, HasMetadata}; +use libafl_qemu_sys::{GuestAddr, GuestUsize}; use libafl_targets::drcov::{DrCovBasicBlock, DrCovWriter}; use rangemap::RangeMap; use serde::{Deserialize, Serialize}; use crate::{ - emu::{GuestAddr, GuestUsize}, - helper::{ + helpers::{ HasInstrumentationFilter, IsFilter, QemuHelper, QemuHelperTuple, QemuInstrumentationAddressRangeFilter, }, hooks::{Hook, QemuHooks}, - Emulator, + Qemu, }; static DRCOV_IDS: Mutex>> = Mutex::new(None); @@ -78,7 +76,11 @@ impl QemuDrCovHelper { } } -impl HasInstrumentationFilter for QemuDrCovHelper { +impl HasInstrumentationFilter + for QemuDrCovHelper +where + S: UsesInput, +{ fn filter(&self) -> &QemuInstrumentationAddressRangeFilter { &self.filter } @@ -103,11 +105,11 @@ where ); } - fn pre_exec(&mut self, _emulator: &Emulator, _input: &S::Input) {} + fn pre_exec(&mut self, _qemu: Qemu, _input: &S::Input) {} fn post_exec( &mut self, - _emulator: &Emulator, + _qemu: Qemu, _input: &S::Input, _observers: &mut OT, _exit_kind: &mut ExitKind, @@ -200,8 +202,7 @@ pub fn gen_unique_block_ids( pc: GuestAddr, ) -> Option where - S: HasMetadata, - S: UsesInput, + S: UsesInput + HasMetadata, QT: QemuHelperTuple, { let drcov_helper = hooks @@ -255,8 +256,7 @@ pub fn gen_block_lengths( pc: GuestAddr, block_length: GuestUsize, ) where - S: HasMetadata, - S: UsesInput, + S: UsesInput + HasMetadata, QT: QemuHelperTuple, { let drcov_helper = hooks @@ -276,9 +276,8 @@ pub fn gen_block_lengths( pub fn exec_trace_block(hooks: &mut QemuHooks, _state: Option<&mut S>, id: u64) where - S: HasMetadata, - S: UsesInput, QT: QemuHelperTuple, + S: UsesInput + HasMetadata, { if hooks .helpers() diff --git a/libafl_qemu/src/edges.rs b/libafl_qemu/src/helpers/edges.rs similarity index 81% rename from libafl_qemu/src/edges.rs rename to libafl_qemu/src/helpers/edges.rs index c5978dd208..097d4d3c4c 100644 --- a/libafl_qemu/src/edges.rs +++ b/libafl_qemu/src/helpers/edges.rs @@ -1,24 +1,26 @@ use std::{cell::UnsafeCell, cmp::max}; use hashbrown::{hash_map::Entry, HashMap}; -use libafl::{inputs::UsesInput, state::HasMetadata}; +use libafl::{inputs::UsesInput, HasMetadata}; +use libafl_qemu_sys::GuestAddr; +#[cfg(emulation_mode = "systemmode")] +use libafl_qemu_sys::GuestPhysAddr; pub use libafl_targets::{ - edges_map_mut_ptr, edges_map_mut_slice, edges_max_num, std_edges_map_observer, EDGES_MAP, - EDGES_MAP_PTR, EDGES_MAP_PTR_NUM, EDGES_MAP_SIZE, MAX_EDGES_NUM, + edges_map_mut_ptr, EDGES_MAP, EDGES_MAP_PTR, EDGES_MAP_SIZE_IN_USE, EDGES_MAP_SIZE_MAX, + MAX_EDGES_FOUND, }; use serde::{Deserialize, Serialize}; +#[cfg(emulation_mode = "systemmode")] +use crate::helpers::QemuInstrumentationPagingFilter; use crate::{ - emu::GuestAddr, - helper::{ + helpers::{ hash_me, HasInstrumentationFilter, QemuHelper, QemuHelperTuple, QemuInstrumentationAddressRangeFilter, }, hooks::{Hook, QemuHooks}, IsFilter, }; -#[cfg(emulation_mode = "systemmode")] -use crate::{helper::QemuInstrumentationPagingFilter, GuestPhysAddr}; #[cfg_attr( any(not(feature = "serdeany_autoreg"), miri), @@ -130,7 +132,9 @@ impl Default for QemuEdgeCoverageHelper { } } -impl HasInstrumentationFilter for QemuEdgeCoverageHelper { +impl HasInstrumentationFilter + for QemuEdgeCoverageHelper +{ fn filter(&self) -> &QemuInstrumentationAddressRangeFilter { &self.address_filter } @@ -141,7 +145,9 @@ impl HasInstrumentationFilter for QemuEdg } #[cfg(emulation_mode = "systemmode")] -impl HasInstrumentationFilter for QemuEdgeCoverageHelper { +impl HasInstrumentationFilter + for QemuEdgeCoverageHelper +{ fn filter(&self) -> &QemuInstrumentationPagingFilter { &self.paging_filter } @@ -277,7 +283,7 @@ impl Default for QemuEdgeCoverageChildHelper { } } -impl HasInstrumentationFilter +impl HasInstrumentationFilter for QemuEdgeCoverageChildHelper { fn filter(&self) -> &QemuInstrumentationAddressRangeFilter { @@ -290,7 +296,9 @@ impl HasInstrumentationFilter } #[cfg(emulation_mode = "systemmode")] -impl HasInstrumentationFilter for QemuEdgeCoverageChildHelper { +impl HasInstrumentationFilter + for QemuEdgeCoverageChildHelper +{ fn filter(&self) -> &QemuInstrumentationPagingFilter { &self.paging_filter } @@ -302,8 +310,7 @@ impl HasInstrumentationFilter for QemuEdgeCover impl QemuHelper for QemuEdgeCoverageChildHelper where - S: UsesInput, - S: HasMetadata, + S: UsesInput + HasMetadata, { const HOOKS_DO_SIDE_EFFECTS: bool = false; @@ -330,6 +337,7 @@ where pub struct QemuEdgeCoverageClassicHelper { address_filter: QemuInstrumentationAddressRangeFilter, use_hitcounts: bool, + use_jit: bool, } #[cfg(emulation_mode = "systemmode")] @@ -338,23 +346,29 @@ pub struct QemuEdgeCoverageClassicHelper { address_filter: QemuInstrumentationAddressRangeFilter, paging_filter: QemuInstrumentationPagingFilter, use_hitcounts: bool, + use_jit: bool, } #[cfg(emulation_mode = "usermode")] impl QemuEdgeCoverageClassicHelper { #[must_use] - pub fn new(address_filter: QemuInstrumentationAddressRangeFilter) -> Self { + pub fn new(address_filter: QemuInstrumentationAddressRangeFilter, use_jit: bool) -> Self { Self { address_filter, use_hitcounts: true, + use_jit, } } #[must_use] - pub fn without_hitcounts(address_filter: QemuInstrumentationAddressRangeFilter) -> Self { + pub fn without_hitcounts( + address_filter: QemuInstrumentationAddressRangeFilter, + use_jit: bool, + ) -> Self { Self { address_filter, use_hitcounts: false, + use_jit, } } @@ -370,11 +384,13 @@ impl QemuEdgeCoverageClassicHelper { pub fn new( address_filter: QemuInstrumentationAddressRangeFilter, paging_filter: QemuInstrumentationPagingFilter, + use_jit: bool, ) -> Self { Self { address_filter, paging_filter, use_hitcounts: true, + use_jit, } } @@ -382,11 +398,13 @@ impl QemuEdgeCoverageClassicHelper { pub fn without_hitcounts( address_filter: QemuInstrumentationAddressRangeFilter, paging_filter: QemuInstrumentationPagingFilter, + use_jit: bool, ) -> Self { Self { address_filter, paging_filter, use_hitcounts: false, + use_jit, } } @@ -399,7 +417,7 @@ impl QemuEdgeCoverageClassicHelper { #[cfg(emulation_mode = "usermode")] impl Default for QemuEdgeCoverageClassicHelper { fn default() -> Self { - Self::new(QemuInstrumentationAddressRangeFilter::None) + Self::new(QemuInstrumentationAddressRangeFilter::None, false) } } @@ -409,11 +427,12 @@ impl Default for QemuEdgeCoverageClassicHelper { Self::new( QemuInstrumentationAddressRangeFilter::None, QemuInstrumentationPagingFilter::None, + false, ) } } -impl HasInstrumentationFilter +impl HasInstrumentationFilter for QemuEdgeCoverageClassicHelper { fn filter(&self) -> &QemuInstrumentationAddressRangeFilter { @@ -426,7 +445,9 @@ impl HasInstrumentationFilter } #[cfg(emulation_mode = "systemmode")] -impl HasInstrumentationFilter for QemuEdgeCoverageClassicHelper { +impl HasInstrumentationFilter + for QemuEdgeCoverageClassicHelper +{ fn filter(&self) -> &QemuInstrumentationPagingFilter { &self.paging_filter } @@ -436,10 +457,10 @@ impl HasInstrumentationFilter for QemuEdgeCover } } +#[allow(clippy::collapsible_else_if)] impl QemuHelper for QemuEdgeCoverageClassicHelper where - S: UsesInput, - S: HasMetadata, + S: UsesInput + HasMetadata, { const HOOKS_DO_SIDE_EFFECTS: bool = false; @@ -448,17 +469,47 @@ where QT: QemuHelperTuple, { if self.use_hitcounts { - hooks.blocks( - Hook::Function(gen_hashed_block_ids::), - Hook::Empty, - Hook::Raw(trace_block_transition_hitcount), - ); + if self.use_jit { + let hook_id = hooks.blocks( + Hook::Function(gen_hashed_block_ids::), + Hook::Empty, + Hook::Empty, + ); + + unsafe { + libafl_qemu_sys::libafl_qemu_block_hook_set_jit( + hook_id.0, + Some(libafl_qemu_sys::libafl_jit_trace_block_hitcount), + ); + } + } else { + hooks.blocks( + Hook::Function(gen_hashed_block_ids::), + Hook::Empty, + Hook::Raw(trace_block_transition_hitcount), + ); + } } else { - hooks.blocks( - Hook::Function(gen_hashed_block_ids::), - Hook::Empty, - Hook::Raw(trace_block_transition_single), - ); + if self.use_jit { + let hook_id = hooks.blocks( + Hook::Function(gen_hashed_block_ids::), + Hook::Empty, + Hook::Empty, + ); + + unsafe { + libafl_qemu_sys::libafl_qemu_block_hook_set_jit( + hook_id.0, + Some(libafl_qemu_sys::libafl_jit_trace_block_single), + ); + } + } else { + hooks.blocks( + Hook::Function(gen_hashed_block_ids::), + Hook::Empty, + Hook::Raw(trace_block_transition_single), + ); + } } } } @@ -472,8 +523,7 @@ pub fn gen_unique_edge_ids( dest: GuestAddr, ) -> Option where - S: HasMetadata, - S: UsesInput, + S: UsesInput + HasMetadata, QT: QemuHelperTuple, { if let Some(h) = hooks.helpers().match_first_type::() { @@ -487,9 +537,9 @@ where #[cfg(emulation_mode = "systemmode")] { let paging_id = hooks - .emulator() + .qemu() .current_cpu() - .map(|cpu| cpu.get_current_paging_id()) + .map(|cpu| cpu.current_paging_id()) .flatten(); if !h.must_instrument(src, paging_id) && !h.must_instrument(dest, paging_id) { @@ -498,29 +548,23 @@ where } } let state = state.expect("The gen_unique_edge_ids hook works only for in-process fuzzing"); - if state.metadata_map().get::().is_none() { - state.add_metadata(QemuEdgesMapMetadata::new()); - } - let meta = state - .metadata_map_mut() - .get_mut::() - .unwrap(); + let meta = state.metadata_or_insert_with(QemuEdgesMapMetadata::new); match meta.map.entry((src, dest)) { Entry::Occupied(e) => { let id = *e.get(); - let nxt = (id as usize + 1) & (EDGES_MAP_SIZE - 1); + let nxt = (id as usize + 1) & (EDGES_MAP_SIZE_MAX - 1); unsafe { - MAX_EDGES_NUM = max(MAX_EDGES_NUM, nxt); + MAX_EDGES_FOUND = max(MAX_EDGES_FOUND, nxt); } Some(id) } Entry::Vacant(e) => { let id = meta.current_id; e.insert(id); - meta.current_id = (id + 1) & (EDGES_MAP_SIZE as u64 - 1); + meta.current_id = (id + 1) & (EDGES_MAP_SIZE_MAX as u64 - 1); unsafe { - MAX_EDGES_NUM = meta.current_id as usize; + MAX_EDGES_FOUND = meta.current_id as usize; } // GuestAddress is u32 for 32 bit guests #[allow(clippy::unnecessary_cast)] @@ -563,9 +607,9 @@ where #[cfg(emulation_mode = "systemmode")] { let paging_id = hooks - .emulator() + .qemu() .current_cpu() - .map(|cpu| cpu.get_current_paging_id()) + .map(|cpu| cpu.current_paging_id()) .flatten(); if !h.must_instrument(src, paging_id) && !h.must_instrument(dest, paging_id) { @@ -575,7 +619,7 @@ where } // GuestAddress is u32 for 32 bit guests #[allow(clippy::unnecessary_cast)] - Some((hash_me(src as u64) ^ hash_me(dest as u64)) & (unsafe { EDGES_MAP_PTR_NUM } as u64 - 1)) + Some((hash_me(src as u64) ^ hash_me(dest as u64)) & (EDGES_MAP_SIZE_MAX as u64 - 1)) } pub extern "C" fn trace_edge_hitcount_ptr(_: *const (), id: u64) { @@ -630,9 +674,9 @@ where #[cfg(emulation_mode = "systemmode")] { let paging_id = hooks - .emulator() + .qemu() .current_cpu() - .map(|cpu| cpu.get_current_paging_id()) + .map(|cpu| cpu.current_paging_id()) .flatten(); if !h.must_instrument(pc, paging_id) { @@ -648,7 +692,7 @@ where pub extern "C" fn trace_block_transition_hitcount(_: *const (), id: u64) { unsafe { PREV_LOC.with(|prev_loc| { - let x = ((*prev_loc.get() ^ id) as usize) & (EDGES_MAP_PTR_NUM - 1); + let x = ((*prev_loc.get() ^ id) as usize) & (EDGES_MAP_SIZE_MAX - 1); let entry = EDGES_MAP_PTR.add(x); *entry = (*entry).wrapping_add(1); *prev_loc.get() = id.overflowing_shr(1).0; @@ -659,7 +703,7 @@ pub extern "C" fn trace_block_transition_hitcount(_: *const (), id: u64) { pub extern "C" fn trace_block_transition_single(_: *const (), id: u64) { unsafe { PREV_LOC.with(|prev_loc| { - let x = ((*prev_loc.get() ^ id) as usize) & (EDGES_MAP_PTR_NUM - 1); + let x = ((*prev_loc.get() ^ id) as usize) & (EDGES_MAP_SIZE_MAX - 1); let entry = EDGES_MAP_PTR.add(x); *entry = 1; *prev_loc.get() = id.overflowing_shr(1).0; diff --git a/libafl_qemu/src/injections.rs b/libafl_qemu/src/helpers/injections.rs similarity index 96% rename from libafl_qemu/src/injections.rs rename to libafl_qemu/src/helpers/injections.rs index 775ce67227..be410cb07f 100644 --- a/libafl_qemu/src/injections.rs +++ b/libafl_qemu/src/helpers/injections.rs @@ -15,13 +15,14 @@ use std::{ffi::CStr, fmt::Display, fs, os::raw::c_char, path::Path}; use hashbrown::HashMap; use libafl::{inputs::UsesInput, Error}; +use libafl_qemu_sys::GuestAddr; use serde::{Deserialize, Serialize}; #[cfg(not(cpu_target = "hexagon"))] use crate::SYS_execve; use crate::{ - elf::EasyElf, emu::ArchExtras, CallingConvention, Emulator, GuestAddr, Hook, QemuHelper, - QemuHelperTuple, QemuHooks, SyscallHookResult, + elf::EasyElf, emu::ArchExtras, CallingConvention, Hook, Qemu, QemuHelper, QemuHelperTuple, + QemuHooks, SyscallHookResult, }; #[cfg(cpu_target = "hexagon")] /// Hexagon syscalls are not currently supported by the `syscalls` crate, so we just paste this here for now. @@ -211,8 +212,8 @@ impl QemuInjectionHelper { id: usize, parameter: u8, ) { - let emu = hooks.emulator(); - let reg: GuestAddr = emu + let qemu = hooks.qemu(); + let reg: GuestAddr = qemu .current_cpu() .unwrap() .read_function_argument(CallingConvention::Cdecl, parameter) @@ -266,12 +267,13 @@ where where QT: QemuHelperTuple, { - let emu = hooks.emulator(); + let qemu = *hooks.qemu(); let mut libs: Vec = Vec::new(); - for region in emu.mappings() { + for region in qemu.mappings() { if let Some(path) = region.path().map(ToOwned::to_owned) { - if !path.is_empty() { + // skip [heap], [vdso] and friends + if !path.is_empty() && !path.starts_with('[') { LibInfo::add_unique( &mut libs, LibInfo { @@ -300,7 +302,7 @@ where vec![func_pc] } else { libs.iter() - .filter_map(|lib| find_function(emu, &lib.name, name, lib.off).unwrap()) + .filter_map(|lib| find_function(qemu, &lib.name, name, lib.off).unwrap()) .map(|func_pc| { log::info!("Injections: Function {name} found at {func_pc:#x}",); func_pc @@ -393,8 +395,8 @@ where } fn find_function( - emu: &Emulator, - file: &String, + qemu: Qemu, + file: &str, function: &str, loadaddr: GuestAddr, ) -> Result, Error> { @@ -403,7 +405,7 @@ fn find_function( let offset = if loadaddr > 0 { loadaddr } else { - emu.load_addr() + qemu.load_addr() }; Ok(elf.resolve_symbol(function, offset)) } diff --git a/libafl_qemu/src/helper.rs b/libafl_qemu/src/helpers/mod.rs similarity index 57% rename from libafl_qemu/src/helper.rs rename to libafl_qemu/src/helpers/mod.rs index ab421085da..2eaab7b140 100644 --- a/libafl_qemu/src/helper.rs +++ b/libafl_qemu/src/helpers/mod.rs @@ -1,14 +1,49 @@ use core::{fmt::Debug, ops::Range}; -use std::{collections::HashSet, hash}; +use std::{collections::HashSet, hash::BuildHasher}; use libafl::{executors::ExitKind, inputs::UsesInput, observers::ObserversTuple}; use libafl_bolts::tuples::{MatchFirstType, SplitBorrowExtractFirstType}; +use libafl_qemu_sys::{GuestAddr, GuestPhysAddr}; -use crate::{ - emu::{Emulator, GuestAddr}, - hooks::QemuHooks, - GuestPhysAddr, -}; +use crate::{hooks::QemuHooks, Qemu}; + +pub mod edges; +pub use edges::QemuEdgeCoverageHelper; + +#[cfg(not(cpu_target = "hexagon"))] +pub mod calls; +#[cfg(not(cpu_target = "hexagon"))] +pub use calls::QemuCallTracerHelper; + +#[cfg(not(cpu_target = "hexagon"))] +pub mod drcov; +#[cfg(not(cpu_target = "hexagon"))] +pub use drcov::QemuDrCovHelper; + +#[cfg(not(any(cpu_target = "mips", cpu_target = "hexagon")))] +pub mod cmplog; +#[cfg(not(any(cpu_target = "mips", cpu_target = "hexagon")))] +pub use cmplog::QemuCmpLogHelper; + +#[cfg(all(emulation_mode = "usermode", feature = "injections"))] +pub mod injections; +#[cfg(all(emulation_mode = "usermode", feature = "injections"))] +pub use injections::QemuInjectionHelper; + +#[cfg(all(emulation_mode = "usermode", not(cpu_target = "hexagon")))] +pub mod snapshot; +#[cfg(all(emulation_mode = "usermode", not(cpu_target = "hexagon")))] +pub use snapshot::QemuSnapshotHelper; + +#[cfg(all(emulation_mode = "usermode", not(cpu_target = "hexagon")))] +pub mod asan; +#[cfg(all(emulation_mode = "usermode", not(cpu_target = "hexagon")))] +pub use asan::{init_qemu_with_asan, QemuAsanHelper}; + +#[cfg(all(emulation_mode = "usermode", not(cpu_target = "hexagon")))] +pub mod asan_guest; +#[cfg(all(emulation_mode = "usermode", not(cpu_target = "hexagon")))] +pub use asan_guest::{init_qemu_with_asan_guest, QemuAsanGuestHelper}; /// A helper for `libafl_qemu`. // TODO remove 'static when specialization will be stable @@ -30,11 +65,11 @@ where { } - fn pre_exec(&mut self, _emulator: &Emulator, _input: &S::Input) {} + fn pre_exec(&mut self, _qemu: Qemu, _input: &S::Input) {} fn post_exec( &mut self, - _emulator: &Emulator, + _qemu: Qemu, _input: &S::Input, _observers: &mut OT, _exit_kind: &mut ExitKind, @@ -58,11 +93,11 @@ where where QT: QemuHelperTuple; - fn pre_exec_all(&mut self, _emulator: &Emulator, input: &S::Input); + fn pre_exec_all(&mut self, _qemu: Qemu, input: &S::Input); fn post_exec_all( &mut self, - _emulator: &Emulator, + _qemu: Qemu, input: &S::Input, _observers: &mut OT, _exit_kind: &mut ExitKind, @@ -88,11 +123,11 @@ where { } - fn pre_exec_all(&mut self, _emulator: &Emulator, _input: &S::Input) {} + fn pre_exec_all(&mut self, _qemu: Qemu, _input: &S::Input) {} fn post_exec_all( &mut self, - _emulator: &Emulator, + _qemu: Qemu, _input: &S::Input, _observers: &mut OT, _exit_kind: &mut ExitKind, @@ -102,6 +137,21 @@ where } } +impl HasInstrumentationFilter for (Head, ()) +where + Head: QemuHelper + HasInstrumentationFilter, + S: UsesInput, + F: IsFilter, +{ + fn filter(&self) -> &F { + self.0.filter() + } + + fn filter_mut(&mut self) -> &mut F { + self.0.filter_mut() + } +} + impl QemuHelperTuple for (Head, Tail) where Head: QemuHelper, @@ -126,27 +176,27 @@ where self.1.first_exec_all(hooks); } - fn pre_exec_all(&mut self, emulator: &Emulator, input: &S::Input) { - self.0.pre_exec(emulator, input); - self.1.pre_exec_all(emulator, input); + fn pre_exec_all(&mut self, qemu: Qemu, input: &S::Input) { + self.0.pre_exec(qemu, input); + self.1.pre_exec_all(qemu, input); } fn post_exec_all( &mut self, - emulator: &Emulator, + qemu: Qemu, input: &S::Input, observers: &mut OT, exit_kind: &mut ExitKind, ) where OT: ObserversTuple, { - self.0.post_exec(emulator, input, observers, exit_kind); - self.1.post_exec_all(emulator, input, observers, exit_kind); + self.0.post_exec(qemu, input, observers, exit_kind); + self.1.post_exec_all(qemu, input, observers, exit_kind); } } -#[derive(Debug)] -pub enum QemuFilterList { +#[derive(Debug, Clone)] +pub enum QemuFilterList { AllowList(T), DenyList(T), None, @@ -154,7 +204,7 @@ pub enum QemuFilterList { impl IsFilter for QemuFilterList where - T: IsFilter, + T: IsFilter + Clone, { type FilterParameter = T::FilterParameter; @@ -169,7 +219,10 @@ where pub type QemuInstrumentationPagingFilter = QemuFilterList>; -impl IsFilter for HashSet { +impl IsFilter for HashSet +where + H: BuildHasher, +{ type FilterParameter = Option; fn allowed(&self, paging_id: Self::FilterParameter) -> bool { @@ -192,7 +245,7 @@ impl IsFilter for Vec> { } } -pub trait HasInstrumentationFilter +pub trait HasInstrumentationFilter where F: IsFilter, { @@ -200,12 +253,43 @@ where fn filter_mut(&mut self) -> &mut F; - fn update_filter(&mut self, filter: F, emu: &Emulator) { + fn update_filter(&mut self, filter: F, emu: &Qemu) { *self.filter_mut() = filter; emu.flush_jit(); } } +#[cfg(emulation_mode = "usermode")] +pub trait StdInstrumentationFilter: + HasInstrumentationFilter +{ +} + +#[cfg(emulation_mode = "systemmode")] +pub trait StdInstrumentationFilter: + HasInstrumentationFilter + + HasInstrumentationFilter +{ +} + +#[cfg(emulation_mode = "systemmode")] +impl StdInstrumentationFilter for (Head, ()) +where + Head: QemuHelper + + HasInstrumentationFilter + + HasInstrumentationFilter, + S: UsesInput, +{ +} + +#[cfg(emulation_mode = "usermode")] +impl StdInstrumentationFilter for (Head, ()) +where + Head: QemuHelper + HasInstrumentationFilter, + S: UsesInput, +{ +} + pub trait IsFilter: Debug { type FilterParameter; diff --git a/libafl_qemu/src/snapshot.rs b/libafl_qemu/src/helpers/snapshot.rs similarity index 74% rename from libafl_qemu/src/snapshot.rs rename to libafl_qemu/src/helpers/snapshot.rs index 8cac85cba1..0eb544fb78 100644 --- a/libafl_qemu/src/snapshot.rs +++ b/libafl_qemu/src/helpers/snapshot.rs @@ -1,10 +1,12 @@ use std::{ cell::UnsafeCell, collections::{HashMap, HashSet}, + mem::MaybeUninit, sync::Mutex, }; -use libafl::{inputs::UsesInput, state::HasMetadata}; +use libafl::{inputs::UsesInput, HasMetadata}; +use libafl_qemu_sys::{GuestAddr, MmapPerms}; use meminterval::{Interval, IntervalTree}; use thread_local::ThreadLocal; @@ -23,20 +25,20 @@ use crate::SYS_mmap2; use crate::SYS_newfstatat; use crate::{ asan::QemuAsanHelper, - emu::{Emulator, MmapPerms, SyscallHookResult}, - helper::{QemuHelper, QemuHelperTuple}, + emu::SyscallHookResult, + helpers::{QemuHelper, QemuHelperTuple}, hooks::{Hook, QemuHooks}, - GuestAddr, SYS_fstat, SYS_fstatfs, SYS_futex, SYS_getrandom, SYS_mprotect, SYS_mremap, - SYS_munmap, SYS_pread64, SYS_read, SYS_readlinkat, SYS_statfs, + Qemu, SYS_fstat, SYS_fstatfs, SYS_futex, SYS_getrandom, SYS_mprotect, SYS_mremap, SYS_munmap, + SYS_pread64, SYS_read, SYS_readlinkat, SYS_statfs, }; -// TODO use the functions provided by Emulator +// TODO use the functions provided by Qemu pub const SNAPSHOT_PAGE_SIZE: usize = 4096; pub const SNAPSHOT_PAGE_MASK: GuestAddr = !(SNAPSHOT_PAGE_SIZE as GuestAddr - 1); -pub type StopExecutionCallback = Box; +pub type StopExecutionCallback = Box; -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct SnapshotPageInfo { pub addr: GuestAddr, pub perms: MmapPerms, @@ -136,11 +138,12 @@ impl QemuSnapshotHelper { } #[allow(clippy::uninit_assumed_init)] - pub fn snapshot(&mut self, emulator: &Emulator) { - self.brk = emulator.get_brk(); - self.mmap_start = emulator.get_mmap_start(); + pub fn snapshot(&mut self, qemu: Qemu) { + log::info!("Start snapshot"); + self.brk = qemu.get_brk(); + self.mmap_start = qemu.get_mmap_start(); self.pages.clear(); - for map in emulator.mappings() { + for map in qemu.mappings() { let mut addr = map.start(); while addr < map.end() { let mut info = SnapshotPageInfo { @@ -149,11 +152,11 @@ impl QemuSnapshotHelper { private: map.is_priv(), data: None, }; - if map.flags().is_r() { + if map.flags().readable() { // TODO not just for R pages unsafe { info.data = Some(Box::new(core::mem::zeroed())); - emulator.read_mem(addr, &mut info.data.as_mut().unwrap()[..]); + qemu.read_mem(addr, &mut info.data.as_mut().unwrap()[..]); } } self.pages.insert(addr, info); @@ -171,6 +174,7 @@ impl QemuSnapshotHelper { } self.empty = false; *self.new_maps.lock().unwrap() = self.maps.clone(); + log::info!("End snapshot"); } pub fn page_access(&mut self, page: GuestAddr) { @@ -199,7 +203,7 @@ impl QemuSnapshotHelper { pub fn access(&mut self, addr: GuestAddr, size: usize) { // ASSUMPTION: the access can only cross 2 pages - debug_assert!(size > 0); + debug_assert!(size > 0 && size < SNAPSHOT_PAGE_SIZE); let page = addr & SNAPSHOT_PAGE_MASK; self.page_access(page); let second_page = (addr + size as GuestAddr - 1) & SNAPSHOT_PAGE_MASK; @@ -208,10 +212,122 @@ impl QemuSnapshotHelper { } } - pub fn reset(&mut self, emulator: &Emulator) { + pub fn check_snapshot(&self, qemu: Qemu) { + let mut saved_pages_list = self.pages.clone(); + + log::info!("Checking snapshot correctness"); + + let mut perm_errors: Vec<(GuestAddr, MmapPerms, MmapPerms)> = Vec::new(); + let mut content_mismatch = false; + + for map in qemu.mappings() { + let mut addr = map.start(); + // assert_eq!(addr & SNAPSHOT_PAGE_MASK, 0); + while addr < map.end() { + if let Some(saved_page) = saved_pages_list.remove(&addr) { + if saved_page.perms.readable() { + let mut current_page_content: MaybeUninit<[u8; SNAPSHOT_PAGE_SIZE]> = + MaybeUninit::uninit(); + + if saved_page.perms != map.flags() { + perm_errors.push((addr, saved_page.perms, map.flags())); + log::warn!( + "\t0x{:x}: Flags do not match: saved is {:?} and current is {:?}", + addr, + saved_page.perms, + map.flags() + ); + } + + unsafe { + qemu.read_mem( + addr, + current_page_content.as_mut_ptr().as_mut().unwrap(), + ); + } + + let current_page_content: &mut [u8; SNAPSHOT_PAGE_SIZE] = + unsafe { &mut current_page_content.assume_init() }; + + if saved_page.data.as_ref().unwrap().as_ref() + != current_page_content.as_ref() + { + let mut offsets = Vec::new(); + for (i, (saved_page_byte, current_page_byte)) in saved_page + .data + .unwrap() + .iter() + .zip(current_page_content.iter()) + .enumerate() + { + if saved_page_byte != current_page_byte { + offsets.push(i); + } + } + log::warn!( + "Faulty restore at {}", + offsets.iter().fold(String::new(), |acc, offset| format!( + "{}, 0x{:x}", + acc, + addr + *offset as GuestAddr + )) + ); + content_mismatch = true; + } + } + } else { + log::warn!("\tpage not found @addr 0x{:x}", addr); + } + + addr += SNAPSHOT_PAGE_SIZE as GuestAddr; + } + } + + assert!(saved_pages_list.is_empty()); + + if !perm_errors.is_empty() { + let mut perm_error_ranges: Vec<(GuestAddr, GuestAddr, MmapPerms, MmapPerms)> = + Vec::new(); + + for error in perm_errors { + if let Some(last_range) = perm_error_ranges.last_mut() { + if last_range.1 + SNAPSHOT_PAGE_SIZE as GuestAddr == error.0 as GuestAddr + && error.1 == last_range.2 + && error.2 == last_range.3 + { + last_range.1 += SNAPSHOT_PAGE_SIZE as GuestAddr; + } else { + perm_error_ranges.push((error.0, error.0, error.1, error.2)); + } + } else { + perm_error_ranges.push((error.0, error.0, error.1, error.2)); + } + } + + for error_range in perm_error_ranges { + log::error!( + "0x{:x} -> 0x{:x}: saved is {:?} but current is {:?}", + error_range.0, + error_range.1, + error_range.2, + error_range.3 + ); + } + + content_mismatch = true; + } + + assert!(!content_mismatch, "Error found, stopping..."); + + log::info!("Snapshot check OK"); + } + + pub fn reset(&mut self, qemu: Qemu) { { let new_maps = self.new_maps.get_mut().unwrap(); + log::info!("Start restore"); + for acc in &mut self.accesses { unsafe { &mut (*acc.get()) }.dirty.retain(|page| { if let Some(info) = self.pages.get_mut(page) { @@ -223,8 +339,8 @@ impl QemuSnapshotHelper { .tree .query_mut(*page..(page + SNAPSHOT_PAGE_SIZE as GuestAddr)) { - if !entry.value.perms.unwrap_or(MmapPerms::None).is_w() { - drop(emulator.mprotect( + if !entry.value.perms.unwrap_or(MmapPerms::None).writable() { + drop(qemu.mprotect( entry.interval.start, (entry.interval.end - entry.interval.start) as usize, MmapPerms::ReadWrite, @@ -239,7 +355,7 @@ impl QemuSnapshotHelper { return true; // Restore later } - unsafe { emulator.write_mem(*page, &data[..]) }; + unsafe { qemu.write_mem(*page, &data[..]) }; } else { panic!("Cannot restored a dirty but unsaved page"); } @@ -249,7 +365,7 @@ impl QemuSnapshotHelper { } } - self.reset_maps(emulator); + self.reset_maps(qemu); // This one is after that we remapped potential regions mapped at snapshot time but unmapped during execution for acc in &mut self.accesses { @@ -259,9 +375,10 @@ impl QemuSnapshotHelper { .tree .query_mut(*page..(page + SNAPSHOT_PAGE_SIZE as GuestAddr)) { - if !entry.value.perms.unwrap_or(MmapPerms::None).is_w() && !entry.value.changed + if !entry.value.perms.unwrap_or(MmapPerms::None).writable() + && !entry.value.changed { - drop(emulator.mprotect( + drop(qemu.mprotect( entry.interval.start, (entry.interval.end - entry.interval.start) as usize, MmapPerms::ReadWrite, @@ -273,7 +390,7 @@ impl QemuSnapshotHelper { if let Some(info) = self.pages.get_mut(page) { // TODO avoid duplicated memcpy if let Some(data) = info.data.as_ref() { - unsafe { emulator.write_mem(*page, &data[..]) }; + unsafe { qemu.write_mem(*page, &data[..]) }; } else { panic!("Cannot restored a dirty but unsaved page"); } @@ -284,7 +401,7 @@ impl QemuSnapshotHelper { for entry in self.maps.tree.query_mut(0..GuestAddr::MAX) { if entry.value.changed { - drop(emulator.mprotect( + drop(qemu.mprotect( entry.interval.start, (entry.interval.end - entry.interval.start) as usize, entry.value.perms.unwrap(), @@ -293,8 +410,13 @@ impl QemuSnapshotHelper { } } - emulator.set_brk(self.brk); - emulator.set_mmap_start(self.mmap_start); + qemu.set_brk(self.brk); + qemu.set_mmap_start(self.mmap_start); + + #[cfg(feature = "paranoid_debug")] + self.check_snapshot(qemu); + + log::info!("End restore"); } pub fn is_unmap_allowed(&mut self, start: GuestAddr, mut size: usize) -> bool { @@ -331,8 +453,8 @@ impl QemuSnapshotHelper { if self.mmap_limit != 0 && total_size > self.mmap_limit { let mut cb = self.stop_execution.take().unwrap(); - let emu = Emulator::get().unwrap(); - (cb)(self, &emu); + let qemu = Qemu::get().unwrap(); + cb(self, &qemu); self.stop_execution = Some(cb); } } @@ -425,7 +547,7 @@ impl QemuSnapshotHelper { } } - pub fn reset_maps(&mut self, emulator: &Emulator) { + pub fn reset_maps(&mut self, qemu: Qemu) { let new_maps = self.new_maps.get_mut().unwrap(); for entry in self.maps.tree.query(0..GuestAddr::MAX) { @@ -440,14 +562,14 @@ impl QemuSnapshotHelper { if found.is_empty() { //panic!("A pre-snapshot memory region was unmapped"); - drop(emulator.map_fixed( + drop(qemu.map_fixed( entry.interval.start, (entry.interval.end - entry.interval.start) as usize, entry.value.perms.unwrap(), )); } else if found.len() == 1 && found[0].0 == *entry.interval { if found[0].1 && found[0].2 != entry.value.perms { - drop(emulator.mprotect( + drop(qemu.mprotect( entry.interval.start, (entry.interval.end - entry.interval.start) as usize, entry.value.perms.unwrap(), @@ -455,20 +577,20 @@ impl QemuSnapshotHelper { } } else { // TODO check for holes - drop(emulator.mprotect( + drop(qemu.mprotect( entry.interval.start, (entry.interval.end - entry.interval.start) as usize, entry.value.perms.unwrap(), )); } - for (i, _, _) in found { - new_maps.tree.delete(i); + for (interval, ..) in found { + new_maps.tree.delete(interval); } } for entry in new_maps.tree.query(0..GuestAddr::MAX) { - drop(emulator.unmap( + drop(qemu.unmap( entry.interval.start, (entry.interval.end - entry.interval.start) as usize, )); @@ -496,10 +618,10 @@ where // The ASan helper, if present, will call the tracer hook for the snapshot helper as opt hooks.writes( Hook::Empty, - Hook::Function(trace_write1_snapshot::), - Hook::Function(trace_write2_snapshot::), - Hook::Function(trace_write4_snapshot::), - Hook::Function(trace_write8_snapshot::), + Hook::Function(trace_write_snapshot::), + Hook::Function(trace_write_snapshot::), + Hook::Function(trace_write_snapshot::), + Hook::Function(trace_write_snapshot::), Hook::Function(trace_write_n_snapshot::), ); } @@ -510,55 +632,16 @@ where hooks.after_syscalls(Hook::Function(trace_mmap_snapshot::)); } - fn pre_exec(&mut self, emulator: &Emulator, _input: &S::Input) { + fn pre_exec(&mut self, qemu: Qemu, _input: &S::Input) { if self.empty { - self.snapshot(emulator); + self.snapshot(qemu); } else { - self.reset(emulator); + self.reset(qemu); } } } -pub fn trace_write1_snapshot( - hooks: &mut QemuHooks, - _state: Option<&mut S>, - _id: u64, - addr: GuestAddr, -) where - S: UsesInput, - QT: QemuHelperTuple, -{ - let h = hooks.match_helper_mut::().unwrap(); - h.access(addr, 1); -} - -pub fn trace_write2_snapshot( - hooks: &mut QemuHooks, - _state: Option<&mut S>, - _id: u64, - addr: GuestAddr, -) where - S: UsesInput, - QT: QemuHelperTuple, -{ - let h = hooks.match_helper_mut::().unwrap(); - h.access(addr, 2); -} - -pub fn trace_write4_snapshot( - hooks: &mut QemuHooks, - _state: Option<&mut S>, - _id: u64, - addr: GuestAddr, -) where - S: UsesInput, - QT: QemuHelperTuple, -{ - let h = hooks.match_helper_mut::().unwrap(); - h.access(addr, 4); -} - -pub fn trace_write8_snapshot( +pub fn trace_write_snapshot( hooks: &mut QemuHooks, _state: Option<&mut S>, _id: u64, @@ -568,7 +651,7 @@ pub fn trace_write8_snapshot( QT: QemuHelperTuple, { let h = hooks.match_helper_mut::().unwrap(); - h.access(addr, 8); + h.access(addr, SIZE); } pub fn trace_write_n_snapshot( @@ -708,7 +791,7 @@ where } else if sys_const == SYS_mprotect { if let Ok(prot) = MmapPerms::try_from(a2 as i32) { let h = hooks.match_helper_mut::().unwrap(); - h.add_mapped(a0, a1 as usize, Some(prot)); + h.change_mapped(a0, a1 as usize, Some(prot)); } } else if sys_const == SYS_munmap { let h = hooks.match_helper_mut::().unwrap(); diff --git a/libafl_qemu/src/hooks.rs b/libafl_qemu/src/hooks.rs index a1a6aad8be..6b51ddda11 100644 --- a/libafl_qemu/src/hooks.rs +++ b/libafl_qemu/src/hooks.rs @@ -1,25 +1,34 @@ //! The high-level hooks -#![allow(clippy::type_complexity)] +#![allow(clippy::type_complexity, clippy::missing_transmute_annotations)] +#[cfg(emulation_mode = "usermode")] +use core::ptr::addr_of_mut; use core::{ ffi::c_void, fmt::{self, Debug, Formatter}, marker::PhantomData, mem::transmute, pin::Pin, - ptr::{self, addr_of, addr_of_mut}, + ptr::{self, addr_of}, }; -use libafl::{executors::hooks::inprocess::inprocess_get_state, inputs::UsesInput}; +use libafl::{ + executors::{hooks::inprocess::inprocess_get_state, ExitKind}, + inputs::UsesInput, + state::NopState, +}; +use libafl_qemu_sys::{CPUArchStatePtr, FatPtr, GuestAddr, GuestUsize}; pub use crate::emu::SyscallHookResult; use crate::{ - emu::{Emulator, FatPtr, MemAccessInfo, SKIP_EXEC_HOOK}, - helper::QemuHelperTuple, - BackdoorHookId, BlockHookId, CmpHookId, EdgeHookId, GuestAddr, GuestUsize, HookId, - InstructionHookId, NewThreadHookId, PostSyscallHookId, PreSyscallHookId, ReadHookId, + emu::{MemAccessInfo, Qemu, SKIP_EXEC_HOOK}, + helpers::QemuHelperTuple, + sys::TCGTemp, + BackdoorHookId, BlockHookId, CmpHookId, EdgeHookId, HookId, InstructionHookId, ReadHookId, WriteHookId, }; +#[cfg(emulation_mode = "usermode")] +use crate::{NewThreadHookId, PostSyscallHookId, PreSyscallHookId}; /* // all kinds of hooks @@ -108,7 +117,7 @@ macro_rules! create_wrapper { { unsafe { let hooks = get_qemu_hooks::(); - let func: fn(&mut QemuHooks, Option<&mut S>, $($param_type),*) = transmute(hook as *mut c_void); + let func: fn(&mut QemuHooks, Option<&mut S>, $($param_type),*) = transmute(ptr::from_mut::(hook)); func(hooks, inprocess_get_state::(), $($param),*); } } @@ -135,7 +144,7 @@ macro_rules! create_wrapper { { unsafe { let hooks = get_qemu_hooks::(); - let func: fn(&mut QemuHooks, Option<&mut S>, $($param_type),*) -> $ret_type= transmute(hook as *mut c_void); + let func: fn(&mut QemuHooks, Option<&mut S>, $($param_type),*) -> $ret_type= transmute(ptr::from_mut::(hook)); func(hooks, inprocess_get_state::(), $($param),*) } } @@ -246,32 +255,45 @@ macro_rules! create_exec_wrapper { static mut GENERIC_HOOKS: Vec>> = vec![]; create_wrapper!(generic, (pc: GuestAddr)); static mut BACKDOOR_HOOKS: Vec>> = vec![]; -create_wrapper!(backdoor, (pc: GuestAddr)); +create_wrapper!(backdoor, (cpu: CPUArchStatePtr, pc: GuestAddr)); #[cfg(emulation_mode = "usermode")] static mut PRE_SYSCALL_HOOKS: Vec>> = vec![]; #[cfg(emulation_mode = "usermode")] -create_wrapper!(pre_syscall, (sys_num: i32, - a0: GuestAddr, - a1: GuestAddr, - a2: GuestAddr, - a3: GuestAddr, - a4: GuestAddr, - a5: GuestAddr, - a6: GuestAddr, - a7: GuestAddr), SyscallHookResult); +create_wrapper!( + pre_syscall, + ( + sys_num: i32, + a0: GuestAddr, + a1: GuestAddr, + a2: GuestAddr, + a3: GuestAddr, + a4: GuestAddr, + a5: GuestAddr, + a6: GuestAddr, + a7: GuestAddr + ), + SyscallHookResult +); #[cfg(emulation_mode = "usermode")] static mut POST_SYSCALL_HOOKS: Vec>> = vec![]; #[cfg(emulation_mode = "usermode")] -create_wrapper!(post_syscall, (res: GuestAddr, sys_num: i32, - a0: GuestAddr, - a1: GuestAddr, - a2: GuestAddr, - a3: GuestAddr, - a4: GuestAddr, - a5: GuestAddr, - a6: GuestAddr, - a7: GuestAddr), GuestAddr); +create_wrapper!( + post_syscall, + ( + res: GuestAddr, + sys_num: i32, + a0: GuestAddr, + a1: GuestAddr, + a2: GuestAddr, + a3: GuestAddr, + a4: GuestAddr, + a5: GuestAddr, + a6: GuestAddr, + a7: GuestAddr + ), + GuestAddr +); #[cfg(emulation_mode = "usermode")] static mut NEW_THREAD_HOOKS: Vec>> = vec![]; #[cfg(emulation_mode = "usermode")] @@ -287,20 +309,32 @@ create_post_gen_wrapper!(block, (addr: GuestAddr, len: GuestUsize), 1, BlockHook create_exec_wrapper!(block, (id: u64), 0, 1, BlockHookId); static mut READ_HOOKS: Vec>>> = vec![]; -create_gen_wrapper!(read, (pc: GuestAddr, info: MemAccessInfo), u64, 5, ReadHookId); +create_gen_wrapper!(read, (pc: GuestAddr, addr: *mut TCGTemp, info: MemAccessInfo), u64, 5, ReadHookId); create_exec_wrapper!(read, (id: u64, addr: GuestAddr), 0, 5, ReadHookId); create_exec_wrapper!(read, (id: u64, addr: GuestAddr), 1, 5, ReadHookId); create_exec_wrapper!(read, (id: u64, addr: GuestAddr), 2, 5, ReadHookId); create_exec_wrapper!(read, (id: u64, addr: GuestAddr), 3, 5, ReadHookId); -create_exec_wrapper!(read, (id: u64, addr: GuestAddr, size: usize), 4, 5, ReadHookId); +create_exec_wrapper!( + read, + (id: u64, addr: GuestAddr, size: usize), + 4, + 5, + ReadHookId +); static mut WRITE_HOOKS: Vec>>> = vec![]; -create_gen_wrapper!(write, (pc: GuestAddr, info: MemAccessInfo), u64, 5, WriteHookId); +create_gen_wrapper!(write, (pc: GuestAddr, addr: *mut TCGTemp, info: MemAccessInfo), u64, 5, WriteHookId); create_exec_wrapper!(write, (id: u64, addr: GuestAddr), 0, 5, WriteHookId); create_exec_wrapper!(write, (id: u64, addr: GuestAddr), 1, 5, WriteHookId); create_exec_wrapper!(write, (id: u64, addr: GuestAddr), 2, 5, WriteHookId); create_exec_wrapper!(write, (id: u64, addr: GuestAddr), 3, 5, WriteHookId); -create_exec_wrapper!(write, (id: u64, addr: GuestAddr, size: usize), 4, 5, WriteHookId); +create_exec_wrapper!( + write, + (id: u64, addr: GuestAddr, size: usize), + 4, + 5, + WriteHookId +); static mut CMP_HOOKS: Vec>>> = vec![]; create_gen_wrapper!(cmp, (pc: GuestAddr, size: usize), u64, 4, CmpHookId); @@ -337,6 +371,7 @@ where } static mut HOOKS_IS_INITIALIZED: bool = false; +static mut FIRST_EXEC: bool = true; pub struct QemuHooks where @@ -344,7 +379,7 @@ where S: UsesInput, { helpers: QT, - emulator: Emulator, + qemu: Qemu, phantom: PhantomData, } @@ -356,17 +391,47 @@ where fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { f.debug_struct("QemuHooks") .field("helpers", &self.helpers) - .field("emulator", &self.emulator) + .field("emulator", &self.qemu) .finish() } } +impl QemuHooks> +where + QT: QemuHelperTuple>, + NopState: UsesInput, +{ + pub fn reproducer(qemu: Qemu, helpers: QT) -> Box { + Self::new(qemu, helpers) + } + + pub fn repro_run(&mut self, harness: &mut H, input: &I) -> ExitKind + where + H: FnMut(&I) -> ExitKind, + { + unsafe { + if FIRST_EXEC { + self.helpers.first_exec_all(self); + FIRST_EXEC = false; + } + } + self.helpers.pre_exec_all(self.qemu, input); + + let mut exit_kind = harness(input); + + self.helpers + .post_exec_all(self.qemu, input, &mut (), &mut exit_kind); + + exit_kind + } +} + impl QemuHooks where QT: QemuHelperTuple, S: UsesInput, { - pub fn new(emulator: Emulator, helpers: QT) -> Box { + pub fn new(qemu: Qemu, helpers: QT) -> Box { unsafe { assert!( !HOOKS_IS_INITIALIZED, @@ -375,9 +440,9 @@ where HOOKS_IS_INITIALIZED = true; } // re-translate blocks with hooks - emulator.flush_jit(); + qemu.flush_jit(); let slf = Box::new(Self { - emulator, + qemu, helpers, phantom: PhantomData, }); @@ -404,8 +469,8 @@ where self.helpers.match_first_type_mut::() } - pub fn emulator(&self) -> &Emulator { - &self.emulator + pub fn qemu(&self) -> &Qemu { + &self.qemu } pub fn helpers(&self) -> &QT { @@ -431,7 +496,7 @@ where Hook::Closure(c) => self.instruction_closure(addr, c, invalidate_block), Hook::Raw(r) => { let z: *const () = ptr::null::<()>(); - self.emulator.set_hook(z, addr, r, invalidate_block) + self.qemu.set_hook(z, addr, r, invalidate_block) } Hook::Empty => InstructionHookId(0), // TODO error type } @@ -444,7 +509,7 @@ where invalidate_block: bool, ) -> InstructionHookId { unsafe { - self.emulator.set_hook( + self.qemu.set_hook( transmute(hook), addr, func_generic_hook_wrapper::, @@ -462,7 +527,7 @@ where unsafe { let fat: FatPtr = transmute(hook); GENERIC_HOOKS.push(Box::pin((InstructionHookId(0), fat))); - let id = self.emulator.set_hook( + let id = self.qemu.set_hook( &mut GENERIC_HOOKS .last_mut() .unwrap() @@ -524,7 +589,7 @@ where post_gen: HookRepr::Empty, execs: [hook_to_repr!(execution_hook)], })); - let id = self.emulator.add_edge_hooks( + let id = self.qemu.add_edge_hooks( EDGE_HOOKS.last_mut().unwrap().as_mut().get_unchecked_mut(), gen, exec, @@ -583,7 +648,7 @@ where post_gen: hook_to_repr!(post_generation_hook), execs: [hook_to_repr!(execution_hook)], })); - let id = self.emulator.add_block_hooks( + let id = self.qemu.add_block_hooks( BLOCK_HOOKS.last_mut().unwrap().as_mut().get_unchecked_mut(), gen, postgen, @@ -603,16 +668,23 @@ where pub fn reads( &self, generation_hook: Hook< - fn(&mut Self, Option<&mut S>, pc: GuestAddr, info: MemAccessInfo) -> Option, + fn( + &mut Self, + Option<&mut S>, + pc: GuestAddr, + addr: *mut TCGTemp, + info: MemAccessInfo, + ) -> Option, Box< dyn for<'a> FnMut( &'a mut Self, Option<&'a mut S>, GuestAddr, + *mut TCGTemp, MemAccessInfo, ) -> Option, >, - extern "C" fn(*const (), pc: GuestAddr, info: MemAccessInfo) -> u64, + extern "C" fn(*const (), pc: GuestAddr, addr: *mut TCGTemp, info: MemAccessInfo) -> u64, >, execution_hook_1: Hook< fn(&mut Self, Option<&mut S>, id: u64, addr: GuestAddr), @@ -647,6 +719,7 @@ where extern "C" fn( &mut HookState<5, ReadHookId>, pc: GuestAddr, + addr: *mut TCGTemp, info: MemAccessInfo, ) -> u64 ); @@ -687,7 +760,7 @@ where hook_to_repr!(execution_hook_n), ], })); - let id = self.emulator.add_read_hooks( + let id = self.qemu.add_read_hooks( READ_HOOKS.last_mut().unwrap().as_mut().get_unchecked_mut(), gen, exec1, @@ -710,16 +783,23 @@ where pub fn writes( &self, generation_hook: Hook< - fn(&mut Self, Option<&mut S>, pc: GuestAddr, info: MemAccessInfo) -> Option, + fn( + &mut Self, + Option<&mut S>, + pc: GuestAddr, + addr: *mut TCGTemp, + info: MemAccessInfo, + ) -> Option, Box< dyn for<'a> FnMut( &'a mut Self, Option<&'a mut S>, GuestAddr, + *mut TCGTemp, MemAccessInfo, ) -> Option, >, - extern "C" fn(*const (), pc: GuestAddr, info: MemAccessInfo) -> u64, + extern "C" fn(*const (), pc: GuestAddr, addr: *mut TCGTemp, info: MemAccessInfo) -> u64, >, execution_hook_1: Hook< fn(&mut Self, Option<&mut S>, id: u64, addr: GuestAddr), @@ -754,6 +834,7 @@ where extern "C" fn( &mut HookState<5, WriteHookId>, pc: GuestAddr, + addr: *mut TCGTemp, info: MemAccessInfo, ) -> u64 ); @@ -799,7 +880,7 @@ where hook_to_repr!(execution_hook_n), ], })); - let id = self.emulator.add_write_hooks( + let id = self.qemu.add_write_hooks( WRITE_HOOKS.last_mut().unwrap().as_mut().get_unchecked_mut(), gen, exec1, @@ -885,7 +966,7 @@ where hook_to_repr!(execution_hook_8), ], })); - let id = self.emulator.add_cmp_hooks( + let id = self.qemu.add_cmp_hooks( CMP_HOOKS.last_mut().unwrap().as_mut().get_unchecked_mut(), gen, exec1, @@ -906,9 +987,9 @@ where pub fn backdoor( &self, hook: Hook< - fn(&mut Self, Option<&mut S>, GuestAddr), + fn(&mut Self, Option<&mut S>, cpu: CPUArchStatePtr, GuestAddr), Box FnMut(&'a mut Self, Option<&'a mut S>, GuestAddr)>, - extern "C" fn(*const (), pc: GuestAddr), + extern "C" fn(*const (), cpu: CPUArchStatePtr, pc: GuestAddr), >, ) -> BackdoorHookId { match hook { @@ -916,7 +997,7 @@ where Hook::Closure(c) => self.backdoor_closure(c), Hook::Raw(r) => { let z: *const () = ptr::null::<()>(); - self.emulator.add_backdoor_hook(z, r) + self.qemu.add_backdoor_hook(z, r) } Hook::Empty => BackdoorHookId(0), // TODO error type } @@ -924,10 +1005,10 @@ where pub fn backdoor_function( &self, - hook: fn(&mut Self, Option<&mut S>, pc: GuestAddr), + hook: fn(&mut Self, Option<&mut S>, cpu: CPUArchStatePtr, pc: GuestAddr), ) -> BackdoorHookId { unsafe { - self.emulator + self.qemu .add_backdoor_hook(transmute(hook), func_backdoor_hook_wrapper::) } } @@ -939,7 +1020,7 @@ where unsafe { let fat: FatPtr = transmute(hook); BACKDOOR_HOOKS.push(Box::pin((BackdoorHookId(0), fat))); - let id = self.emulator.add_backdoor_hook( + let id = self.qemu.add_backdoor_hook( &mut BACKDOOR_HOOKS .last_mut() .unwrap() @@ -1010,7 +1091,7 @@ where Hook::Closure(c) => self.syscalls_closure(c), Hook::Raw(r) => { let z: *const () = ptr::null::<()>(); - self.emulator.add_pre_syscall_hook(z, r) + self.qemu.add_pre_syscall_hook(z, r) } Hook::Empty => PreSyscallHookId(0), // TODO error type } @@ -1035,7 +1116,7 @@ where ) -> SyscallHookResult, ) -> PreSyscallHookId { unsafe { - self.emulator + self.qemu .add_pre_syscall_hook(transmute(hook), func_pre_syscall_hook_wrapper::) } } @@ -1063,7 +1144,7 @@ where unsafe { let fat: FatPtr = transmute(hook); PRE_SYSCALL_HOOKS.push(Box::pin((PreSyscallHookId(0), fat))); - let id = self.emulator.add_pre_syscall_hook( + let id = self.qemu.add_pre_syscall_hook( &mut PRE_SYSCALL_HOOKS .last_mut() .unwrap() @@ -1137,7 +1218,7 @@ where Hook::Closure(c) => self.after_syscalls_closure(c), Hook::Raw(r) => { let z: *const () = ptr::null::<()>(); - self.emulator.add_post_syscall_hook(z, r) + self.qemu.add_post_syscall_hook(z, r) } Hook::Empty => PostSyscallHookId(0), // TODO error type } @@ -1163,7 +1244,7 @@ where ) -> GuestAddr, ) -> PostSyscallHookId { unsafe { - self.emulator + self.qemu .add_post_syscall_hook(transmute(hook), func_post_syscall_hook_wrapper::) } } @@ -1192,7 +1273,7 @@ where unsafe { let fat: FatPtr = transmute(hook); POST_SYSCALL_HOOKS.push(Box::pin((PostSyscallHookId(0), fat))); - let id = self.emulator.add_post_syscall_hook( + let id = self.qemu.add_post_syscall_hook( &mut POST_SYSCALL_HOOKS .last_mut() .unwrap() @@ -1225,7 +1306,7 @@ where Hook::Closure(c) => self.thread_creation_closure(c), Hook::Raw(r) => { let z: *const () = ptr::null::<()>(); - self.emulator.add_new_thread_hook(z, r) + self.qemu.add_new_thread_hook(z, r) } Hook::Empty => NewThreadHookId(0), // TODO error type } @@ -1237,7 +1318,7 @@ where hook: fn(&mut Self, Option<&mut S>, tid: u32) -> bool, ) -> NewThreadHookId { unsafe { - self.emulator + self.qemu .add_new_thread_hook(transmute(hook), func_new_thread_hook_wrapper::) } } @@ -1250,7 +1331,7 @@ where unsafe { let fat: FatPtr = transmute(hook); NEW_THREAD_HOOKS.push(Box::pin((NewThreadHookId(0), fat))); - let id = self.emulator.add_new_thread_hook( + let id = self.qemu.add_new_thread_hook( &mut NEW_THREAD_HOOKS .last_mut() .unwrap() @@ -1272,7 +1353,7 @@ where #[cfg(emulation_mode = "usermode")] pub fn crash_function(&self, hook: fn(&mut Self, target_signal: i32)) { unsafe { - self.emulator.set_crash_hook(crash_hook_wrapper::); + self.qemu.set_crash_hook(crash_hook_wrapper::); CRASH_HOOKS.push(HookRepr::Function(hook as *const libc::c_void)); } } @@ -1280,7 +1361,7 @@ where #[cfg(emulation_mode = "usermode")] pub fn crash_closure(&self, hook: Box) { unsafe { - self.emulator.set_crash_hook(crash_hook_wrapper::); + self.qemu.set_crash_hook(crash_hook_wrapper::); CRASH_HOOKS.push(HookRepr::Closure(transmute(hook))); } } diff --git a/libafl_qemu/src/lib.rs b/libafl_qemu/src/lib.rs index 7c5719d3a3..de3ef10c88 100644 --- a/libafl_qemu/src/lib.rs +++ b/libafl_qemu/src/lib.rs @@ -1,6 +1,7 @@ #![cfg_attr(nightly, feature(used_with_arg))] //! Welcome to `LibAFL` QEMU //! +//! __Warning__: The documentation is built by default for `x86_64` in `usermode`. To access the documentation of other architectures or `systemmode`, the documentation must be rebuilt with the right features. #![doc = include_str!("../../README.md")] /*! */ #![cfg_attr(feature = "document-features", doc = document_features::document_features!())] @@ -32,76 +33,17 @@ use std::env; pub use libafl_qemu_sys as sys; pub use strum::IntoEnumIterator; -#[cfg(cpu_target = "aarch64")] -pub mod aarch64; -#[cfg(all(cpu_target = "aarch64", not(feature = "clippy")))] -pub use aarch64::*; - -#[cfg(cpu_target = "arm")] -pub mod arm; -#[cfg(all(cpu_target = "arm", not(feature = "clippy")))] -pub use arm::*; - -#[cfg(cpu_target = "i386")] -pub mod i386; -#[cfg(all(cpu_target = "i386", not(feature = "clippy")))] -pub use i386::*; - -#[cfg(cpu_target = "x86_64")] -pub mod x86_64; -#[cfg(cpu_target = "x86_64")] -pub use x86_64::*; - -#[cfg(cpu_target = "mips")] -pub mod mips; -#[cfg(cpu_target = "mips")] -pub use mips::*; - -#[cfg(cpu_target = "ppc")] -pub mod ppc; -#[cfg(cpu_target = "ppc")] -pub use ppc::*; - -#[cfg(cpu_target = "hexagon")] -pub mod hexagon; -#[cfg(cpu_target = "hexagon")] -pub use hexagon::*; +pub mod arch; +pub use arch::*; pub mod elf; -pub mod helper; -pub use helper::*; +pub mod helpers; +pub use helpers::*; + pub mod hooks; pub use hooks::*; -pub mod edges; -pub use edges::QemuEdgeCoverageHelper; - -#[cfg(not(any(cpu_target = "mips", cpu_target = "hexagon")))] -pub mod cmplog; -#[cfg(not(any(cpu_target = "mips", cpu_target = "hexagon")))] -pub use cmplog::QemuCmpLogHelper; - -#[cfg(all(emulation_mode = "usermode", feature = "injections"))] -pub mod injections; -#[cfg(all(emulation_mode = "usermode", feature = "injections"))] -pub use injections::QemuInjectionHelper; - -#[cfg(all(emulation_mode = "usermode", not(cpu_target = "hexagon")))] -pub mod snapshot; -#[cfg(all(emulation_mode = "usermode", not(cpu_target = "hexagon")))] -pub use snapshot::QemuSnapshotHelper; - -#[cfg(all(emulation_mode = "usermode", not(cpu_target = "hexagon")))] -pub mod asan; -#[cfg(all(emulation_mode = "usermode", not(cpu_target = "hexagon")))] -pub use asan::{init_with_asan, QemuAsanHelper}; - -#[cfg(not(cpu_target = "hexagon"))] -pub mod calls; -#[cfg(not(cpu_target = "hexagon"))] -pub mod drcov; - pub mod executor; pub use executor::QemuExecutor; #[cfg(feature = "fork")] @@ -110,7 +52,9 @@ pub use executor::QemuForkExecutor; pub mod emu; pub use emu::*; -pub mod sync_backdoor; +pub mod breakpoint; +pub mod command; +pub mod sync_exit; #[must_use] pub fn filter_qemu_args() -> Vec { @@ -154,7 +98,7 @@ pub fn python_module(py: Python, m: &PyModule) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; - m.add_class::()?; + m.add_class::()?; Ok(()) } diff --git a/libafl_qemu/src/sync_backdoor.rs b/libafl_qemu/src/sync_backdoor.rs deleted file mode 100644 index a3cfb1c9d9..0000000000 --- a/libafl_qemu/src/sync_backdoor.rs +++ /dev/null @@ -1,246 +0,0 @@ -use std::{ - fmt::{Display, Formatter}, - sync::OnceLock, -}; - -use enum_map::{enum_map, Enum, EnumMap}; -use libafl::executors::ExitKind; -use num_enum::{TryFromPrimitive, TryFromPrimitiveError}; - -use crate::{ - get_sync_backdoor_arch_regs, Emulator, GuestAddrKind, GuestPhysAddr, GuestReg, GuestVirtAddr, - Regs, -}; - -#[derive(Debug, Clone)] -pub enum SyncBackdoorError { - UnknownCommand(GuestReg), - RegError(String), -} - -impl From for SyncBackdoorError { - fn from(error_string: String) -> Self { - SyncBackdoorError::RegError(error_string) - } -} - -#[derive(Debug, Clone, Enum)] -pub enum SyncBackdoorArgs { - Ret, - Cmd, - Arg1, - Arg2, - Arg3, - Arg4, - Arg5, - Arg6, -} - -// TODO: Move in a separate header file to have a central definition of native definitions, -// reusable in targets directly. -#[derive(Debug, Clone, TryFromPrimitive)] -#[repr(u64)] -pub enum NativeSyncBackdoorCommand { - Save = 0, // Save the VM - Load = 1, // Reload the target without ending the run? - InputVirt = 2, // The address is a virtual address using the paging currently running in the VM. - InputPhys = 3, // The address is a physical address - End = 4, // Implies reloading of the target. The first argument gives the exit status. - StartVirt = 5, // Shortcut for Save + InputVirt - StartPhys = 6, // Shortcut for Save + InputPhys -} - -#[derive(Debug, Clone, Enum, TryFromPrimitive)] -#[repr(u64)] -pub enum NativeExitKind { - Unknown = 0, // Should not be used - Ok = 1, // Normal exit - Crash = 2, // Crash reported in the VM -} - -static EMU_EXIT_KIND_MAP: OnceLock>> = OnceLock::new(); - -impl From> for SyncBackdoorError { - fn from(error: TryFromPrimitiveError) -> Self { - SyncBackdoorError::UnknownCommand(error.number.try_into().unwrap()) - } -} - -#[derive(Debug, Clone)] -pub struct CommandInput { - addr: GuestAddrKind, - max_input_size: GuestReg, -} - -impl CommandInput { - pub fn exec(&self, emu: &Emulator, backdoor: &SyncBackdoor, input: &[u8]) { - match self.addr { - GuestAddrKind::Physical(hwaddr) => unsafe { - #[cfg(emulation_mode = "usermode")] - { - // For now the default behaviour is to fall back to virtual addresses - emu.write_mem(hwaddr.try_into().unwrap(), input); - } - #[cfg(emulation_mode = "systemmode")] - { - emu.write_phys_mem(hwaddr, input); - } - }, - GuestAddrKind::Virtual(vaddr) => unsafe { - emu.write_mem(vaddr.try_into().unwrap(), input); - }, - }; - - backdoor.ret(emu, input.len().try_into().unwrap()).unwrap(); - } -} - -impl Display for CommandInput { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{} ({:x} max nb bytes)", self.addr, self.max_input_size) - } -} - -#[derive(Debug, Clone)] -pub enum Command { - Save, - Load, - Input(CommandInput), - Start(CommandInput), - Exit(Option), -} - -impl Display for Command { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - match self { - Command::Save => write!(f, "Save VM"), - Command::Load => write!(f, "Reload VM"), - Command::Input(command_input) => write!(f, "Set fuzzing input @{command_input}"), - Command::Start(command_input) => { - write!(f, "Start fuzzing with input @{command_input}") - } - Command::Exit(exit_kind) => write!(f, "Exit of kind {exit_kind:?}"), - } - } -} - -#[derive(Debug, Clone)] -pub struct SyncBackdoor { - command: Command, - arch_regs_map: &'static EnumMap, -} - -impl SyncBackdoor { - #[must_use] - pub fn command(&self) -> &Command { - &self.command - } - - pub fn ret(&self, emu: &Emulator, value: GuestReg) -> Result<(), SyncBackdoorError> { - Ok(emu.write_reg(self.arch_regs_map[SyncBackdoorArgs::Ret], value)?) - } -} - -impl Display for SyncBackdoor { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.command) - } -} - -impl TryFrom<&Emulator> for SyncBackdoor { - type Error = SyncBackdoorError; - - fn try_from(emu: &Emulator) -> Result { - let arch_regs_map: &'static EnumMap = get_sync_backdoor_arch_regs(); - let cmd_id: GuestReg = - emu.read_reg::(arch_regs_map[SyncBackdoorArgs::Cmd])?; - - Ok(match u64::from(cmd_id).try_into()? { - NativeSyncBackdoorCommand::Save => SyncBackdoor { - command: Command::Save, - arch_regs_map, - }, - NativeSyncBackdoorCommand::Load => SyncBackdoor { - command: Command::Load, - arch_regs_map, - }, - NativeSyncBackdoorCommand::InputVirt => { - let virt_addr: GuestVirtAddr = - emu.read_reg(arch_regs_map[SyncBackdoorArgs::Arg1])?; - let max_input_size: GuestReg = - emu.read_reg(arch_regs_map[SyncBackdoorArgs::Arg2])?; - - SyncBackdoor { - command: Command::Input(CommandInput { - addr: GuestAddrKind::Virtual(virt_addr), - max_input_size, - }), - arch_regs_map, - } - } - NativeSyncBackdoorCommand::InputPhys => { - let phys_addr: GuestPhysAddr = - emu.read_reg(arch_regs_map[SyncBackdoorArgs::Arg1])?; - let max_input_size: GuestReg = - emu.read_reg(arch_regs_map[SyncBackdoorArgs::Arg2])?; - - SyncBackdoor { - command: Command::Input(CommandInput { - addr: GuestAddrKind::Physical(phys_addr), - max_input_size, - }), - arch_regs_map, - } - } - NativeSyncBackdoorCommand::End => { - let native_exit_kind: GuestReg = - emu.read_reg(arch_regs_map[SyncBackdoorArgs::Arg1])?; - let native_exit_kind: Result = - u64::from(native_exit_kind).try_into(); - - let exit_kind = native_exit_kind.ok().and_then(|k| { - EMU_EXIT_KIND_MAP.get_or_init(|| { - enum_map! { - NativeExitKind::Unknown => None, - NativeExitKind::Ok => Some(ExitKind::Ok), - NativeExitKind::Crash => Some(ExitKind::Crash) - } - })[k] - }); - - SyncBackdoor { - command: Command::Exit(exit_kind), - arch_regs_map, - } - } - NativeSyncBackdoorCommand::StartPhys => { - let input_phys_addr: GuestPhysAddr = - emu.read_reg(arch_regs_map[SyncBackdoorArgs::Arg1])?; - let max_input_size: GuestReg = - emu.read_reg(arch_regs_map[SyncBackdoorArgs::Arg2])?; - - SyncBackdoor { - command: Command::Start(CommandInput { - addr: GuestAddrKind::Physical(input_phys_addr), - max_input_size, - }), - arch_regs_map, - } - } - NativeSyncBackdoorCommand::StartVirt => { - let input_virt_addr: GuestVirtAddr = - emu.read_reg(arch_regs_map[SyncBackdoorArgs::Arg1])?; - let max_input_size: GuestReg = - emu.read_reg(arch_regs_map[SyncBackdoorArgs::Arg2])?; - - SyncBackdoor { - command: Command::Start(CommandInput { - addr: GuestAddrKind::Virtual(input_virt_addr), - max_input_size, - }), - arch_regs_map, - } - } - }) - } -} diff --git a/libafl_qemu/src/sync_exit.rs b/libafl_qemu/src/sync_exit.rs new file mode 100644 index 0000000000..7697232dba --- /dev/null +++ b/libafl_qemu/src/sync_exit.rs @@ -0,0 +1,221 @@ +use std::{ + fmt::{Display, Formatter}, + sync::OnceLock, +}; + +use enum_map::{enum_map, Enum, EnumMap}; +use libafl::{ + executors::ExitKind, + state::{HasExecutions, State}, +}; +use libafl_qemu_sys::{GuestAddr, GuestPhysAddr, GuestVirtAddr}; +use num_enum::TryFromPrimitiveError; + +use crate::{ + command::{ + Command, EmulatorMemoryChunk, EndCommand, FilterCommand, InputCommand, LoadCommand, + NativeBackdoorCommand, NativeExitKind, SaveCommand, StartCommand, VersionCommand, + }, + get_backdoor_arch_regs, EmuExitHandler, Emulator, GuestReg, QemuHelperTuple, + QemuInstrumentationAddressRangeFilter, Regs, CPU, +}; + +#[derive(Debug, Clone)] +pub enum SyncBackdoorError { + UnknownCommand(GuestReg), + RegError(String), + VersionDifference(u64), +} + +impl From for SyncBackdoorError { + fn from(error_string: String) -> Self { + SyncBackdoorError::RegError(error_string) + } +} + +#[derive(Debug, Clone, Enum)] +pub enum BackdoorArgs { + Ret, + Cmd, + Arg1, + Arg2, + Arg3, + Arg4, + Arg5, + Arg6, +} + +static EMU_EXIT_KIND_MAP: OnceLock>> = OnceLock::new(); + +impl From> for SyncBackdoorError { + fn from(error: TryFromPrimitiveError) -> Self { + SyncBackdoorError::UnknownCommand(error.number.try_into().unwrap()) + } +} + +#[derive(Debug, Clone)] +pub struct SyncBackdoor { + command: Command, + arch_regs_map: &'static EnumMap, +} + +impl SyncBackdoor { + #[must_use] + pub fn command(&self) -> &Command { + &self.command + } + + pub fn ret(&self, cpu: &CPU, value: GuestReg) -> Result<(), SyncBackdoorError> { + Ok(cpu.write_reg(self.arch_regs_map[BackdoorArgs::Ret], value)?) + } + + #[must_use] + pub fn ret_reg(&self) -> Regs { + self.arch_regs_map[BackdoorArgs::Ret] + } +} + +impl Display for SyncBackdoor { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.command) + } +} + +impl TryFrom<&Emulator> for SyncBackdoor +where + E: EmuExitHandler, + QT: QemuHelperTuple, + S: State + HasExecutions, +{ + type Error = SyncBackdoorError; + + #[allow(clippy::too_many_lines)] + fn try_from(emu: &Emulator) -> Result { + let arch_regs_map: &'static EnumMap = get_backdoor_arch_regs(); + let cmd_id: GuestReg = emu + .qemu() + .read_reg::(arch_regs_map[BackdoorArgs::Cmd])?; + + Ok(match u64::from(cmd_id).try_into()? { + NativeBackdoorCommand::Save => SyncBackdoor { + command: Command::SaveCommand(SaveCommand), + arch_regs_map, + }, + NativeBackdoorCommand::Load => SyncBackdoor { + command: Command::LoadCommand(LoadCommand), + arch_regs_map, + }, + NativeBackdoorCommand::InputVirt => { + let virt_addr: GuestVirtAddr = + emu.qemu().read_reg(arch_regs_map[BackdoorArgs::Arg1])?; + let max_input_size: GuestReg = + emu.qemu().read_reg(arch_regs_map[BackdoorArgs::Arg2])?; + + SyncBackdoor { + command: Command::InputCommand(InputCommand::new( + EmulatorMemoryChunk::virt( + virt_addr, + max_input_size, + emu.qemu().current_cpu().unwrap().clone(), + ), + emu.qemu().current_cpu().unwrap(), + )), + arch_regs_map, + } + } + NativeBackdoorCommand::InputPhys => { + let phys_addr: GuestPhysAddr = + emu.qemu().read_reg(arch_regs_map[BackdoorArgs::Arg1])?; + let max_input_size: GuestReg = + emu.qemu().read_reg(arch_regs_map[BackdoorArgs::Arg2])?; + + SyncBackdoor { + command: Command::InputCommand(InputCommand::new( + EmulatorMemoryChunk::phys( + phys_addr, + max_input_size, + Some(emu.qemu().current_cpu().unwrap().clone()), + ), + emu.qemu().current_cpu().unwrap(), + )), + arch_regs_map, + } + } + NativeBackdoorCommand::End => { + let native_exit_kind: GuestReg = + emu.qemu().read_reg(arch_regs_map[BackdoorArgs::Arg1])?; + let native_exit_kind: Result = + u64::from(native_exit_kind).try_into(); + + let exit_kind = native_exit_kind.ok().and_then(|k| { + EMU_EXIT_KIND_MAP.get_or_init(|| { + enum_map! { + NativeExitKind::Unknown => None, + NativeExitKind::Ok => Some(ExitKind::Ok), + NativeExitKind::Crash => Some(ExitKind::Crash) + } + })[k] + }); + + SyncBackdoor { + command: Command::EndCommand(EndCommand::new(exit_kind)), + arch_regs_map, + } + } + NativeBackdoorCommand::StartPhys => { + let input_phys_addr: GuestPhysAddr = + emu.qemu().read_reg(arch_regs_map[BackdoorArgs::Arg1])?; + let max_input_size: GuestReg = + emu.qemu().read_reg(arch_regs_map[BackdoorArgs::Arg2])?; + + SyncBackdoor { + command: Command::StartCommand(StartCommand::new(EmulatorMemoryChunk::phys( + input_phys_addr, + max_input_size, + Some(emu.qemu().current_cpu().unwrap().clone()), + ))), + arch_regs_map, + } + } + NativeBackdoorCommand::StartVirt => { + let input_virt_addr: GuestVirtAddr = + emu.qemu().read_reg(arch_regs_map[BackdoorArgs::Arg1])?; + let max_input_size: GuestReg = + emu.qemu().read_reg(arch_regs_map[BackdoorArgs::Arg2])?; + + SyncBackdoor { + command: Command::StartCommand(StartCommand::new(EmulatorMemoryChunk::virt( + input_virt_addr, + max_input_size, + emu.qemu().current_cpu().unwrap().clone(), + ))), + arch_regs_map, + } + } + NativeBackdoorCommand::Version => { + let client_version = emu.qemu().read_reg(arch_regs_map[BackdoorArgs::Arg1])?; + + SyncBackdoor { + command: Command::VersionCommand(VersionCommand::new(client_version)), + arch_regs_map, + } + } + NativeBackdoorCommand::VaddrFilterAllowRange => { + let vaddr_start: GuestAddr = + emu.qemu().read_reg(arch_regs_map[BackdoorArgs::Arg1])?; + let vaddr_end: GuestAddr = + emu.qemu().read_reg(arch_regs_map[BackdoorArgs::Arg2])?; + + SyncBackdoor { + command: Command::AddressRangeFilterCommand(FilterCommand::new( + #[allow(clippy::single_range_in_vec_init)] + QemuInstrumentationAddressRangeFilter::AllowList(vec![ + vaddr_start..vaddr_end, + ]), + )), + arch_regs_map, + } + } + }) + } +} diff --git a/libafl_sugar/Cargo.toml b/libafl_sugar/Cargo.toml index 18e9c8c10e..f11446fc50 100644 --- a/libafl_sugar/Cargo.toml +++ b/libafl_sugar/Cargo.toml @@ -33,15 +33,17 @@ hexagon = ["libafl_qemu/hexagon"] # build qemu for hexagon pyo3-build-config = { version = "0.18", optional = true } [dependencies] -libafl = { path = "../libafl", version = "0.11.2" } -libafl_bolts = { path = "../libafl_bolts", version = "0.11.2" } -libafl_targets = { path = "../libafl_targets", version = "0.11.2" } -libafl_qemu = { path = "../libafl_qemu", version = "0.11.2" } +libafl = { path = "../libafl", version = "0.12.0", features = ["adaptive_serialization"] } +libafl_bolts = { path = "../libafl_bolts", version = "0.12.0" } +libafl_targets = { path = "../libafl_targets", version = "0.12.0" } typed-builder = "0.16" # Implement the builder pattern at compiletime pyo3 = { version = "0.18", optional = true } log = "0.4.20" +[target.'cfg(target_os = "linux")'.dependencies] +libafl_qemu = { path = "../libafl_qemu", version = "0.12.0" } + [lib] name = "libafl_sugar" crate-type = ["cdylib", "rlib"] diff --git a/libafl_sugar/src/forkserver.rs b/libafl_sugar/src/forkserver.rs index 74c87ce14f..cc3a4d7b65 100644 --- a/libafl_sugar/src/forkserver.rs +++ b/libafl_sugar/src/forkserver.rs @@ -15,27 +15,23 @@ use libafl::{ scheduled::{havoc_mutations, tokens_mutations, StdScheduledMutator}, token_mutations::Tokens, }, - observers::{HitcountsMapObserver, StdMapObserver, TimeObserver}, + observers::{CanTrack, HitcountsMapObserver, StdMapObserver, TimeObserver}, schedulers::{IndexesLenTimeMinimizerScheduler, QueueScheduler}, stages::StdMutationalStage, - state::{HasCorpus, HasMetadata, StdState}, - Error, + state::{HasCorpus, StdState}, + Error, HasMetadata, }; use libafl_bolts::{ core_affinity::Cores, - current_nanos, rands::StdRand, shmem::{ShMem, ShMemProvider, UnixShMemProvider}, - tuples::{tuple_list, Merge}, - AsMutSlice, + tuples::{tuple_list, Handler, Merge}, + AsSliceMut, }; use typed_builder::TypedBuilder; use crate::{CORPUS_CACHE_SIZE, DEFAULT_TIMEOUT_SECS}; -/// The default coverage map size to use for forkserver targets -pub const DEFAULT_MAP_SIZE: usize = 65536; - /// Creates a Forkserver-based fuzzer. #[derive(Debug, TypedBuilder)] pub struct ForkserverBytesCoverageSugar<'a> { @@ -86,7 +82,7 @@ impl<'a> ForkserverBytesCoverageSugar<'a> { // a large initial map size that should be enough // to house all potential coverage maps for our targets // (we will eventually reduce the used size according to the actual map) - const MAP_SIZE: usize = 2_621_440; + const MAP_SIZE: usize = 65_536; let conf = match self.configuration.as_ref() { Some(name) => EventConfig::from_name(name), @@ -117,31 +113,36 @@ impl<'a> ForkserverBytesCoverageSugar<'a> { let monitor = MultiMonitor::new(|s| log::info!("{s}")); + // Create an observation channel to keep track of the execution time + let time_observer = TimeObserver::new("time"); + let time_ref = time_observer.handle(); + let mut run_client = |state: Option<_>, - mut mgr: LlmpRestartingEventManager<_, _>, + mut mgr: LlmpRestartingEventManager<_, _, _>, _core_id| { + let time_observer = time_observer.clone(); + // Coverage map shared between target and fuzzer let mut shmem = shmem_provider_client.new_shmem(MAP_SIZE).unwrap(); shmem.write_to_env("__AFL_SHM_ID").unwrap(); - let shmem_map = shmem.as_mut_slice(); + let shmem_map = shmem.as_slice_mut(); // To let know the AFL++ binary that we have a big map std::env::set_var("AFL_MAP_SIZE", format!("{MAP_SIZE}")); // Create an observation channel using the coverage map - let edges_observer = - unsafe { HitcountsMapObserver::new(StdMapObserver::new("shared_mem", shmem_map)) }; - - // Create an observation channel to keep track of the execution time - let time_observer = TimeObserver::new("time"); + let edges_observer = unsafe { + HitcountsMapObserver::new(StdMapObserver::new("shared_mem", shmem_map)) + .track_indices() + }; // Feedback to rate the interestingness of an input // This one is composed by two Feedbacks in OR let mut feedback = feedback_or!( // New maximization map feedback linked to the edges observer and the feedback state - MaxMapFeedback::tracking(&edges_observer, true, false), + MaxMapFeedback::new(&edges_observer), // Time feedback, this one does not need a feedback state - TimeFeedback::with_observer(&time_observer) + TimeFeedback::new(&time_observer) ); // A feedback to choose if an input is a solution or not @@ -151,7 +152,7 @@ impl<'a> ForkserverBytesCoverageSugar<'a> { let mut state = state.unwrap_or_else(|| { StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep a part in memory for performance CachedOnDiskCorpus::new(out_dir.clone(), CORPUS_CACHE_SIZE).unwrap(), // Corpus in which we store solutions (crashes in this example), @@ -167,7 +168,8 @@ impl<'a> ForkserverBytesCoverageSugar<'a> { let mut tokens = Tokens::new(); // A minimization+queue policy to get testcasess from the corpus - let scheduler = IndexesLenTimeMinimizerScheduler::new(QueueScheduler::new()); + let scheduler = + IndexesLenTimeMinimizerScheduler::new(&edges_observer, QueueScheduler::new()); // A fuzzer with feedbacks and a corpus scheduler let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); @@ -292,7 +294,8 @@ impl<'a> ForkserverBytesCoverageSugar<'a> { .run_client(&mut run_client) .cores(self.cores) .broker_port(self.broker_port) - .remote_broker_addr(self.remote_broker_addr); + .remote_broker_addr(self.remote_broker_addr) + .time_ref(time_ref); #[cfg(unix)] let launcher = launcher.stdout_file(Some("/dev/null")); match launcher.build().launch() { diff --git a/libafl_sugar/src/inmemory.rs b/libafl_sugar/src/inmemory.rs index c799534e75..49269d1230 100644 --- a/libafl_sugar/src/inmemory.rs +++ b/libafl_sugar/src/inmemory.rs @@ -18,21 +18,21 @@ use libafl::{ scheduled::{havoc_mutations, tokens_mutations, StdScheduledMutator}, token_mutations::{I2SRandReplace, Tokens}, }, - observers::{HitcountsMapObserver, TimeObserver}, + observers::{CanTrack, HitcountsMapObserver, StdMapObserver, TimeObserver}, schedulers::{IndexesLenTimeMinimizerScheduler, QueueScheduler}, stages::{ShadowTracingStage, StdMutationalStage}, - state::{HasCorpus, HasMetadata, StdState}, - Error, + state::{HasCorpus, StdState}, + Error, HasMetadata, }; use libafl_bolts::{ core_affinity::Cores, - current_nanos, + ownedref::OwnedMutSlice, rands::StdRand, shmem::{ShMemProvider, StdShMemProvider}, - tuples::{tuple_list, Merge}, + tuples::{tuple_list, Handler, Merge}, AsSlice, }; -use libafl_targets::{std_edges_map_observer, CmpLogObserver}; +use libafl_targets::{edges_map_mut_ptr, CmpLogObserver}; use typed_builder::TypedBuilder; use crate::{CORPUS_CACHE_SIZE, DEFAULT_TIMEOUT_SECS}; @@ -57,7 +57,7 @@ where /// Dictionary #[builder(default = None)] tokens_file: Option, - /// Flag if use CmpLog + /// Flag if use `CmpLog` #[builder(default = None)] use_cmplog: Option, /// The port used for communication between this fuzzer node and other fuzzer nodes @@ -72,6 +72,9 @@ where /// Bytes harness #[builder(setter(strip_option))] harness: Option, + /// The map size used for the fuzzer + #[builder(default = 65536usize)] + map_size: usize, /// Fuzz `iterations` number of times, instead of indefinitely; implies use of `fuzz_loop_for` #[builder(default = None)] iterations: Option, @@ -138,15 +141,23 @@ where let monitor = MultiMonitor::new(|s| println!("{s}")); + // Create an observation channel to keep track of the execution time + let time_observer = TimeObserver::new("time"); + let time_ref = time_observer.handle(); + let mut run_client = |state: Option<_>, - mut mgr: LlmpRestartingEventManager<_, _>, + mut mgr: LlmpRestartingEventManager<_, _, _>, _core_id| { - // Create an observation channel using the coverage map - let edges_observer = - HitcountsMapObserver::new(unsafe { std_edges_map_observer("edges") }); + let time_observer = time_observer.clone(); - // Create an observation channel to keep track of the execution time - let time_observer = TimeObserver::new("time"); + // Create an observation channel using the coverage map + let edges_observer = HitcountsMapObserver::new(unsafe { + StdMapObserver::from_mut_slice( + "edges", + OwnedMutSlice::from_raw_parts_mut(edges_map_mut_ptr(), self.map_size), + ) + }) + .track_indices(); let cmplog_observer = CmpLogObserver::new("cmplog", true); @@ -154,9 +165,9 @@ where // This one is composed by two Feedbacks in OR let mut feedback = feedback_or!( // New maximization map feedback linked to the edges observer and the feedback state - MaxMapFeedback::tracking(&edges_observer, true, false), + MaxMapFeedback::new(&edges_observer), // Time feedback, this one does not need a feedback state - TimeFeedback::with_observer(&time_observer) + TimeFeedback::new(&time_observer) ); // A feedback to choose if an input is a solution or not @@ -166,7 +177,7 @@ where let mut state = state.unwrap_or_else(|| { StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep a part in memory for performance CachedOnDiskCorpus::new(out_dir.clone(), CORPUS_CACHE_SIZE).unwrap(), // Corpus in which we store solutions (crashes in this example), @@ -186,7 +197,8 @@ where } // A minimization+queue policy to get testcasess from the corpus - let scheduler = IndexesLenTimeMinimizerScheduler::new(QueueScheduler::new()); + let scheduler = + IndexesLenTimeMinimizerScheduler::new(&edges_observer, QueueScheduler::new()); // A fuzzer with feedbacks and a corpus scheduler let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); @@ -338,7 +350,8 @@ where .run_client(&mut run_client) .cores(self.cores) .broker_port(self.broker_port) - .remote_broker_addr(self.remote_broker_addr); + .remote_broker_addr(self.remote_broker_addr) + .time_ref(time_ref); #[cfg(unix)] let launcher = launcher.stdout_file(Some("/dev/null")); match launcher.build().launch() { diff --git a/libafl_sugar/src/lib.rs b/libafl_sugar/src/lib.rs index fb41e4a5aa..2851795e95 100644 --- a/libafl_sugar/src/lib.rs +++ b/libafl_sugar/src/lib.rs @@ -27,14 +27,12 @@ ))] #![cfg_attr(test, deny( missing_debug_implementations, - missing_docs, //trivial_casts, trivial_numeric_casts, unused_extern_crates, unused_import_braces, unused_qualifications, unused_must_use, - missing_docs, //unused_results ))] #![cfg_attr( diff --git a/libafl_sugar/src/qemu.rs b/libafl_sugar/src/qemu.rs index 092a88fb3a..a4ec440c37 100644 --- a/libafl_sugar/src/qemu.rs +++ b/libafl_sugar/src/qemu.rs @@ -21,24 +21,25 @@ use libafl::{ token_mutations::Tokens, I2SRandReplace, }, - observers::{HitcountsMapObserver, TimeObserver, VariableMapObserver}, + observers::{CanTrack, HitcountsMapObserver, TimeObserver, VariableMapObserver}, schedulers::{IndexesLenTimeMinimizerScheduler, QueueScheduler}, stages::{ShadowTracingStage, StdMutationalStage}, - state::{HasCorpus, HasMetadata, StdState}, + state::{HasCorpus, StdState}, + HasMetadata, }; use libafl_bolts::{ core_affinity::Cores, - current_nanos, + ownedref::OwnedMutSlice, rands::StdRand, shmem::{ShMemProvider, StdShMemProvider}, - tuples::{tuple_list, Merge}, + tuples::{tuple_list, Handler, Merge}, AsSlice, }; -pub use libafl_qemu::emu::Emulator; +pub use libafl_qemu::emu::Qemu; #[cfg(not(any(feature = "mips", feature = "hexagon")))] use libafl_qemu::QemuCmpLogHelper; use libafl_qemu::{edges, QemuEdgeCoverageHelper, QemuExecutor, QemuHooks}; -use libafl_targets::{edges_map_mut_slice, CmpLogObserver}; +use libafl_targets::{edges_map_mut_ptr, CmpLogObserver}; use typed_builder::TypedBuilder; use crate::{CORPUS_CACHE_SIZE, DEFAULT_TIMEOUT_SECS}; @@ -63,7 +64,7 @@ where /// Dictionary #[builder(default = None)] tokens_file: Option, - /// Flag if use CmpLog + /// Flag if use `CmpLog` #[builder(default = None)] use_cmplog: Option, /// The port the fuzzing nodes communicate over @@ -118,7 +119,7 @@ where { /// Run the fuzzer #[allow(clippy::too_many_lines, clippy::similar_names)] - pub fn run(&mut self, emulator: &Emulator) { + pub fn run(&mut self, qemu: &Qemu) { let conf = match self.configuration.as_ref() { Some(name) => EventConfig::from_name(name), None => EventConfig::AlwaysUnique, @@ -145,21 +146,28 @@ where let monitor = MultiMonitor::new(|s| log::info!("{s}")); + // Create an observation channel to keep track of the execution time + let time_observer = TimeObserver::new("time"); + let time_ref = time_observer.handle(); + let mut run_client = |state: Option<_>, - mut mgr: LlmpRestartingEventManager<_, _>, + mut mgr: LlmpRestartingEventManager<_, _, _>, _core_id| { + let time_observer = time_observer.clone(); + // Create an observation channel using the coverage map let edges_observer = unsafe { HitcountsMapObserver::new(VariableMapObserver::from_mut_slice( "edges", - edges_map_mut_slice(), - addr_of_mut!(edges::MAX_EDGES_NUM), + OwnedMutSlice::from_raw_parts_mut( + edges_map_mut_ptr(), + edges::EDGES_MAP_SIZE_IN_USE, + ), + addr_of_mut!(edges::MAX_EDGES_FOUND), )) + .track_indices() }; - // Create an observation channel to keep track of the execution time - let time_observer = TimeObserver::new("time"); - // Keep tracks of CMPs let cmplog_observer = CmpLogObserver::new("cmplog", true); @@ -167,9 +175,9 @@ where // This one is composed by two Feedbacks in OR let mut feedback = feedback_or!( // New maximization map feedback linked to the edges observer and the feedback state - MaxMapFeedback::tracking(&edges_observer, true, false), + MaxMapFeedback::new(&edges_observer), // Time feedback, this one does not need a feedback state - TimeFeedback::with_observer(&time_observer) + TimeFeedback::new(&time_observer) ); // A feedback to choose if an input is a solution or not @@ -179,7 +187,7 @@ where let mut state = state.unwrap_or_else(|| { StdState::new( // RNG - StdRand::with_seed(current_nanos()), + StdRand::new(), // Corpus that will be evolved, we keep a part in memory for performance CachedOnDiskCorpus::new(out_dir.clone(), CORPUS_CACHE_SIZE).unwrap(), // Corpus in which we store solutions (crashes in this example), @@ -199,7 +207,8 @@ where } // A minimization+queue policy to get testcasess from the corpus - let scheduler = IndexesLenTimeMinimizerScheduler::new(QueueScheduler::new()); + let scheduler = + IndexesLenTimeMinimizerScheduler::new(&edges_observer, QueueScheduler::new()); // A fuzzer with feedbacks and a corpus scheduler let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); @@ -208,13 +217,13 @@ where let mut harness = |input: &BytesInput| { let target = input.target_bytes(); let buf = target.as_slice(); - (harness_bytes)(buf); + harness_bytes(buf); ExitKind::Ok }; if self.use_cmplog.unwrap_or(false) { let mut hooks = QemuHooks::new( - emulator.clone(), + *qemu, #[cfg(not(any(feature = "mips", feature = "hexagon")))] tuple_list!( QemuEdgeCoverageHelper::default(), @@ -325,10 +334,8 @@ where } } } else { - let mut hooks = QemuHooks::new( - emulator.clone(), - tuple_list!(QemuEdgeCoverageHelper::default()), - ); + let mut hooks = + QemuHooks::new(*qemu, tuple_list!(QemuEdgeCoverageHelper::default())); let mut executor = QemuExecutor::new( &mut hooks, @@ -432,9 +439,11 @@ where .run_client(&mut run_client) .cores(self.cores) .broker_port(self.broker_port) - .remote_broker_addr(self.remote_broker_addr); + .remote_broker_addr(self.remote_broker_addr) + .time_ref(time_ref); #[cfg(unix)] let launcher = launcher.stdout_file(Some("/dev/null")); + launcher.build().launch().expect("Launcher failed"); } } @@ -445,7 +454,7 @@ pub mod pybind { use std::path::PathBuf; use libafl_bolts::core_affinity::Cores; - use libafl_qemu::emu::pybind::Emulator; + use libafl_qemu::emu::pybind::Qemu; use pyo3::{prelude::*, types::PyBytes}; use crate::qemu; @@ -492,7 +501,7 @@ pub mod pybind { /// Run the fuzzer #[allow(clippy::needless_pass_by_value)] - pub fn run(&self, emulator: &Emulator, harness: PyObject) { + pub fn run(&self, qemu: &Qemu, harness: PyObject) { qemu::QemuBytesCoverageSugar::builder() .input_dirs(&self.input_dirs) .output_dir(self.output_dir.clone()) @@ -511,7 +520,7 @@ pub mod pybind { .tokens_file(self.tokens_file.clone()) .iterations(self.iterations) .build() - .run(&emulator.emu); + .run(&qemu.qemu); } } diff --git a/libafl_targets/Cargo.toml b/libafl_targets/Cargo.toml index d9af7efde4..8cf541ce33 100644 --- a/libafl_targets/Cargo.toml +++ b/libafl_targets/Cargo.toml @@ -31,7 +31,7 @@ default = [ "common", ] std = ["libafl/std"] -libfuzzer = ["std", "common", "sanitizer_interfaces"] +libfuzzer = ["std", "common"] libfuzzer_no_link_main = ["libfuzzer"] libfuzzer_define_run_driver = ["libfuzzer"] libfuzzer_interceptors = ["libfuzzer", "sancov_cmplog"] @@ -42,6 +42,9 @@ sancov_pcguard_edges = ["coverage"] sancov_pcguard_hitcounts = ["coverage"] sancov_value_profile = ["common"] sancov_8bit = [] +sancov_ngram4 = ["coverage"] +sancov_ngram8 = ["coverage"] +sancov_ctx = ["coverage"] sancov_cmplog = ["common"] # Defines cmp and __sanitizer_weak_hook functions. Use libfuzzer_interceptors to define interceptors (only compatible with Linux) sancov_pcguard = ["sancov_pcguard_hitcounts"] sanitizer_interfaces = [] @@ -56,15 +59,16 @@ whole_archive = [] # use +whole-archive to ensure the presence of weak symbols cmplog_extended_instrumentation = [] # support for aflpp cmplog map, we will remove this once aflpp and libafl cmplog shares the same LLVM passes. [build-dependencies] -bindgen = "0.68" +bindgen = "0.69.4" cc = { version = "1.0", features = ["parallel"] } rustversion = "1.0" [dependencies] -libafl = { path = "../libafl", version = "0.11.2", default-features = false, features = [] } -libafl_bolts = { path = "../libafl_bolts", version = "0.11.2", default-features = false, features = [] } +libafl = { path = "../libafl", version = "0.12.0", default-features = false, features = [] } +libafl_bolts = { path = "../libafl_bolts", version = "0.12.0", default-features = false, features = [] } libc = "0.2" log = "0.4.20" +rustversion = "1.0" rangemap = "1.3" serde = { version = "1.0", default-features = false, features = ["alloc"] } # serialization lib diff --git a/libafl_targets/build.rs b/libafl_targets/build.rs index c49d05c6aa..bccbb07a1f 100644 --- a/libafl_targets/build.rs +++ b/libafl_targets/build.rs @@ -5,8 +5,17 @@ use std::{env, fs::File, io::Write, path::Path}; const TWO_MB: usize = 2_621_440; const SIXTY_FIVE_KB: usize = 65_536; +#[rustversion::nightly] +fn enable_nightly() { + println!("cargo:rustc-cfg=nightly"); +} + +#[rustversion::not(nightly)] +fn enable_nightly() {} + #[allow(clippy::too_many_lines)] fn main() { + enable_nightly(); let out_dir = env::var_os("OUT_DIR").unwrap(); let out_dir = out_dir.to_string_lossy().to_string(); //let out_dir_path = Path::new(&out_dir); @@ -16,9 +25,12 @@ fn main() { let dest_path = Path::new(&out_dir).join("constants.rs"); let mut constants_file = File::create(dest_path).expect("Could not create file"); - let edges_map_size: usize = option_env!("LIBAFL_EDGES_MAP_SIZE") + let edges_map_size_max: usize = option_env!("LIBAFL_EDGES_MAP_SIZE_MAX") .map_or(Ok(TWO_MB), str::parse) - .expect("Could not parse LIBAFL_EDGES_MAP_SIZE"); + .expect("Could not parse LIBAFL_EDGES_MAP_SIZE_MAX"); + let edges_map_size_in_use: usize = option_env!("LIBAFL_EDGES_MAP_SIZE_IN_USE") + .map_or(Ok(SIXTY_FIVE_KB), str::parse) + .expect("Could not parse LIBAFL_EDGES_MAP_SIZE_IN_USE"); let cmp_map_size: usize = option_env!("LIBAFL_CMP_MAP_SIZE") .map_or(Ok(SIXTY_FIVE_KB), str::parse) .expect("Could not parse LIBAFL_CMP_MAP_SIZE"); @@ -31,13 +43,18 @@ fn main() { let acc_map_size: usize = option_env!("LIBAFL_ACCOUNTING_MAP_SIZE") .map_or(Ok(SIXTY_FIVE_KB), str::parse) .expect("Could not parse LIBAFL_ACCOUNTING_MAP_SIZE"); + let ddg_map_size: usize = option_env!("LIBAFL_DDG_MAP_SIZE") + .map_or(Ok(SIXTY_FIVE_KB), str::parse) + .expect("Could not parse LIBAFL_DDG_MAP_SIZE"); write!( constants_file, "// These constants are autogenerated by build.rs - /// The size of the edges map - pub const EDGES_MAP_SIZE: usize = {edges_map_size}; + /// The default size of the edges map the fuzzer uses + pub const EDGES_MAP_SIZE_IN_USE: usize = {edges_map_size_in_use}; + /// The real allocated size of the edges map + pub const EDGES_MAP_SIZE_MAX: usize = {edges_map_size_max}; /// The size of the cmps map pub const CMP_MAP_SIZE: usize = {cmp_map_size}; /// The width of the `CmpLog` map @@ -46,15 +63,18 @@ fn main() { pub const CMPLOG_MAP_H: usize = {cmplog_map_h}; /// The size of the accounting maps pub const ACCOUNTING_MAP_SIZE: usize = {acc_map_size}; + /// The size of the accounting maps + pub const DDG_MAP_SIZE: usize = {ddg_map_size}; " ) .expect("Could not write file"); - println!("cargo:rerun-if-env-changed=LIBAFL_EDGES_MAP_SIZE"); + println!("cargo:rerun-if-env-changed=LIBAFL_EDGES_MAP_SIZE_IN_USE"); println!("cargo:rerun-if-env-changed=LIBAFL_CMP_MAP_SIZE"); println!("cargo:rerun-if-env-changed=LIBAFL_CMPLOG_MAP_W"); println!("cargo:rerun-if-env-changed=LIBAFL_CMPLOG_MAP_H"); println!("cargo:rerun-if-env-changed=LIBAFL_ACCOUNTING_MAP_SIZE"); + println!("cargo:rerun-if-env-changed=LIBAFL_DDG_MAP_SIZE"); #[cfg(feature = "common")] { @@ -138,8 +158,12 @@ fn main() { cc::Build::new() .file(src_dir.join("coverage.c")) - .define("EDGES_MAP_SIZE", Some(&*format!("{edges_map_size}"))) + .define( + "EDGES_MAP_SIZE_MAX", + Some(&*format!("{edges_map_size_max}")), + ) .define("ACCOUNTING_MAP_SIZE", Some(&*format!("{acc_map_size}"))) + .define("DDG_MAP_SIZE", Some(&*format!("{ddg_map_size}"))) .compile("coverage"); } @@ -206,7 +230,7 @@ fn main() { .header("src/sanitizer_interfaces.h") .use_core() .generate_comments(true) - .parse_callbacks(Box::new(bindgen::CargoCallbacks)) + .parse_callbacks(Box::new(bindgen::CargoCallbacks::new())) .generate() .expect("Couldn't generate the sanitizer headers!"); diff --git a/libafl_targets/src/cmplog.c b/libafl_targets/src/cmplog.c index 65871c68dc..ffaec50f3b 100644 --- a/libafl_targets/src/cmplog.c +++ b/libafl_targets/src/cmplog.c @@ -50,70 +50,10 @@ CmpLogMap *libafl_cmplog_map_ptr = &libafl_cmplog_map; CmpLogMapExtended *libafl_cmplog_map_extended_ptr = &libafl_cmplog_map_extended; #endif -void __libafl_targets_cmplog_instructions(uintptr_t k, uint8_t shape, - uint64_t arg1, uint64_t arg2) { - if (!libafl_cmplog_enabled) { return; } - libafl_cmplog_enabled = false; - - uint16_t hits; - if (libafl_cmplog_map_ptr->headers[k].kind != CMPLOG_KIND_INS) { - libafl_cmplog_map_ptr->headers[k].kind = CMPLOG_KIND_INS; - libafl_cmplog_map_ptr->headers[k].hits = 1; - libafl_cmplog_map_ptr->headers[k].shape = shape; - hits = 0; - } else { - hits = libafl_cmplog_map_ptr->headers[k].hits++; - if (libafl_cmplog_map_ptr->headers[k].shape < shape) { - libafl_cmplog_map_ptr->headers[k].shape = shape; - } - } - - hits &= CMPLOG_MAP_H - 1; - libafl_cmplog_map_ptr->vals.operands[k][hits].v0 = arg1; - libafl_cmplog_map_ptr->vals.operands[k][hits].v1 = arg2; - libafl_cmplog_enabled = true; -} - -void __libafl_targets_cmplog_instructions_extended(uintptr_t k, uint8_t shape, - uint64_t arg1, uint64_t arg2, - uint8_t attr) { -#ifdef CMPLOG_EXTENDED - if (!libafl_cmplog_enabled) { return; } - libafl_cmplog_enabled = false; - - // printf("%ld %ld %ld\n", k, arg1, arg2); - uint16_t hits; - if (libafl_cmplog_map_extended_ptr->headers[k].type != AFL_CMP_TYPE_INS) { - libafl_cmplog_map_extended_ptr->headers[k].type = AFL_CMP_TYPE_INS; - libafl_cmplog_map_extended_ptr->headers[k].hits = 1; - libafl_cmplog_map_extended_ptr->headers[k].shape = shape; - hits = 0; - } else { - hits = libafl_cmplog_map_extended_ptr->headers[k].hits++; - if (libafl_cmplog_map_extended_ptr->headers[k].shape < shape) { - libafl_cmplog_map_extended_ptr->headers[k].shape = shape; - } - } - - hits &= CMPLOG_MAP_H - 1; - libafl_cmplog_map_extended_ptr->vals.operands[k][hits].v0 = arg1; - libafl_cmplog_map_extended_ptr->vals.operands[k][hits].v1 = arg2; - libafl_cmplog_map_extended_ptr->headers[k].attribute = attr; - libafl_cmplog_enabled = true; -#else - // just do nothing - (void)k; - (void)shape; - (void)arg1; - (void)arg2; - (void)attr; -#endif -} - // POSIX shenanigan to see if an area is mapped. // If it is mapped as X-only, we have a problem, so maybe we should add a check // to avoid to call it on .text addresses -static long area_is_valid(const void *ptr, size_t len) { +static inline long area_is_valid(const void *ptr, size_t len) { if (!ptr || __asan_region_is_poisoned(ptr, len)) { return 0; } long valid_len; @@ -129,7 +69,7 @@ static long area_is_valid(const void *ptr, size_t len) { dymmy_initialized = 1; } - valid_len = syscall(SYS_write, dummy_fd[1], ptr, len); + valid_len = write(dummy_fd[1], ptr, len); if (valid_len <= 0 || valid_len > (long)len) { return 0; } #endif @@ -156,66 +96,10 @@ static long area_is_valid(const void *ptr, size_t len) { } } -// cmplog routines after area check -void __libafl_targets_cmplog_routines_checked(uintptr_t k, const uint8_t *ptr1, - const uint8_t *ptr2, size_t len) { - libafl_cmplog_enabled = false; - uint32_t hits; - - if (libafl_cmplog_map_ptr->headers[k].kind != CMPLOG_KIND_RTN) { - libafl_cmplog_map_ptr->headers[k].kind = CMPLOG_KIND_RTN; - libafl_cmplog_map_ptr->headers[k].hits = 1; - libafl_cmplog_map_ptr->headers[k].shape = len; - hits = 0; - } else { - hits = libafl_cmplog_map_ptr->headers[k].hits++; - if (libafl_cmplog_map_ptr->headers[k].shape < len) { - libafl_cmplog_map_ptr->headers[k].shape = - len; // TODO; adjust len for AFL++'s cmplog protocol - } - } - - hits &= CMPLOG_MAP_RTN_H - 1; - MEMCPY(libafl_cmplog_map_ptr->vals.routines[k][hits].v0, ptr1, len); - MEMCPY(libafl_cmplog_map_ptr->vals.routines[k][hits].v1, ptr2, len); - libafl_cmplog_enabled = true; -} - -// cmplog routines after area check -void __libafl_targets_cmplog_routines_checked_extended(uintptr_t k, - const uint8_t *ptr1, - const uint8_t *ptr2, - size_t len) { -#ifdef CMPLOG_EXTENDED - libafl_cmplog_enabled = false; - uint32_t hits; - // printf("RTN: %ld %ld %ld %ld\n", k, *ptr1, *ptr2, len); - if (libafl_cmplog_map_extended_ptr->headers[k].type != AFL_CMP_TYPE_RTN) { - libafl_cmplog_map_extended_ptr->headers[k].type = AFL_CMP_TYPE_RTN; - libafl_cmplog_map_extended_ptr->headers[k].hits = 1; - libafl_cmplog_map_extended_ptr->headers[k].shape = len; - hits = 0; - } else { - hits = libafl_cmplog_map_extended_ptr->headers[k].hits++; - if (libafl_cmplog_map_extended_ptr->headers[k].shape < len) { - libafl_cmplog_map_extended_ptr->headers[k].shape = - len; // TODO; adjust len for AFL++'s cmplog protocol - } - } - - hits &= CMPLOG_MAP_RTN_H - 1; - libafl_cmplog_map_extended_ptr->vals.routines[k][hits].v0_len = len; - libafl_cmplog_map_extended_ptr->vals.routines[k][hits].v1_len = len; - MEMCPY(libafl_cmplog_map_extended_ptr->vals.routines[k][hits].v0, ptr1, len); - MEMCPY(libafl_cmplog_map_extended_ptr->vals.routines[k][hits].v1, ptr2, len); - libafl_cmplog_enabled = true; -#else - // just do nothing - (void)k; - (void)ptr1; - (void)ptr2; - (void)len; -#endif +// Very generic cmplog instructions callback +void __libafl_targets_cmplog_instructions(uintptr_t k, uint8_t shape, + uint64_t arg1, uint64_t arg2) { + cmplog_instructions_checked(k, shape, arg1, arg2); } // Very generic cmplog routines callback @@ -230,7 +114,7 @@ void __libafl_targets_cmplog_routines(uintptr_t k, const uint8_t *ptr1, } int len = MIN(l1, l2); - __libafl_targets_cmplog_routines_checked(k, ptr1, ptr2, len); + cmplog_routines_checked(k, ptr1, ptr2, len); } // cmplog routines but with len specified @@ -243,7 +127,7 @@ void __libafl_targets_cmplog_routines_len(uintptr_t k, const uint8_t *ptr1, return; } - __libafl_targets_cmplog_routines_checked(k, ptr1, ptr2, len); + cmplog_routines_checked(k, ptr1, ptr2, len); } /* CMPLOG Callback for instructions @@ -254,14 +138,14 @@ void __cmplog_ins_hook1_extended(uint8_t arg1, uint8_t arg2, uint8_t attr) { k = (k >> 4) ^ (k << 8); k &= CMPLOG_MAP_W - 1; - __libafl_targets_cmplog_instructions_extended(k, 0, arg1, arg2, attr); + cmplog_instructions_extended_checked(k, 0, arg1, arg2, attr); } void __cmplog_ins_hook1(uint8_t arg1, uint8_t arg2) { uintptr_t k = RETADDR; k = (k >> 4) ^ (k << 8); k &= CMPLOG_MAP_W - 1; - __libafl_targets_cmplog_instructions(k, 1, arg1, arg2); + cmplog_instructions_checked(k, 1, arg1, arg2); } void __cmplog_ins_hook2_extended(uint16_t arg1, uint16_t arg2, uint8_t attr) { @@ -269,14 +153,14 @@ void __cmplog_ins_hook2_extended(uint16_t arg1, uint16_t arg2, uint8_t attr) { k = (k >> 4) ^ (k << 8); k &= CMPLOG_MAP_W - 1; - __libafl_targets_cmplog_instructions_extended(k, 1, arg1, arg2, attr); + cmplog_instructions_extended_checked(k, 1, arg1, arg2, attr); } void __cmplog_ins_hook2(uint16_t arg1, uint16_t arg2) { uintptr_t k = RETADDR; k = (k >> 4) ^ (k << 8); k &= CMPLOG_MAP_W - 1; - __libafl_targets_cmplog_instructions(k, 2, arg1, arg2); + cmplog_instructions_checked(k, 2, arg1, arg2); } void __cmplog_ins_hook4_extended(uint32_t arg1, uint32_t arg2, uint8_t attr) { @@ -284,14 +168,14 @@ void __cmplog_ins_hook4_extended(uint32_t arg1, uint32_t arg2, uint8_t attr) { k = (k >> 4) ^ (k << 8); k &= CMPLOG_MAP_W - 1; - __libafl_targets_cmplog_instructions_extended(k, 3, arg1, arg2, attr); + cmplog_instructions_extended_checked(k, 3, arg1, arg2, attr); } void __cmplog_ins_hook4(uint32_t arg1, uint32_t arg2) { uintptr_t k = RETADDR; k = (k >> 4) ^ (k << 8); k &= CMPLOG_MAP_W - 1; - __libafl_targets_cmplog_instructions(k, 4, arg1, arg2); + cmplog_instructions_checked(k, 4, arg1, arg2); } void __cmplog_ins_hook8_extended(uint64_t arg1, uint64_t arg2, uint8_t attr) { @@ -299,14 +183,14 @@ void __cmplog_ins_hook8_extended(uint64_t arg1, uint64_t arg2, uint8_t attr) { k = (k >> 4) ^ (k << 8); k &= CMPLOG_MAP_W - 1; - __libafl_targets_cmplog_instructions_extended(k, 7, arg1, arg2, attr); + cmplog_instructions_extended_checked(k, 7, arg1, arg2, attr); } void __cmplog_ins_hook8(uint64_t arg1, uint64_t arg2) { uintptr_t k = RETADDR; k = (k >> 4) ^ (k << 8); k &= CMPLOG_MAP_W - 1; - __libafl_targets_cmplog_instructions(k, 8, arg1, arg2); + cmplog_instructions_checked(k, 8, arg1, arg2); } #if !defined(_WIN32) && defined(__SIZEOF_INT128__) @@ -316,14 +200,14 @@ void __cmplog_ins_hook16_extended(uint128_t arg1, uint128_t arg2, k = (k >> 4) ^ (k << 8); k &= CMPLOG_MAP_W - 1; - __libafl_targets_cmplog_instructions_extended(k, 15, arg1, arg2, attr); + cmplog_instructions_extended_checked(k, 15, arg1, arg2, attr); } void __cmplog_ins_hook16(uint128_t arg1, uint128_t arg2) { uintptr_t k = RETADDR; k = (k >> 4) ^ (k << 8); k &= CMPLOG_MAP_W - 1; - __libafl_targets_cmplog_instructions(k, 16, arg1, arg2); + cmplog_instructions_checked(k, 16, arg1, arg2); } void __cmplog_ins_hookN_extended(uint128_t arg1, uint128_t arg2, uint8_t attr, @@ -332,14 +216,14 @@ void __cmplog_ins_hookN_extended(uint128_t arg1, uint128_t arg2, uint8_t attr, k = (k >> 4) ^ (k << 8); k &= CMPLOG_MAP_W - 1; - __libafl_targets_cmplog_instructions_extended(k, size - 1, arg1, arg2, attr); + cmplog_instructions_extended_checked(k, size - 1, arg1, arg2, attr); } void __cmplog_ins_hookN(uint128_t arg1, uint128_t arg2, uint8_t size) { uintptr_t k = RETADDR; k = (k >> 4) ^ (k << 8); k &= CMPLOG_MAP_W - 1; - __libafl_targets_cmplog_instructions(k, size, arg1, arg2); + cmplog_instructions_checked(k, size, arg1, arg2); } #endif /* @@ -360,7 +244,7 @@ void __cmplog_rtn_hook(const uint8_t *ptr1, const uint8_t *ptr2) { k = (k >> 4) ^ (k << 8); k &= CMPLOG_MAP_W - 1; - __libafl_targets_cmplog_routines_checked(k, ptr1, ptr2, len); + cmplog_routines_checked(k, ptr1, ptr2, len); } void __cmplog_rtn_hook_extended(const uint8_t *ptr1, const uint8_t *ptr2) { @@ -377,7 +261,7 @@ void __cmplog_rtn_hook_extended(const uint8_t *ptr1, const uint8_t *ptr2) { k = (k >> 4) ^ (k << 8); k &= CMPLOG_MAP_W - 1; - __libafl_targets_cmplog_routines_checked_extended(k, ptr1, ptr2, len); + cmplog_routines_checked_extended(k, ptr1, ptr2, len); } void __cmplog_rtn_hook_n(const uint8_t *ptr1, const uint8_t *ptr2, @@ -412,7 +296,7 @@ void __cmplog_rtn_hook_str(const uint8_t *ptr1, uint8_t *ptr2) { k = (k >> 4) ^ (k << 8); k &= CMPLOG_MAP_W - 1; - __libafl_targets_cmplog_routines_checked(k, ptr1, ptr2, l); + cmplog_routines_checked(k, ptr1, ptr2, l); } /* hook for string functions, eg. strcmp, strcasecmp etc. */ void __cmplog_rtn_hook_str_extended(const uint8_t *ptr1, uint8_t *ptr2) { @@ -434,7 +318,7 @@ void __cmplog_rtn_hook_str_extended(const uint8_t *ptr1, uint8_t *ptr2) { k = (k >> 4) ^ (k << 8); k &= CMPLOG_MAP_W - 1; - __libafl_targets_cmplog_routines_checked_extended(k, ptr1, ptr2, l); + cmplog_routines_checked_extended(k, ptr1, ptr2, l); } /* hook for string with length functions, eg. strncmp, strncasecmp etc. @@ -459,7 +343,7 @@ void __cmplog_rtn_hook_strn(uint8_t *ptr1, uint8_t *ptr2, uint64_t len) { k = (k >> 4) ^ (k << 8); k &= CMPLOG_MAP_W - 1; - __libafl_targets_cmplog_routines_checked(k, ptr1, ptr2, l); + cmplog_routines_checked(k, ptr1, ptr2, l); } /* hook for string with length functions, eg. strncmp, strncasecmp etc. Note that we ignore the len parameter and take longer strings if present. */ @@ -484,7 +368,7 @@ void __cmplog_rtn_hook_strn_extended(uint8_t *ptr1, uint8_t *ptr2, k = (k >> 4) ^ (k << 8); k &= CMPLOG_MAP_W - 1; - __libafl_targets_cmplog_routines_checked_extended(k, ptr1, ptr2, l); + cmplog_routines_checked_extended(k, ptr1, ptr2, l); } // gcc libstdc++ @@ -533,7 +417,7 @@ void __cmplog_rtn_gcc_stdstring_cstring(const uint8_t *stdstring, uintptr_t k = RETADDR; k = (k >> 4) ^ (k << 8); k &= CMPLOG_MAP_W - 1; - __libafl_targets_cmplog_routines_checked(k, string_ptr, cstring, len); + cmplog_routines_checked(k, string_ptr, cstring, len); } void __cmplog_rtn_gcc_stdstring_cstring_extended(const uint8_t *stdstring, const uint8_t *cstring) { @@ -554,8 +438,7 @@ void __cmplog_rtn_gcc_stdstring_cstring_extended(const uint8_t *stdstring, uintptr_t k = RETADDR; k = (k >> 4) ^ (k << 8); k &= CMPLOG_MAP_W - 1; - __libafl_targets_cmplog_routines_checked_extended(k, string_ptr, cstring, - len); + cmplog_routines_checked_extended(k, string_ptr, cstring, len); } void __cmplog_rtn_gcc_stdstring_stdstring(const uint8_t *stdstring1, @@ -578,7 +461,7 @@ void __cmplog_rtn_gcc_stdstring_stdstring(const uint8_t *stdstring1, uintptr_t k = RETADDR; k = (k >> 4) ^ (k << 8); k &= CMPLOG_MAP_W - 1; - __libafl_targets_cmplog_routines_checked(k, string_ptr1, string_ptr2, len); + cmplog_routines_checked(k, string_ptr1, string_ptr2, len); } void __cmplog_rtn_gcc_stdstring_stdstring_extended(const uint8_t *stdstring1, const uint8_t *stdstring2) { @@ -600,8 +483,7 @@ void __cmplog_rtn_gcc_stdstring_stdstring_extended(const uint8_t *stdstring1, uintptr_t k = RETADDR; k = (k >> 4) ^ (k << 8); k &= CMPLOG_MAP_W - 1; - __libafl_targets_cmplog_routines_checked_extended(k, string_ptr1, string_ptr2, - len); + cmplog_routines_checked_extended(k, string_ptr1, string_ptr2, len); } void __cmplog_rtn_llvm_stdstring_cstring(const uint8_t *stdstring, @@ -622,7 +504,7 @@ void __cmplog_rtn_llvm_stdstring_cstring(const uint8_t *stdstring, uintptr_t k = RETADDR; k = (k >> 4) ^ (k << 8); k &= CMPLOG_MAP_W - 1; - __libafl_targets_cmplog_routines_checked(k, string_ptr, cstring, len); + cmplog_routines_checked(k, string_ptr, cstring, len); } void __cmplog_rtn_llvm_stdstring_cstring_extended(const uint8_t *stdstring, const uint8_t *cstring) { @@ -642,8 +524,7 @@ void __cmplog_rtn_llvm_stdstring_cstring_extended(const uint8_t *stdstring, uintptr_t k = RETADDR; k = (k >> 4) ^ (k << 8); k &= CMPLOG_MAP_W - 1; - __libafl_targets_cmplog_routines_checked_extended(k, string_ptr, cstring, - len); + cmplog_routines_checked_extended(k, string_ptr, cstring, len); } void __cmplog_rtn_llvm_stdstring_stdstring(const uint8_t *stdstring1, @@ -666,7 +547,7 @@ void __cmplog_rtn_llvm_stdstring_stdstring(const uint8_t *stdstring1, uintptr_t k = RETADDR; k = (k >> 4) ^ (k << 8); k &= CMPLOG_MAP_W - 1; - __libafl_targets_cmplog_routines_checked(k, string_ptr1, string_ptr2, len); + cmplog_routines_checked(k, string_ptr1, string_ptr2, len); } void __cmplog_rtn_llvm_stdstring_stdstring_extended(const uint8_t *stdstring1, const uint8_t *stdstring2) { @@ -688,6 +569,5 @@ void __cmplog_rtn_llvm_stdstring_stdstring_extended(const uint8_t *stdstring1, uintptr_t k = RETADDR; k = (k >> 4) ^ (k << 8); k &= CMPLOG_MAP_W - 1; - __libafl_targets_cmplog_routines_checked_extended(k, string_ptr1, string_ptr2, - len); + cmplog_routines_checked_extended(k, string_ptr1, string_ptr2, len); } \ No newline at end of file diff --git a/libafl_targets/src/cmplog.h b/libafl_targets/src/cmplog.h index ae37f2559f..292954fa80 100644 --- a/libafl_targets/src/cmplog.h +++ b/libafl_targets/src/cmplog.h @@ -108,6 +108,136 @@ extern CmpLogMapExtended *libafl_cmplog_map_extended_ptr; extern uint8_t libafl_cmplog_enabled; +// 5 of CMPLOG inner APIs, we static inline everything +// area_is_valid, cmplog_instructions_checked, +// cmplog_instructions_extended_checked, +// cmplog_routines_checked, +// cmplog_routines_checked_extended + +static inline void cmplog_instructions_checked(uintptr_t k, uint8_t shape, + uint64_t arg1, uint64_t arg2) { + if (!libafl_cmplog_enabled) { return; } + libafl_cmplog_enabled = false; + + uint16_t hits; + if (libafl_cmplog_map_ptr->headers[k].kind != CMPLOG_KIND_INS) { + libafl_cmplog_map_ptr->headers[k].kind = CMPLOG_KIND_INS; + libafl_cmplog_map_ptr->headers[k].hits = 1; + libafl_cmplog_map_ptr->headers[k].shape = shape; + hits = 0; + } else { + hits = libafl_cmplog_map_ptr->headers[k].hits++; + if (libafl_cmplog_map_ptr->headers[k].shape < shape) { + libafl_cmplog_map_ptr->headers[k].shape = shape; + } + } + + hits &= CMPLOG_MAP_H - 1; + libafl_cmplog_map_ptr->vals.operands[k][hits].v0 = arg1; + libafl_cmplog_map_ptr->vals.operands[k][hits].v1 = arg2; + libafl_cmplog_enabled = true; +} + +static inline void cmplog_instructions_extended_checked( + uintptr_t k, uint8_t shape, uint64_t arg1, uint64_t arg2, uint8_t attr) { +#ifdef CMPLOG_EXTENDED + if (!libafl_cmplog_enabled) { return; } + libafl_cmplog_enabled = false; + + // printf("%ld %ld %ld\n", k, arg1, arg2); + uint16_t hits; + if (libafl_cmplog_map_extended_ptr->headers[k].type != AFL_CMP_TYPE_INS) { + libafl_cmplog_map_extended_ptr->headers[k].type = AFL_CMP_TYPE_INS; + libafl_cmplog_map_extended_ptr->headers[k].hits = 1; + libafl_cmplog_map_extended_ptr->headers[k].shape = shape; + hits = 0; + } else { + hits = libafl_cmplog_map_extended_ptr->headers[k].hits++; + if (libafl_cmplog_map_extended_ptr->headers[k].shape < shape) { + libafl_cmplog_map_extended_ptr->headers[k].shape = shape; + } + } + + hits &= CMPLOG_MAP_H - 1; + libafl_cmplog_map_extended_ptr->vals.operands[k][hits].v0 = arg1; + libafl_cmplog_map_extended_ptr->vals.operands[k][hits].v1 = arg2; + libafl_cmplog_map_extended_ptr->headers[k].attribute = attr; + libafl_cmplog_enabled = true; +#else + // just do nothing + (void)k; + (void)shape; + (void)arg1; + (void)arg2; + (void)attr; +#endif +} + +// cmplog routines after area check +static inline void cmplog_routines_checked(uintptr_t k, const uint8_t *ptr1, + const uint8_t *ptr2, size_t len) { + libafl_cmplog_enabled = false; + uint32_t hits; + + if (libafl_cmplog_map_ptr->headers[k].kind != CMPLOG_KIND_RTN) { + libafl_cmplog_map_ptr->headers[k].kind = CMPLOG_KIND_RTN; + libafl_cmplog_map_ptr->headers[k].hits = 1; + libafl_cmplog_map_ptr->headers[k].shape = len; + hits = 0; + } else { + hits = libafl_cmplog_map_ptr->headers[k].hits++; + if (libafl_cmplog_map_ptr->headers[k].shape < len) { + libafl_cmplog_map_ptr->headers[k].shape = + len; // TODO; adjust len for AFL++'s cmplog protocol + } + } + + hits &= CMPLOG_MAP_RTN_H - 1; + MEMCPY(libafl_cmplog_map_ptr->vals.routines[k][hits].v0, ptr1, len); + MEMCPY(libafl_cmplog_map_ptr->vals.routines[k][hits].v1, ptr2, len); + libafl_cmplog_enabled = true; +} + +// cmplog routines after area check +static inline void cmplog_routines_checked_extended(uintptr_t k, + const uint8_t *ptr1, + const uint8_t *ptr2, + size_t len) { +#ifdef CMPLOG_EXTENDED + libafl_cmplog_enabled = false; + uint32_t hits; + // printf("RTN: %ld %ld %ld %ld\n", k, *ptr1, *ptr2, len); + if (libafl_cmplog_map_extended_ptr->headers[k].type != AFL_CMP_TYPE_RTN) { + libafl_cmplog_map_extended_ptr->headers[k].type = AFL_CMP_TYPE_RTN; + libafl_cmplog_map_extended_ptr->headers[k].hits = 1; + libafl_cmplog_map_extended_ptr->headers[k].shape = len; + hits = 0; + } else { + hits = libafl_cmplog_map_extended_ptr->headers[k].hits++; + if (libafl_cmplog_map_extended_ptr->headers[k].shape < len) { + libafl_cmplog_map_extended_ptr->headers[k].shape = + len; // TODO; adjust len for AFL++'s cmplog protocol + } + } + + hits &= CMPLOG_MAP_RTN_H - 1; + libafl_cmplog_map_extended_ptr->vals.routines[k][hits].v0_len = len; + libafl_cmplog_map_extended_ptr->vals.routines[k][hits].v1_len = len; + MEMCPY(libafl_cmplog_map_extended_ptr->vals.routines[k][hits].v0, ptr1, len); + MEMCPY(libafl_cmplog_map_extended_ptr->vals.routines[k][hits].v1, ptr2, len); + libafl_cmplog_enabled = true; +#else + // just do nothing + (void)k; + (void)ptr1; + (void)ptr2; + (void)len; +#endif +} + +// Expose these three APIs so that you can still call into them from outside +// libafl_targets + void __libafl_targets_cmplog_instructions(uintptr_t k, uint8_t shape, uint64_t arg1, uint64_t arg2); diff --git a/libafl_targets/src/cmps/mod.rs b/libafl_targets/src/cmps/mod.rs index 095103484d..b79f086f62 100644 --- a/libafl_targets/src/cmps/mod.rs +++ b/libafl_targets/src/cmps/mod.rs @@ -8,6 +8,7 @@ use alloc::{alloc::alloc_zeroed, boxed::Box, vec::Vec}; use core::{ alloc::Layout, fmt::{self, Debug, Formatter}, + mem, ptr, slice, }; use libafl::{ @@ -28,13 +29,12 @@ pub const CMPLOG_MAP_SIZE: usize = CMPLOG_MAP_W * CMPLOG_MAP_H; pub const CMPLOG_RTN_LEN: usize = 32; /// The hight of a cmplog routine map -pub const CMPLOG_MAP_RTN_H: usize = (CMPLOG_MAP_H * core::mem::size_of::()) - / core::mem::size_of::(); +pub const CMPLOG_MAP_RTN_H: usize = + (CMPLOG_MAP_H * mem::size_of::()) / mem::size_of::(); /// The height of extended rountine map -pub const CMPLOG_MAP_RTN_EXTENDED_H: usize = CMPLOG_MAP_H - * core::mem::size_of::() - / core::mem::size_of::(); +pub const CMPLOG_MAP_RTN_EXTENDED_H: usize = + CMPLOG_MAP_H * mem::size_of::() / mem::size_of::(); /// `CmpLog` instruction kind pub const CMPLOG_KIND_INS: u8 = 0; @@ -276,14 +276,14 @@ pub union AFLppCmpLogVals { } impl Debug for AFLppCmpLogVals { - fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { f.debug_struct("AFLppCmpLogVals").finish_non_exhaustive() } } impl AFLppCmpLogVals { #[must_use] - /// Reference comparison values as comparison operands + /// Handle comparison values as comparison operands pub fn operands(&self) -> &[[AFLppCmpLogOperands; CMPLOG_MAP_H]; CMPLOG_MAP_W] { unsafe { &self.operands } } @@ -295,7 +295,7 @@ impl AFLppCmpLogVals { } #[must_use] - /// Reference comparison values as comparison function operands + /// Handle comparison values as comparison function operands pub fn fn_operands( &self, ) -> &[[AFLppCmpLogFnOperands; CMPLOG_MAP_RTN_EXTENDED_H]; CMPLOG_MAP_W] { @@ -323,7 +323,7 @@ pub struct CmpLogMap { impl Default for CmpLogMap { fn default() -> Self { - unsafe { core::mem::zeroed() } + unsafe { mem::zeroed() } } } @@ -450,7 +450,7 @@ impl AFLppCmpLogMap { } #[must_use] - /// Reference the headers for the map + /// Handle the headers for the map pub fn headers(&self) -> &[AFLppCmpLogHeader] { &self.headers } @@ -462,7 +462,7 @@ impl AFLppCmpLogMap { } #[must_use] - /// Reference the values for the map + /// Handle the values for the map pub fn values(&self) -> &AFLppCmpLogVals { &self.vals } @@ -480,10 +480,7 @@ impl Serialize for AFLppCmpLogMap { S: Serializer, { let slice = unsafe { - core::slice::from_raw_parts( - (core::ptr::from_ref::(self)) as *const u8, - core::mem::size_of::(), - ) + slice::from_raw_parts(ptr::from_ref(self) as *const u8, mem::size_of::()) }; serializer.serialize_bytes(slice) } @@ -495,7 +492,7 @@ impl<'de> Deserialize<'de> for AFLppCmpLogMap { D: Deserializer<'de>, { let bytes = Vec::::deserialize(deserializer)?; - let map: Self = unsafe { core::ptr::read(bytes.as_ptr() as *const _) }; + let map: Self = unsafe { ptr::read(bytes.as_ptr() as *const _) }; Ok(map) } } diff --git a/libafl_targets/src/cmps/observers/aflpp.rs b/libafl_targets/src/cmps/observers/aflpp.rs index 7dd125e2bd..4405072c3b 100644 --- a/libafl_targets/src/cmps/observers/aflpp.rs +++ b/libafl_targets/src/cmps/observers/aflpp.rs @@ -1,7 +1,4 @@ -use alloc::{ - string::{String, ToString}, - vec::Vec, -}; +use alloc::{borrow::Cow, vec::Vec}; use core::{fmt::Debug, marker::PhantomData}; use libafl::{ @@ -11,8 +8,7 @@ use libafl::{ cmp::{AFLppCmpValuesMetadata, CmpMap, CmpObserver, CmpObserverMetadata, CmpValues}, Observer, }, - state::HasMetadata, - Error, + Error, HasMetadata, }; use libafl_bolts::{ownedref::OwnedRefMut, Named}; use serde::{Deserialize, Serialize}; @@ -69,13 +65,10 @@ struct cmp_map { /// A [`CmpObserver`] observer for AFL++ redqueen #[derive(Serialize, Deserialize, Debug)] -pub struct AFLppCmpLogObserver<'a, S> -where - S: UsesInput + HasMetadata, -{ +pub struct AFLppCmpLogObserver<'a, S> { cmp_map: OwnedRefMut<'a, AFLppCmpLogMap>, size: Option>, - name: String, + name: Cow<'static, str>, add_meta: bool, original: >::Data, phantom: PhantomData, @@ -84,7 +77,7 @@ where impl<'a, S> CmpObserver<'a, AFLppCmpLogMap, S, AFLppCmpValuesMetadata> for AFLppCmpLogObserver<'a, S> where - S: UsesInput + Debug + HasMetadata, + S: UsesInput + HasMetadata, { /// Get the number of usable cmps (all by default) fn usable_count(&self) -> usize { @@ -146,7 +139,7 @@ where impl<'a, S> Observer for AFLppCmpLogObserver<'a, S> where - S: UsesInput + Debug + HasMetadata, + S: UsesInput + HasMetadata, { fn pre_exec(&mut self, _state: &mut S, _input: &S::Input) -> Result<(), Error> { #[cfg(feature = "cmplog_extended_instrumentation")] @@ -180,19 +173,13 @@ where } } -impl<'a, S> Named for AFLppCmpLogObserver<'a, S> -where - S: UsesInput + HasMetadata, -{ - fn name(&self) -> &str { +impl<'a, S> Named for AFLppCmpLogObserver<'a, S> { + fn name(&self) -> &Cow<'static, str> { &self.name } } -impl<'a, S> AFLppCmpLogObserver<'a, S> -where - S: UsesInput + HasMetadata, -{ +impl<'a, S> AFLppCmpLogObserver<'a, S> { /// Creates a new [`AFLppCmpLogObserver`] with the given name and map. #[must_use] pub fn new( @@ -201,7 +188,7 @@ where add_meta: bool, ) -> Self { Self { - name: name.to_string(), + name: Cow::from(name), size: None, cmp_map, add_meta, @@ -224,7 +211,7 @@ where size: OwnedRefMut<'a, usize>, ) -> Self { Self { - name: name.to_string(), + name: Cow::from(name), size: Some(size), cmp_map, add_meta, diff --git a/libafl_targets/src/cmps/observers/cmplog.rs b/libafl_targets/src/cmps/observers/cmplog.rs index f66cb074f8..f8583e8c23 100644 --- a/libafl_targets/src/cmps/observers/cmplog.rs +++ b/libafl_targets/src/cmps/observers/cmplog.rs @@ -2,15 +2,14 @@ //! The values will then be used in subsequent mutations. //! -use alloc::string::{String, ToString}; +use alloc::borrow::Cow; use core::fmt::Debug; use libafl::{ executors::ExitKind, inputs::UsesInput, observers::{cmp::CmpValuesMetadata, CmpMap, CmpObserver, Observer}, - state::HasMetadata, - Error, + Error, HasMetadata, }; use libafl_bolts::{ownedref::OwnedMutPtr, Named}; @@ -23,7 +22,7 @@ pub struct CmpLogObserver { map: OwnedMutPtr, size: Option>, add_meta: bool, - name: String, + name: Cow<'static, str>, } impl<'a, S> CmpObserver<'a, CmpLogMap, S, CmpValuesMetadata> for CmpLogObserver @@ -79,7 +78,7 @@ where } impl Named for CmpLogObserver { - fn name(&self) -> &str { + fn name(&self) -> &Cow<'static, str> { &self.name } } @@ -92,7 +91,7 @@ impl CmpLogObserver { #[must_use] pub unsafe fn with_map_ptr(name: &'static str, map: *mut CmpLogMap, add_meta: bool) -> Self { Self { - name: name.to_string(), + name: Cow::from(name), size: None, add_meta, map: OwnedMutPtr::Ptr(map), diff --git a/libafl_targets/src/cmps/stages/aflpptracing.rs b/libafl_targets/src/cmps/stages/aflpptracing.rs index 9243e43ae0..56284643b0 100644 --- a/libafl_targets/src/cmps/stages/aflpptracing.rs +++ b/libafl_targets/src/cmps/stages/aflpptracing.rs @@ -1,47 +1,61 @@ -use alloc::string::{String, ToString}; +use alloc::borrow::Cow; use core::marker::PhantomData; #[cfg(feature = "introspection")] use libafl::state::HasClientPerfMonitor; use libafl::{ - corpus::{Corpus, HasCurrentCorpusIdx}, executors::{Executor, HasObservers}, inputs::{BytesInput, UsesInput}, observers::ObserversTuple, - stages::{colorization::TaintMetadata, Stage}, - state::{HasCorpus, HasExecutions, HasMetadata, UsesState}, - Error, + stages::{colorization::TaintMetadata, RetryRestartHelper, Stage}, + state::{HasCorpus, HasCurrentTestcase, HasExecutions, UsesState}, + Error, HasMetadata, HasNamedMetadata, +}; +use libafl_bolts::{ + tuples::{Handle, MatchNameRef}, + Named, }; -use libafl_bolts::tuples::MatchName; use crate::cmps::observers::AFLppCmpLogObserver; /// Trace with tainted input #[derive(Clone, Debug)] -pub struct AFLppCmplogTracingStage { +pub struct AFLppCmplogTracingStage<'a, EM, TE, Z> +where + TE: UsesState, +{ tracer_executor: TE, - cmplog_observer_name: Option, + cmplog_observer_ref: Option>>, #[allow(clippy::type_complexity)] phantom: PhantomData<(EM, TE, Z)>, } -impl UsesState for AFLppCmplogTracingStage +impl UsesState for AFLppCmplogTracingStage<'_, EM, TE, Z> where TE: UsesState, { type State = TE::State; } -impl Stage for AFLppCmplogTracingStage +impl Named for AFLppCmplogTracingStage<'_, EM, TE, Z> +where + TE: UsesState, +{ + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("AFLppCmplogTracingStage"); + &NAME + } +} + +impl Stage for AFLppCmplogTracingStage<'_, EM, TE, Z> where E: UsesState, TE: Executor + HasObservers, - TE::State: HasExecutions + HasCorpus + HasMetadata + UsesInput, + TE::State: + HasExecutions + HasCorpus + HasMetadata + UsesInput + HasNamedMetadata, EM: UsesState, Z: UsesState, { - type Progress = (); // TODO this needs resumption - #[inline] fn perform( &mut self, @@ -51,18 +65,10 @@ where manager: &mut EM, ) -> Result<(), Error> { // First run with the un-mutated input - let corpus_idx = state.current_corpus_idx()?.ok_or_else(|| { - Error::illegal_state("state is not currently processing a corpus index") - })?; - - let unmutated_input = state.corpus().cloned_input_for_id(corpus_idx)?; - - if let Some(name) = &self.cmplog_observer_name { - if let Some(ob) = self - .tracer_executor - .observers_mut() - .match_name_mut::>(name) - { + let unmutated_input = state.current_input_cloned()?; + + if let Some(obs_ref) = &self.cmplog_observer_ref { + if let Some(ob) = self.tracer_executor.observers_mut().get_mut(obs_ref) { // This is not the original input, // Set it to false ob.set_original(true); @@ -91,12 +97,8 @@ where None => return Err(Error::unknown("No metadata found")), }; - if let Some(name) = &self.cmplog_observer_name { - if let Some(ob) = self - .tracer_executor - .observers_mut() - .match_name_mut::>(name) - { + if let Some(obs_ref) = &self.cmplog_observer_ref { + if let Some(ob) = self.tracer_executor.observers_mut().get_mut(obs_ref) { // This is not the original input, // Set it to false ob.set_original(false); @@ -121,22 +123,38 @@ where Ok(()) } + + fn restart_progress_should_run(&mut self, state: &mut Self::State) -> Result { + // TODO: this may need better resumption? (Or is it always used with a forkserver?) + RetryRestartHelper::restart_progress_should_run(state, self, 3) + } + + fn clear_restart_progress(&mut self, state: &mut Self::State) -> Result<(), Error> { + // TODO: this may need better resumption? (Or is it always used with a forkserver?) + RetryRestartHelper::clear_restart_progress(state, self) + } } -impl AFLppCmplogTracingStage { +impl<'a, EM, TE, Z> AFLppCmplogTracingStage<'a, EM, TE, Z> +where + TE: UsesState, +{ /// Creates a new default stage pub fn new(tracer_executor: TE) -> Self { Self { - cmplog_observer_name: None, + cmplog_observer_ref: None, tracer_executor, phantom: PhantomData, } } /// With cmplog observer - pub fn with_cmplog_observer_name(tracer_executor: TE, name: &'static str) -> Self { + pub fn with_cmplog_observer( + tracer_executor: TE, + obs_ref: Handle>, + ) -> Self { Self { - cmplog_observer_name: Some(name.to_string()), + cmplog_observer_ref: Some(obs_ref), tracer_executor, phantom: PhantomData, } diff --git a/libafl_targets/src/common.h b/libafl_targets/src/common.h index 69d1a0c184..0852f94c20 100644 --- a/libafl_targets/src/common.h +++ b/libafl_targets/src/common.h @@ -82,6 +82,7 @@ typedef uint128_t u128; }) #define MEMCPY __builtin_memcpy #else + #include // needed to use memcpy on windows #define MAX(a, b) (((a) > (b)) ? (a) : (b)) #define MIN(a, b) (((a) < (b)) ? (a) : (b)) #define MEMCPY memcpy @@ -140,14 +141,11 @@ typedef uint128_t u128; #else #if defined(__APPLE__) - // On Apple, weak_import and weak attrs behave differently to linux. - - #define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ - __attribute__((weak, visibility("default"))) RETURN_TYPE NAME FUNC_SIG { \ - return (RETURN_TYPE)0; \ - } - #define EXT_FUNC_IMPL(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ + EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) + + // Declare these symbols as weak to allow them to be optionally defined. + #define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ __attribute__((weak, visibility("default"))) RETURN_TYPE NAME FUNC_SIG // Weakly defined globals diff --git a/libafl_targets/src/coverage.c b/libafl_targets/src/coverage.c index 7f4987a15b..727d5e1567 100644 --- a/libafl_targets/src/coverage.c +++ b/libafl_targets/src/coverage.c @@ -8,9 +8,12 @@ typedef uint32_t prev_loc_t; /* Maximum K for top-K context sensitivity */ #define CTX_MAX_K 32U -extern uint8_t __afl_area_ptr_local[EDGES_MAP_SIZE]; +extern uint8_t __afl_area_ptr_local[EDGES_MAP_SIZE_MAX]; uint8_t *__afl_area_ptr = __afl_area_ptr_local; +extern uint8_t __ddg_area_ptr_local[DDG_MAP_SIZE]; +uint8_t *__ddg_area_ptr = __ddg_area_ptr_local; + extern uint32_t __afl_acc_memop_ptr_local[ACCOUNTING_MAP_SIZE]; uint32_t *__afl_acc_memop_ptr = __afl_acc_memop_ptr_local; @@ -33,7 +36,5 @@ uint8_t *__token_stop = &__stop_libafl_token; #endif // #if defined(__ANDROID__) || defined(__HAIKU__) -MAYBE_THREAD_LOCAL prev_loc_t __afl_prev_loc[NGRAM_SIZE_MAX]; -MAYBE_THREAD_LOCAL prev_loc_t __afl_prev_caller[CTX_MAX_K]; -MAYBE_THREAD_LOCAL uint32_t __afl_prev_ctx; +uint32_t __afl_prev_ctx; MAYBE_THREAD_LOCAL prev_loc_t __afl_acc_prev_loc; diff --git a/libafl_targets/src/coverage.rs b/libafl_targets/src/coverage.rs index 4d82ea7ea1..1d7b88f0e0 100644 --- a/libafl_targets/src/coverage.rs +++ b/libafl_targets/src/coverage.rs @@ -1,29 +1,45 @@ //! Coverage maps as static mut array -use alloc::string::String; +#[cfg(any( + feature = "sancov_pcguard_edges", + feature = "sancov_pcguard_hitcounts", + feature = "sancov_ngram4", + feature = "sancov_ctx" +))] +use alloc::borrow::Cow; #[cfg(any(target_os = "linux", target_vendor = "apple"))] use libafl::{mutators::Tokens, Error}; -use crate::{ACCOUNTING_MAP_SIZE, EDGES_MAP_SIZE}; +use crate::{ACCOUNTING_MAP_SIZE, DDG_MAP_SIZE, EDGES_MAP_SIZE_IN_USE, EDGES_MAP_SIZE_MAX}; /// The map for edges. #[no_mangle] -pub static mut __afl_area_ptr_local: [u8; EDGES_MAP_SIZE] = [0; EDGES_MAP_SIZE]; +pub static mut __afl_area_ptr_local: [u8; EDGES_MAP_SIZE_MAX] = [0; EDGES_MAP_SIZE_MAX]; pub use __afl_area_ptr_local as EDGES_MAP; +/// The map for data dependency +#[no_mangle] +pub static mut __ddg_area_ptr_local: [u8; DDG_MAP_SIZE] = [0; DDG_MAP_SIZE]; +pub use __ddg_area_ptr_local as DDG_MAP; + /// The map for accounting mem writes. #[no_mangle] pub static mut __afl_acc_memop_ptr_local: [u32; ACCOUNTING_MAP_SIZE] = [0; ACCOUNTING_MAP_SIZE]; pub use __afl_acc_memop_ptr_local as ACCOUNTING_MEMOP_MAP; -/// The max count of edges tracked. -pub static mut MAX_EDGES_NUM: usize = 0; +/// The max count of edges found. +/// This is either computed during the compilation time or at runtime (in this case this is used to shrink the map). +/// You can use this for the initial map size for the observer only if you compute this time at compilation time. +pub static mut MAX_EDGES_FOUND: usize = 0; extern "C" { /// The area pointer points to the edges map. pub static mut __afl_area_ptr: *mut u8; + /// The area pointer points to the data flow map + pub static mut __ddg_area_ptr: *mut u8; + /// The area pointer points to the accounting mem operations map. pub static mut __afl_acc_memop_ptr: *mut u32; @@ -37,6 +53,7 @@ extern "C" { } pub use __afl_acc_memop_ptr as ACCOUNTING_MEMOP_MAP_PTR; pub use __afl_area_ptr as EDGES_MAP_PTR; +pub use __ddg_area_ptr as DDG_MAP_PTR; /// Return Tokens from the compile-time token section #[cfg(any(target_os = "linux", target_vendor = "apple"))] @@ -51,21 +68,40 @@ pub fn autotokens() -> Result { } } -/// The size of the map for edges. +/// The actual size we use for the map of edges. +/// This is used for forkserver backend #[no_mangle] -pub static mut __afl_map_size: usize = EDGES_MAP_SIZE; -pub use __afl_map_size as EDGES_MAP_PTR_NUM; +pub static mut __afl_map_size: usize = EDGES_MAP_SIZE_IN_USE; + +#[cfg(any( + feature = "sancov_pcguard_edges", + feature = "sancov_pcguard_hitcounts", + feature = "sancov_ngram4", + feature = "sancov_ctx" +))] use libafl::observers::StdMapObserver; +#[cfg(any( + feature = "sancov_pcguard_edges", + feature = "sancov_pcguard_hitcounts", + feature = "sancov_ngram4", + feature = "sancov_ctx" +))] use libafl_bolts::ownedref::OwnedMutSlice; /// Gets the edges map from the `EDGES_MAP_PTR` raw pointer. -/// Assumes a `len` of `EDGES_MAP_PTR_NUM`. +/// Assumes a `len` of at least `EDGES_MAP_PTR_MAX`. /// /// # Safety /// /// This function will crash if `edges_map_mut_ptr` is not a valid pointer. /// The [`edges_max_num`] needs to be smaller than, or equal to the size of the map. #[must_use] +#[cfg(any( + feature = "sancov_pcguard_edges", + feature = "sancov_pcguard_hitcounts", + feature = "sancov_ngram4", + feature = "sancov_ctx" +))] pub unsafe fn edges_map_mut_slice<'a>() -> OwnedMutSlice<'a, u8> { OwnedMutSlice::from_raw_parts_mut(edges_map_mut_ptr(), edges_max_num()) } @@ -75,11 +111,11 @@ pub unsafe fn edges_map_mut_slice<'a>() -> OwnedMutSlice<'a, u8> { /// /// ```rust,ignore /// use libafl::observers::StdMapObserver; -/// use libafl_targets::{EDGES_MAP, MAX_EDGES_NUM}; +/// use libafl_targets::{EDGES_MAP, EDGES_MAP_SIZE_IN_USE}; /// /// #[cfg(not(feature = "pointer_maps"))] /// let observer = unsafe { -/// StdMapObserver::from_mut_ptr("edges", EDGES_MAP.as_mut_ptr(), MAX_EDGES_NUM) +/// StdMapObserver::from_mut_ptr("edges", EDGES_MAP.as_mut_ptr(), EDGES_MAP_SIZE_IN_USE) /// }; /// ``` /// @@ -97,9 +133,15 @@ pub unsafe fn edges_map_mut_slice<'a>() -> OwnedMutSlice<'a, u8> { /// /// # Safety /// This will dereference [`edges_map_mut_ptr`] and crash if it is not a valid address. +#[cfg(any( + feature = "sancov_pcguard_edges", + feature = "sancov_pcguard_hitcounts", + feature = "sancov_ngram4", + feature = "sancov_ctx" +))] pub unsafe fn std_edges_map_observer<'a, S>(name: S) -> StdMapObserver<'a, u8, false> where - S: Into, + S: Into>, { StdMapObserver::from_mut_slice(name, edges_map_mut_slice()) } @@ -120,15 +162,21 @@ pub fn edges_map_mut_ptr() -> *mut u8 { } /// Gets the current maximum number of edges tracked. +#[cfg(any( + feature = "sancov_pcguard_edges", + feature = "sancov_pcguard_hitcounts", + feature = "sancov_ngram4", + feature = "sancov_ctx" +))] #[must_use] pub fn edges_max_num() -> usize { unsafe { - if MAX_EDGES_NUM > 0 { - MAX_EDGES_NUM + if MAX_EDGES_FOUND > 0 { + MAX_EDGES_FOUND } else { #[cfg(feature = "pointer_maps")] { - EDGES_MAP_PTR_NUM + EDGES_MAP_SIZE_MAX // the upper bound } #[cfg(not(feature = "pointer_maps"))] { @@ -143,7 +191,7 @@ pub use swap::*; #[cfg(feature = "pointer_maps")] mod swap { - use alloc::string::{String, ToString}; + use alloc::borrow::Cow; use core::fmt::Debug; use libafl::{ @@ -151,10 +199,10 @@ mod swap { observers::{DifferentialObserver, Observer, ObserversTuple, StdMapObserver}, Error, }; - use libafl_bolts::{ownedref::OwnedMutSlice, AsMutSlice, Named}; + use libafl_bolts::{ownedref::OwnedMutSlice, AsSliceMut, Named}; use serde::{Deserialize, Serialize}; - use super::{EDGES_MAP_PTR, EDGES_MAP_PTR_NUM}; + use super::EDGES_MAP_PTR; /// Observer to be used with `DiffExecutor`s when executing a differential target that shares /// the AFL map in order to swap out the maps (and thus allow for map observing the two targets @@ -164,9 +212,9 @@ mod swap { pub struct DifferentialAFLMapSwapObserver<'a, 'b> { first_map: OwnedMutSlice<'a, u8>, second_map: OwnedMutSlice<'b, u8>, - first_name: String, - second_name: String, - name: String, + first_name: Cow<'static, str>, + second_name: Cow<'static, str>, + name: Cow<'static, str>, } impl<'a, 'b> DifferentialAFLMapSwapObserver<'a, 'b> { @@ -176,15 +224,15 @@ mod swap { second: &mut StdMapObserver<'b, u8, D2>, ) -> Self { Self { - first_name: first.name().to_string(), - second_name: second.name().to_string(), - name: format!("differential_{}_{}", first.name(), second.name()), + first_name: first.name().clone(), + second_name: second.name().clone(), + name: Cow::from(format!("differential_{}_{}", first.name(), second.name())), first_map: unsafe { - let slice = first.map_mut().as_mut_slice(); + let slice = first.map_mut().as_slice_mut(); OwnedMutSlice::from_raw_parts_mut(slice.as_mut_ptr(), slice.len()) }, second_map: unsafe { - let slice = second.map_mut().as_mut_slice(); + let slice = second.map_mut().as_slice_mut(); OwnedMutSlice::from_raw_parts_mut(slice.as_mut_ptr(), slice.len()) }, } @@ -216,7 +264,7 @@ mod swap { } impl<'a, 'b> Named for DifferentialAFLMapSwapObserver<'a, 'b> { - fn name(&self) -> &str { + fn name(&self) -> &Cow<'static, str> { &self.name } } @@ -231,19 +279,17 @@ mod swap { S: UsesInput, { fn pre_observe_first(&mut self, _: &mut OTA) -> Result<(), Error> { - let slice = self.first_map.as_mut_slice(); + let slice = self.first_map.as_slice_mut(); unsafe { EDGES_MAP_PTR = slice.as_mut_ptr(); - EDGES_MAP_PTR_NUM = slice.len(); } Ok(()) } fn pre_observe_second(&mut self, _: &mut OTB) -> Result<(), Error> { - let slice = self.second_map.as_mut_slice(); + let slice = self.second_map.as_slice_mut(); unsafe { EDGES_MAP_PTR = slice.as_mut_ptr(); - EDGES_MAP_PTR_NUM = slice.len(); } Ok(()) } diff --git a/libafl_targets/src/lib.rs b/libafl_targets/src/lib.rs index 34df1f1ce6..5ced10b1de 100644 --- a/libafl_targets/src/lib.rs +++ b/libafl_targets/src/lib.rs @@ -3,6 +3,7 @@ #![deny(rustdoc::broken_intra_doc_links)] #![deny(clippy::all)] #![deny(clippy::pedantic)] +#![cfg_attr(nightly, feature(portable_simd))] #![allow( clippy::unreadable_literal, clippy::type_repetition_in_bounds, @@ -27,14 +28,12 @@ ))] #![cfg_attr(test, deny( missing_debug_implementations, - missing_docs, //trivial_casts, trivial_numeric_casts, unused_extern_crates, unused_import_braces, unused_qualifications, unused_must_use, - missing_docs, //unused_results ))] #![cfg_attr( @@ -66,9 +65,19 @@ extern crate alloc; include!(concat!(env!("OUT_DIR"), "/constants.rs")); -#[cfg(any(feature = "sancov_pcguard_edges", feature = "sancov_pcguard_hitcounts",))] +#[cfg(any( + feature = "sancov_pcguard_edges", + feature = "sancov_pcguard_hitcounts", + feature = "sancov_ngram4", + feature = "sancov_ctx" +))] pub mod sancov_pcguard; -#[cfg(any(feature = "sancov_pcguard_edges", feature = "sancov_pcguard_hitcounts",))] +#[cfg(any( + feature = "sancov_pcguard_edges", + feature = "sancov_pcguard_hitcounts", + feature = "sancov_ngram4", + feature = "sancov_ctx" +))] pub use sancov_pcguard::*; #[cfg(any(feature = "sancov_cmplog", feature = "sancov_value_profile"))] @@ -78,6 +87,7 @@ pub use sancov_cmp::*; /// Module containing bindings to the various sanitizer interface headers #[cfg(feature = "sanitizer_interfaces")] +#[allow(clippy::mixed_attributes_style)] pub mod sanitizer_ifaces { #![allow(non_snake_case)] #![allow(non_camel_case_types)] @@ -87,6 +97,7 @@ pub mod sanitizer_ifaces { #![allow(clippy::unreadable_literal)] #![allow(missing_docs)] #![allow(missing_debug_implementations)] + #![allow(unused_qualifications)] include!(concat!(env!("OUT_DIR"), "/sanitizer_interfaces.rs")); } diff --git a/libafl_targets/src/libfuzzer.c b/libafl_targets/src/libfuzzer.c index eea3228090..ff6e2968ee 100644 --- a/libafl_targets/src/libfuzzer.c +++ b/libafl_targets/src/libfuzzer.c @@ -48,7 +48,11 @@ int main(int argc, char **argv) { libafl_main(); return 0; } + #ifdef FUZZER_DEFINE_RUN_DRIVER return LLVMFuzzerRunDriver(&argc, &argv, &LLVMFuzzerTestOneInput); + #else + return 0; + #endif } #endif #endif diff --git a/libafl_targets/src/libfuzzer/mutators.rs b/libafl_targets/src/libfuzzer/mutators.rs index ed2f81d772..44324425fc 100644 --- a/libafl_targets/src/libfuzzer/mutators.rs +++ b/libafl_targets/src/libfuzzer/mutators.rs @@ -1,4 +1,7 @@ -use alloc::rc::{Rc, Weak}; +use alloc::{ + borrow::Cow, + rc::{Rc, Weak}, +}; use std::{ cell::RefCell, marker::PhantomData, @@ -12,7 +15,7 @@ use libafl::{ mutators::{ ComposedByMutations, MutationId, MutationResult, Mutator, MutatorsTuple, ScheduledMutator, }, - random_corpus_id, + random_corpus_id_with_disabled, state::{HasCorpus, HasMaxSize, HasRand}, Error, }; @@ -87,7 +90,6 @@ struct MutatorProxy<'a, M, MT, S> { /// The result of mutation, to be propagated to the mutational stage result: Rc>>, /// Stage index, which is used by libafl mutator implementations - stage_idx: i32, phantom: PhantomData<(&'a mut (), MT)>, } @@ -97,13 +99,11 @@ impl<'a, M, MT, S> MutatorProxy<'a, M, MT, S> { state: &'a mut S, mutator: &Rc>, result: &Rc>>, - stage_idx: i32, ) -> Self { Self { state: Rc::new(RefCell::new(state)), mutator: Rc::downgrade(mutator), result: result.clone(), - stage_idx, phantom: PhantomData, } } @@ -126,7 +126,6 @@ impl<'a, M, MT, S> MutatorProxy<'a, M, MT, S> { false }, mutator: self.mutator.clone(), - stage_idx: self.stage_idx, result: self.result.clone(), phantom: PhantomData, } @@ -143,7 +142,7 @@ struct WeakMutatorProxy { /// A weak reference to the mutator mutator: Weak>, /// The stage index to provide to the mutator, when executed. - stage_idx: i32, + /// The result of mutation, to be propagated to the mutational stage result: Rc>>, phantom: PhantomData<(MT, S)>, @@ -165,7 +164,7 @@ where BytesInput::from(unsafe { core::slice::from_raw_parts(data, size) }); let old = state.max_size(); state.set_max_size(max_size); - let res = mutator.scheduled_mutate(state, &mut intermediary, self.stage_idx); + let res = mutator.scheduled_mutate(state, &mut intermediary); state.set_max_size(old); let succeeded = res.is_ok(); @@ -283,8 +282,9 @@ where } impl Named for LLVMCustomMutator { - fn name(&self) -> &str { - "LLVMCustomMutator" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("LLVMCustomMutator"); + &NAME } } @@ -295,13 +295,8 @@ where SM: ScheduledMutator + 'static, { #[inline] - fn mutate( - &mut self, - state: &mut S, - input: &mut S::Input, - stage_idx: i32, - ) -> Result { - self.scheduled_mutate(state, input, stage_idx) + fn mutate(&mut self, state: &mut S, input: &mut S::Input) -> Result { + self.scheduled_mutate(state, input) } } @@ -325,7 +320,6 @@ where &mut self, state: &mut S, input: &mut S::Input, - stage_idx: i32, ) -> Result { let seed = state.rand_mut().next(); let target = input.bytes(); @@ -335,7 +329,7 @@ where // we assume that the fuzzer did not use this mutator, but instead utilised their own let result = Rc::new(RefCell::new(Ok(MutationResult::Mutated))); - let proxy = MutatorProxy::new(state, &self.mutator, &result, stage_idx); + let proxy = MutatorProxy::new(state, &self.mutator, &result); let old = MUTATOR.with(|mutator| { let mut mutator = mutator.borrow_mut(); mutator.replace(Box::new(proxy.weak())) @@ -363,8 +357,9 @@ where } impl Named for LLVMCustomMutator { - fn name(&self) -> &str { - "LLVMCustomCrossover" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("LLVMCustomCrossover"); + &NAME } } @@ -375,13 +370,8 @@ where SM: ScheduledMutator + 'static, { #[inline] - fn mutate( - &mut self, - state: &mut S, - input: &mut S::Input, - stage_idx: i32, - ) -> Result { - self.scheduled_mutate(state, input, stage_idx) + fn mutate(&mut self, state: &mut S, input: &mut S::Input) -> Result { + self.scheduled_mutate(state, input) } } @@ -405,17 +395,16 @@ where &mut self, state: &mut S, input: &mut S::Input, - stage_idx: i32, ) -> Result { // We don't want to use the testcase we're already using for splicing - let idx = random_corpus_id!(state.corpus(), state.rand_mut()); + let idx = random_corpus_id_with_disabled!(state.corpus(), state.rand_mut()); if let Some(cur) = state.corpus().current() { if idx == *cur { return Ok(MutationResult::Skipped); } } - let mut other_testcase = state.corpus().get(idx)?.borrow_mut(); + let mut other_testcase = state.corpus().get_from_all(idx)?.borrow_mut(); let other = other_testcase.load_input(state.corpus())?; let data2 = Vec::from(other.bytes()); drop(other_testcase); @@ -426,7 +415,7 @@ where // we assume that the fuzzer did not use this mutator, but instead utilised their own let result = Rc::new(RefCell::new(Ok(MutationResult::Mutated))); - let proxy = MutatorProxy::new(state, &self.mutator, &result, stage_idx); + let proxy = MutatorProxy::new(state, &self.mutator, &result); let old = MUTATOR.with(|mutator| { let mut mutator = mutator.borrow_mut(); mutator.replace(Box::new(proxy.weak())) diff --git a/libafl_targets/src/libfuzzer/observers/oom.rs b/libafl_targets/src/libfuzzer/observers/oom.rs index e24c5169c9..78de4c6764 100644 --- a/libafl_targets/src/libfuzzer/observers/oom.rs +++ b/libafl_targets/src/libfuzzer/observers/oom.rs @@ -1,3 +1,4 @@ +use alloc::borrow::Cow; use core::{ffi::c_void, fmt::Debug}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; @@ -68,7 +69,7 @@ pub unsafe extern "C" fn __sanitizer_free_hook(ptr: *const c_void) { } } -const OOM_OBS_NAME: &str = "libfuzzer-like-oom"; +static OOM_OBS_NAME: Cow<'static, str> = Cow::Borrowed("libfuzzer-like-oom"); /// Observer which detects if the target would run out of memory or otherwise violate the permissible usage of malloc #[derive(Debug, Serialize, Deserialize)] @@ -88,8 +89,8 @@ impl OomObserver { impl Named for OomObserver { // strictly one name to prevent two from being registered - fn name(&self) -> &str { - OOM_OBS_NAME + fn name(&self) -> &Cow<'static, str> { + &OOM_OBS_NAME } } @@ -142,8 +143,9 @@ impl OomFeedback { } impl Named for OomFeedback { - fn name(&self) -> &str { - "oom" + fn name(&self) -> &Cow<'static, str> { + static NAME: Cow<'static, str> = Cow::Borrowed("oom"); + &NAME } } diff --git a/libafl_targets/src/sancov_8bit.rs b/libafl_targets/src/sancov_8bit.rs index 96751eda57..89f7118a2c 100644 --- a/libafl_targets/src/sancov_8bit.rs +++ b/libafl_targets/src/sancov_8bit.rs @@ -2,7 +2,7 @@ use alloc::vec::Vec; use core::ptr::addr_of_mut; -use libafl_bolts::{ownedref::OwnedMutSlice, AsMutSlice, AsSlice}; +use libafl_bolts::{ownedref::OwnedMutSlice, AsSlice, AsSliceMut}; /// A [`Vec`] of `8-bit-counters` maps for multiple modules. /// They are initialized by calling [`__sanitizer_cov_8bit_counters_init`]( @@ -32,9 +32,9 @@ pub unsafe fn extra_counters() -> Vec> { pub extern "C" fn __sanitizer_cov_8bit_counters_init(start: *mut u8, stop: *mut u8) { unsafe { for existing in &mut *addr_of_mut!(COUNTERS_MAPS) { - let range = existing.as_mut_slice().as_mut_ptr() + let range = existing.as_slice_mut().as_mut_ptr() ..=existing - .as_mut_slice() + .as_slice_mut() .as_mut_ptr() .add(existing.as_slice().len()); if range.contains(&start) || range.contains(&stop) { @@ -59,13 +59,10 @@ pub use self::observers::{counters_maps_observer, CountersMultiMapObserver}; #[cfg(feature = "observers")] mod observers { - use alloc::{ - string::{String, ToString}, - vec::Vec, - }; + use alloc::{borrow::Cow, vec::Vec}; use core::{ fmt::Debug, - hash::{BuildHasher, Hasher}, + hash::{Hash, Hasher}, iter::Flatten, ptr::{addr_of, addr_of_mut}, slice::{from_raw_parts, Iter, IterMut}, @@ -78,7 +75,7 @@ mod observers { Error, }; use libafl_bolts::{ - ownedref::OwnedMutSlice, AsIter, AsIterMut, AsMutSlice, AsSlice, HasLen, Named, + ownedref::OwnedMutSlice, AsIter, AsIterMut, AsSlice, AsSliceMut, HasLen, Named, }; use meminterval::IntervalTree; use serde::{Deserialize, Serialize}; @@ -122,7 +119,7 @@ mod observers { intervals: IntervalTree, len: usize, initial: u8, - name: String, + name: Cow<'static, str>, iter_idx: usize, } @@ -147,8 +144,8 @@ mod observers { impl Named for CountersMultiMapObserver { #[inline] - fn name(&self) -> &str { - self.name.as_str() + fn name(&self) -> &Cow<'static, str> { + &self.name } } @@ -159,23 +156,48 @@ mod observers { } } + impl Hash for CountersMultiMapObserver { + fn hash(&self, hasher: &mut H) { + for map in unsafe { &*addr_of!(COUNTERS_MAPS) } { + let slice = map.as_slice(); + let ptr = slice.as_ptr(); + let map_size = slice.len() / core::mem::size_of::(); + unsafe { + hasher.write(from_raw_parts(ptr, map_size)); + } + } + } + } + + impl AsRef for CountersMultiMapObserver { + fn as_ref(&self) -> &Self { + self + } + } + + impl AsMut for CountersMultiMapObserver { + fn as_mut(&mut self) -> &mut Self { + self + } + } + impl MapObserver for CountersMultiMapObserver { type Entry = u8; #[inline] - fn get(&self, idx: usize) -> &u8 { + fn get(&self, idx: usize) -> u8 { let elem = self.intervals.query(idx..=idx).next().unwrap(); let i = elem.value; let j = idx - elem.interval.start; - unsafe { &(*addr_of!(COUNTERS_MAPS[*i])).as_slice()[j] } + unsafe { (*addr_of!(COUNTERS_MAPS[*i])).as_slice()[j] } } #[inline] - fn get_mut(&mut self, idx: usize) -> &mut u8 { + fn set(&mut self, idx: usize, val: u8) { let elem = self.intervals.query_mut(idx..=idx).next().unwrap(); let i = elem.value; let j = idx - elem.interval.start; - unsafe { &mut (*addr_of_mut!(COUNTERS_MAPS[*i])).as_mut_slice()[j] } + unsafe { (*addr_of_mut!(COUNTERS_MAPS[*i])).as_slice_mut()[j] = val }; } #[inline] @@ -196,23 +218,15 @@ mod observers { res } - fn hash(&self) -> u64 { - let mut hasher = RandomState::with_seeds(0, 0, 0, 0).build_hasher(); - for map in unsafe { &*addr_of!(COUNTERS_MAPS) } { - let slice = map.as_slice(); - let ptr = slice.as_ptr(); - let map_size = slice.len() / core::mem::size_of::(); - unsafe { - hasher.write(from_raw_parts(ptr, map_size)); - } - } - hasher.finish() + #[inline] + fn hash_simple(&self) -> u64 { + RandomState::with_seeds(0, 0, 0, 0).hash_one(self) } fn reset_map(&mut self) -> Result<(), Error> { let initial = self.initial(); for map in unsafe { &mut *addr_of_mut!(COUNTERS_MAPS) } { - for x in map.as_mut_slice() { + for x in map.as_slice_mut() { *x = initial; } } @@ -227,7 +241,7 @@ mod observers { let cnt = self.usable_count(); let mut res = Vec::with_capacity(cnt); for i in 0..cnt { - res.push(*self.get(i)); + res.push(self.get(i)); } res } @@ -238,7 +252,7 @@ mod observers { let cnt = self.usable_count(); let mut res = 0; for i in indexes { - if *i < cnt && *self.get(*i) != initial { + if *i < cnt && self.get(*i) != initial { res += 1; } } @@ -260,7 +274,7 @@ mod observers { Self { intervals, len: idx, - name: name.to_string(), + name: Cow::from(name), initial: u8::default(), iter_idx: 0, } @@ -291,7 +305,7 @@ mod observers { unsafe { &mut *addr_of_mut!(COUNTERS_MAPS) } .iter_mut() .for_each(|m| { - let l = m.as_mut_slice().len(); + let l = m.as_slice_mut().len(); intervals.insert(idx..(idx + l), v); idx += l; v += 1; @@ -299,7 +313,7 @@ mod observers { Self { intervals, len: idx, - name: name.to_string(), + name: Cow::from(name), initial: u8::default(), iter_idx: 0, } @@ -308,6 +322,7 @@ mod observers { impl<'it, const DIFFERENTIAL: bool> AsIter<'it> for CountersMultiMapObserver { type Item = u8; + type Ref = &'it Self::Item; type IntoIter = Flatten>>; fn as_iter(&'it self) -> Self::IntoIter { @@ -316,10 +331,10 @@ mod observers { } impl<'it, const DIFFERENTIAL: bool> AsIterMut<'it> for CountersMultiMapObserver { - type Item = u8; - type IntoIter = Flatten>>; + type RefMut = &'it mut Self::Item; + type IntoIterMut = Flatten>>; - fn as_iter_mut(&'it mut self) -> Self::IntoIter { + fn as_iter_mut(&'it mut self) -> Self::IntoIterMut { unsafe { COUNTERS_MAPS.iter_mut().flatten() } } } diff --git a/libafl_targets/src/sancov_cmp.c b/libafl_targets/src/sancov_cmp.c index b2e1b9cc97..26dfb10651 100644 --- a/libafl_targets/src/sancov_cmp.c +++ b/libafl_targets/src/sancov_cmp.c @@ -21,7 +21,7 @@ void __sanitizer_cov_trace_cmp1(uint8_t arg1, uint8_t arg2) { #endif #ifdef SANCOV_CMPLOG k &= CMPLOG_MAP_W - 1; - __libafl_targets_cmplog_instructions(k, 1, (uint64_t)arg1, (uint64_t)arg2); + cmplog_instructions_checked(k, 1, (uint64_t)arg1, (uint64_t)arg2); #endif } @@ -35,7 +35,7 @@ void __sanitizer_cov_trace_cmp2(uint16_t arg1, uint16_t arg2) { #endif #ifdef SANCOV_CMPLOG k &= CMPLOG_MAP_W - 1; - __libafl_targets_cmplog_instructions(k, 2, (uint64_t)arg1, (uint64_t)arg2); + cmplog_instructions_checked(k, 2, (uint64_t)arg1, (uint64_t)arg2); #endif } @@ -49,7 +49,7 @@ void __sanitizer_cov_trace_cmp4(uint32_t arg1, uint32_t arg2) { #endif #ifdef SANCOV_CMPLOG k &= CMPLOG_MAP_W - 1; - __libafl_targets_cmplog_instructions(k, 4, (uint64_t)arg1, (uint64_t)arg2); + cmplog_instructions_checked(k, 4, (uint64_t)arg1, (uint64_t)arg2); #endif } @@ -63,7 +63,7 @@ void __sanitizer_cov_trace_cmp8(uint64_t arg1, uint64_t arg2) { #endif #ifdef SANCOV_CMPLOG k &= CMPLOG_MAP_W - 1; - __libafl_targets_cmplog_instructions(k, 8, (uint64_t)arg1, (uint64_t)arg2); + cmplog_instructions_checked(k, 8, (uint64_t)arg1, (uint64_t)arg2); #endif } @@ -97,7 +97,7 @@ void __sanitizer_cov_trace_switch(uint64_t val, uint64_t *cases) { #endif #ifdef SANCOV_CMPLOG k &= CMPLOG_MAP_W - 1; - __libafl_targets_cmplog_instructions(k, cases[1] / 8, val, cases[i + 2]); + cmplog_instructions_checked(k, cases[1] / 8, val, cases[i + 2]); #endif } } @@ -118,69 +118,6 @@ void __sanitizer_cov_trace_const_cmp8(uint64_t arg1, uint64_t arg2) { __sanitizer_cov_trace_cmp8(arg1, arg2); } -#ifdef SANCOV_CMPLOG - -void __sanitizer_weak_hook_memcmp(void *called_pc, const void *s1, - const void *s2, size_t n, int result) { - if (result != 0) { - uintptr_t k = (uintptr_t)called_pc; - k = (k >> 4) ^ (k << 8); - k &= CMPLOG_MAP_W - 1; - - __libafl_targets_cmplog_routines_len(k, s1, s2, MIN(n, 32)); - } -} - -void __sanitizer_weak_hook_strncmp(void *called_pc, const char *s1, - const char *s2, size_t n, int result) { - if (result != 0) { - n = MIN(n, 32); - - uintptr_t k = (uintptr_t)called_pc; - k = (k >> 4) ^ (k << 8); - k &= CMPLOG_MAP_W - 1; - - size_t actual_len; - for (actual_len = 0; actual_len < n; actual_len++) { - if (s1[actual_len] == 0 || s2[actual_len] == 0) { break; } - } - - __libafl_targets_cmplog_routines_len(k, (const uint8_t *)s1, - (const uint8_t *)s2, actual_len); - } -} - -void __sanitizer_weak_hook_strncasecmp(void *called_pc, const char *s1, - const char *s2, size_t n, int result) { - __sanitizer_weak_hook_strncmp(called_pc, s1, s2, n, result); -} - -void __sanitizer_weak_hook_strcmp(void *called_pc, const char *s1, - const char *s2, int result) { - if (result != 0) { - uintptr_t k = (uintptr_t)called_pc; - k = (k >> 4) ^ (k << 8); - k &= CMPLOG_MAP_W - 1; - - size_t actual_len; - for (actual_len = 0; actual_len < 32; actual_len++) { - if (s1[actual_len] == 0 || s2[actual_len] == 0) { break; } - } - - __libafl_targets_cmplog_routines_len(k, (const uint8_t *)s1, - (const uint8_t *)s2, actual_len); - } -} - -void __sanitizer_weak_hook_strcasecmp(void *called_pc, const char *s1, - const char *s2, int result) { - __sanitizer_weak_hook_strcmp(called_pc, s1, s2, result); -} - -// strstr, strcasestr, memmem unhandled - -#endif - #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" diff --git a/libafl_targets/src/sancov_cmp.rs b/libafl_targets/src/sancov_cmp.rs index 0d71e17176..3cd796dd58 100644 --- a/libafl_targets/src/sancov_cmp.rs +++ b/libafl_targets/src/sancov_cmp.rs @@ -1,9 +1,12 @@ //! Sanitizer Coverage comparison functions -use core::{mem, ptr, slice}; +use core::{ + cmp, + ffi::{c_char, c_int, c_void}, + ptr, +}; -static mut PCS_BEG: *const usize = ptr::null(); -static mut PCS_END: *const usize = ptr::null(); +use crate::CMPLOG_MAP_W; extern "C" { @@ -28,73 +31,110 @@ extern "C" { /// Trace a switch statement pub fn __sanitizer_cov_trace_switch(val: u64, cases: *const u64); + /// cmplog internal api + pub fn __libafl_targets_cmplog_routines_len(k: usize, s1: *const u8, s2: *const u8, len: usize); } +/// overriding `__sanitizer_weak_hook_memcmp` +/// # Safety +/// this function has raw pointer access #[no_mangle] -unsafe extern "C" fn __sanitizer_cov_pcs_init(pcs_beg: *const usize, pcs_end: *const usize) { - // "The Unsafe Code Guidelines also notably defines that usize and isize are respectively compatible with uintptr_t and intptr_t defined in C." - assert!( - PCS_BEG.is_null(), - "__sanitizer_cov_pcs_init can be called only once." - ); - assert!( - PCS_END.is_null(), - "__sanitizer_cov_pcs_init can be called only once." - ); - - PCS_BEG = pcs_beg; - PCS_END = pcs_end; +pub unsafe extern "C" fn __sanitizer_weak_hook_memcmp( + called_pc: *const c_void, + s1: *const c_void, + s2: *const c_void, + n: usize, + result: c_int, +) { + if result != 0 { + let k: usize = called_pc as usize; + let k = (k >> 4) ^ (k << 8); + let k = k & (CMPLOG_MAP_W - 1); + __libafl_targets_cmplog_routines_len(k, s1 as *const u8, s2 as *const u8, cmp::min(n, 32)); + } } -/// An entry to the `sanitizer_cov` `pc_table` -#[repr(C, packed)] -#[derive(Debug, PartialEq, Eq)] -pub struct PcTableEntry { - addr: usize, - flags: usize, -} +#[no_mangle] +/// overriding `__sanitizer_weak_hook_strncmp` +/// # Safety +/// this function has raw pointer access +pub unsafe extern "C" fn __sanitizer_weak_hook_strncmp( + called_pc: *const c_void, + s1: *const c_char, + s2: *const c_char, + n: usize, + result: c_int, +) { + if result != 0 { + let n = cmp::min(n, 32); + let k: usize = called_pc as usize; + let k = (k >> 4) ^ (k << 8); + let k = k & (CMPLOG_MAP_W - 1); + let mut actual_len = 0; + while actual_len < n { + let c1 = ptr::read(s1.add(actual_len)); + let c2 = ptr::read(s2.add(actual_len)); -impl PcTableEntry { - /// Returns whether the PC corresponds to a function entry point. - #[must_use] - pub fn is_function_entry(&self) -> bool { - self.flags == 0x1 + if c1 == 0 || c2 == 0 { + break; + } + actual_len += 1; + } + __libafl_targets_cmplog_routines_len(k, s1 as *const u8, s2 as *const u8, actual_len); } +} - /// Returns the address associated with this PC. - #[must_use] - pub fn addr(&self) -> usize { - self.addr - } +#[no_mangle] +/// overriding `__sanitizer_weak_hook_strncasecmps` +/// # Safety +/// this function has raw pointer access +pub unsafe extern "C" fn __sanitizer_weak_hook_strncasecmp( + called_pc: *const c_void, + s1: *const c_char, + s2: *const c_char, + n: usize, + result: c_int, +) { + __sanitizer_weak_hook_strncmp(called_pc, s1, s2, n, result); } -/// Returns a slice containing the PC table. -#[must_use] -pub fn sanitizer_cov_pc_table() -> Option<&'static [PcTableEntry]> { - // SAFETY: Once PCS_BEG and PCS_END have been initialized, will not be written to again. So - // there's no TOCTOU issue. - unsafe { - if PCS_BEG.is_null() || PCS_END.is_null() { - return None; +#[no_mangle] +/// overriding `__sanitizer_weak_hook_strcmp` +/// # Safety +/// this function has raw pointer access +pub unsafe extern "C" fn __sanitizer_weak_hook_strcmp( + called_pc: *const c_void, + s1: *const c_char, + s2: *const c_char, + result: c_int, +) { + if result != 0 { + let k: usize = called_pc as usize; + let k = (k >> 4) ^ (k << 8); + let k = k & (CMPLOG_MAP_W - 1); + let mut actual_len = 0; + while actual_len < 32 { + let c1 = ptr::read(s1.add(actual_len)); + let c2 = ptr::read(s2.add(actual_len)); + + if c1 == 0 || c2 == 0 { + break; + } + actual_len += 1; } - let len = PCS_END.offset_from(PCS_BEG); - assert!( - len > 0, - "Invalid PC Table bounds - start: {PCS_BEG:x?} end: {PCS_END:x?}" - ); - assert_eq!( - len % 2, - 0, - "PC Table size is not evens - start: {PCS_BEG:x?} end: {PCS_END:x?}" - ); - assert_eq!( - (PCS_BEG as usize) % mem::align_of::(), - 0, - "Unaligned PC Table - start: {PCS_BEG:x?} end: {PCS_END:x?}" - ); - Some(slice::from_raw_parts( - PCS_BEG as *const PcTableEntry, - (len / 2).try_into().unwrap(), - )) + __libafl_targets_cmplog_routines_len(k, s1 as *const u8, s2 as *const u8, actual_len); } } + +#[no_mangle] +/// overriding `__sanitizer_weak_hook_strcmp` +/// # Safety +/// this function has raw pointer access +pub unsafe extern "C" fn __sanitizer_weak_hook_strcasecmp( + called_pc: *const c_void, + s1: *const c_char, + s2: *const c_char, + result: c_int, +) { + __sanitizer_weak_hook_strcmp(called_pc, s1, s2, result); +} diff --git a/libafl_targets/src/sancov_pcguard.rs b/libafl_targets/src/sancov_pcguard.rs index aa18211309..0d09dd084e 100644 --- a/libafl_targets/src/sancov_pcguard.rs +++ b/libafl_targets/src/sancov_pcguard.rs @@ -1,8 +1,26 @@ //! [`LLVM` `PcGuard`](https://clang.llvm.org/docs/SanitizerCoverage.html#tracing-pcs-with-guards) runtime for `LibAFL`. -use crate::coverage::{EDGES_MAP, MAX_EDGES_NUM}; +#[rustversion::nightly] +#[cfg(feature = "sancov_ngram4")] +use core::simd::num::SimdUint; +use core::{mem, ptr, slice}; + +#[cfg(any(feature = "sancov_ngram4", feature = "sancov_ctx"))] +use libafl::executors::{hooks::ExecutorHook, HasObservers}; + +#[cfg(any( + feature = "pointer_maps", + feature = "sancov_pcguard_edges", + feature = "sancov_pcguard_hitcounts", + feature = "sancov_ctx", + feature = "sancov_ngram4", +))] +use crate::coverage::EDGES_MAP; +use crate::coverage::MAX_EDGES_FOUND; +#[cfg(feature = "sancov_ngram4")] +use crate::EDGES_MAP_SIZE_IN_USE; #[cfg(feature = "pointer_maps")] -use crate::coverage::{EDGES_MAP_PTR, EDGES_MAP_PTR_NUM}; +use crate::{coverage::EDGES_MAP_PTR, EDGES_MAP_SIZE_MAX}; #[cfg(all(feature = "sancov_pcguard_edges", feature = "sancov_pcguard_hitcounts"))] #[cfg(not(any(doc, feature = "clippy")))] @@ -10,14 +28,205 @@ compile_error!( "the libafl_targets `sancov_pcguard_edges` and `sancov_pcguard_hitcounts` features are mutually exclusive." ); +#[cfg(any(feature = "sancov_ngram4", feature = "sancov_ngram8"))] +use core::ops::ShlAssign; + +#[cfg(feature = "sancov_ngram4")] +#[rustversion::nightly] +type Ngram4 = core::simd::u32x4; + +#[cfg(feature = "sancov_ngram8")] +#[rustversion::nightly] +type Ngram8 = core::simd::u32x8; + +/// The array holding the previous locs. This is required for NGRAM-4 instrumentation +#[cfg(feature = "sancov_ngram4")] +#[rustversion::nightly] +pub static mut PREV_ARRAY_4: Ngram4 = Ngram4::from_array([0, 0, 0, 0]); + +/// The array holding the previous locs. This is required for NGRAM-4 instrumentation +#[cfg(feature = "sancov_ngram8")] +#[rustversion::nightly] +pub static mut PREV_ARRAY_8: Ngram8 = Ngram8::from_array([0, 0, 0, 0, 0, 0, 0, 0]); + +/// We shift each of the values in ngram4 everytime we see new edges +#[cfg(feature = "sancov_ngram4")] +#[rustversion::nightly] +pub static SHR_4: Ngram4 = Ngram4::from_array([1, 1, 1, 1]); + +/// We shift each of the values in ngram8 everytime we see new edges +#[cfg(feature = "sancov_ngram8")] +#[rustversion::nightly] +pub static SHR_8: Ngram8 = Ngram8::from_array([1, 1, 1, 1, 1, 1, 1, 1]); + +#[cfg(any( + feature = "sancov_ngram4", + feature = "sancov_ngram8", + feature = "sancov_ctx" +))] +use core::marker::PhantomData; + +/// The hook to initialize ngram everytime we run the harness +#[cfg(any(feature = "sancov_ngram4", feature = "sancov_ngram8"))] +#[rustversion::nightly] +#[derive(Debug, Clone, Copy)] +pub struct NgramHook +where + S: libafl::inputs::UsesInput, +{ + phantom: PhantomData, +} + +/// The hook to initialize ctx everytime we run the harness +#[cfg(feature = "sancov_ctx")] +#[derive(Debug, Clone, Copy)] +pub struct CtxHook { + phantom: PhantomData, +} + +#[cfg(feature = "sancov_ctx")] +impl CtxHook +where + S: libafl::inputs::UsesInput, +{ + /// The constructor for this struct + #[must_use] + pub fn new() -> Self { + Self { + phantom: PhantomData, + } + } +} + +#[cfg(feature = "sancov_ctx")] +impl Default for CtxHook +where + S: libafl::inputs::UsesInput, +{ + fn default() -> Self { + Self::new() + } +} + +#[cfg(any(feature = "sancov_ngram4", feature = "sancov_ngram8"))] +#[rustversion::nightly] +impl ExecutorHook for NgramHook +where + S: libafl::inputs::UsesInput, +{ + fn init(&mut self, _state: &mut S) {} + fn pre_exec(&mut self, _state: &mut S, _input: &S::Input) { + #[cfg(feature = "sancov_ngram4")] + unsafe { + PREV_ARRAY_4 = Ngram4::from_array([0, 0, 0, 0]); + } + + #[cfg(feature = "sancov_ngram8")] + unsafe { + PREV_ARRAY_8 = Ngram8::from_array([0, 0, 0, 0, 0, 0, 0, 0]); + } + } + fn post_exec(&mut self, _state: &mut S, _input: &S::Input) {} +} + +#[cfg(any(feature = "sancov_ngram4", feature = "sancov_ngram8"))] +#[rustversion::nightly] +impl NgramHook +where + S: libafl::inputs::UsesInput, +{ + /// The constructor for this struct + #[must_use] + pub fn new() -> Self { + Self { + phantom: PhantomData, + } + } +} + +#[cfg(any(feature = "sancov_ngram4", feature = "sancov_ngram8"))] +#[rustversion::nightly] +impl Default for NgramHook +where + S: libafl::inputs::UsesInput, +{ + fn default() -> Self { + Self::new() + } +} + +#[cfg(feature = "sancov_ctx")] +impl ExecutorHook for CtxHook +where + S: libafl::inputs::UsesInput, +{ + fn init(&mut self, _state: &mut S) {} + fn pre_exec(&mut self, _state: &mut S, _input: &S::Input) { + unsafe { + __afl_prev_ctx = 0; + } + } + fn post_exec(&mut self, _state: &mut S, _input: &S::Input) {} +} + +#[rustversion::nightly] +#[allow(unused)] +#[inline] +#[cfg(any(feature = "sancov_ngram4", feature = "sancov_ngram8"))] +unsafe fn update_ngram(pos: usize) -> usize { + let mut reduced = pos; + #[cfg(feature = "sancov_ngram4")] + { + PREV_ARRAY_4 = PREV_ARRAY_4.rotate_elements_right::<1>(); + PREV_ARRAY_4.shl_assign(SHR_4); + PREV_ARRAY_4.as_mut_array()[0] = pos as u32; + reduced = PREV_ARRAY_4.reduce_xor() as usize; + } + #[cfg(feature = "sancov_ngram8")] + { + PREV_ARRAY_8 = PREV_ARRAY_8.rotate_elements_right::<1>(); + PREV_ARRAY_8.shl_assign(SHR_8); + PREV_ARRAY_8.as_mut_array()[0] = pos as u32; + reduced = PREV_ARRAY_8.reduce_xor() as usize; + } + reduced %= EDGES_MAP_SIZE_IN_USE; + reduced +} + +#[rustversion::not(nightly)] +#[cfg(any(feature = "sancov_ngram4", feature = "sancov_ngram8"))] +unsafe fn update_ngram(pos: usize) -> usize { + pos +} + +extern "C" { + /// The ctx variable + pub static mut __afl_prev_ctx: u32; +} + /// Callback for sancov `pc_guard` - usually called by `llvm` on each block or edge. /// /// # Safety /// Dereferences `guard`, reads the position from there, then dereferences the [`EDGES_MAP`] at that position. /// Should usually not be called directly. #[no_mangle] +#[allow(unused_assignments)] pub unsafe extern "C" fn __sanitizer_cov_trace_pc_guard(guard: *mut u32) { - let pos = *guard as usize; + #[allow(unused_mut)] + let mut pos = *guard as usize; + + #[cfg(any(feature = "sancov_ngram4", feature = "sancov_ngram8"))] + { + pos = update_ngram(pos); + // println!("Wrinting to {} {}", pos, EDGES_MAP_SIZE_IN_USE); + } + + #[cfg(feature = "sancov_ctx")] + { + pos ^= __afl_prev_ctx as usize; + // println!("Wrinting to {} {}", pos, EDGES_MAP_SIZE_IN_USE); + } + #[cfg(feature = "pointer_maps")] { #[cfg(feature = "sancov_pcguard_edges")] @@ -54,7 +263,6 @@ pub unsafe extern "C" fn __sanitizer_cov_trace_pc_guard_init(mut start: *mut u32 #[cfg(feature = "pointer_maps")] if EDGES_MAP_PTR.is_null() { EDGES_MAP_PTR = EDGES_MAP.as_mut_ptr(); - EDGES_MAP_PTR_NUM = EDGES_MAP.len(); } if start == stop || *start != 0 { @@ -62,17 +270,89 @@ pub unsafe extern "C" fn __sanitizer_cov_trace_pc_guard_init(mut start: *mut u32 } while start < stop { - *start = MAX_EDGES_NUM as u32; + *start = MAX_EDGES_FOUND as u32; start = start.offset(1); #[cfg(feature = "pointer_maps")] { - MAX_EDGES_NUM = MAX_EDGES_NUM.wrapping_add(1) % EDGES_MAP_PTR_NUM; + MAX_EDGES_FOUND = MAX_EDGES_FOUND.wrapping_add(1) % EDGES_MAP_SIZE_MAX; } #[cfg(not(feature = "pointer_maps"))] { - MAX_EDGES_NUM = MAX_EDGES_NUM.wrapping_add(1); - assert!((MAX_EDGES_NUM <= EDGES_MAP.len()), "The number of edges reported by SanitizerCoverage exceed the size of the edges map ({}). Use the LIBAFL_EDGES_MAP_SIZE env to increase it at compile time.", EDGES_MAP.len()); + MAX_EDGES_FOUND = MAX_EDGES_FOUND.wrapping_add(1); + assert!((MAX_EDGES_FOUND <= EDGES_MAP.len()), "The number of edges reported by SanitizerCoverage exceed the size of the edges map ({}). Use the LIBAFL_EDGES_MAP_SIZE_IN_USE env to increase it at compile time.", EDGES_MAP.len()); + } + } +} + +static mut PCS_BEG: *const usize = ptr::null(); +static mut PCS_END: *const usize = ptr::null(); + +#[no_mangle] +unsafe extern "C" fn __sanitizer_cov_pcs_init(pcs_beg: *const usize, pcs_end: *const usize) { + // "The Unsafe Code Guidelines also notably defines that usize and isize are respectively compatible with uintptr_t and intptr_t defined in C." + assert!( + pcs_beg == PCS_BEG || PCS_BEG.is_null(), + "__sanitizer_cov_pcs_init can be called only once." + ); + assert!( + pcs_end == PCS_END || PCS_END.is_null(), + "__sanitizer_cov_pcs_init can be called only once." + ); + + PCS_BEG = pcs_beg; + PCS_END = pcs_end; +} + +/// An entry to the `sanitizer_cov` `pc_table` +#[repr(C, packed)] +#[derive(Debug, PartialEq, Eq)] +pub struct PcTableEntry { + addr: usize, + flags: usize, +} + +impl PcTableEntry { + /// Returns whether the PC corresponds to a function entry point. + #[must_use] + pub fn is_function_entry(&self) -> bool { + self.flags == 0x1 + } + + /// Returns the address associated with this PC. + #[must_use] + pub fn addr(&self) -> usize { + self.addr + } +} + +/// Returns a slice containing the PC table. +#[must_use] +pub fn sanitizer_cov_pc_table() -> Option<&'static [PcTableEntry]> { + // SAFETY: Once PCS_BEG and PCS_END have been initialized, will not be written to again. So + // there's no TOCTOU issue. + unsafe { + if PCS_BEG.is_null() || PCS_END.is_null() { + return None; } + let len = PCS_END.offset_from(PCS_BEG); + assert!( + len > 0, + "Invalid PC Table bounds - start: {PCS_BEG:x?} end: {PCS_END:x?}" + ); + assert_eq!( + len % 2, + 0, + "PC Table size is not evens - start: {PCS_BEG:x?} end: {PCS_END:x?}" + ); + assert_eq!( + (PCS_BEG as usize) % mem::align_of::(), + 0, + "Unaligned PC Table - start: {PCS_BEG:x?} end: {PCS_END:x?}" + ); + Some(slice::from_raw_parts( + PCS_BEG as *const PcTableEntry, + (len / 2).try_into().unwrap(), + )) } } diff --git a/libafl_tinyinst/Cargo.toml b/libafl_tinyinst/Cargo.toml index 7697fda9bf..d8060a5826 100644 --- a/libafl_tinyinst/Cargo.toml +++ b/libafl_tinyinst/Cargo.toml @@ -12,11 +12,11 @@ description = "TinyInst backend for libafl" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -libafl = { path = "../libafl", version = "0.11.2", features = [ +libafl = { path = "../libafl", version = "0.12.0", features = [ "std", "libafl_derive", ] } -libafl_bolts = { path = "../libafl_bolts", version = "0.11.2", features = [ +libafl_bolts = { path = "../libafl_bolts", version = "0.12.0", features = [ "std", "libafl_derive", ] } diff --git a/libafl_tinyinst/src/executor.rs b/libafl_tinyinst/src/executor.rs index d99930bdc9..d2a63c3715 100644 --- a/libafl_tinyinst/src/executor.rs +++ b/libafl_tinyinst/src/executor.rs @@ -11,7 +11,8 @@ use libafl::{ use libafl_bolts::{ fs::{InputFile, INPUTFILE_STD}, shmem::{ShMem, ShMemProvider, StdShMemProvider}, - AsMutSlice, AsSlice, + tuples::RefIndexable, + AsSlice, AsSliceMut, }; use tinyinst::tinyinst::{litecov::RunResult, TinyInst}; @@ -65,9 +66,9 @@ where let size = target_bytes.as_slice().len(); let size_in_bytes = size.to_ne_bytes(); // The first four bytes tells the size of the shmem. - shmem.as_mut_slice()[..SHMEM_FUZZ_HDR_SIZE] + shmem.as_slice_mut()[..SHMEM_FUZZ_HDR_SIZE] .copy_from_slice(&size_in_bytes[..SHMEM_FUZZ_HDR_SIZE]); - shmem.as_mut_slice()[SHMEM_FUZZ_HDR_SIZE..(SHMEM_FUZZ_HDR_SIZE + size)] + shmem.as_slice_mut()[SHMEM_FUZZ_HDR_SIZE..(SHMEM_FUZZ_HDR_SIZE + size)] .copy_from_slice(target_bytes.as_slice()); } None => { @@ -240,7 +241,7 @@ where // shmem.write_to_env("__TINY_SHM_FUZZ_ID")?; let size_in_bytes = (MAX_FILE + SHMEM_FUZZ_HDR_SIZE).to_ne_bytes(); - shmem.as_mut_slice()[..4].clone_from_slice(&size_in_bytes[..4]); + shmem.as_slice_mut()[..4].clone_from_slice(&size_in_bytes[..4]); (Some(shmem), Some(shmem_id)) } @@ -300,12 +301,12 @@ where SP: ShMemProvider, OT: ObserversTuple, { - fn observers(&self) -> &OT { - &self.observers + fn observers(&self) -> RefIndexable<&Self::Observers, Self::Observers> { + RefIndexable::from(&self.observers) } - fn observers_mut(&mut self) -> &mut OT { - &mut self.observers + fn observers_mut(&mut self) -> RefIndexable<&mut Self::Observers, Self::Observers> { + RefIndexable::from(&mut self.observers) } } impl<'a, S, SP, OT> UsesState for TinyInstExecutor<'a, S, SP, OT> diff --git a/libafl_tinyinst/src/lib.rs b/libafl_tinyinst/src/lib.rs index e68c4607d0..6827b8112b 100644 --- a/libafl_tinyinst/src/lib.rs +++ b/libafl_tinyinst/src/lib.rs @@ -32,14 +32,12 @@ The tinyinst module for `LibAFL`. ))] #![cfg_attr(test, deny( missing_debug_implementations, - missing_docs, //trivial_casts, trivial_numeric_casts, unused_extern_crates, unused_import_braces, unused_qualifications, unused_must_use, - missing_docs, //unused_results ))] #![cfg_attr( diff --git a/scripts/autofix.sh b/scripts/autofix.sh index 705b210fff..b253c90d2d 100755 --- a/scripts/autofix.sh +++ b/scripts/autofix.sh @@ -11,13 +11,13 @@ fi echo echo "[+] Fixing build" -cargo +nightly fix --release --workspace --all-features +cargo +nightly fix --release --workspace --all-features --allow-dirty --allow-staged echo "[+] Done fixing build" echo echo 'Fixing clippy (might need a "git commit" and a rerun, if "cargo fix" changed the source)' -RUST_BACKTRACE=full cargo +nightly clippy --fix --release --all --all-features --tests --examples --benches -- -Z macro-backtrace \ +RUST_BACKTRACE=full cargo +nightly clippy --fix --release --all --all-features --tests --examples --benches --allow-dirty --allow-staged -- -Z macro-backtrace \ -D clippy::all \ -D clippy::pedantic \ -W clippy::similar_names \ diff --git a/scripts/build_all_fuzzers.sh b/scripts/build_all_fuzzers.sh new file mode 100755 index 0000000000..08c59bfa10 --- /dev/null +++ b/scripts/build_all_fuzzers.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +cd "$SCRIPT_DIR/.." || exit 1 +# TODO: This should be rewritten in rust, a Makefile, or some platform-independent language + +if [[ -z "${RUN_ON_CI}" ]]; then + fuzzers=$(find ./fuzzers -mindepth 1 -maxdepth 1 -type d) + backtrace_fuzzers=$(find ./fuzzers/backtrace_baby_fuzzers -mindepth 1 -maxdepth 1 -type d) +else + cargo build -p build_and_test_fuzzers + fuzzers=$(cargo run -p build_and_test_fuzzers -- "remotes/origin/main" "HEAD^") + backtrace_fuzzers="" + export PROFILE=dev + export PROFILE_DIR=debug +fi + +fuzzers=$(echo "$fuzzers" | tr ' ' '\n') +backtrace_fuzzers=$(echo "$backtrace_fuzzers" | tr ' ' '\n') + +libafl=$(pwd) + +# build with a shared target dir for all fuzzers. this should speed up +# compilation a bit, and allows for easier artifact management (caching and +# cargo clean). +export CARGO_TARGET_DIR="$libafl/target" +mkdir -p "$CARGO_TARGET_DIR" + +git submodule init && git submodule update + +# override default profile settings for speed +# export RUSTFLAGS="-C prefer-dynamic" +for profile in DEV RELEASE; # loop for all profiles +do + export CARGO_PROFILE_"$profile"_OPT_LEVEL=z # optimize for size + # runs into shared target dir bug: + # [pid 351769] openat(AT_FDCWD, "LibAFL/target/release/deps/libc-dbff77a14da5d893.libc.5deb7d4a-cgu.0.rcgu.dwo", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + # error: failed to build archive: No such file or directory + # export CARGO_PROFILE_"$profile"_SPLIT_DEBUGINFO=unpacked # minimize debug info + # export CARGO_PROFILE_"$profile"_PANIC=abort + # export CARGO_PROFILE_"$profile"_INCREMENTAL=true +done + +# shellcheck disable=SC2116 +for fuzzer in $(echo "$fuzzers" "$backtrace_fuzzers"); +do + # skip nyx test on non-linux platforms + if [[ $fuzzer == *"nyx_"* ]]; then + continue + fi + + cd "$fuzzer" || exit 1 + # Clippy checks + echo "[*] Checking fmt for $fuzzer" + cargo +nightly fmt --all || exit 1 + + if [ -e ./Makefile.toml ]; then + echo "[*] Building $fuzzer" + cargo make build || exit 1 + echo "[+] Done building $fuzzer" + else + echo "[*] Building $fuzzer" + cargo build || exit 1 + echo "[+] Done building $fuzzer" + fi + + # no cleaning -- this is a local test, we want to cache here + cd "$libafl" || exit 1 + echo "" +done diff --git a/scripts/check_tested_fuzzers.sh b/scripts/check_tested_fuzzers.sh new file mode 100755 index 0000000000..8b33c42931 --- /dev/null +++ b/scripts/check_tested_fuzzers.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +ret=0 + +while read -r fuzzdir; do + if ! grep -qa "$fuzzdir" .github/workflows/build_and_test.yml; then + ret=1 + echo "Missing fuzzer ${fuzzdir} in CI tests!" + fi + if grep -qa "# - $fuzzdir" .github/workflows/build_and_test.yml; then + echo "Fuzzer ${fuzzdir} is explicitly ignored" + fi +done < <( + find ./fuzzers -mindepth 1 -maxdepth 1 -type d + find ./fuzzers/backtrace_baby_fuzzers -mindepth 1 -maxdepth 1 -type d + ) + +exit $ret \ No newline at end of file diff --git a/scripts/clippy.sh b/scripts/clippy.sh index 6f7395ef57..e37554a8f5 100755 --- a/scripts/clippy.sh +++ b/scripts/clippy.sh @@ -4,7 +4,7 @@ cd "$SCRIPT_DIR/.." || exit 1 set -e -RUST_BACKTRACE=full cargo +nightly clippy --all --all-features --tests --examples --benches -- -Z macro-backtrace \ +RUST_BACKTRACE=full cargo +nightly clippy --all --all-features --exclude libafl_nyx --exclude symcc_runtime --exclude runtime_test --no-deps --tests --examples --benches -- -Z macro-backtrace \ -D clippy::all \ -D clippy::pedantic \ -W clippy::similar_names \ @@ -21,7 +21,7 @@ RUST_BACKTRACE=full cargo +nightly clippy --all --all-features --tests --example if [[ "$OSTYPE" == "linux-gnu"* ]]; then cd libafl_libfuzzer/libafl_libfuzzer_runtime - RUST_BACKTRACE=full cargo +nightly clippy --all --all-features --tests --examples --benches -- -Z macro-backtrace \ + RUST_BACKTRACE=full cargo +nightly clippy --all --all-features --exclude libafl_nyx --exclude symcc_runtime --exclude runtime_test --no-deps --tests --examples --benches -- -Z macro-backtrace \ -D clippy::all \ -D clippy::pedantic \ -W clippy::similar_names \ diff --git a/scripts/fmt_all.sh b/scripts/fmt_all.sh index e73018c042..545a65dc44 100755 --- a/scripts/fmt_all.sh +++ b/scripts/fmt_all.sh @@ -11,9 +11,7 @@ cargo +nightly fmt echo "[*] Formatting C(pp) files" # shellcheck disable=SC2046 -clang-format-17 -i --style=file $(find . -type f \( -name '*.cpp' -o -iname '*.hpp' -o -name '*.cc' -o -name '*.cxx' -o -name '*.cc' -o -name '*.c' -o -name '*.h' \) | grep -v '/target/' | grep -v 'libpng-1\.6\.37' | grep -v 'stb_image\.h' | grep -v 'dlmalloc\.c') - - +clang-format-18 -i --style=file $(find . -type f \( -name '*.cpp' -o -iname '*.hpp' -o -name '*.cc' -o -name '*.cxx' -o -name '*.cc' -o -name '*.c' -o -name '*.h' \) | grep -v '/target/' | grep -v 'libpng-1\.6\.37' | grep -v 'stb_image\.h' | grep -v 'dlmalloc\.c') fuzzers=$(find ./fuzzers -maxdepth 1 -type d) backtrace_fuzzers=$(find ./fuzzers/backtrace_baby_fuzzers -maxdepth 1 -type d) @@ -26,3 +24,10 @@ do cargo +nightly fmt --all popd || exit 1 done + +echo "[*] Formatting libafl_libfuzzer_runtime" +pushd "libafl_libfuzzer/libafl_libfuzzer_runtime" || exit 1 +cargo +nightly fmt --all +popd || exit 1 + +echo "[*] Done :)" diff --git a/scripts/parallellize_cargo_check.py b/scripts/parallellize_cargo_check.py new file mode 100755 index 0000000000..92901b33a7 --- /dev/null +++ b/scripts/parallellize_cargo_check.py @@ -0,0 +1,26 @@ +#!/usr/bin/python3 +import subprocess +import os +import sys +import math +# Current CI Runner +ci_instances = 18 + +if len(sys.argv) != 2: + exit(1) + +instance_idx = int(sys.argv[1]) +# set llvm config +os.environ["LLVM_CONFIG"] = "llvm-config" +command = "cargo hack check --workspace --each-feature --clean-per-run --exclude-features=prelude,agpl,nautilus,python,sancov_pcguard_edges,arm,aarch64,i386,be,systemmode,whole_archive --no-dev-deps --exclude libafl_libfuzzer --print-command-list" + +# Run the command and capture the output +output = subprocess.check_output(command, shell=True, text=True) +output = output.strip().split('\n')[0:] +all_task_cnt = len(output) // 2 # by 2 cuz one task has two lines +task_per_core = math.ceil(all_task_cnt // ci_instances) +print(task_per_core, "tasks assigned to this instance") + +for task in output[instance_idx * 2 * task_per_core: (instance_idx + 1) * 2 * task_per_core]: + print("Running ", task) + cargo_check = subprocess.check_output(task, shell=True, text=True) \ No newline at end of file diff --git a/scripts/test_all_fuzzers.sh b/scripts/test_fuzzer.sh similarity index 75% rename from scripts/test_all_fuzzers.sh rename to scripts/test_fuzzer.sh index 3b2668b131..e293948b79 100755 --- a/scripts/test_all_fuzzers.sh +++ b/scripts/test_fuzzer.sh @@ -4,33 +4,20 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" cd "$SCRIPT_DIR/.." || exit 1 # TODO: This should be rewritten in rust, a Makefile, or some platform-independent language + if [[ -z "${RUN_ON_CI}" ]]; then fuzzers=$(find ./fuzzers -mindepth 1 -maxdepth 1 -type d) backtrace_fuzzers=$(find ./fuzzers/backtrace_baby_fuzzers -mindepth 1 -maxdepth 1 -type d) + fuzzer_to_test="$fuzzers $backtrace_fuzzers" else - cargo build -p build_and_test_fuzzers - fuzzers=$(cargo run -p build_and_test_fuzzers -- "remotes/origin/main" "HEAD^") - backtrace_fuzzers="" + fuzzer_to_test="$1" export PROFILE=dev export PROFILE_DIR=debug fi -if [[ -n "${RUN_QEMU_FUZZER}" ]]; then - fuzzers=$(echo "$fuzzers" | tr ' ' '\n' | grep "qemu") - backtrace_fuzzers=$(echo "$backtrace_fuzzers" | tr ' ' '\n' | grep "qemu") -elif [[ -n "${RUN_BABY_FUZZER}" ]]; then - fuzzers=$(echo "$fuzzers" | tr ' ' '\n' | grep "baby") - backtrace_fuzzers=$(echo "$backtrace_fuzzers" | tr ' ' '\n' | grep "baby") -elif [[ -n "${RUN_LIBPNG_FUZZER}" ]]; then - fuzzers=$(echo "$fuzzers" | tr ' ' '\n' | grep "libpng") - backtrace_fuzzers=$(echo "$backtrace_fuzzers" | tr ' ' '\n' | grep "libpng") -else - fuzzers=$(echo "$fuzzers" | tr ' ' '\n' | grep -v "qemu" | grep -v "baby" | grep -v "libpng") - backtrace_fuzzers=$(echo "$backtrace_fuzzers" | tr ' ' '\n' | grep -v "qemu" | grep -v "baby" | grep -v "libpng") -fi - libafl=$(pwd) +echo "Testing" "$fuzzer_to_test" # build with a shared target dir for all fuzzers. this should speed up # compilation a bit, and allows for easier artifact management (caching and # cargo clean). @@ -53,7 +40,7 @@ do done # shellcheck disable=SC2116 -for fuzzer in $(echo "$fuzzers" "$backtrace_fuzzers"); +for fuzzer in $(echo "$fuzzer_to_test"); do # skip nyx test on non-linux platforms if [[ $fuzzer == *"nyx_"* ]] && [[ $(uname -s) != "Linux" ]]; then diff --git a/utils/gramatron/construct_automata/src/main.rs b/utils/gramatron/construct_automata/src/main.rs index dd3dd403e0..50d5fabda3 100644 --- a/utils/gramatron/construct_automata/src/main.rs +++ b/utils/gramatron/construct_automata/src/main.rs @@ -7,7 +7,7 @@ use std::{ sync::OnceLock, }; -use clap::{self, Parser}; +use clap::Parser; use libafl::generators::gramatron::{Automaton, Trigger}; use regex::Regex; use serde_json::Value; diff --git a/utils/libafl_benches/benches/rand_speeds.rs b/utils/libafl_benches/benches/rand_speeds.rs index 53f30b855e..378fb09ff8 100644 --- a/utils/libafl_benches/benches/rand_speeds.rs +++ b/utils/libafl_benches/benches/rand_speeds.rs @@ -2,16 +2,19 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion}; use libafl_bolts::rands::{ - Lehmer64Rand, Rand, RomuDuoJrRand, RomuTrioRand, XorShift64Rand, Xoshiro256StarRand, + Lehmer64Rand, Rand, RomuDuoJrRand, RomuTrioRand, Sfc64Rand, XorShift64Rand, + Xoshiro256PlusPlusRand, }; fn criterion_benchmark(c: &mut Criterion) { + let mut sfc64 = Sfc64Rand::with_seed(1); let mut xorshift = XorShift64Rand::with_seed(1); - let mut xoshiro = Xoshiro256StarRand::with_seed(1); + let mut xoshiro = Xoshiro256PlusPlusRand::with_seed(1); let mut romu = RomuDuoJrRand::with_seed(1); let mut lehmer = Lehmer64Rand::with_seed(1); let mut romu_trio = RomuTrioRand::with_seed(1); + c.bench_function("sfc64", |b| b.iter(|| black_box(sfc64.next()))); c.bench_function("xorshift", |b| b.iter(|| black_box(xorshift.next()))); c.bench_function("xoshiro", |b| b.iter(|| black_box(xoshiro.next()))); c.bench_function("romu", |b| b.iter(|| black_box(romu.next()))); diff --git a/utils/noaslr/libnoaslr/src/lib.rs b/utils/noaslr/libnoaslr/src/lib.rs index 2517166b4c..777396628d 100644 --- a/utils/noaslr/libnoaslr/src/lib.rs +++ b/utils/noaslr/libnoaslr/src/lib.rs @@ -65,7 +65,7 @@ fn libnoaslr() -> Result<()> { libc::P_PID, 0, libc::PROC_ASLR_CTL, - &mut status as *mut i32 as *mut libc::c_void, + &mut core::ptr::from_mut(status) as *mut libc::c_void, ) < 0 { return Err(anyhow!("Failed to set aslr control")); diff --git a/utils/noaslr/noaslr/src/main.rs b/utils/noaslr/noaslr/src/main.rs index a00be2982f..1b9f146b92 100644 --- a/utils/noaslr/noaslr/src/main.rs +++ b/utils/noaslr/noaslr/src/main.rs @@ -36,7 +36,7 @@ fn disable_aslr() -> Result<()> { libc::P_PID, 0, libc::PROC_ASLR_CTL, - &mut status as *mut i32 as *mut libc::c_void, + &mut core::ptr::from_mut(status) as *mut libc::c_void, ) }; if r < 0 {