Fix CI after Rust 1.80, update to Ubuntu 24.04 (AFLplusplus#2449)

* 0.13.2 (for real this time) * More 0.13.2 * Make sure this never bites us * Update ahash * Fix CI after rust 1.80 * Does 18 work? * Trying to fix this? * lol regex * noble * works? * fix llvm-config * fmt * Windows? * more fix? * more fix * remove pip * Add find_python3_version support to libafl_cc * llvm-config * more curl * windows has no apt lol * more
cube0x8 · Jul 29, 2024 · 75af67f · 75af67f
1 parent bab50f2
commit 75af67f
Show file tree

Hide file tree

Showing 15 changed files with 165 additions and 137 deletions.
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
@@ -10,7 +10,7 @@ on:
 env:
   CARGO_TERM_COLOR: always
   CARGO_NET_GIT_FETCH_WITH_CLI: true
-  MAIN_LLVM_VERSION: 19
+  MAIN_LLVM_VERSION: 18
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
@@ -20,12 +20,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ ubuntu-latest, windows-latest, macOS-latest ]
+        os: [ ubuntu-24.04, windows-latest, macOS-latest ]
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v4
-      - if: runner.os == 'Linux'
-        uses: ./.github/workflows/ubuntu-prepare
       - name: Install mimetype
         if: runner.os == 'Linux'
         run: sudo apt-get install -y libfile-mimeinfo-perl
@@ -41,6 +38,9 @@ jobs:
           #       => https://github.com/Michael-F-Bryan/mdbook-linkcheck/pull/82#issuecomment-2241058491
           git: https://github.com/Michael-F-Bryan/mdbook-linkcheck.git
           rev: 8c783c5d754d83bcd50c28fb4174854b04ece990
+      - uses: actions/checkout@v4
+      - if: runner.os == 'Linux'
+        uses: ./.github/workflows/ubuntu-prepare
       - uses: Swatinem/rust-cache@v2
         with: { shared-key: "ubuntu" }
         if: runner.os == 'Linux'
@@ -67,27 +67,29 @@ jobs:
         run: cd libafl_targets && cargo test --no-default-features
 
   llvm-tester:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     continue-on-error: true
     strategy:
       matrix:
         llvm-version: [ "16", "17" ] # Add 18 when KyleMayes/install-llvm-action enables it
     steps:
       - name: Remove Dotnet & Haskell
         run: rm -rf /usr/share/dotnet && rm -rf /opt/ghc
+      - name: Install curl
+        run: sudo apt-get install clang
       - uses: dtolnay/rust-toolchain@stable
-      - uses: actions/checkout@v4
-      - uses: Swatinem/rust-cache@v2
-        with: { shared-key: "llvm-tester" }
       - name: Install LLVM and Clang
         uses: KyleMayes/install-llvm-action@v2
         with:
           version: "${{matrix.llvm-version}}"
+      - uses: actions/checkout@v4
+      - uses: Swatinem/rust-cache@v2
+        with: { shared-key: "llvm-tester" }
       - name: Build and test with llvm-${{ matrix.llvm-version }}
         run: pwd && ls & cd libafl_cc && cargo build --release
 
   ubuntu-doc-build:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/workflows/ubuntu-prepare
@@ -97,7 +99,7 @@ jobs:
         run: RUSTFLAGS="--cfg docsrs" cargo +nightly doc --all-features --no-deps
 
   ubuntu-doc-test:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/workflows/ubuntu-prepare
@@ -107,7 +109,7 @@ jobs:
         run: RUSTFLAGS="--cfg docsrs" cargo +nightly test --doc --all-features
 
   ubuntu-miri:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     needs: ubuntu
     steps:
       - uses: actions/checkout@v4
@@ -120,14 +122,14 @@ jobs:
         run: RUST_BACKTRACE=1 MIRIFLAGS="-Zmiri-disable-isolation" cargo +nightly miri test
 
   ubuntu:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
+      - name: Remove Dotnet & Haskell
+        run: rm -rf /usr/share/dotnet && rm -rf /opt/ghc
       - uses: actions/checkout@v4
       - uses: ./.github/workflows/ubuntu-prepare
       - uses: Swatinem/rust-cache@v2
         with: { shared-key: "ubuntu" }
-      - name: Remove Dotnet & Haskell
-        run: rm -rf /usr/share/dotnet && rm -rf /opt/ghc
       # pcguard edges and pcguard hitcounts are not compatible and we need to build them seperately
       - name: Check pcguard edges
         run: cargo check --features=sancov_pcguard_edges
@@ -140,16 +142,16 @@ jobs:
         run: cargo build --examples --verbose
 
   ubuntu-clippy:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/workflows/ubuntu-prepare
-      - uses: Swatinem/rust-cache@v2
-        with: { shared-key: "ubuntu" }
       - name: Remove Dotnet & Haskell
         run: rm -rf /usr/share/dotnet && rm -rf /opt/ghc
       - name: Add nightly clippy
         run: rustup toolchain install nightly --component clippy --allow-downgrade && rustup default nightly
+      - uses: actions/checkout@v4
+      - uses: ./.github/workflows/ubuntu-prepare
+      - uses: Swatinem/rust-cache@v2
+        with: { shared-key: "ubuntu" }
       - name: Run clippy
         run: ./scripts/clippy.sh
     # --- test embedding the libafl_libfuzzer_runtime library
@@ -158,7 +160,7 @@ jobs:
     #  run: cargo +nightly test --features=embed-runtime --manifest-path libafl_libfuzzer/Cargo.toml
 
   ubuntu-check:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     needs: ubuntu
     strategy:
       matrix:
@@ -176,9 +178,11 @@ jobs:
         run: python3 ./scripts/parallellize_cargo_check.py ${{ matrix.instance_idx }}
 
   ubuntu-concolic:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     needs: ubuntu
     steps:
+      - name: Install curl
+        run: sudo apt-get install clang
       - uses: dtolnay/rust-toolchain@stable
       - uses: actions/checkout@v4
       - uses: Swatinem/rust-cache@v2
@@ -189,23 +193,23 @@ jobs:
         run: ./libafl_concolic/test/smoke_test.sh
 
   python-bindings:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
+      - name: Install deps
+        run: sudo apt-get install -y lsb-release wget software-properties-common gnupg ninja-build python3-dev python3-pip python3-venv libz3-dev
+      - name: Install maturin
+        run: cargo install --locked maturin
       - uses: actions/checkout@v4
       - uses: ./.github/workflows/ubuntu-prepare
       - uses: Swatinem/rust-cache@v2
         with: { shared-key: "ubuntu" }
-      - name: Install deps
-        run: sudo apt-get install -y lsb-release wget software-properties-common gnupg ninja-build python3-dev python3-pip python3-venv libz3-dev
-      - name: Install maturin
-        run: python3 -m pip install maturin
       - name: Run a maturin build
-        run: export LLVM_CONFIG=llvm-config-16 && cd ./bindings/pylibafl && python3 -m venv .env && . .env/bin/activate && pip install --upgrade --force-reinstall . && ./test.sh
+        run: export LLVM_CONFIG=llvm-config-${{env.MAIN_LLVM_VERSION}} && cd ./bindings/pylibafl && python3 -m venv .env && . .env/bin/activate && pip install --upgrade --force-reinstall . && ./test.sh
       - name: Run python test
         run: . ./bindings/pylibafl/.env/bin/activate # && cd ./fuzzers/qemu/python_qemu/ && python3 fuzzer.py 2>&1 | grep "Bye"
 
   cargo-fmt:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/workflows/ubuntu-prepare
@@ -218,7 +222,7 @@ jobs:
         run: ./scripts/fmt_all.sh check
 
   fuzzers-preflight:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v4
       - name: Fuzzer in CI Check
@@ -232,7 +236,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        os: [ ubuntu-latest ]
+        os: [ ubuntu-24.04 ]
         fuzzer:
           # Baby
           - ./fuzzers/baby/baby_fuzzer_with_forkexecutor
@@ -311,10 +315,10 @@ jobs:
       - name: Build and run example fuzzers (Linux)
         if: runner.os == 'Linux'
         shell: bash
-        run: RUN_ON_CI=1 LLVM_CONFIG=llvm-config ./scripts/test_fuzzer.sh ${{ matrix.fuzzer }}
+        run: RUN_ON_CI=1 LLVM_CONFIG=llvm-config-${{env.MAIN_LLVM_VERSION}} ./scripts/test_fuzzer.sh ${{ matrix.fuzzer }}
 
   changes:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     permissions:
       pull-requests: read
     outputs:
@@ -340,7 +344,7 @@ jobs:
     if: ${{ needs.changes.outputs.qemu == 'true' }}
     strategy:
       matrix:
-        os: [ubuntu-latest]
+        os: [ubuntu-24.04]
         fuzzer:
           - ./fuzzers/qemu/qemu_cmin
           - ./fuzzers/qemu/qemu_systemmode
@@ -355,18 +359,18 @@ jobs:
       - name: Build and run example QEMU fuzzers (Linux)
         if: runner.os == 'Linux'
         shell: bash
-        run: RUN_ON_CI=1 LLVM_CONFIG=llvm-config ./scripts/test_fuzzer.sh ${{ matrix.fuzzer }}
+        run: RUN_ON_CI=1 LLVM_CONFIG=llvm-config-${{env.MAIN_LLVM_VERSION}} ./scripts/test_fuzzer.sh ${{ matrix.fuzzer }}
 
   nostd-build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - uses: dtolnay/rust-toolchain@nightly
         with:
           components: rust-src
-      - uses: actions/checkout@v4
-      - uses: Swatinem/rust-cache@v2
       - name: Add targets
         run: rustup target add arm-linux-androideabi && rustup target add thumbv6m-none-eabi
+      - uses: actions/checkout@v4
+      - uses: Swatinem/rust-cache@v2
       - name: Build aarch64-unknown-none
         run: cd ./fuzzers/baby/baby_no_std && cargo +nightly build -Zbuild-std=core,alloc --target aarch64-unknown-none -v --release && cd ../..
       - name: run x86_64 until panic!
@@ -375,15 +379,15 @@ jobs:
         run: cd ./libafl && cargo test --no-default-features
 
   nostd-clippy:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - uses: dtolnay/rust-toolchain@nightly
         with:
           components: clippy, rust-src
-      - uses: actions/checkout@v4
-      - uses: Swatinem/rust-cache@v2
       - name: Add targets
         run: rustup target add arm-linux-androideabi && rustup target add thumbv6m-none-eabi
+      - uses: actions/checkout@v4
+      - uses: Swatinem/rust-cache@v2
       - name: libafl armv6m-none-eabi (32 bit no_std) clippy
         run: cd ./libafl && cargo clippy --target thumbv6m-none-eabi --no-default-features
       - name: Build no_std no_alloc bolts
@@ -399,7 +403,7 @@ jobs:
         run: taplo format --check
 
   build-docker:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v4
       - name: Build docker
@@ -493,8 +497,10 @@ jobs:
         run: cargo build --target aarch64-apple-ios && cd libafl_frida && cargo build --target aarch64-apple-ios && cd ..
 
   android:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
+      - name: Install curl
+        run: sudo apt-get install clang
       - uses: dtolnay/rust-toolchain@stable
       - uses: nttld/setup-ndk@v1
         with:

diff --git a/.github/workflows/fuzzer-tester-prepare/action.yml b/.github/workflows/fuzzer-tester-prepare/action.yml
@@ -12,10 +12,7 @@ runs:
       with: { shared-key: "${{ runner.os }}-shared-fuzzer-cache" }
     - name: Install fuzzers deps
       shell: bash
-      run: sudo apt-get update && sudo apt-get install -y nasm nlohmann-json3-dev gcc-aarch64-linux-gnu g++-aarch64-linux-gnu gcc-mipsel-linux-gnu g++-mipsel-linux-gnu gcc-powerpc-linux-gnu g++-powerpc-linux-gnu libc6-dev-i386-cross libc6-dev libc6-dev-i386 lib32gcc-11-dev lib32stdc++-11-dev libgtk-3-dev pax-utils
-    - name: pip install
-      shell: bash
-      run: python3 -m pip install msgpack jinja2 find_libpython
+      run: sudo apt-get update && sudo apt-get install -y nasm nlohmann-json3-dev gcc-aarch64-linux-gnu g++-aarch64-linux-gnu gcc-mipsel-linux-gnu g++-mipsel-linux-gnu gcc-powerpc-linux-gnu g++-powerpc-linux-gnu libc6-dev-i386-cross libc6-dev libc6-dev-i386 lib32gcc-11-dev lib32stdc++-11-dev libgtk-3-dev pax-utils python3-msgpack python3-jinja2 
     - name: enable mult-thread for `make`
       shell: bash
       run: export MAKEFLAGS="-j$(expr $(nproc) \+ 1)"

diff --git a/.github/workflows/qemu-fuzzer-tester-prepare/action.yml b/.github/workflows/qemu-fuzzer-tester-prepare/action.yml
@@ -3,24 +3,21 @@ description: Sets up the QEMU fuzzers environment
 runs:
   using: composite
   steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-        fetch-depth: 0
     - name: Install QEMU deps
       shell: bash
-      run: apt-get update && apt-get install -y qemu-utils sudo
-    - uses: ./.github/workflows/ubuntu-prepare
-    - uses: Swatinem/rust-cache@v2
-      with: { shared-key: "${{ runner.os }}-shared-fuzzer-cache" }
+      run: apt-get update && apt-get install -y qemu-utils sudo python3-msgpack python3-jinja2 curl
     - uses: dtolnay/rust-toolchain@stable
-    - name: pip install
-      shell: bash
-      run: python3 -m pip install msgpack jinja2 find_libpython
     - name: enable mult-thread for `make`
       shell: bash
       run: export MAKEFLAGS="-j$(expr $(nproc) \+ 1)"
     - name: install cargo-make
       uses: baptiste0928/cargo-install@v3
       with:
         crate: cargo-make
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+        fetch-depth: 0
+    - uses: ./.github/workflows/ubuntu-prepare
+    - uses: Swatinem/rust-cache@v2
+      with: { shared-key: "${{ runner.os }}-shared-fuzzer-cache" }
diff --git a/.github/workflows/ubuntu-prepare/action.yml b/.github/workflows/ubuntu-prepare/action.yml
@@ -28,16 +28,9 @@ runs:
     - name: Add LLVM in sources list
       shell: bash
       run: |
-        sudo tee -a /etc/apt/sources.list.d/llvm.list << EOF
-        deb https://apt.llvm.org/jammy/ llvm-toolchain-jammy main
-        deb-src https://apt.llvm.org/jammy/ llvm-toolchain-jammy main
-        EOF
-    - name: Add LLVM archive signature
-      shell: bash
-      run: wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
-    - name: Install LLVM and Clang (Version ${{ env.MAIN_LLVM_VERSION }})
-      shell: bash
-      run: sudo apt-get update && sudo apt-get install -y llvm llvm-dev clang libclang-cpp-dev llvm-${{ env.MAIN_LLVM_VERSION }} clang-${{ env.MAIN_LLVM_VERSION }} llvm-${{ env.MAIN_LLVM_VERSION }}-dev libclang-cpp${{ env.MAIN_LLVM_VERSION }} libclang-cpp${{ env.MAIN_LLVM_VERSION }}-dev
+        wget https://apt.llvm.org/llvm.sh
+        chmod +x llvm.sh
+        sudo ./llvm.sh ${{env.MAIN_LLVM_VERSION}} all
     - name: Symlink Headers
       shell: bash
       run: sudo ln -s /usr/include/asm-generic /usr/include/asm
diff --git a/fuzzers/others/libafl-fuzz/test/test.sh b/fuzzers/others/libafl-fuzz/test/test.sh
@@ -17,7 +17,7 @@ cargo build --release
 
 AFL_PATH=$AFL_DIR_NAME $AFL_CC_PATH $AFL_DIR_NAME/test-instr.c -o out-instr
 
-AFL_CORES=1 LLVM_CONFIG=llvm-config-18 AFL_STATS_INTERVAL=1 AFL_NUM_CORES=1 timeout 5 $LIBAFL_FUZZ_PATH -i ./seeds -o ./output $(pwd)/out-instr
+AFL_CORES=1 LLVM_CONFIG=${LLVM_CONFIG} AFL_STATS_INTERVAL=1 AFL_NUM_CORES=1 timeout 5 $LIBAFL_FUZZ_PATH -i ./seeds -o ./output $(pwd)/out-instr
 test -n "$( ls output/fuzzer_main/queue/id:000002* 2>/dev/null )" || exit 1
 test -n "$( ls output/fuzzer_main/fuzzer_stats 2>/dev/null )" || exit 1
 test -n "$( ls output/fuzzer_main/plot_data 2>/dev/null )" || exit 1

diff --git a/fuzzers/others/libfuzzer_libmozjpeg/src/lib.rs b/fuzzers/others/libfuzzer_libmozjpeg/src/lib.rs
@@ -80,10 +80,17 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re
     let edges_observer = unsafe { std_edges_map_observer("edges") };
 
     // Create an observation channel using the cmp map
-    let cmps_observer = unsafe { StdMapObserver::new("cmps", &mut CMP_MAP) };
+    let cmps_observer =
+        unsafe { StdMapObserver::from_mut_ptr("cmps", CMP_MAP.as_mut_ptr(), CMP_MAP.len()) };
 
     // Create an observation channel using the allocations map
-    let allocs_observer = unsafe { StdMapObserver::new("allocs", &mut libafl_alloc_map) };
+    let allocs_observer = unsafe {
+        StdMapObserver::from_mut_ptr(
+            "allocs",
+            libafl_alloc_map.as_mut_ptr(),
+            libafl_alloc_map.len(),
+        )
+    };
 
     // Feedback to rate the interestingness of an input
     let mut feedback = feedback_or!(
@@ -152,7 +159,7 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re
     // Call LLVMFUzzerInitialize() if present.
     let args: Vec<String> = env::args().collect();
     if libfuzzer_initialize(&args) == -1 {
-        println!("Warning: LLVMFuzzerInitialize failed with -1")
+        println!("Warning: LLVMFuzzerInitialize failed with -1");
     }
 
     // In case the corpus is empty (on first run), reset