rust.yml

name: Rust
on:
  push:
    branches: ["main"]
    paths-ignore:
      - "README.md"
      - "cla.md"
  pull_request:
    branches: ["main"]
    paths-ignore:
      - "README.md"
      - "cla.md"

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  CARGO_TERM_COLOR: always

jobs:
  build:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4
      - uses: actions-rs/toolchain@v1
        with:
          toolchain: nightly-2023-08-24
          override: true
          components: rustfmt, clippy
      - name: Build
        run: cargo build --verbose

  docs:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions-rs/toolchain@v1
        with:
          toolchain: nightly-2023-08-24
          override: true
          components: rustfmt, clippy
      - name: Docs
        run: cargo doc --verbose

  library-tests:
    runs-on: ubuntu-latest-32-cores
    steps:
      - uses: actions/checkout@v4
      - uses: actions-rs/toolchain@v1
        with:
          toolchain: nightly-2023-08-24
          override: true
          components: rustfmt, clippy
      - uses: baptiste0928/cargo-install@v1
        with:
          crate: cargo-nextest
          locked: true
      - name: Doc tests
        # nextest doesn't support --doc tests
        run: cargo test --doc --verbose
      - name: Library tests
        run: cargo nextest run --lib --verbose -- --include-ignored

  model-serialization:
    runs-on: 256gb
    steps:
      - uses: actions/checkout@v4
      - uses: actions-rs/toolchain@v1
        with:
          toolchain: nightly-2023-08-24
          override: true
          components: rustfmt, clippy
      - uses: baptiste0928/cargo-install@v1
        with:
          crate: cargo-nextest
          locked: true
      - name: Model serialization
        run: cargo nextest run native_tests::tests::model_serialization_::t
      - name: Model serialization different binary ID
        run: cargo nextest run native_tests::tests::model_serialization_different_binaries_::t  --test-threads 1

  wasm32-tests:
    runs-on: ubuntu-latest
    # needs: [build,  library-tests, docs]
    steps:
      - uses: actions/checkout@v4
      - uses: actions-rs/toolchain@v1
        with:
          toolchain: nightly-2023-08-24
          override: true
          components: rustfmt, clippy
      - uses: jetli/wasm-pack-action@v0.4.0
      - uses: nanasess/setup-chromedriver@v2
        with:
          chromedriver-version: "115.0.5790.102"
      - name: Install wasm32-unknown-unknown
        run: rustup target add wasm32-unknown-unknown
      - name: Install wasm runner
        run: cargo install wasm-server-runner
      - name: Add rust-src
        run: rustup component add rust-src --toolchain nightly-2023-08-24-x86_64-unknown-linux-gnu
      - name: Run wasm verifier tests
        run: wasm-pack test --chrome --headless -- -Z build-std="panic_abort,std" --features web

  tutorial:
    runs-on: ubuntu-latest
    needs: [build, library-tests, docs]
    steps:
      - uses: actions/checkout@v4
      - uses: actions-rs/toolchain@v1
        with:
          toolchain: nightly-2023-08-24
          override: true
          components: rustfmt, clippy
      - uses: baptiste0928/cargo-install@v1
        with:
          crate: cargo-nextest
          locked: true
      - name: Circuit Render
        run: cargo nextest run --release --verbose tests::tutorial_

  mock-proving-tests:
    runs-on: self-hosted
    # needs: [build,  library-tests, docs]
    steps:
      - uses: actions/checkout@v4
      - uses: actions-rs/toolchain@v1
        with:
          toolchain: nightly-2023-08-24
          override: true
          components: rustfmt, clippy
      - uses: baptiste0928/cargo-install@v1
        with:
          crate: cargo-nextest
          locked: true
      - name: Mock accuracy calibration
        run: cargo nextest run --release --verbose tests::mock_accuracy_cal_tests::a
      - name: hashed inputs
        run: cargo nextest run --release --verbose tests::mock_hashed_input_::t  --test-threads 32
      - name: hashed params
        run: cargo nextest run --release --verbose tests::mock_hashed_params_::t  --test-threads 32
      - name: hashed outputs
        run: cargo nextest run --release --verbose tests::mock_hashed_output_::t  --test-threads 32
      - name: hashed inputs + params + outputs
        run: cargo nextest run --release --verbose tests::mock_hashed_all_::t  --test-threads 32
      - name: encrypted inputs
        run: cargo nextest run --release --verbose tests::mock_encrypted_input_::t  --test-threads 32
      - name: encrypted params
        run: cargo nextest run --release --verbose tests::mock_encrypted_params_::t  --test-threads 32
      - name: encrypted outputs
        run: cargo nextest run --release --verbose tests::mock_encrypted_output_::t  --test-threads 32
      - name: encrypted inputs + params
        run: cargo nextest run --release --verbose tests::mock_encrypted_input_params_::t  --test-threads 32
      - name: encrypted inputs + params + outputs
        run: cargo nextest run --release --verbose tests::mock_encrypted_all_::t  --test-threads 32
      - name: encrypted inputs + hashed params
        run: cargo nextest run --release --verbose tests::mock_encrypted_input_hashed_params_::t  --test-threads 32
      - name: MNIST Gan Mock
        run: cargo nextest run --release --verbose tests::large_mock_::large_tests_4_expects -- --include-ignored
      - name: NanoGPT Mock
        run: cargo nextest run --release --verbose tests::large_mock_::large_tests_1_expects -- --include-ignored
      - name: Self Attention Mock
        run: cargo nextest run --release --verbose tests::large_mock_::large_tests_0_expects -- --include-ignored
      - name: Multihead Attention Mock
        run: cargo nextest run --release --verbose tests::large_mock_::large_tests_2_expects -- --include-ignored
      - name: public outputs
        run: cargo nextest run --release --verbose tests::mock_public_outputs_ --test-threads 32
      - name: public outputs + batch size == 10
        run: cargo nextest run --release --verbose tests::mock_large_batch_public_outputs_ --test-threads 32
      - name: public inputs
        run: cargo nextest run --release --verbose tests::mock_public_inputs_ --test-threads 32
      - name: public params
        run: cargo nextest run --release --verbose tests::mock_public_params_ --test-threads 32

  prove-and-verify-evm-tests:
    runs-on: ubuntu-latest-16-cores
    needs: [build, library-tests, python-tests]
    steps:
      - uses: actions/checkout@v4
      - uses: actions-rs/toolchain@v1
        with:
          toolchain: nightly-2023-08-24
          override: true
          components: rustfmt, clippy
      - uses: baptiste0928/cargo-install@v1
        with:
          crate: cargo-nextest
          locked: true
      - name: Install solc
        run: (hash svm 2>/dev/null || cargo install svm-rs) && svm install 0.8.20 && solc --version
      - name: Install Anvil
        run: cargo install --git https://github.com/foundry-rs/foundry --rev 95a93cd397f25f3f8d49d2851eb52bc2d52dd983 --profile local --locked anvil --force
      - name: KZG prove and verify tests (EVM)
        run: cargo nextest run --release --verbose tests_evm::kzg_evm_prove_and_verify --test-threads 1
      - name: KZG prove and verify tests (EVM + hashed inputs)
        run: cargo nextest run --release --verbose tests_evm::kzg_evm_hashed_input_prove_and_verify --test-threads 1
      - name: KZG prove and verify tests (EVM + hashed params)
        run: cargo nextest run --release --verbose tests_evm::kzg_evm_hashed_params_prove_and_verify --test-threads 1
      - name: KZG prove and verify tests (EVM + hashed outputs)
        run: cargo nextest run --release --verbose tests_evm::kzg_evm_hashed_output_prove_and_verify --test-threads 1
      - name: KZG prove and verify tests (EVM + on chain inputs)
        run: cargo nextest run --release --verbose tests_evm::kzg_evm_on_chain_input_prove_and_verify --test-threads 1
      - name: KZG prove and verify tests (EVM + on chain outputs)
        run: cargo nextest run --release --verbose tests_evm::kzg_evm_on_chain_output_prove_and_verify --test-threads 1
      - name: KZG prove and verify tests (EVM + on chain inputs & outputs)
        run: cargo nextest run --release --verbose tests_evm::kzg_evm_on_chain_input_output_prove_and_verify --test-threads 1

  prove-and-verify-tests:
    runs-on: ubuntu-latest-32-cores
    needs: [build, library-tests, python-tests]
    steps:
      - uses: actions/checkout@v4
      - uses: actions-rs/toolchain@v1
        with:
          toolchain: nightly-2023-08-24
          override: true
          components: rustfmt, clippy
      - uses: baptiste0928/cargo-install@v1
        with:
          crate: cargo-nextest
          locked: true
      - name: KZG prove and verify tests (public outputs)
        run: cargo nextest run --release --verbose tests::kzg_prove_and_verify_::t --test-threads 8
      - name: KZG prove and verify tests (public outputs + column overflow)
        run: cargo nextest run --release --verbose tests::kzg_prove_and_verify_::t --test-threads 8
      - name: KZG prove and verify tests (public inputs)
        run: cargo nextest run --release --verbose tests::kzg_prove_and_verify_public_inputs --test-threads 8
      - name: KZG prove and verify tests (public params)
        run: cargo nextest run --release --verbose tests::kzg_prove_and_verify_public_params --test-threads 8
      - name: KZG prove and verify tests (hashed outputs)
        run: cargo nextest run --release --verbose tests::kzg_prove_and_verify_hashed --test-threads 8
      - name: KZG prove and verify tests (encrypted outputs)
        run: cargo nextest run --release --verbose tests::kzg_prove_and_verify_encrypted --test-threads 8

  fuzz-tests:
    runs-on: ubuntu-latest-32-cores
    needs: [build, library-tests, python-tests]
    steps:
      - uses: actions/checkout@v4
      - uses: actions-rs/toolchain@v1
        with:
          toolchain: nightly-2023-08-24
          override: true
          components: rustfmt, clippy
      - uses: baptiste0928/cargo-install@v1
        with:
          crate: cargo-nextest
          locked: true
      - name: Install solc
        run: (hash svm 2>/dev/null || cargo install svm-rs) && svm install 0.8.20 && solc --version
      - name: Install Anvil
        run: cargo install --git https://github.com/foundry-rs/foundry --rev 95a93cd397f25f3f8d49d2851eb52bc2d52dd983 --profile local --locked anvil --force
      - name: fuzz tests (EVM)
        run: cargo nextest run --release --verbose tests_evm::kzg_evm_fuzz_ --test-threads 2
      - name: fuzz tests
        run: cargo nextest run --release --verbose tests::kzg_fuzz_ --test-threads 6

  prove-and-verify-mock-aggr-tests:
    runs-on: ubuntu-latest-32-cores
    needs: [build, library-tests]
    steps:
      - uses: actions/checkout@v4
      - uses: actions-rs/toolchain@v1
        with:
          toolchain: nightly-2023-08-24
          override: true
          components: rustfmt, clippy
      - uses: baptiste0928/cargo-install@v1
        with:
          crate: cargo-nextest
          locked: true
      - name: Mock aggr tests
        run: cargo nextest run --release --verbose tests_aggr::kzg_aggr_mock_prove_and_verify_ --test-threads 8

  prove-and-verify-aggr-tests:
    runs-on: large-self-hosted
    needs: [build, library-tests, python-tests]
    steps:
      - uses: actions/checkout@v4
      - uses: actions-rs/toolchain@v1
        with:
          toolchain: nightly-2023-08-24
          override: true
          components: rustfmt, clippy
      - uses: baptiste0928/cargo-install@v1
        with:
          crate: cargo-nextest
          locked: true
      - name: KZG )tests
        run: cargo nextest run --release --verbose tests_aggr::kzg_aggr_prove_and_verify_ --test-threads 8 -- --include-ignored

  prove-and-verify-aggr-evm-tests:
    runs-on: large-self-hosted
    needs:
      [
        build,
        library-tests,
        python-tests,
      ]
    steps:
      - uses: actions/checkout@v4
      - uses: actions-rs/toolchain@v1
        with:
          toolchain: nightly-2023-08-24
          override: true
          components: rustfmt, clippy
      - uses: baptiste0928/cargo-install@v1
        with:
          crate: cargo-nextest
          locked: true
      - name: Install solc
        run: (hash svm 2>/dev/null || cargo install svm-rs) && svm install 0.8.20 && solc --version
      - name: Install Anvil
        run: cargo install --git https://github.com/foundry-rs/foundry --rev 95a93cd397f25f3f8d49d2851eb52bc2d52dd983 --profile local --locked anvil --force
      - name: KZG prove and verify aggr tests
        run: cargo nextest run  --release --verbose tests_evm::kzg_evm_aggr_prove_and_verify_::t --test-threads 4 -- --include-ignored
      - name: KZG prove and verify aggr tests (encrypted input)
        run: cargo nextest run  --release --verbose tests_evm::kzg_evm_aggr_prove_and_verify_encrypted --test-threads 4 -- --include-ignored

  examples:
    runs-on: ubuntu-latest-32-cores
    needs: [build, library-tests, docs]
    steps:
      - uses: actions/checkout@v4
      - uses: actions-rs/toolchain@v1
        with:
          toolchain: nightly-2023-08-24
          override: true
          components: rustfmt, clippy
      - uses: baptiste0928/cargo-install@v1
        with:
          crate: cargo-nextest
          locked: true
      - name: Download MNIST
        run: sh data.sh
      - name: Examples
        run: cargo nextest run --release tests_examples

  neg-tests:
    runs-on: ubuntu-latest
    needs: [build, library-tests, docs]
    steps:
      - uses: actions/checkout@v4
      - uses: actions-rs/toolchain@v1
        with:
          toolchain: nightly-2023-08-24
          override: true
          components: rustfmt, clippy
      - uses: baptiste0928/cargo-install@v1
        with:
          crate: cargo-nextest
          locked: true
      - name: should fail)
        run: cargo nextest run neg_tests::neg_examples_

  python-tests:
    runs-on: 256gb
    needs: [build, library-tests, docs]
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v4
        with:
          python-version: "3.7"
      - uses: actions-rs/toolchain@v1
        with:
          toolchain: nightly-2023-08-24
          override: true
          components: rustfmt, clippy
      - name: Install solc
        run: (hash svm 2>/dev/null || cargo install svm-rs) && svm install 0.8.20 && solc --version
      - name: Setup Virtual Env and Install python dependencies
        run: python -m venv .env; source .env/bin/activate; pip install -r requirements.txt;
      - name: Install Anvil
        run: cargo install --git https://github.com/foundry-rs/foundry --rev 95a93cd397f25f3f8d49d2851eb52bc2d52dd983 --profile local --locked anvil --force
      - name: Build python ezkl
        run: source .env/bin/activate; maturin develop --features python-bindings --release
      - name: Run pytest
        run: source .env/bin/activate; pytest -v

  accuracy-measurement-tests:
    runs-on: ubuntu-latest-32-cores
    # needs: [build, library-tests, docs]
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v4
        with:
          python-version: "3.7"
      - uses: actions-rs/toolchain@v1
        with:
          toolchain: nightly-2023-08-24
          override: true
          components: rustfmt, clippy
      - uses: baptiste0928/cargo-install@v1
        with:
          crate: cargo-nextest
          locked: true
      - name: Setup Virtual Env and Install python dependencies
        run: python -m venv .env; source .env/bin/activate; pip install -r requirements.txt;
      - name: Build python ezkl
        run: source .env/bin/activate; maturin develop --features python-bindings --release
      - name: Public inputs
        run: source .env/bin/activate; cargo nextest run --release --verbose tests::accuracy_measurement_public_inputs_ --no-capture
      - name: Public params
        run: source .env/bin/activate; cargo nextest run --release --verbose tests::accuracy_measurement_public_params_ --no-capture
      - name: Public outputs
        run: source .env/bin/activate; cargo nextest run --release --verbose tests::accuracy_measurement_public_outputs_ --no-capture

  python-integration-tests:
    runs-on:
      large-self-hosted
      # Service containers to run with `container-job`
    services:
      # Label used to access the service container
      postgres:
        # Docker Hub image
        image: postgres
        env:
          POSTGRES_USER: ubuntu
          POSTGRES_HOST_AUTH_METHOD: trust
        # Set health checks to wait until postgres has started
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
          # Maps tcp port 5432 on service container to the host
          - 5432:5432
    # needs: [build,  library-tests, docs]
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v4
        with:
          python-version: "3.9"
      - uses: actions-rs/toolchain@v1
        with:
          toolchain: nightly-2023-08-24
          override: true
          components: rustfmt, clippy
      - uses: baptiste0928/cargo-install@v1
        with:
          crate: cargo-nextest
          locked: true
      - name: Install solc
        run: (hash svm 2>/dev/null || cargo install svm-rs) && svm install 0.8.20 && solc --version
      - name: Install Anvil
        run: cargo install --git https://github.com/foundry-rs/foundry --rev 95a93cd397f25f3f8d49d2851eb52bc2d52dd983 --profile local --locked anvil --force
      - name: Setup Virtual Env and Install python dependencies
        run: python -m venv .env; source .env/bin/activate; pip install -r requirements.txt;
      - name: Build python ezkl
        run: source .env/bin/activate; maturin develop --features python-bindings --release
      # - name: authenticate-kaggle-cli
      #   shell: bash
      #   env:
      #     KAGGLE_API_KEY: ${{ secrets.KAGGLE_API_KEY }}
      #   run: |
      #     mkdir /home/ubuntu/.kaggle
      #     # now dump the contents of the file into a file called kaggle.json
      #     echo $KAGGLE_API_KEY > /home/ubuntu/.kaggle/kaggle.json
      #     chmod 600 /home/ubuntu/.kaggle/kaggle.json
      - name: LightGBM
        run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_17_expects
      - name: XGBoost
        run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_16_expects
      - name: Gradient boosted tress
        run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_15_expects
      - name: Random Forest
        run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_14_expects
      - name: Decision Tree
        run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_13_expects
      - name: Set membership
        run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_12_expects
      - name: Voice tutorial
        run: source .env/bin/activate; cargo nextest run py_tests::tests::voice_
      - name: Mean tutorial
        run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_7_expects --no-capture
      - name: End to end demo tutorial
        run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_10_expects
      - name: LSTM tutorial
        run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_11_expects
      - name: Keras tutorial
        run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_1_expects
      - name: Simple aggregation tutorial
        run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_9_expects
      - name: Little transformer tutorial
        run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_8_expects --no-capture
      - name: Encrypted tutorial
        run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_2_expects
      - name: Hashed tutorial
        run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_3_expects
      - name: Simple tutorial
        run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_4_expects
      - name: Data attestation tutorial
        run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_5_expects
      - name: Variance tutorial
        run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_6_expects
      - name: Gan tutorial
        run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_0_expects