Skip to content
Rust

refactor: reduce all circuits to a single instance col #2465

Sign in to view logs

refactor: reduce all circuits to a single instance col

refactor: reduce all circuits to a single instance col #2465

Workflow file for this run

.github/workflows/rust.yml at 6631ec4
name: Rust
on:
push:
branches: ["main"]
paths-ignore:
- "README.md"
- "cla.md"
pull_request:
branches: ["main"]
paths-ignore:
- "README.md"
- "cla.md"
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
env:
CARGO_TERM_COLOR: always
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions-rs/toolchain@v1
with:
toolchain: nightly-2023-08-24
override: true
components: rustfmt, clippy
- name: Build
run: cargo build --verbose
docs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions-rs/toolchain@v1
with:
toolchain: nightly-2023-08-24
override: true
components: rustfmt, clippy
- name: Docs
run: cargo doc --verbose
library-tests:
runs-on: ubuntu-latest-32-cores
steps:
- uses: actions/checkout@v4
- uses: actions-rs/toolchain@v1
with:
toolchain: nightly-2023-08-24
override: true
components: rustfmt, clippy
- uses: baptiste0928/cargo-install@v1
with:
crate: cargo-nextest
locked: true
- name: Doc tests
# nextest doesn't support --doc tests
run: cargo test --doc --verbose
- name: Library tests
run: cargo nextest run --lib --verbose
ultra-overflow-tests:
runs-on: self-hosted
steps:
- uses: actions/checkout@v4
- uses: actions-rs/toolchain@v1
with:
toolchain: nightly-2023-08-24
override: true
components: rustfmt, clippy
- uses: baptiste0928/cargo-install@v1
with:
crate: cargo-nextest
locked: true
- uses: mwilliamson/setup-wasmtime-action@v2
with:
wasmtime-version: "3.0.1"
- name: Install wasm32-wasi
run: rustup target add wasm32-wasi
- name: Install cargo-wasi
run: cargo install cargo-wasi
# - name: Matmul overflow (wasi)
# run: cargo wasi test matmul_col_ultra_overflow -- --include-ignored --nocapture
# - name: Conv overflow (wasi)
# run: cargo wasi test conv_col_ultra_overflow -- --include-ignored --nocapture
- name: Conv + relu overflow (wasi)
run: cargo wasi test conv_relu_col_ultra_overflow -- --include-ignored --nocapture
# - name: Matmul overflow
# run: RUST_LOG=debug cargo nextest run matmul_col_ultra_overflow --no-capture -- --include-ignored
# - name: Conv overflow
# run: RUST_LOG=debug cargo nextest run conv_col_ultra_overflow --no-capture -- --include-ignored
- name: Conv + relu overflow
run: RUST_LOG=debug cargo nextest run conv_relu_col_ultra_overflow --no-capture -- --include-ignored
model-serialization:
runs-on: ubuntu-latest-16-cores
steps:
- uses: actions/checkout@v4
- uses: actions-rs/toolchain@v1
with:
toolchain: nightly-2023-08-24
override: true
components: rustfmt, clippy
- uses: baptiste0928/cargo-install@v1
with:
crate: cargo-nextest
locked: true
- name: Model serialization
run: cargo nextest run native_tests::tests::model_serialization_::t
- name: Model serialization different binary ID
run: cargo nextest run native_tests::tests::model_serialization_different_binaries_ --test-threads 1
wasm32-tests:
runs-on: ubuntu-latest
# needs: [build, library-tests, docs]
steps:
- uses: actions/checkout@v4
- uses: actions-rs/toolchain@v1
with:
toolchain: nightly-2023-08-24
override: true
components: rustfmt, clippy
- uses: jetli/[email protected]
- uses: nanasess/setup-chromedriver@v2
# with:
# chromedriver-version: "115.0.5790.102"
- name: Install wasm32-unknown-unknown
run: rustup target add wasm32-unknown-unknown
- name: Install wasm runner
run: cargo install wasm-server-runner
- name: Add rust-src
run: rustup component add rust-src --toolchain nightly-2023-08-24-x86_64-unknown-linux-gnu
- name: Run wasm verifier tests
run: wasm-pack test --chrome --headless -- -Z build-std="panic_abort,std" --features web
tutorial:
runs-on: ubuntu-latest
needs: [build, library-tests, docs]
steps:
- uses: actions/checkout@v4
- uses: actions-rs/toolchain@v1
with:
toolchain: nightly-2023-08-24
override: true
components: rustfmt, clippy
- uses: baptiste0928/cargo-install@v1
with:
crate: cargo-nextest
locked: true
- name: Circuit Render
run: cargo nextest run --release --verbose tests::tutorial_
mock-proving-tests:
runs-on: self-hosted
# needs: [build, library-tests, docs]
steps:
- uses: actions/checkout@v4
- uses: actions-rs/toolchain@v1
with:
toolchain: nightly-2023-08-24
override: true
components: rustfmt, clippy
- uses: baptiste0928/cargo-install@v1
with:
crate: cargo-nextest
locked: true
- name: Mock fixed inputs
run: cargo nextest run --release --verbose tests::mock_fixed_inputs_ --test-threads 32
- name: Mock fixed outputs
run: cargo nextest run --release --verbose tests::mock_fixed_outputs --test-threads 32
- name: Mock accuracy calibration
run: cargo nextest run --release --verbose tests::mock_accuracy_cal_tests::a
- name: hashed inputs
run: cargo nextest run --release --verbose tests::mock_hashed_input_::t --test-threads 32
- name: hashed params
run: cargo nextest run --release --verbose tests::mock_hashed_params_::t --test-threads 32
- name: hashed outputs
run: cargo nextest run --release --verbose tests::mock_hashed_output_::t --test-threads 32
- name: hashed inputs + params + outputs
run: cargo nextest run --release --verbose tests::mock_hashed_all_::t --test-threads 32
- name: hashed inputs + fixed params
run: cargo nextest run --release --verbose tests::mock_hashed_output_fixed_params_::t --test-threads 32
- name: encrypted inputs
run: cargo nextest run --release --verbose tests::mock_encrypted_input_::t --test-threads 32
- name: encrypted params
run: cargo nextest run --release --verbose tests::mock_encrypted_params_::t --test-threads 32
- name: encrypted outputs
run: cargo nextest run --release --verbose tests::mock_encrypted_output_::t --test-threads 32
- name: encrypted inputs + params
run: cargo nextest run --release --verbose tests::mock_encrypted_input_params_::t --test-threads 32
- name: encrypted inputs + params + outputs
run: cargo nextest run --release --verbose tests::mock_encrypted_all_::t --test-threads 32
- name: encrypted inputs + hashed params
run: cargo nextest run --release --verbose tests::mock_encrypted_input_hashed_params_::t --test-threads 32
- name: MNIST Gan Mock
run: cargo nextest run --release --verbose tests::large_mock_::large_tests_4_expects -- --include-ignored
- name: NanoGPT Mock
run: cargo nextest run --release --verbose tests::large_mock_::large_tests_1_expects -- --include-ignored
- name: Self Attention Mock
run: cargo nextest run --release --verbose tests::large_mock_::large_tests_0_expects -- --include-ignored
- name: Multihead Attention Mock
run: cargo nextest run --release --verbose tests::large_mock_::large_tests_2_expects -- --include-ignored
- name: public outputs
run: cargo nextest run --release --verbose tests::mock_public_outputs_ --test-threads 32
- name: public outputs + batch size == 10
run: cargo nextest run --release --verbose tests::mock_large_batch_public_outputs_ --test-threads 32
- name: public inputs
run: cargo nextest run --release --verbose tests::mock_public_inputs_ --test-threads 32
- name: fixed params
run: cargo nextest run --release --verbose tests::mock_fixed_params_ --test-threads 32
prove-and-verify-evm-tests:
runs-on: ubuntu-latest-16-cores
needs: [build, library-tests, python-tests]
steps:
- uses: actions/checkout@v4
- uses: actions-rs/toolchain@v1
with:
toolchain: nightly-2023-08-24
override: true
components: rustfmt, clippy
- uses: baptiste0928/cargo-install@v1
with:
crate: cargo-nextest
locked: true
- name: Install solc
run: (hash svm 2>/dev/null || cargo install svm-rs) && svm install 0.8.20 && solc --version
- name: Install Anvil
run: cargo install --git https://github.com/foundry-rs/foundry --rev 95a93cd397f25f3f8d49d2851eb52bc2d52dd983 --profile local --locked anvil --force
- name: KZG prove and verify tests (EVM)
run: cargo nextest run --release --verbose tests_evm::kzg_evm_prove_and_verify --test-threads 1
- name: KZG prove and verify tests (EVM + hashed inputs)
run: cargo nextest run --release --verbose tests_evm::kzg_evm_hashed_input_prove_and_verify --test-threads 1
- name: KZG prove and verify tests (EVM + hashed params)
run: cargo nextest run --release --verbose tests_evm::kzg_evm_hashed_params_prove_and_verify --test-threads 1
- name: KZG prove and verify tests (EVM + hashed outputs)
run: cargo nextest run --release --verbose tests_evm::kzg_evm_hashed_output_prove_and_verify --test-threads 1
- name: KZG prove and verify tests (EVM + on chain inputs)
run: cargo nextest run --release --verbose tests_evm::kzg_evm_on_chain_input_prove_and_verify --test-threads 1
- name: KZG prove and verify tests (EVM + on chain outputs)
run: cargo nextest run --release --verbose tests_evm::kzg_evm_on_chain_output_prove_and_verify --test-threads 1
- name: KZG prove and verify tests (EVM + on chain inputs & outputs)
run: cargo nextest run --release --verbose tests_evm::kzg_evm_on_chain_input_output_prove_and_verify --test-threads 1
prove-and-verify-tests:
runs-on: ubuntu-latest-32-cores
needs: [build, library-tests, python-tests]
steps:
- uses: actions/checkout@v4
- uses: actions-rs/toolchain@v1
with:
toolchain: nightly-2023-08-24
override: true
components: rustfmt, clippy
- uses: jetli/[email protected]
- name: Add wasm32-unknown-unknown target
run: rustup target add wasm32-unknown-unknown
- name: Install wasm-server-runner
run: cargo install wasm-server-runner
- name: Add rust-src
run: rustup component add rust-src --toolchain nightly-2023-08-24-x86_64-unknown-linux-gnu
- name: Build wasm files for both web and nodejs compilation targets
run: |
wasm-pack build --target nodejs --out-dir ./tests/wasm/nodejs . -- -Z build-std="panic_abort,std"
- name: Replace memory definition in nodejs
run: |
sed -i "3s|.*|imports['env'] = {memory: new WebAssembly.Memory({initial:20,maximum:65536,shared:true})}|" tests/wasm/nodejs/ezkl.js
- uses: actions/checkout@v3
- name: Use pnpm 8
uses: pnpm/action-setup@v2
with:
version: 8
- name: Use Node.js 18.12.1
uses: actions/setup-node@v3
with:
node-version: '18.12.1'
cache: 'pnpm'
- name: Install dependencies
run: |
pnpm install --no-frozen-lockfile
env:
CI: false
NODE_ENV: development
- uses: baptiste0928/cargo-install@v1
with:
crate: cargo-nextest
locked: true
- name: Build wasm package for nodejs target.
run: |
wasm-pack build --release --target nodejs --out-dir ./tests/wasm/nodejs . -- -Z build-std="panic_abort,std"
- name: Replace memory definition in nodejs
run: |
sed -i "3s|.*|imports['env'] = {memory: new WebAssembly.Memory({initial:20,maximum:65536,shared:true})}|" tests/wasm/nodejs/ezkl.js
- name: KZG prove and verify tests (public outputs + column overflow)
run: cargo nextest run --release --verbose tests::kzg_prove_and_verify_with_overflow_::w --test-threads 8
- name: KZG prove and verify tests (public outputs + fixed params + column overflow)
run: cargo nextest run --release --verbose tests::kzg_prove_and_verify_with_overflow_fixed_params_
- name: KZG prove and verify tests (public outputs)
run: cargo nextest run --release --verbose tests::kzg_prove_and_verify_::t --test-threads 8
- name: KZG prove and verify tests (public outputs + column overflow)
run: cargo nextest run --release --verbose tests::kzg_prove_and_verify_::t --test-threads 8
- name: KZG prove and verify tests (public inputs)
run: cargo nextest run --release --verbose tests::kzg_prove_and_verify_public_input --test-threads 8
- name: KZG prove and verify tests (fixed params)
run: cargo nextest run --release --verbose tests::kzg_prove_and_verify_fixed_params --test-threads 8
- name: KZG prove and verify tests (hashed outputs)
run: cargo nextest run --release --verbose tests::kzg_prove_and_verify_hashed --test-threads 8
- name: KZG prove and verify tests (encrypted outputs)
run: cargo nextest run --release --verbose tests::kzg_prove_and_verify_encrypted --test-threads 8
fuzz-tests:
runs-on: ubuntu-latest-32-cores
needs: [build, library-tests, python-tests]
steps:
- uses: actions/checkout@v4
- uses: actions-rs/toolchain@v1
with:
toolchain: nightly-2023-08-24
override: true
components: rustfmt, clippy
- uses: baptiste0928/cargo-install@v1
with:
crate: cargo-nextest
locked: true
- name: Install solc
run: (hash svm 2>/dev/null || cargo install svm-rs) && svm install 0.8.20 && solc --version
- name: Install Anvil
run: cargo install --git https://github.com/foundry-rs/foundry --rev 95a93cd397f25f3f8d49d2851eb52bc2d52dd983 --profile local --locked anvil --force
- name: fuzz tests (EVM)
run: cargo nextest run --release --verbose tests_evm::kzg_evm_fuzz_ --test-threads 2
# - name: fuzz tests
# run: cargo nextest run --release --verbose tests::kzg_fuzz_ --test-threads 6
prove-and-verify-mock-aggr-tests:
runs-on: ubuntu-latest-32-cores
needs: [build, library-tests]
steps:
- uses: actions/checkout@v4
- uses: actions-rs/toolchain@v1
with:
toolchain: nightly-2023-08-24
override: true
components: rustfmt, clippy
- uses: baptiste0928/cargo-install@v1
with:
crate: cargo-nextest
locked: true
- name: Mock aggr tests
run: cargo nextest run --release --verbose tests_aggr::kzg_aggr_mock_prove_and_verify_ --test-threads 8
prove-and-verify-aggr-tests:
runs-on: large-self-hosted
needs: [build, library-tests, python-tests]
steps:
- uses: actions/checkout@v4
- uses: actions-rs/toolchain@v1
with:
toolchain: nightly-2023-08-24
override: true
components: rustfmt, clippy
- uses: baptiste0928/cargo-install@v1
with:
crate: cargo-nextest
locked: true
- name: KZG )tests
run: cargo nextest run --release --verbose tests_aggr::kzg_aggr_prove_and_verify_ --test-threads 8 -- --include-ignored
prove-and-verify-aggr-evm-tests:
runs-on: large-self-hosted
needs: [build, library-tests, python-tests]
steps:
- uses: actions/checkout@v4
- uses: actions-rs/toolchain@v1
with:
toolchain: nightly-2023-08-24
override: true
components: rustfmt, clippy
- uses: baptiste0928/cargo-install@v1
with:
crate: cargo-nextest
locked: true
- name: Install solc
run: (hash svm 2>/dev/null || cargo install svm-rs) && svm install 0.8.20 && solc --version
- name: Install Anvil
run: cargo install --git https://github.com/foundry-rs/foundry --rev 95a93cd397f25f3f8d49d2851eb52bc2d52dd983 --profile local --locked anvil --force
- name: KZG prove and verify aggr tests
run: cargo nextest run --release --verbose tests_evm::kzg_evm_aggr_prove_and_verify_::t --test-threads 4 -- --include-ignored
- name: KZG prove and verify aggr tests (encrypted input)
run: cargo nextest run --release --verbose tests_evm::kzg_evm_aggr_prove_and_verify_encrypted --test-threads 4 -- --include-ignored
examples:
runs-on: ubuntu-latest-32-cores
needs: [build, library-tests, docs]
steps:
- uses: actions/checkout@v4
- uses: actions-rs/toolchain@v1
with:
toolchain: nightly-2023-08-24
override: true
components: rustfmt, clippy
- uses: baptiste0928/cargo-install@v1
with:
crate: cargo-nextest
locked: true
- name: Download MNIST
run: sh data.sh
- name: Examples
run: cargo nextest run --release tests_examples
neg-tests:
runs-on: ubuntu-latest
needs: [build, library-tests, docs]
steps:
- uses: actions/checkout@v4
- uses: actions-rs/toolchain@v1
with:
toolchain: nightly-2023-08-24
override: true
components: rustfmt, clippy
- uses: baptiste0928/cargo-install@v1
with:
crate: cargo-nextest
locked: true
- name: should fail)
run: cargo nextest run neg_tests::neg_examples_
python-tests:
runs-on: 256gb
needs: [build, library-tests, docs]
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v4
with:
python-version: "3.7"
- uses: actions-rs/toolchain@v1
with:
toolchain: nightly-2023-08-24
override: true
components: rustfmt, clippy
- name: Install solc
run: (hash svm 2>/dev/null || cargo install svm-rs) && svm install 0.8.20 && solc --version
- name: Setup Virtual Env and Install python dependencies
run: python -m venv .env; source .env/bin/activate; pip install -r requirements.txt;
- name: Install Anvil
run: cargo install --git https://github.com/foundry-rs/foundry --rev 95a93cd397f25f3f8d49d2851eb52bc2d52dd983 --profile local --locked anvil --force
- name: Build python ezkl
run: source .env/bin/activate; maturin develop --features python-bindings --release
- name: Run pytest
run: source .env/bin/activate; pytest -v
accuracy-measurement-tests:
runs-on: ubuntu-latest-32-cores
# needs: [build, library-tests, docs]
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v4
with:
python-version: "3.7"
- uses: actions-rs/toolchain@v1
with:
toolchain: nightly-2023-08-24
override: true
components: rustfmt, clippy
- uses: baptiste0928/cargo-install@v1
with:
crate: cargo-nextest
locked: true
- name: Setup Virtual Env and Install python dependencies
run: python -m venv .env; source .env/bin/activate; pip install -r requirements.txt;
- name: Build python ezkl
run: source .env/bin/activate; maturin develop --features python-bindings --release
- name: Public inputs
run: source .env/bin/activate; cargo nextest run --release --verbose tests::accuracy_measurement_public_inputs_ --no-capture
- name: fixed params
run: source .env/bin/activate; cargo nextest run --release --verbose tests::accuracy_measurement_fixed_params_ --no-capture
- name: Public outputs
run: source .env/bin/activate; cargo nextest run --release --verbose tests::accuracy_measurement_public_outputs_ --no-capture
python-integration-tests:
runs-on:
large-self-hosted
# Service containers to run with `container-job`
services:
# Label used to access the service container
postgres:
# Docker Hub image
image: postgres
env:
POSTGRES_USER: ubuntu
POSTGRES_HOST_AUTH_METHOD: trust
# Set health checks to wait until postgres has started
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
# Maps tcp port 5432 on service container to the host
- 5432:5432
# needs: [build, library-tests, docs]
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v4
with:
python-version: "3.9"
- uses: actions-rs/toolchain@v1
with:
toolchain: nightly-2023-08-24
override: true
components: rustfmt, clippy
- uses: baptiste0928/cargo-install@v1
with:
crate: cargo-nextest
locked: true
- name: Install solc
run: (hash svm 2>/dev/null || cargo install svm-rs) && svm install 0.8.20 && solc --version
- name: Install Anvil
run: cargo install --git https://github.com/foundry-rs/foundry --rev 95a93cd397f25f3f8d49d2851eb52bc2d52dd983 --profile local --locked anvil --force
- name: Setup Virtual Env and Install python dependencies
run: python -m venv .env; source .env/bin/activate; pip install -r requirements.txt;
- name: Build python ezkl
run: source .env/bin/activate; maturin develop --features python-bindings --release
# - name: authenticate-kaggle-cli
# shell: bash
# env:
# KAGGLE_API_KEY: ${{ secrets.KAGGLE_API_KEY }}
# run: |
# mkdir /home/ubuntu/.kaggle
# # now dump the contents of the file into a file called kaggle.json
# echo $KAGGLE_API_KEY > /home/ubuntu/.kaggle/kaggle.json
# chmod 600 /home/ubuntu/.kaggle/kaggle.json
- name: Simple tutorial (public network and outputs)
run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_20_expects
- name: Simple tutorial (public inputs and outputs)
run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_19_expects
- name: Simple tutorial (all public)
run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_4_expects
- name: NBEATS tutorial
run: source .env/bin/activate; cargo nextest run py_tests::tests::nbeats_
- name: SVM
run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_18_expects
- name: LightGBM
run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_17_expects
- name: XGBoost
run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_16_expects
- name: Gradient boosted tres
run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_15_expects
- name: Random Forest
run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_14_expects
- name: Decision Tree
run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_13_expects
- name: Set membership
run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_12_expects
- name: Voice tutorial
run: source .env/bin/activate; cargo nextest run py_tests::tests::voice_
# - name: Mean tutorial
# run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_7_expects --no-capture
- name: End to end demo tutorial
run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_10_expects
- name: LSTM tutorial
run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_11_expects
- name: Keras tutorial
run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_1_expects
- name: Simple aggregation tutorial
run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_9_expects
- name: Little transformer tutorial
run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_8_expects --no-capture
- name: Encrypted tutorial
run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_2_expects
- name: Hashed tutorial
run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_3_expects
- name: Data attestation tutorial
run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_5_expects
- name: Variance tutorial
run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_6_expects
- name: Gan tutorial
run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_::tests_0_expects