Implement aggregation prover (Compression Circuit part) (#158)

* Temporarily replace with zkevm-circuits `proof-aggregation-circuit` branch (and make build work).

* Add struct `Prover` and functions.

* Update Prover functions.

* Implement `gen_agg_proof` function.

* Add compression tests.

* Refactor `Proof`.

* Implement `Verifier` and finish `compression_tests` (testing on super).

* Use test branch `scroll-prover-test-agg-circuit` (merging latest develop).

* Fix `Cargo.toml`.

* Set default AGG degree to `26`.

* Add missing.

* Replace `load_params` with `load_or_download_params` in zkevm prover.

* Update aggregation-tests.

* Small fix in `Proof`.

* Reuse `Params`.

* fail..

* Fix issue.

* Add `aggregation-tests` (testing).

* Add aggregation prover and verifier to FFI.

* testing

* Add comments.

* Fix compression-tests.

* fix

* switch back to develop

* Update aggregation-tests.

* Update README.

* Fix to use develop branch of zkevm-circuits.

* Delete aggrestion-circuit related code.

* Rename `VERIFY_CONFIG` to `COMPRESSION_CONFIG`.

* Ignore unit-test `test_capacity_checker` (will fix further).

* Fix CI.

---------

Co-authored-by: Zhuo Zhang <[email protected]>

.github/workflows/unit_test.yml

@@ -35,7 +35,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           submodules: true
-      - run: git config --system --add safe.directory /__w/scroll-zkevm/scroll-zkevm
+      - run: git config --system --add safe.directory /__w/scroll-prover/scroll-prover
       - uses: actions/setup-go@v3
         with:
           go-version: '>=1.18.0'

Makefile

@@ -26,9 +26,6 @@ bridge-test:
 	cargo build --release
 	./target/release/prove --params=./test_params --trace=prover/tests/traces/bridge
 
-test-inner-prove: ## Test inner circuit with real trace
-	cargo test --features prove_verify --release test_prove_verify
-
 mock:
 	@cargo test --features prove_verify --release test_mock_prove -- --exact --nocapture
 
@@ -38,8 +35,17 @@ mock-debug:
 mock-testnet:
 	@cargo run --bin mock_testnet --release
 
+test-inner-prove:
+	@cargo test --features prove_verify --release test_inner_prove_verify
+
 test-chunk-prove:
-	@cargo test --features prove_verify --release test_aggregation_api
+	@cargo test --features prove_verify --release test_chunk_prove_verify
+
+test-comp-prove:
+	@cargo test --features prove_verify --release test_comp_prove_verify
+
+test-agg-prove:
+	@cargo test --features prove_verify --release test_agg_prove_verify
 
 rows:
 	@cargo test --features prove_verify --release estimate_circuit_rows

README.md

@@ -12,11 +12,13 @@ git submodule init
 git submodule update --checkout
 ```
 
-Download setup params
+Download all setup params, degree `20` and `25` are used in [config.rs](https://github.com/scroll-tech/scroll-prover/tree/main/prover/src/config.rs).
+Could only download params of degree `25`, but it may affect performance (when dowsizing to `20`).
 ```shell
-make download-setup
+make download-setup -e degree=20
+make download-setup -e degree=25
 ```
-Or specify degree and target directory to download
+Or specify other degree and target directory to download.
 ```shell
 # As default `degree=25` and `params_dir=./prover/test_params`.
 make download-setup -e degree=DEGREE params_dir=PARAMS_DIR
@@ -26,7 +28,7 @@ make download-setup -e degree=DEGREE params_dir=PARAMS_DIR
 
 `make test-chunk-prove` is the main testing entry point for the multi-level circuit constraint system of scroll-zkevm. Developers could understand how the system works by reading the codes of this test.
 
-Besides it, `make test-inner-prove` could be used to test the first-level circuit.
+Besides it, `make test-inner-prove` could be used to test the first-level circuit, and `make-comp-prove` could be used to test two-layers compression circuits.
 
 ### Binaries
 

batch.sh

@@ -1,23 +1,33 @@
 set -x
 set -u
-#set -e 
+set -e
 set -o pipefail
 
+#export RUST_LOG=debug
 export RUST_LOG=trace
 
+function simple_tests() {
+	for mode in sushi multiple #nft dao native empty # pack
+	#for mode in native empty # pack
+	do
+		MODE=$mode make mock 2>&1 | tee /tmp/mock_${mode}.log
+	done
+}
+
 function check_batch() {
 	TRACE_VER="0317-alpha"
 	for d in $(ls ~/zip-traces/${TRACE_VER}/traces); do
-		export TRACE_PATH=$(realpath ~/zip-traces/${TRACE_VER}/traces/${d}/traces-data) 
+		export TRACE_PATH=$(realpath ~/zip-traces/${TRACE_VER}/traces/${d}/traces-data)
 		make mock 2>&1 | tee /tmp/mock_${d}.log
 	done
 }
 
 function check_block() {
 	for t in prover/tests/extra_traces/tx_storage_proof.json prover/tests/extra_traces/hash_precompile_2.json prover/tests/extra_traces/hash_precompile_1.json prover/tests/traces/sushi/sushi_chef-withdraw.json prover/tests/traces/erc20/erc20_10_transfer.json; do
-		TRACE_PATH=`realpath $t` make mock 2>&1 | tee /tmp/mock_`basename $t`.log
+		TRACE_PATH=$(realpath $t) make mock 2>&1 | tee /tmp/mock_$(basename $t).log
 	done
 }
 
-check_block
+simple_tests
+#check_block
 #check_batch

bin/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "bin"
-version = "0.3.0"
+version = "0.4.0"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

bin/src/prove.rs

@@ -1,8 +1,8 @@
 use clap::Parser;
 use log::info;
 use prover::{
-    utils::{get_block_trace_from_file, init_env_and_log, load_or_download_params},
-    zkevm::{circuit::AGG_DEGREE, Prover},
+    utils::{get_block_trace_from_file, init_env_and_log},
+    zkevm::Prover,
 };
 use std::{fs, path::PathBuf, time::Instant};
 
@@ -26,10 +26,7 @@ fn main() {
     std::env::set_var("VERIFY_CONFIG", "./prover/configs/verify_circuit.config");
 
     let args = Args::parse();
-    let agg_params = load_or_download_params(&args.params_path, *AGG_DEGREE)
-        .expect("failed to load or create params");
-
-    let mut prover = Prover::from_params(agg_params);
+    let mut prover = Prover::from_params_dir(&args.params_path);
 
     let mut traces = Vec::new();
     let trace_path = PathBuf::from(&args.trace_path);

bin/src/verify.rs

@@ -1,13 +1,6 @@
 use clap::Parser;
 use log::info;
-use prover::{
-    utils::{init_env_and_log, load_or_download_params},
-    zkevm::{
-        circuit::{AGG_DEGREE, DEGREE},
-        Verifier,
-    },
-    Proof,
-};
+use prover::{utils::init_env_and_log, zkevm::Verifier, Proof};
 use std::{fs::File, io::Read, path::PathBuf};
 
 #[derive(Parser, Debug)]
@@ -26,13 +19,8 @@ fn main() {
     std::env::set_var("VERIFY_CONFIG", "./prover/configs/verify_circuit.config");
 
     let args = Args::parse();
-    let params = load_or_download_params(&args.params_path, *DEGREE)
-        .expect("failed to load or create params");
-    let agg_params = load_or_download_params(&args.params_path, *AGG_DEGREE)
-        .expect("failed to load or create params");
-    let agg_vk = read_from_file(&args.vk_path);
-
-    let v = Verifier::from_params(params, agg_params, Some(agg_vk));
+    let chunk_vk = read_from_file(&args.vk_path);
+    let v = Verifier::from_params_dir(&args.params_path, Some(chunk_vk));
 
     let proof_path = PathBuf::from("proof_data").join("chunk_full_proof.json");
     let proof_vec = read_from_file(&proof_path.to_string_lossy());

ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ffi"
-version = "0.3.0"
+version = "0.4.0"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

ffi/src/prove.rs

@@ -12,7 +12,7 @@ pub unsafe extern "C" fn init_prover(params_path: *const c_char, _seed_path: *co
     init_env_and_log("ffi_prove");
 
     let params_path = c_char_to_str(params_path);
-    let p = zkevm::Prover::from_param_dir(params_path);
+    let p = zkevm::Prover::from_params_dir(params_path);
     PROVER.set(p).unwrap();
 }
 

ffi/src/verify.rs

@@ -16,7 +16,7 @@ pub unsafe extern "C" fn init_verifier(params_path: *const c_char, agg_vk_path:
     let mut agg_vk = vec![];
     f.read_to_end(&mut agg_vk).unwrap();
 
-    let v = Box::new(zkevm::Verifier::from_fpath(params_path, Some(agg_vk)));
+    let v = Box::new(zkevm::Verifier::from_params_dir(params_path, Some(agg_vk)));
     VERIFIER = Some(Box::leak(v))
 }