Refactor and simplify zkevm prover (#156)

* refactor zkevm prover

* Replace `--agg` with `--chunk` in a command of Makefile.

* Update traces.

* Rename `test-super-trace` to `test-inner-prove` and `test-agg` to `test-chunk-prove`.

* Delete bin `setup`, and update README.

* remove redundant files

* Replace `load_or_create_params` with `load_or_download_params` (log a warning if download).

* Add `rustfmt.toml` from zkevm-circuits, and format code.

* Fix downloading setup params.

* Update `prove`, `verify` bins and README.

* Small fixes in Makefile and README.

---------

Co-authored-by: Zhuo Zhang <[email protected]>
Co-authored-by: Steven Gu <[email protected]>

Makefile

@@ -24,9 +24,9 @@ test: ## Run tests for all the workspace members
 
 bridge-test:
 	cargo build --release
-	./target/release/prove --params=./test_params --seed=./test_seed --trace=prover/tests/traces/bridge --agg=true
+	./target/release/prove --params=./test_params --trace=prover/tests/traces/bridge
 
-test-super-trace: ## test super circuit with real trace
+test-inner-prove: ## Test inner circuit with real trace
 	cargo test --features prove_verify --release test_prove_verify
 
 mock:
@@ -38,7 +38,7 @@ mock-debug:
 mock-testnet:
 	@cargo run --bin mock_testnet --release
 
-test-agg:
+test-chunk-prove:
 	@cargo test --features prove_verify --release test_aggregation_api
 
 rows:

README.md

@@ -4,39 +4,60 @@
 
 ## Usage
 
+### Prerequisite
+
+Fetch git-submodule of test traces
+```shell
+git submodule init
+git submodule update --checkout
+```
+
+Download setup params
+```shell
+make download-setup
+```
+Or specify degree and target directory to download
+```shell
+# As default `degree=25` and `params_dir=./prover/test_params`.
+make download-setup -e degree=DEGREE params_dir=PARAMS_DIR
+```
 
 ### Testing
 
-`make test-agg` is the main testing entry point for the multi-level circuit constraint system of scroll-zkevm. Developers can understand how the system works by reading the codes of this test.
+`make test-chunk-prove` is the main testing entry point for the multi-level circuit constraint system of scroll-zkevm. Developers could understand how the system works by reading the codes of this test.
 
-Besides, `make test-super-trace` can be used to test the first-level circuit.
+Besides it, `make test-inner-prove` could be used to test the first-level circuit.
 
 ### Binaries
 
 This repository is designed to be used as a Rust crate, rather than a standalone running process. However, you can still use the following command to run binaries locally.
 
-Setup 
-```shell
-cargo build --release --bin setup
-
-./target/release/setup --params <params-file-path>
-```
-
-If you run into linking issues during setup you may need to run
+If you run into linking issues you may need to run
 ```shell
 cp `find ./target/release/ | grep libzktrie.so` /usr/local/lib/
 ```
-to move the zktrielib into a path where your linker can locate it
+To move the zktrielib into a path where your linker could locate it.
 
-Prove
+Run prover
 ```shell
 cargo build --release --bin prove
 
 ./target/release/prove --help
 ```
-Or you can use it like the following:
+Could specify arguments as
+```shell
+cargo run --release --bin prove -- --params=./prover/test_params --trace=./prover/tests/traces/erc20/10_transfer.json
+```
+
+Run verifier
+```shell
+cargo build --release --bin verify
+
+./target/release/verify --help
+```
+Could specify arguments as
 ```shell
-cargo run --release --bin prove -- --params=./params --trace=./test.json
+cargo run --release --bin verify -- --params=./prover/test_params --vk=./proof_data/chunk.vkey
 ```
 
 ## License

bin/Cargo.toml

@@ -23,10 +23,6 @@ tokio = { version = "1", features = ["full"] }
 types = { path = "../types" }
 prover = { path = "../prover" }
 
-[[bin]]
-name = "setup"
-path = "src/setup.rs"
-
 [[bin]]
 name = "prove"
 path = "src/prove.rs"

bin/src/mock_testnet.rs

@@ -1,12 +1,16 @@
 use anyhow::Result;
 use ethers_providers::{Http, Provider};
 use itertools::Itertools;
-use prover::utils::init_env_and_log;
-use prover::zkevm::circuit::{
-    block_traces_to_witness_block, calculate_row_usage_of_witness_block, SuperCircuit,
-    SUB_CIRCUIT_NAMES,
+use prover::{
+    utils::init_env_and_log,
+    zkevm::{
+        circuit::{
+            block_traces_to_witness_block, calculate_row_usage_of_witness_block, SuperCircuit,
+            SUB_CIRCUIT_NAMES,
+        },
+        Prover,
+    },
 };
-use prover::zkevm::Prover;
 use reqwest::Url;
 use serde::Deserialize;
 use std::env;
@@ -97,7 +101,8 @@ async fn get_traces_by_batch_api(
     ))
 }
 
-/// Request block traces by API `scroll_getBlockTraceByNumberOrHash`. Return None for no more batches.
+/// Request block traces by API `scroll_getBlockTraceByNumberOrHash`. Return None for no more
+/// batches.
 async fn get_traces_by_block_api(
     provider: &Provider<Http>,
     setting: &Setting,

bin/src/prove.rs

@@ -1,34 +1,38 @@
 use clap::Parser;
 use log::info;
-use prover::utils::{get_block_trace_from_file, init_env_and_log, load_or_create_params};
-use prover::zkevm::{circuit::AGG_DEGREE, Prover};
-use std::fs;
-use std::path::PathBuf;
-use std::time::Instant;
+use prover::{
+    utils::{get_block_trace_from_file, init_env_and_log, load_or_download_params},
+    zkevm::{circuit::AGG_DEGREE, Prover},
+};
+use std::{fs, path::PathBuf, time::Instant};
 
 #[derive(Parser, Debug)]
 #[clap(author, version, about, long_about = None)]
 struct Args {
     /// Get params and write into file.
-    #[clap(short, long = "params")]
-    params_path: Option<String>,
+    #[clap(short, long = "params", default_value = "prover/test_params")]
+    params_path: String,
     /// Get BlockTrace from file or dir.
-    #[clap(short, long = "trace")]
-    trace_path: Option<String>,
+    #[clap(
+        short,
+        long = "trace",
+        default_value = "prover/tests/traces/empty.json"
+    )]
+    trace_path: String,
 }
 
 fn main() {
     init_env_and_log("prove");
-    std::env::set_var("VERIFY_CONFIG", "./zkevm/configs/verify_circuit.config");
+    std::env::set_var("VERIFY_CONFIG", "./prover/configs/verify_circuit.config");
 
     let args = Args::parse();
-    let agg_params = load_or_create_params(&args.params_path.unwrap(), *AGG_DEGREE)
+    let agg_params = load_or_download_params(&args.params_path, *AGG_DEGREE)
         .expect("failed to load or create params");
 
     let mut prover = Prover::from_params(agg_params);
 
     let mut traces = Vec::new();
-    let trace_path = PathBuf::from(&args.trace_path.unwrap());
+    let trace_path = PathBuf::from(&args.trace_path);
     if trace_path.is_dir() {
         for entry in fs::read_dir(trace_path).unwrap() {
             let path = entry.unwrap().path();
@@ -42,14 +46,17 @@ fn main() {
         traces.push(block_trace);
     }
 
-    let mut proof_path = PathBuf::from("agg.proof");
+    let mut proof_dir = PathBuf::from("proof_data");
 
     let now = Instant::now();
-    let agg_proof = prover
-        .create_agg_circuit_proof_batch(traces.as_slice())
-        .expect("cannot generate agg_proof");
-    info!("finish generating agg proof, elapsed: {:?}", now.elapsed());
-
-    fs::create_dir_all(&proof_path).unwrap();
-    agg_proof.dump(&mut proof_path).unwrap();
+    let chunk_proof = prover
+        .gen_chunk_proof(traces.as_slice())
+        .expect("cannot generate chunk proof");
+    info!(
+        "finish generating chunk proof, elapsed: {:?}",
+        now.elapsed()
+    );
+
+    fs::create_dir_all(&proof_dir).unwrap();
+    chunk_proof.dump(&mut proof_dir, "chunk").unwrap();
 }

bin/src/verify.rs

@@ -1,38 +1,43 @@
 use clap::Parser;
 use log::info;
-use prover::utils::{init_env_and_log, load_or_create_params};
-use prover::zkevm::circuit::{AGG_DEGREE, DEGREE};
-use prover::zkevm::{AggCircuitProof, Verifier};
-use std::fs::File;
-use std::io::Read;
+use prover::{
+    utils::{init_env_and_log, load_or_download_params},
+    zkevm::{
+        circuit::{AGG_DEGREE, DEGREE},
+        Verifier,
+    },
+    Proof,
+};
+use std::{fs::File, io::Read, path::PathBuf};
 
 #[derive(Parser, Debug)]
 #[clap(author, version, about, long_about = None)]
 struct Args {
     /// Get params from the file.
-    #[clap(short, long = "params")]
-    params_path: Option<String>,
+    #[clap(short, long = "params", default_value = "prover/test_params")]
+    params_path: String,
     /// Get vk from the file.
-    #[clap(long = "vk")]
-    vk_path: Option<String>,
+    #[clap(long = "vk", default_value = "proof_data/chunk.vkey")]
+    vk_path: String,
 }
 
 fn main() {
     init_env_and_log("verify");
-    std::env::set_var("VERIFY_CONFIG", "./zkevm/configs/verify_circuit.config");
+    std::env::set_var("VERIFY_CONFIG", "./prover/configs/verify_circuit.config");
 
     let args = Args::parse();
-    let params = load_or_create_params(&args.params_path.clone().unwrap(), *DEGREE)
+    let params = load_or_download_params(&args.params_path, *DEGREE)
         .expect("failed to load or create params");
-    let agg_params = load_or_create_params(&args.params_path.unwrap(), *AGG_DEGREE)
+    let agg_params = load_or_download_params(&args.params_path, *AGG_DEGREE)
         .expect("failed to load or create params");
-    let agg_vk = read_from_file(&args.vk_path.unwrap());
+    let agg_vk = read_from_file(&args.vk_path);
 
     let v = Verifier::from_params(params, agg_params, Some(agg_vk));
 
-    let proof_vec = read_from_file("agg.proof");
-    let proof = serde_json::from_slice::<AggCircuitProof>(proof_vec.as_slice()).unwrap();
-    let verified = v.verify_agg_circuit_proof(proof).is_ok();
+    let proof_path = PathBuf::from("proof_data").join("chunk_full_proof.json");
+    let proof_vec = read_from_file(&proof_path.to_string_lossy());
+    let proof = serde_json::from_slice::<Proof>(proof_vec.as_slice()).unwrap();
+    let verified = v.verify_chunk_proof(proof).is_ok();
     info!("verify agg proof: {}", verified)
 }
 

ffi/src/lib.rs

@@ -4,8 +4,10 @@ pub mod prove;
 pub mod verify;
 
 pub(crate) mod utils {
-    use std::ffi::{CStr, CString};
-    use std::os::raw::c_char;
+    use std::{
+        ffi::{CStr, CString},
+        os::raw::c_char,
+    };
 
     pub(crate) fn c_char_to_str(c: *const c_char) -> &'static str {
         let cstr = unsafe { CStr::from_ptr(c) };

ffi/src/prove.rs

@@ -1,7 +1,6 @@
 use crate::utils::{c_char_to_str, c_char_to_vec, vec_to_c_char};
 use libc::c_char;
-use prover::utils::init_env_and_log;
-use prover::zkevm;
+use prover::{utils::init_env_and_log, zkevm};
 use std::cell::OnceCell;
 use types::eth::BlockTrace;
 
@@ -19,27 +18,23 @@ pub unsafe extern "C" fn init_prover(params_path: *const c_char, _seed_path: *co
 
 /// # Safety
 #[no_mangle]
-pub unsafe extern "C" fn create_agg_proof(trace_char: *const c_char) -> *const c_char {
+pub unsafe extern "C" fn create_block_proof(trace_char: *const c_char) -> *const c_char {
     let trace_vec = c_char_to_vec(trace_char);
     let trace = serde_json::from_slice::<BlockTrace>(&trace_vec).unwrap();
-    let proof = PROVER
-        .get_mut()
-        .unwrap()
-        .create_agg_circuit_proof(&trace)
-        .unwrap();
+    let proof = PROVER.get_mut().unwrap().gen_chunk_proof(&[trace]).unwrap();
     let proof_bytes = serde_json::to_vec(&proof).unwrap();
     vec_to_c_char(proof_bytes)
 }
 
 /// # Safety
 #[no_mangle]
-pub unsafe extern "C" fn create_agg_proof_multi(trace_char: *const c_char) -> *const c_char {
+pub unsafe extern "C" fn create_chunk_proof(trace_char: *const c_char) -> *const c_char {
     let trace_vec = c_char_to_vec(trace_char);
     let traces = serde_json::from_slice::<Vec<BlockTrace>>(&trace_vec).unwrap();
     let proof = PROVER
         .get_mut()
         .unwrap()
-        .create_agg_circuit_proof_batch(traces.as_slice())
+        .gen_chunk_proof(traces.as_slice())
         .unwrap();
     let proof_bytes = serde_json::to_vec(&proof).unwrap();
     vec_to_c_char(proof_bytes)

ffi/src/verify.rs

@@ -1,9 +1,7 @@
 use crate::utils::{c_char_to_str, c_char_to_vec};
 use libc::c_char;
-use prover::utils::init_env_and_log;
-use prover::zkevm;
-use std::fs::File;
-use std::io::Read;
+use prover::{utils::init_env_and_log, zkevm, Proof};
+use std::{fs::File, io::Read};
 
 static mut VERIFIER: Option<&zkevm::Verifier> = None;
 
@@ -24,12 +22,9 @@ pub unsafe extern "C" fn init_verifier(params_path: *const c_char, agg_vk_path:
 
 /// # Safety
 #[no_mangle]
-pub unsafe extern "C" fn verify_agg_proof(proof: *const c_char) -> c_char {
+pub unsafe extern "C" fn verify_chunk_proof(proof: *const c_char) -> c_char {
     let proof_vec = c_char_to_vec(proof);
-    let agg_proof = serde_json::from_slice::<zkevm::AggCircuitProof>(proof_vec.as_slice()).unwrap();
-    let verified = VERIFIER
-        .unwrap()
-        .verify_agg_circuit_proof(agg_proof)
-        .is_ok();
+    let chunk_proof = serde_json::from_slice::<Proof>(proof_vec.as_slice()).unwrap();
+    let verified = VERIFIER.unwrap().verify_chunk_proof(chunk_proof).is_ok();
     verified as c_char
 }