From fdf58328daa946c62fcfab1eb4bdc189aedb36f4 Mon Sep 17 00:00:00 2001 From: Steven Gu Date: Fri, 7 Jul 2023 08:56:09 +0800 Subject: [PATCH 01/16] Add functions to generate aggregation snark, and test cases. --- Cargo.lock | 16 ++- README.md | 2 +- ffi/Cargo.toml | 2 +- ffi/src/prove.rs | 75 +++++++++++-- ffi/src/verify.rs | 60 +++++++--- prover/src/aggregator/prover.rs | 71 +++++++++++- prover/src/aggregator/prover/aggregation.rs | 20 ++++ prover/src/aggregator/prover/compression.rs | 12 +- prover/src/test_util/aggregator.rs | 25 +++++ prover/tests/aggregation_tests.rs | 116 ++++++++++++++++++++ types/Cargo.toml | 2 +- 11 files changed, 366 insertions(+), 35 deletions(-) create mode 100644 prover/src/aggregator/prover/aggregation.rs create mode 100644 prover/tests/aggregation_tests.rs diff --git a/Cargo.lock b/Cargo.lock index 2d84107c2..75e34d129 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -25,7 +25,7 @@ name = "aggregator" version = "0.1.0" source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=develop#ba4f7fe610e920c173e2acf3c276e8d048b97854" dependencies = [ - "ark-std", + "ark-std 0.4.0", "env_logger 0.10.0", "eth-types", "ethers-core 0.17.0", @@ -102,6 +102,16 @@ dependencies = [ "rand", ] +[[package]] +name = "ark-std" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94893f1e0c6eeab764ade8dc4c0db24caf4fe7cbbaafc0eba0a9030f447b5185" +dependencies = [ + "num-traits", + "rand", +] + [[package]] name = "array-init" version = "2.1.0" @@ -1437,7 +1447,7 @@ dependencies = [ [[package]] name = "ffi" -version = "0.4.0" +version = "0.5.0" dependencies = [ "libc", "log", @@ -1809,7 +1819,7 @@ name = "halo2_proofs" version = "0.2.0" source = "git+https://github.com/scroll-tech/halo2.git?branch=develop#9bf3562083dd9bed8a19f651b52bc810f5e2235f" dependencies = [ - "ark-std", + "ark-std 0.3.0", "blake2b_simd", "cfg-if 0.1.10", "crossbeam", diff --git a/README.md b/README.md index 052e4ca0e..4eb43a466 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ make download-setup -e degree=DEGREE params_dir=PARAMS_DIR ### Testing -`make test-chunk-prove` is the main testing entry point for the multi-level circuit constraint system of scroll-zkevm. Developers could understand how the system works by reading the codes of this test. +`make test-chunk-prove` and `make test-agg-prove` are the main testing entries for multi-level circuit constraint system of scroll-prover. Developers could understand how the system works by reading the codes of these tests. Besides it, `make test-inner-prove` could be used to test the first-level circuit, and `make-comp-prove` could be used to test two-layers compression circuits. diff --git a/ffi/Cargo.toml b/ffi/Cargo.toml index b8de7144a..35f2282ad 100644 --- a/ffi/Cargo.toml +++ b/ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ffi" -version = "0.4.0" +version = "0.5.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/ffi/src/prove.rs b/ffi/src/prove.rs index 3f8a5819b..801ab7e94 100644 --- a/ffi/src/prove.rs +++ b/ffi/src/prove.rs @@ -1,19 +1,32 @@ use crate::utils::{c_char_to_str, c_char_to_vec, vec_to_c_char}; use libc::c_char; -use prover::{utils::init_env_and_log, zkevm}; +use prover::{aggregator, config::ALL_AGG_DEGREES, utils::init_env_and_log, zkevm}; use std::cell::OnceCell; use types::eth::BlockTrace; -static mut PROVER: OnceCell = OnceCell::new(); +static mut CHUNK_PROVER: OnceCell = OnceCell::new(); +static mut AGG_PROVER: OnceCell = OnceCell::new(); +static mut AGG_CHUNK_TRACES: OnceCell>> = OnceCell::new(); /// # Safety #[no_mangle] -pub unsafe extern "C" fn init_prover(params_path: *const c_char, _seed_path: *const c_char) { - init_env_and_log("ffi_prove"); +pub unsafe extern "C" fn init_chunk_prover(params_dir: *const c_char) { + init_env_and_log("ffi_chunk_prove"); - let params_path = c_char_to_str(params_path); - let p = zkevm::Prover::from_params_dir(params_path); - PROVER.set(p).unwrap(); + let params_dir = c_char_to_str(params_dir); + let prover = zkevm::Prover::from_params_dir(params_dir); + CHUNK_PROVER.set(prover).unwrap(); +} + +/// # Safety +#[no_mangle] +pub unsafe extern "C" fn init_agg_prover(params_dir: *const c_char) { + init_env_and_log("ffi_agg_prove"); + + let params_dir = c_char_to_str(params_dir); + + let prover = aggregator::Prover::from_params_dir(params_dir, &ALL_AGG_DEGREES); + AGG_PROVER.set(prover).unwrap(); } /// # Safety @@ -21,7 +34,11 @@ pub unsafe extern "C" fn init_prover(params_path: *const c_char, _seed_path: *co pub unsafe extern "C" fn create_block_proof(trace_char: *const c_char) -> *const c_char { let trace_vec = c_char_to_vec(trace_char); let trace = serde_json::from_slice::(&trace_vec).unwrap(); - let proof = PROVER.get_mut().unwrap().gen_chunk_proof(&[trace]).unwrap(); + let proof = CHUNK_PROVER + .get_mut() + .unwrap() + .gen_chunk_proof(&[trace]) + .unwrap(); let proof_bytes = serde_json::to_vec(&proof).unwrap(); vec_to_c_char(proof_bytes) } @@ -31,7 +48,7 @@ pub unsafe extern "C" fn create_block_proof(trace_char: *const c_char) -> *const pub unsafe extern "C" fn create_chunk_proof(trace_char: *const c_char) -> *const c_char { let trace_vec = c_char_to_vec(trace_char); let traces = serde_json::from_slice::>(&trace_vec).unwrap(); - let proof = PROVER + let proof = CHUNK_PROVER .get_mut() .unwrap() .gen_chunk_proof(traces.as_slice()) @@ -39,3 +56,43 @@ pub unsafe extern "C" fn create_chunk_proof(trace_char: *const c_char) -> *const let proof_bytes = serde_json::to_vec(&proof).unwrap(); vec_to_c_char(proof_bytes) } + +/// # Safety +#[no_mangle] +pub unsafe extern "C" fn add_agg_chunk_trace(trace_char: *const c_char) { + let trace_vec = c_char_to_vec(trace_char); + let trace = serde_json::from_slice::>(&trace_vec).unwrap(); + + AGG_CHUNK_TRACES + .get_mut() + .or_else(|| { + AGG_CHUNK_TRACES.set(vec![]).unwrap(); + AGG_CHUNK_TRACES.get_mut() + }) + .unwrap() + .push(trace); +} + +/// # Safety +#[no_mangle] +pub unsafe extern "C" fn clear_agg_chunk_traces() { + if let Some(chunk_traces) = AGG_CHUNK_TRACES.get_mut() { + chunk_traces.clear(); + } +} + +/// # Safety +#[no_mangle] +pub unsafe extern "C" fn create_agg_proof() -> *const c_char { + // Consume the chunk traces (take and clear). + let chunk_traces = AGG_CHUNK_TRACES.take().unwrap(); + + let proof = AGG_PROVER + .get_mut() + .unwrap() + .gen_agg_proof(chunk_traces) + .unwrap(); + + let proof_bytes = serde_json::to_vec(&proof).unwrap(); + vec_to_c_char(proof_bytes) +} diff --git a/ffi/src/verify.rs b/ffi/src/verify.rs index 30b3f149b..ab919dd8f 100644 --- a/ffi/src/verify.rs +++ b/ffi/src/verify.rs @@ -1,30 +1,64 @@ use crate::utils::{c_char_to_str, c_char_to_vec}; use libc::c_char; -use prover::{utils::init_env_and_log, zkevm, Proof}; +use prover::{aggregator, config::AGG_LAYER4_DEGREE, utils::init_env_and_log, zkevm, Proof}; use std::{fs::File, io::Read}; -static mut VERIFIER: Option<&zkevm::Verifier> = None; +static mut CHUNK_VERIFIER: Option<&zkevm::Verifier> = None; +static mut AGG_VERIFIER: Option<&aggregator::Verifier> = None; /// # Safety #[no_mangle] -pub unsafe extern "C" fn init_verifier(params_path: *const c_char, agg_vk_path: *const c_char) { - init_env_and_log("ffi_verify"); +pub unsafe extern "C" fn init_chunk_verifier(params_dir: *const c_char, vk_path: *const c_char) { + init_env_and_log("ffi_chunk_verify"); - let params_path = c_char_to_str(params_path); - let agg_vk_path = c_char_to_str(agg_vk_path); - let mut f = File::open(agg_vk_path).unwrap(); - let mut agg_vk = vec![]; - f.read_to_end(&mut agg_vk).unwrap(); + let vk_path = c_char_to_str(vk_path); + let mut f = File::open(vk_path).unwrap(); + let mut vk = vec![]; + f.read_to_end(&mut vk).unwrap(); - let v = Box::new(zkevm::Verifier::from_params_dir(params_path, Some(agg_vk))); - VERIFIER = Some(Box::leak(v)) + let params_dir = c_char_to_str(params_dir); + let verifier = Box::new(zkevm::Verifier::from_params_dir(params_dir, Some(vk))); + + CHUNK_VERIFIER = Some(Box::leak(verifier)); +} + +/// # Safety +#[no_mangle] +pub unsafe extern "C" fn init_agg_verifier(params_dir: *const c_char, vk_path: *const c_char) { + init_env_and_log("ffi_agg_verify"); + + let vk_path = c_char_to_str(vk_path); + let mut f = File::open(vk_path).unwrap(); + let mut vk = vec![]; + f.read_to_end(&mut vk).unwrap(); + + let params_dir = c_char_to_str(params_dir); + let verifier = Box::new(aggregator::Verifier::from_params_dir( + params_dir, + *AGG_LAYER4_DEGREE, + Some(vk), + )); + + AGG_VERIFIER = Some(Box::leak(verifier)); } /// # Safety #[no_mangle] pub unsafe extern "C" fn verify_chunk_proof(proof: *const c_char) -> c_char { let proof_vec = c_char_to_vec(proof); - let chunk_proof = serde_json::from_slice::(proof_vec.as_slice()).unwrap(); - let verified = VERIFIER.unwrap().verify_chunk_proof(chunk_proof).is_ok(); + let proof = serde_json::from_slice::(proof_vec.as_slice()).unwrap(); + let verified = CHUNK_VERIFIER.unwrap().verify_chunk_proof(proof).is_ok(); + + verified as c_char +} + +/// # Safety +#[no_mangle] +pub unsafe extern "C" fn verify_agg_proof(proof: *const c_char) -> c_char { + let proof_vec = c_char_to_vec(proof); + let proof = serde_json::from_slice::(proof_vec.as_slice()).unwrap(); + + let verified = AGG_VERIFIER.unwrap().verify_agg_proof(proof).is_ok(); + verified as c_char } diff --git a/prover/src/aggregator/prover.rs b/prover/src/aggregator/prover.rs index 023e5c0f4..274eaf120 100644 --- a/prover/src/aggregator/prover.rs +++ b/prover/src/aggregator/prover.rs @@ -1,11 +1,22 @@ -use crate::utils::{load_params, param_path_for_degree}; +use crate::{ + config::{AGG_LAYER1_DEGREE, AGG_LAYER2_DEGREE, AGG_LAYER3_DEGREE, AGG_LAYER4_DEGREE}, + utils::{chunk_trace_to_witness_block, gen_rng, load_params, param_path_for_degree}, + zkevm::circuit::SuperCircuit, + Proof, +}; +use anyhow::Result; use halo2_proofs::{ halo2curves::bn256::{Bn256, G1Affine}, plonk::ProvingKey, poly::{commitment::Params, kzg::commitment::ParamsKZG}, }; -use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::{ + collections::{BTreeMap, BTreeSet, HashMap}, + env::set_var, +}; +use types::eth::BlockTrace; +mod aggregation; mod chunk; mod common; mod compression; @@ -63,4 +74,60 @@ impl Prover { pk_map: HashMap::new(), } } + + pub fn gen_agg_proof(&mut self, chunk_traces: Vec>) -> Result { + // Convert chunk traces to witness blocks. + let witness_blocks = chunk_traces + .into_iter() + .map(chunk_trace_to_witness_block) + .collect::>>()?; + + // Convert witness blocks to chunk hashes. + let chunk_hashes: Vec<_> = witness_blocks.iter().map(Into::into).collect(); + + // Generate chunk snarks. + let chunk_snarks = witness_blocks + .into_iter() + .map(|block| self.gen_chunk_snark::(&block)) + .collect::>>()?; + + // Generate compression wide snarks (layer-1). + set_var("VERIFY_CONFIG", "./configs/agg_layer1.config"); + let layer1_snarks = chunk_snarks + .into_iter() + .map(|snark| { + let rng = gen_rng(); + self.gen_comp_snark("agg_layer1", true, *AGG_LAYER1_DEGREE, rng, snark) + }) + .collect::>>()?; + + // Generate compression thin snarks (layer-2). + set_var("VERIFY_CONFIG", "./configs/agg_layer2.config"); + let layer2_snarks: Vec<_> = layer1_snarks + .into_iter() + .map(|snark| { + let rng = gen_rng(); + self.gen_comp_snark("agg_layer2", false, *AGG_LAYER2_DEGREE, rng, snark) + }) + .collect::>>()?; + + // Generate aggregation snark (layer-3). + set_var("VERIFY_CONFIG", "./configs/agg_layer3.config"); + let rng = gen_rng(); + let layer3_snark = self.gen_agg_snark( + "agg_layer3", + *AGG_LAYER3_DEGREE, + rng, + &chunk_hashes, + &layer2_snarks, + ); + + // Generate final compression snarks (layer-4). + set_var("VERIFY_CONFIG", "./configs/agg_layer4.config"); + let rng = gen_rng(); + let layer4_snark = + self.gen_comp_snark("agg_layer4", false, *AGG_LAYER4_DEGREE, rng, layer3_snark)?; + + Proof::from_snark(&self.pk_map["agg_layer4"], &layer4_snark) + } } diff --git a/prover/src/aggregator/prover/aggregation.rs b/prover/src/aggregator/prover/aggregation.rs new file mode 100644 index 000000000..4d7483ab5 --- /dev/null +++ b/prover/src/aggregator/prover/aggregation.rs @@ -0,0 +1,20 @@ +use super::Prover; +use aggregator::{AggregationCircuit, ChunkHash}; +use rand::Rng; +use snark_verifier_sdk::Snark; + +impl Prover { + pub fn gen_agg_snark( + &mut self, + id: &str, + degree: u32, + mut rng: impl Rng + Send, + chunk_hashes: &[ChunkHash], + prev_snarks: &[Snark], + ) -> Snark { + let circuit = + AggregationCircuit::new(self.params(degree), prev_snarks, &mut rng, chunk_hashes); + + self.gen_snark(id, degree, &mut rng, circuit) + } +} diff --git a/prover/src/aggregator/prover/compression.rs b/prover/src/aggregator/prover/compression.rs index 868475634..d07e80a54 100644 --- a/prover/src/aggregator/prover/compression.rs +++ b/prover/src/aggregator/prover/compression.rs @@ -1,7 +1,7 @@ use super::Prover; use crate::Proof; use aggregator::CompressionCircuit; -use anyhow::{anyhow, Result}; +use anyhow::Result; use rand::Rng; use snark_verifier_sdk::Snark; @@ -14,8 +14,9 @@ impl Prover { mut rng: impl Rng + Send, prev_snark: Snark, ) -> Result { - let circuit = CompressionCircuit::new(self.params(degree), prev_snark, is_fresh, &mut rng) - .map_err(|err| anyhow!("Failed to construct compression circuit: {err:?}"))?; + // gupeng + let circuit = CompressionCircuit::new(self.params(degree), prev_snark, is_fresh, &mut rng); + // .map_err(|err| anyhow!("Failed to construct compression circuit: {err:?}"))?; Ok(self.gen_snark(id, degree, &mut rng, circuit)) } @@ -28,8 +29,9 @@ impl Prover { mut rng: impl Rng + Send, prev_snark: Snark, ) -> Result { - let circuit = CompressionCircuit::new(self.params(degree), prev_snark, is_fresh, &mut rng) - .map_err(|err| anyhow!("Failed to construct compression circuit: {err:?}"))?; + // gupeng + let circuit = CompressionCircuit::new(self.params(degree), prev_snark, is_fresh, &mut rng); + // .map_err(|err| anyhow!("Failed to construct compression circuit: {err:?}"))?; self.gen_evm_proof(id, degree, &mut rng, circuit) } diff --git a/prover/src/test_util/aggregator.rs b/prover/src/test_util/aggregator.rs index 8f58b7588..ae91f52fd 100644 --- a/prover/src/test_util/aggregator.rs +++ b/prover/src/test_util/aggregator.rs @@ -5,11 +5,32 @@ use crate::{ zkevm::circuit::SuperCircuit, Proof, }; +use aggregator::ChunkHash; use halo2_proofs::halo2curves::bn256::Fr; use snark_verifier_sdk::Snark; use std::{env::set_var, path::PathBuf}; use zkevm_circuits::evm_circuit::witness::Block; +pub fn load_or_gen_agg_snark( + output_dir: &str, + id: &str, + degree: u32, + prover: &mut Prover, + chunk_hashes: &[ChunkHash], + prev_snarks: &[Snark], +) -> Snark { + set_var("VERIFY_CONFIG", format!("./configs/{id}.config")); + let file_path = format!("{output_dir}/{id}_snark.json"); + + load_snark(&file_path).unwrap().unwrap_or_else(|| { + let rng = gen_rng(); + let snark = prover.gen_agg_snark(id, degree, rng, chunk_hashes, prev_snarks); + write_snark(&file_path, &snark); + + snark + }) +} + pub fn gen_comp_evm_proof( output_dir: &str, id: &str, @@ -18,6 +39,8 @@ pub fn gen_comp_evm_proof( prover: &mut Prover, prev_snark: Snark, ) -> Proof { + // gupeng + set_var("VERIFY_CONFIG", format!("./configs/{id}.config")); set_var("COMPRESSION_CONFIG", format!("./configs/{id}.config")); let rng = gen_rng(); @@ -55,6 +78,8 @@ pub fn load_or_gen_comp_snark( prover: &mut Prover, prev_snark: Snark, ) -> Snark { + // gupeng + set_var("VERIFY_CONFIG", format!("./configs/{id}.config")); set_var("COMPRESSION_CONFIG", format!("./configs/{id}.config")); let file_path = format!("{output_dir}/{id}_snark.json"); diff --git a/prover/tests/aggregation_tests.rs b/prover/tests/aggregation_tests.rs new file mode 100644 index 000000000..92ca03397 --- /dev/null +++ b/prover/tests/aggregation_tests.rs @@ -0,0 +1,116 @@ +use aggregator::CompressionCircuit; +use prover::{ + aggregator::{Prover, Verifier}, + config::{ + AGG_LAYER1_DEGREE, AGG_LAYER2_DEGREE, AGG_LAYER3_DEGREE, AGG_LAYER4_DEGREE, ALL_AGG_DEGREES, + }, + test_util::{ + aggregator::{ + gen_comp_evm_proof, load_or_gen_agg_snark, load_or_gen_chunk_snark, + load_or_gen_comp_snark, + }, + load_block_traces_for_test, PARAMS_DIR, + }, + utils::{chunk_trace_to_witness_block, init_env_and_log}, +}; +use std::{env::set_var, path::Path}; + +#[cfg(feature = "prove_verify")] +#[test] +fn test_agg_prove_verify() { + // Init, load block traces and construct prover. + + let output_dir = init_env_and_log("agg_tests"); + log::info!("Initialized ENV and created output-dir {output_dir}"); + + let mut chunk_traces = vec![]; + set_var("TRACE_PATH", "./tests/traces/erc20/1_transfer.json"); + chunk_traces.push(load_block_traces_for_test().1); + set_var("TRACE_PATH", "./tests/traces/erc20/10_transfer.json"); + chunk_traces.push(load_block_traces_for_test().1); + log::info!("Loaded chunk-traces"); + + // Convert chunk traces to witness blocks. + let witness_blocks: Vec<_> = chunk_traces + .into_iter() + .map(|trace| chunk_trace_to_witness_block(trace).unwrap()) + .collect(); + log::info!("Got witness-blocks"); + + // Convert witness blocks to chunk hashes. + let chunk_hashes: Vec<_> = witness_blocks.iter().map(Into::into).collect(); + log::info!("Got chunk-hashes"); + + let mut prover = Prover::from_params_dir(PARAMS_DIR, &*ALL_AGG_DEGREES); + log::info!("Constructed prover"); + + // Load or generate chunk snarks. + let chunk_snarks: Vec<_> = witness_blocks + .into_iter() + .enumerate() + .map(|(i, block)| load_or_gen_chunk_snark(&output_dir, &i.to_string(), &mut prover, block)) + .collect(); + log::info!("Got chunk-snarks"); + + // Load or generate compression wide snarks (layer-1). + let layer1_snarks: Vec<_> = chunk_snarks + .into_iter() + .map(|snark| { + load_or_gen_comp_snark( + &output_dir, + "agg_layer1", + true, + *AGG_LAYER1_DEGREE, + &mut prover, + snark, + ) + }) + .collect(); + log::info!("Got compression wide snarks (layer-1)"); + + // Load or generate compression thin snarks (layer-2). + let layer2_snarks: Vec<_> = layer1_snarks + .into_iter() + .map(|snark| { + load_or_gen_comp_snark( + &output_dir, + "agg_layer2", + false, + *AGG_LAYER2_DEGREE, + &mut prover, + snark, + ) + }) + .collect(); + log::info!("Got compression thin snarks (layer-2)"); + + // Load or generate aggregation snark (layer-3). + let layer3_snark = load_or_gen_agg_snark( + &output_dir, + "agg_layer3", + *AGG_LAYER3_DEGREE, + &mut prover, + &chunk_hashes, + &layer2_snarks, + ); + log::info!("Got aggregation snark (layer-3)"); + + // Load or generate compression EVM proof (layer-4). + let proof = gen_comp_evm_proof( + &output_dir, + "agg_layer4", + false, + *AGG_LAYER4_DEGREE, + &mut prover, + layer3_snark, + ); + log::info!("Got compression EVM proof (layer-4)"); + + // Construct verifier and EVM verify. + let params = prover.params(*AGG_LAYER4_DEGREE).clone(); + let vk = prover.pk("agg_layer4").unwrap().get_vk().clone(); + let verifier = Verifier::new(params, Some(vk)); + let yul_file_path = format!("{output_dir}/agg_verifier.yul"); + verifier.evm_verify::(&proof, Some(Path::new(&yul_file_path))); + log::info!("Finish EVM verify"); +} diff --git a/types/Cargo.toml b/types/Cargo.toml index ab73ebe5b..421373fae 100644 --- a/types/Cargo.toml +++ b/types/Cargo.toml @@ -4,7 +4,7 @@ version = "0.4.0" edition = "2021" [dependencies] -eth-types = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "develop" } +eth-types = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "scroll-prover-test-agg-circuit" } base64 = "0.13.0" blake2 = "0.10.3" ethers-core = "0.17.0" From 50f6eb7958a45d839f9577fc487534e68e54b07d Mon Sep 17 00:00:00 2001 From: Steven Gu Date: Tue, 11 Jul 2023 22:47:58 +0800 Subject: [PATCH 02/16] Update to use zkevm-circuits aggregation-circuit branch. --- Cargo.lock | 34 ++++++++++++---------------------- prover/Cargo.toml | 12 ++++++------ types/Cargo.toml | 2 +- 3 files changed, 19 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 75e34d129..82096694f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,9 +23,9 @@ dependencies = [ [[package]] name = "aggregator" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=develop#ba4f7fe610e920c173e2acf3c276e8d048b97854" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=dynamic-proof-aggregation#e061dd9aeb1a59d22dce6fcd8382be4ae0ef9bae" dependencies = [ - "ark-std 0.4.0", + "ark-std", "env_logger 0.10.0", "eth-types", "ethers-core 0.17.0", @@ -102,16 +102,6 @@ dependencies = [ "rand", ] -[[package]] -name = "ark-std" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94893f1e0c6eeab764ade8dc4c0db24caf4fe7cbbaafc0eba0a9030f447b5185" -dependencies = [ - "num-traits", - "rand", -] - [[package]] name = "array-init" version = "2.1.0" @@ -428,7 +418,7 @@ checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" [[package]] name = "bus-mapping" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=develop#ba4f7fe610e920c173e2acf3c276e8d048b97854" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=dynamic-proof-aggregation#e061dd9aeb1a59d22dce6fcd8382be4ae0ef9bae" dependencies = [ "eth-types", "ethers-core 0.17.0", @@ -1128,7 +1118,7 @@ dependencies = [ [[package]] name = "eth-types" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=develop#ba4f7fe610e920c173e2acf3c276e8d048b97854" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=dynamic-proof-aggregation#e061dd9aeb1a59d22dce6fcd8382be4ae0ef9bae" dependencies = [ "ethers-core 0.17.0", "ethers-signers", @@ -1394,7 +1384,7 @@ dependencies = [ [[package]] name = "external-tracer" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=develop#ba4f7fe610e920c173e2acf3c276e8d048b97854" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=dynamic-proof-aggregation#e061dd9aeb1a59d22dce6fcd8382be4ae0ef9bae" dependencies = [ "eth-types", "geth-utils", @@ -1623,7 +1613,7 @@ dependencies = [ [[package]] name = "gadgets" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=develop#ba4f7fe610e920c173e2acf3c276e8d048b97854" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=dynamic-proof-aggregation#e061dd9aeb1a59d22dce6fcd8382be4ae0ef9bae" dependencies = [ "digest 0.7.6", "eth-types", @@ -1663,7 +1653,7 @@ dependencies = [ [[package]] name = "geth-utils" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=develop#ba4f7fe610e920c173e2acf3c276e8d048b97854" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=dynamic-proof-aggregation#e061dd9aeb1a59d22dce6fcd8382be4ae0ef9bae" dependencies = [ "env_logger 0.9.3", "gobuild 0.1.0-alpha.2 (git+https://github.com/scroll-tech/gobuild.git)", @@ -1819,7 +1809,7 @@ name = "halo2_proofs" version = "0.2.0" source = "git+https://github.com/scroll-tech/halo2.git?branch=develop#9bf3562083dd9bed8a19f651b52bc810f5e2235f" dependencies = [ - "ark-std 0.3.0", + "ark-std", "blake2b_simd", "cfg-if 0.1.10", "crossbeam", @@ -2253,7 +2243,7 @@ dependencies = [ [[package]] name = "keccak256" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=develop#ba4f7fe610e920c173e2acf3c276e8d048b97854" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=dynamic-proof-aggregation#e061dd9aeb1a59d22dce6fcd8382be4ae0ef9bae" dependencies = [ "env_logger 0.9.3", "eth-types", @@ -2453,7 +2443,7 @@ dependencies = [ [[package]] name = "mock" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=develop#ba4f7fe610e920c173e2acf3c276e8d048b97854" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=dynamic-proof-aggregation#e061dd9aeb1a59d22dce6fcd8382be4ae0ef9bae" dependencies = [ "eth-types", "ethers-core 0.17.0", @@ -2468,7 +2458,7 @@ dependencies = [ [[package]] name = "mpt-zktrie" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=develop#ba4f7fe610e920c173e2acf3c276e8d048b97854" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=dynamic-proof-aggregation#e061dd9aeb1a59d22dce6fcd8382be4ae0ef9bae" dependencies = [ "bus-mapping", "eth-types", @@ -4693,7 +4683,7 @@ checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" [[package]] name = "zkevm-circuits" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=develop#ba4f7fe610e920c173e2acf3c276e8d048b97854" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=dynamic-proof-aggregation#e061dd9aeb1a59d22dce6fcd8382be4ae0ef9bae" dependencies = [ "array-init", "bus-mapping", diff --git a/prover/Cargo.toml b/prover/Cargo.toml index 8b19125f2..ce6cf7d86 100644 --- a/prover/Cargo.toml +++ b/prover/Cargo.toml @@ -8,12 +8,12 @@ edition = "2021" [dependencies] halo2_proofs = { git = "https://github.com/privacy-scaling-explorations/halo2.git", tag = "v2023_02_02" } -aggregator = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "develop" } -bus-mapping = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "develop" } -eth-types = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "develop" } -zkevm-circuits = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "develop", default-features = false, features = ["test","scroll","scroll-trace","shanghai"] } -mpt-zktrie = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "develop" } -mock = { git = "https://github.com/scroll-tech/zkevm-circuits", branch = "develop" } +aggregator = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "dynamic-proof-aggregation" } +bus-mapping = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "dynamic-proof-aggregation" } +eth-types = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "dynamic-proof-aggregation" } +zkevm-circuits = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "dynamic-proof-aggregation", default-features = false, features = ["test","scroll","scroll-trace","shanghai"] } +mpt-zktrie = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "dynamic-proof-aggregation" } +mock = { git = "https://github.com/scroll-tech/zkevm-circuits", branch = "dynamic-proof-aggregation" } snark-verifier = { git = "https://github.com/scroll-tech/snark-verifier", branch = "develop" } snark-verifier-sdk = { git = "https://github.com/scroll-tech/snark-verifier", branch = "develop" } diff --git a/types/Cargo.toml b/types/Cargo.toml index 421373fae..82dff8313 100644 --- a/types/Cargo.toml +++ b/types/Cargo.toml @@ -4,7 +4,7 @@ version = "0.4.0" edition = "2021" [dependencies] -eth-types = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "scroll-prover-test-agg-circuit" } +eth-types = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "dynamic-proof-aggregation" } base64 = "0.13.0" blake2 = "0.10.3" ethers-core = "0.17.0" From af076ddba8cf4b985bd44c4b5fa76588798b6253 Mon Sep 17 00:00:00 2001 From: Steven Gu Date: Wed, 12 Jul 2023 07:41:01 +0800 Subject: [PATCH 03/16] Use new zkevm-circuits branch. --- Cargo.lock | 20 ++++++++++---------- prover/Cargo.toml | 12 ++++++------ prover/src/aggregator/prover/aggregation.rs | 4 ++++ prover/src/aggregator/prover/compression.rs | 12 +++++------- types/Cargo.toml | 2 +- 5 files changed, 26 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 82096694f..2d28366e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,7 +23,7 @@ dependencies = [ [[package]] name = "aggregator" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=dynamic-proof-aggregation#e061dd9aeb1a59d22dce6fcd8382be4ae0ef9bae" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#53bfb3e95f526fbe13ac78b73075dad30df18931" dependencies = [ "ark-std", "env_logger 0.10.0", @@ -418,7 +418,7 @@ checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" [[package]] name = "bus-mapping" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=dynamic-proof-aggregation#e061dd9aeb1a59d22dce6fcd8382be4ae0ef9bae" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#53bfb3e95f526fbe13ac78b73075dad30df18931" dependencies = [ "eth-types", "ethers-core 0.17.0", @@ -1118,7 +1118,7 @@ dependencies = [ [[package]] name = "eth-types" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=dynamic-proof-aggregation#e061dd9aeb1a59d22dce6fcd8382be4ae0ef9bae" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#53bfb3e95f526fbe13ac78b73075dad30df18931" dependencies = [ "ethers-core 0.17.0", "ethers-signers", @@ -1384,7 +1384,7 @@ dependencies = [ [[package]] name = "external-tracer" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=dynamic-proof-aggregation#e061dd9aeb1a59d22dce6fcd8382be4ae0ef9bae" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#53bfb3e95f526fbe13ac78b73075dad30df18931" dependencies = [ "eth-types", "geth-utils", @@ -1613,7 +1613,7 @@ dependencies = [ [[package]] name = "gadgets" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=dynamic-proof-aggregation#e061dd9aeb1a59d22dce6fcd8382be4ae0ef9bae" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#53bfb3e95f526fbe13ac78b73075dad30df18931" dependencies = [ "digest 0.7.6", "eth-types", @@ -1653,7 +1653,7 @@ dependencies = [ [[package]] name = "geth-utils" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=dynamic-proof-aggregation#e061dd9aeb1a59d22dce6fcd8382be4ae0ef9bae" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#53bfb3e95f526fbe13ac78b73075dad30df18931" dependencies = [ "env_logger 0.9.3", "gobuild 0.1.0-alpha.2 (git+https://github.com/scroll-tech/gobuild.git)", @@ -2243,7 +2243,7 @@ dependencies = [ [[package]] name = "keccak256" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=dynamic-proof-aggregation#e061dd9aeb1a59d22dce6fcd8382be4ae0ef9bae" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#53bfb3e95f526fbe13ac78b73075dad30df18931" dependencies = [ "env_logger 0.9.3", "eth-types", @@ -2443,7 +2443,7 @@ dependencies = [ [[package]] name = "mock" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=dynamic-proof-aggregation#e061dd9aeb1a59d22dce6fcd8382be4ae0ef9bae" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#53bfb3e95f526fbe13ac78b73075dad30df18931" dependencies = [ "eth-types", "ethers-core 0.17.0", @@ -2458,7 +2458,7 @@ dependencies = [ [[package]] name = "mpt-zktrie" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=dynamic-proof-aggregation#e061dd9aeb1a59d22dce6fcd8382be4ae0ef9bae" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#53bfb3e95f526fbe13ac78b73075dad30df18931" dependencies = [ "bus-mapping", "eth-types", @@ -4683,7 +4683,7 @@ checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" [[package]] name = "zkevm-circuits" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=dynamic-proof-aggregation#e061dd9aeb1a59d22dce6fcd8382be4ae0ef9bae" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#53bfb3e95f526fbe13ac78b73075dad30df18931" dependencies = [ "array-init", "bus-mapping", diff --git a/prover/Cargo.toml b/prover/Cargo.toml index ce6cf7d86..acd73705d 100644 --- a/prover/Cargo.toml +++ b/prover/Cargo.toml @@ -8,12 +8,12 @@ edition = "2021" [dependencies] halo2_proofs = { git = "https://github.com/privacy-scaling-explorations/halo2.git", tag = "v2023_02_02" } -aggregator = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "dynamic-proof-aggregation" } -bus-mapping = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "dynamic-proof-aggregation" } -eth-types = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "dynamic-proof-aggregation" } -zkevm-circuits = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "dynamic-proof-aggregation", default-features = false, features = ["test","scroll","scroll-trace","shanghai"] } -mpt-zktrie = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "dynamic-proof-aggregation" } -mock = { git = "https://github.com/scroll-tech/zkevm-circuits", branch = "dynamic-proof-aggregation" } +aggregator = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "integrate-dynamic-proof-agg" } +bus-mapping = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "integrate-dynamic-proof-agg" } +eth-types = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "integrate-dynamic-proof-agg" } +zkevm-circuits = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "integrate-dynamic-proof-agg", default-features = false, features = ["test","scroll","scroll-trace","shanghai"] } +mpt-zktrie = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "integrate-dynamic-proof-agg" } +mock = { git = "https://github.com/scroll-tech/zkevm-circuits", branch = "integrate-dynamic-proof-agg" } snark-verifier = { git = "https://github.com/scroll-tech/snark-verifier", branch = "develop" } snark-verifier-sdk = { git = "https://github.com/scroll-tech/snark-verifier", branch = "develop" } diff --git a/prover/src/aggregator/prover/aggregation.rs b/prover/src/aggregator/prover/aggregation.rs index 4d7483ab5..54465d613 100644 --- a/prover/src/aggregator/prover/aggregation.rs +++ b/prover/src/aggregator/prover/aggregation.rs @@ -12,9 +12,13 @@ impl Prover { chunk_hashes: &[ChunkHash], prev_snarks: &[Snark], ) -> Snark { + todo!() + +/* let circuit = AggregationCircuit::new(self.params(degree), prev_snarks, &mut rng, chunk_hashes); self.gen_snark(id, degree, &mut rng, circuit) +*/ } } diff --git a/prover/src/aggregator/prover/compression.rs b/prover/src/aggregator/prover/compression.rs index d07e80a54..b990e2361 100644 --- a/prover/src/aggregator/prover/compression.rs +++ b/prover/src/aggregator/prover/compression.rs @@ -1,7 +1,7 @@ use super::Prover; use crate::Proof; use aggregator::CompressionCircuit; -use anyhow::Result; +use anyhow::{anyhow, Result}; use rand::Rng; use snark_verifier_sdk::Snark; @@ -14,9 +14,8 @@ impl Prover { mut rng: impl Rng + Send, prev_snark: Snark, ) -> Result { - // gupeng - let circuit = CompressionCircuit::new(self.params(degree), prev_snark, is_fresh, &mut rng); - // .map_err(|err| anyhow!("Failed to construct compression circuit: {err:?}"))?; + let circuit = CompressionCircuit::new(self.params(degree), prev_snark, is_fresh, &mut rng) + .map_err(|err| anyhow!("Failed to construct compression circuit: {err:?}"))?; Ok(self.gen_snark(id, degree, &mut rng, circuit)) } @@ -29,9 +28,8 @@ impl Prover { mut rng: impl Rng + Send, prev_snark: Snark, ) -> Result { - // gupeng - let circuit = CompressionCircuit::new(self.params(degree), prev_snark, is_fresh, &mut rng); - // .map_err(|err| anyhow!("Failed to construct compression circuit: {err:?}"))?; + let circuit = CompressionCircuit::new(self.params(degree), prev_snark, is_fresh, &mut rng) + .map_err(|err| anyhow!("Failed to construct compression circuit: {err:?}"))?; self.gen_evm_proof(id, degree, &mut rng, circuit) } diff --git a/types/Cargo.toml b/types/Cargo.toml index 82dff8313..6e8fa53c3 100644 --- a/types/Cargo.toml +++ b/types/Cargo.toml @@ -4,7 +4,7 @@ version = "0.4.0" edition = "2021" [dependencies] -eth-types = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "dynamic-proof-aggregation" } +eth-types = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "integrate-dynamic-proof-agg" } base64 = "0.13.0" blake2 = "0.10.3" ethers-core = "0.17.0" From 16b24b9fdf675e7601ec77265516c938e16c9d0b Mon Sep 17 00:00:00 2001 From: Steven Gu Date: Mon, 10 Jul 2023 12:01:13 +0800 Subject: [PATCH 04/16] Update compression-tests to use deserialized vk. --- Cargo.lock | 2 +- Cargo.toml | 2 +- prover/src/aggregator/prover/common.rs | 3 ++- prover/tests/compression_tests.rs | 23 ++++++++++++++++++++--- 4 files changed, 24 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2d28366e9..e6a42384b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1807,7 +1807,7 @@ dependencies = [ [[package]] name = "halo2_proofs" version = "0.2.0" -source = "git+https://github.com/scroll-tech/halo2.git?branch=develop#9bf3562083dd9bed8a19f651b52bc810f5e2235f" +source = "git+https://github.com/scroll-tech/halo2.git?branch=feat/disable_selector_compression#031495f62172c144f392848043da2064261c13dc" dependencies = [ "ark-std", "blake2b_simd", diff --git a/Cargo.toml b/Cargo.toml index d14d3648a..73db3297b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ members = [ ] [patch."https://github.com/privacy-scaling-explorations/halo2.git"] -halo2_proofs = { git = "https://github.com/scroll-tech/halo2.git", branch = "develop" } +halo2_proofs = { git = "https://github.com/scroll-tech/halo2.git", branch = "feat/disable_selector_compression" } [patch."https://github.com/privacy-scaling-explorations/poseidon.git"] poseidon = { git = "https://github.com/scroll-tech/poseidon.git", branch = "scroll-dev-0220" } [patch."https://github.com/privacy-scaling-explorations/halo2wrong.git"] diff --git a/prover/src/aggregator/prover/common.rs b/prover/src/aggregator/prover/common.rs index 139a2ae85..bb5fcf260 100644 --- a/prover/src/aggregator/prover/common.rs +++ b/prover/src/aggregator/prover/common.rs @@ -96,7 +96,8 @@ impl Prover { } tick(&format!("Before generate outer pk of {}", &id)); - let pk = gen_pk(self.params(degree), circuit, None); + // let pk = gen_pk(self.params(degree), circuit, None); + let pk = keygen_pk2(self.params(degree), circuit).unwrap(); tick(&format!("After generate outer pk of {}", &id)); self.pk_map.insert(id.to_string(), pk); diff --git a/prover/tests/compression_tests.rs b/prover/tests/compression_tests.rs index 6258e616a..e3c007635 100644 --- a/prover/tests/compression_tests.rs +++ b/prover/tests/compression_tests.rs @@ -1,14 +1,18 @@ use aggregator::CompressionCircuit; +use halo2_proofs::{halo2curves::bn256::G1Affine, plonk::VerifyingKey, SerdeFormat}; use prover::{ aggregator::{Prover, Verifier}, config::{AGG_LAYER1_DEGREE, AGG_LAYER2_DEGREE, INNER_DEGREE}, + io::serialize_vk, test_util::{ aggregator::{gen_comp_evm_proof, load_or_gen_chunk_snark, load_or_gen_comp_snark}, load_block_traces_for_test, PARAMS_DIR, }, utils::{chunk_trace_to_witness_block, init_env_and_log}, }; -use std::path::Path; +use snark_verifier::pcs::kzg::{Bdfg21, Kzg}; +use snark_verifier_sdk::{evm_verify, gen_evm_verifier, verify_snark_shplonk, CircuitExt}; +use std::{io::Cursor, path::Path}; #[cfg(feature = "prove_verify")] #[test] @@ -56,10 +60,23 @@ fn test_comp_prove_verify() { ); log::info!("Got compression EVM proof (layer-2)"); + // Test vk deserialization. + let vk1 = prover.pk("agg_layer2").unwrap().get_vk().clone(); + let raw_vk1 = serialize_vk(&vk1); + let mut vk2 = VerifyingKey::::read::<_, CompressionCircuit>( + &mut Cursor::new(&raw_vk1), + SerdeFormat::Processed, + ) + .unwrap(); + let raw_vk2 = serialize_vk(&vk2); + assert_eq!(raw_vk1, raw_vk2); + log::error!("test - vk1 = {:#?}", vk1); + log::error!("test - vk2 = {:#?}", vk2); + // Construct verifier and EVM verify. let params = prover.params(*AGG_LAYER2_DEGREE).clone(); - let vk = prover.pk("agg_layer2").unwrap().get_vk().clone(); - let verifier = Verifier::new(params, Some(vk)); + // let vk = prover.pk("agg_layer2").unwrap().get_vk().clone(); + let verifier = Verifier::new(params, Some(vk2)); let yul_file_path = format!("{output_dir}/comp_verifier.yul"); verifier.evm_verify::(&proof, Some(Path::new(&yul_file_path))); log::info!("Finish EVM verify"); From 4a13b826c3299efaa52cf1e5a5d6f8dee7ab4260 Mon Sep 17 00:00:00 2001 From: Steven Gu Date: Wed, 12 Jul 2023 12:08:21 +0800 Subject: [PATCH 05/16] Update aggregation-tests. --- Cargo.lock | 20 ++++---- prover/src/aggregator/prover.rs | 7 +-- prover/src/aggregator/prover/aggregation.rs | 21 ++++---- prover/src/aggregator/prover/chunk.rs | 8 ++- prover/src/aggregator/prover/common.rs | 51 ++++++------------- prover/src/aggregator/prover/compression.rs | 6 +-- prover/src/aggregator/prover/padding.rs | 17 +++++++ prover/src/test_util/aggregator.rs | 55 +++++++++++++-------- prover/tests/aggregation_tests.rs | 42 +++++++++++----- prover/tests/compression_tests.rs | 4 +- 10 files changed, 131 insertions(+), 100 deletions(-) create mode 100644 prover/src/aggregator/prover/padding.rs diff --git a/Cargo.lock b/Cargo.lock index e6a42384b..3d831db03 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,7 +23,7 @@ dependencies = [ [[package]] name = "aggregator" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#53bfb3e95f526fbe13ac78b73075dad30df18931" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d2579a143458d3f3e7c5f0c8aaa1a03537ef6e74" dependencies = [ "ark-std", "env_logger 0.10.0", @@ -418,7 +418,7 @@ checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" [[package]] name = "bus-mapping" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#53bfb3e95f526fbe13ac78b73075dad30df18931" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d2579a143458d3f3e7c5f0c8aaa1a03537ef6e74" dependencies = [ "eth-types", "ethers-core 0.17.0", @@ -1118,7 +1118,7 @@ dependencies = [ [[package]] name = "eth-types" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#53bfb3e95f526fbe13ac78b73075dad30df18931" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d2579a143458d3f3e7c5f0c8aaa1a03537ef6e74" dependencies = [ "ethers-core 0.17.0", "ethers-signers", @@ -1384,7 +1384,7 @@ dependencies = [ [[package]] name = "external-tracer" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#53bfb3e95f526fbe13ac78b73075dad30df18931" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d2579a143458d3f3e7c5f0c8aaa1a03537ef6e74" dependencies = [ "eth-types", "geth-utils", @@ -1613,7 +1613,7 @@ dependencies = [ [[package]] name = "gadgets" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#53bfb3e95f526fbe13ac78b73075dad30df18931" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d2579a143458d3f3e7c5f0c8aaa1a03537ef6e74" dependencies = [ "digest 0.7.6", "eth-types", @@ -1653,7 +1653,7 @@ dependencies = [ [[package]] name = "geth-utils" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#53bfb3e95f526fbe13ac78b73075dad30df18931" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d2579a143458d3f3e7c5f0c8aaa1a03537ef6e74" dependencies = [ "env_logger 0.9.3", "gobuild 0.1.0-alpha.2 (git+https://github.com/scroll-tech/gobuild.git)", @@ -2243,7 +2243,7 @@ dependencies = [ [[package]] name = "keccak256" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#53bfb3e95f526fbe13ac78b73075dad30df18931" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d2579a143458d3f3e7c5f0c8aaa1a03537ef6e74" dependencies = [ "env_logger 0.9.3", "eth-types", @@ -2443,7 +2443,7 @@ dependencies = [ [[package]] name = "mock" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#53bfb3e95f526fbe13ac78b73075dad30df18931" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d2579a143458d3f3e7c5f0c8aaa1a03537ef6e74" dependencies = [ "eth-types", "ethers-core 0.17.0", @@ -2458,7 +2458,7 @@ dependencies = [ [[package]] name = "mpt-zktrie" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#53bfb3e95f526fbe13ac78b73075dad30df18931" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d2579a143458d3f3e7c5f0c8aaa1a03537ef6e74" dependencies = [ "bus-mapping", "eth-types", @@ -4683,7 +4683,7 @@ checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" [[package]] name = "zkevm-circuits" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#53bfb3e95f526fbe13ac78b73075dad30df18931" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d2579a143458d3f3e7c5f0c8aaa1a03537ef6e74" dependencies = [ "array-init", "bus-mapping", diff --git a/prover/src/aggregator/prover.rs b/prover/src/aggregator/prover.rs index 274eaf120..b9a2136be 100644 --- a/prover/src/aggregator/prover.rs +++ b/prover/src/aggregator/prover.rs @@ -20,6 +20,7 @@ mod aggregation; mod chunk; mod common; mod compression; +mod padding; #[derive(Debug)] pub struct Prover { @@ -83,7 +84,7 @@ impl Prover { .collect::>>()?; // Convert witness blocks to chunk hashes. - let chunk_hashes: Vec<_> = witness_blocks.iter().map(Into::into).collect(); + let real_chunk_hashes: Vec<_> = witness_blocks.iter().map(Into::into).collect(); // Generate chunk snarks. let chunk_snarks = witness_blocks @@ -118,9 +119,9 @@ impl Prover { "agg_layer3", *AGG_LAYER3_DEGREE, rng, - &chunk_hashes, + &real_chunk_hashes, &layer2_snarks, - ); + )?; // Generate final compression snarks (layer-4). set_var("VERIFY_CONFIG", "./configs/agg_layer4.config"); diff --git a/prover/src/aggregator/prover/aggregation.rs b/prover/src/aggregator/prover/aggregation.rs index 54465d613..4eb98630a 100644 --- a/prover/src/aggregator/prover/aggregation.rs +++ b/prover/src/aggregator/prover/aggregation.rs @@ -1,5 +1,6 @@ use super::Prover; -use aggregator::{AggregationCircuit, ChunkHash}; +use aggregator::{AggregationCircuit, BatchHash, ChunkHash}; +use anyhow::Result; use rand::Rng; use snark_verifier_sdk::Snark; @@ -9,16 +10,18 @@ impl Prover { id: &str, degree: u32, mut rng: impl Rng + Send, - chunk_hashes: &[ChunkHash], - prev_snarks: &[Snark], - ) -> Snark { - todo!() + real_chunk_hashes: &[ChunkHash], + real_and_padding_snarks: &[Snark], + ) -> Result { + let batch_hash = BatchHash::construct(real_chunk_hashes); -/* - let circuit = - AggregationCircuit::new(self.params(degree), prev_snarks, &mut rng, chunk_hashes); + let circuit = AggregationCircuit::new( + self.params(degree), + real_and_padding_snarks, + &mut rng, + batch_hash, + ); self.gen_snark(id, degree, &mut rng, circuit) -*/ } } diff --git a/prover/src/aggregator/prover/chunk.rs b/prover/src/aggregator/prover/chunk.rs index cd7715700..6f3d1af3d 100644 --- a/prover/src/aggregator/prover/chunk.rs +++ b/prover/src/aggregator/prover/chunk.rs @@ -1,5 +1,6 @@ use super::Prover; use crate::{ + config::INNER_DEGREE, utils::{gen_rng, metric_of_witness_block}, zkevm::circuit::TargetCircuit, }; @@ -19,11 +20,8 @@ impl Prover { ); let (circuit, _instance) = C::from_witness_block(witness_block)?; - log::info!("Create {} proof", C::name()); - - let (params, pk) = self.inner_params_and_pk::(&C::dummy_inner_circuit())?; - - // Generate the SNARK proof for inner circuit. + let (params, pk) = + self.params_and_pk(&C::name(), &C::dummy_inner_circuit(), *INNER_DEGREE)?; let snark = gen_snark_shplonk(params, pk, circuit, &mut gen_rng(), None::); Ok(snark) diff --git a/prover/src/aggregator/prover/common.rs b/prover/src/aggregator/prover/common.rs index bb5fcf260..59ef777bd 100644 --- a/prover/src/aggregator/prover/common.rs +++ b/prover/src/aggregator/prover/common.rs @@ -1,5 +1,5 @@ use super::Prover; -use crate::{config::INNER_DEGREE, utils::tick, zkevm::circuit::TargetCircuit, Proof}; +use crate::Proof; use anyhow::Result; use halo2_proofs::{ halo2curves::bn256::{Bn256, Fr, G1Affine}, @@ -7,7 +7,7 @@ use halo2_proofs::{ poly::{commitment::Params, kzg::commitment::ParamsKZG}, }; use rand::Rng; -use snark_verifier_sdk::{gen_evm_proof_shplonk, gen_pk, gen_snark_shplonk, CircuitExt, Snark}; +use snark_verifier_sdk::{gen_evm_proof_shplonk, gen_snark_shplonk, CircuitExt, Snark}; impl Prover { pub fn gen_snark>( @@ -16,10 +16,10 @@ impl Prover { degree: u32, rng: &mut (impl Rng + Send), circuit: C, - ) -> Snark { - let (params, pk) = self.outer_params_and_pk(id, &circuit, degree); + ) -> Result { + let (params, pk) = self.params_and_pk(id, &circuit, degree)?; - gen_snark_shplonk(params, pk, circuit, rng, None::<&str>) + Ok(gen_snark_shplonk(params, pk, circuit, rng, None::)) } pub fn gen_evm_proof>( @@ -29,7 +29,7 @@ impl Prover { rng: &mut (impl Rng + Send), circuit: C, ) -> Result { - let (params, pk) = self.outer_params_and_pk(id, &circuit, degree); + let (params, pk) = self.params_and_pk(id, &circuit, degree)?; let instances = circuit.instances(); let num_instance = circuit.num_instance(); @@ -38,26 +38,6 @@ impl Prover { Proof::new(pk, proof, &instances, Some(num_instance)) } - // TODO: test if it could use `outer_params_and_pk`. - pub fn inner_params_and_pk( - &mut self, - circuit: &::Inner, - ) -> Result<(&ParamsKZG, &ProvingKey)> { - let id = C::name(); - - // Reuse pk. - if !self.pk_map.contains_key(&id) { - tick(&format!("Before generate inner pk of {}", &id)); - let pk = keygen_pk2(self.params(*INNER_DEGREE), circuit)?; - tick(&format!("After generate inner pk of {}", &id)); - - self.pk_map.insert(id.clone(), pk); - } - assert!(self.params_map.contains_key(&*INNER_DEGREE)); - - Ok((&self.params_map[&*INNER_DEGREE], &self.pk_map[&id])) - } - pub fn params(&mut self, degree: u32) -> &ParamsKZG { if self.params_map.contains_key(°ree) { return &self.params_map[°ree]; @@ -65,7 +45,7 @@ impl Prover { log::warn!("Optimization: download params{degree} to params dir"); - tick(&format!("Before generate params of {degree}")); + log::info!("Before generate params of {degree}"); let mut new_params = self .params_map .range(degree..) @@ -74,7 +54,7 @@ impl Prover { .1 .clone(); new_params.downsize(degree); - tick(&format!("After generate params of {degree}")); + log::info!("After generate params of {degree}"); self.params_map.insert(degree, new_params); &self.params_map[°ree] @@ -84,24 +64,23 @@ impl Prover { self.pk_map.get(id) } - pub fn outer_params_and_pk>( + pub fn params_and_pk>( &mut self, id: &str, circuit: &C, degree: u32, - ) -> (&ParamsKZG, &ProvingKey) { + ) -> Result<(&ParamsKZG, &ProvingKey)> { // Reuse pk. if self.pk_map.contains_key(id) { - return (&self.params_map[°ree], &self.pk_map[id]); + return Ok((&self.params_map[°ree], &self.pk_map[id])); } - tick(&format!("Before generate outer pk of {}", &id)); - // let pk = gen_pk(self.params(degree), circuit, None); - let pk = keygen_pk2(self.params(degree), circuit).unwrap(); - tick(&format!("After generate outer pk of {}", &id)); + log::info!("Before generate pk of {}", &id); + let pk = keygen_pk2(self.params(degree), circuit)?; + log::info!("After generate pk of {}", &id); self.pk_map.insert(id.to_string(), pk); - (&self.params_map[°ree], &self.pk_map[id]) + Ok((&self.params_map[°ree], &self.pk_map[id])) } } diff --git a/prover/src/aggregator/prover/compression.rs b/prover/src/aggregator/prover/compression.rs index b990e2361..9a04df849 100644 --- a/prover/src/aggregator/prover/compression.rs +++ b/prover/src/aggregator/prover/compression.rs @@ -15,9 +15,9 @@ impl Prover { prev_snark: Snark, ) -> Result { let circuit = CompressionCircuit::new(self.params(degree), prev_snark, is_fresh, &mut rng) - .map_err(|err| anyhow!("Failed to construct compression circuit: {err:?}"))?; + .map_err(|err| anyhow!("Failed to construct compression circuit: {err:?}"))?; - Ok(self.gen_snark(id, degree, &mut rng, circuit)) + self.gen_snark(id, degree, &mut rng, circuit) } pub fn gen_comp_evm_proof( @@ -29,7 +29,7 @@ impl Prover { prev_snark: Snark, ) -> Result { let circuit = CompressionCircuit::new(self.params(degree), prev_snark, is_fresh, &mut rng) - .map_err(|err| anyhow!("Failed to construct compression circuit: {err:?}"))?; + .map_err(|err| anyhow!("Failed to construct compression circuit: {err:?}"))?; self.gen_evm_proof(id, degree, &mut rng, circuit) } diff --git a/prover/src/aggregator/prover/padding.rs b/prover/src/aggregator/prover/padding.rs new file mode 100644 index 000000000..8c2c550b0 --- /dev/null +++ b/prover/src/aggregator/prover/padding.rs @@ -0,0 +1,17 @@ +use super::Prover; +use crate::{config::INNER_DEGREE, utils::gen_rng}; +use aggregator::{ChunkHash, DummyChunkHashCircuit}; +use anyhow::Result; +use snark_verifier_sdk::{gen_snark_shplonk, Snark}; + +impl Prover { + pub fn gen_padding_chunk_snark(&mut self, last_real_chunk_hash: &ChunkHash) -> Result { + let chunk_hash = ChunkHash::dummy_chunk_hash(last_real_chunk_hash); + let circuit = DummyChunkHashCircuit::new(chunk_hash); + + let (params, pk) = self.params_and_pk("padding_chunk", &circuit, *INNER_DEGREE)?; + let snark = gen_snark_shplonk(params, pk, circuit, &mut gen_rng(), None::); + + Ok(snark) + } +} diff --git a/prover/src/test_util/aggregator.rs b/prover/src/test_util/aggregator.rs index ae91f52fd..71abffe2a 100644 --- a/prover/src/test_util/aggregator.rs +++ b/prover/src/test_util/aggregator.rs @@ -16,15 +16,17 @@ pub fn load_or_gen_agg_snark( id: &str, degree: u32, prover: &mut Prover, - chunk_hashes: &[ChunkHash], - prev_snarks: &[Snark], + real_chunk_hashes: &[ChunkHash], + real_and_padding_snarks: &[Snark], ) -> Snark { - set_var("VERIFY_CONFIG", format!("./configs/{id}.config")); + set_var("AGGREGATION_CONFIG", format!("./configs/{id}.config")); let file_path = format!("{output_dir}/{id}_snark.json"); load_snark(&file_path).unwrap().unwrap_or_else(|| { let rng = gen_rng(); - let snark = prover.gen_agg_snark(id, degree, rng, chunk_hashes, prev_snarks); + let snark = prover + .gen_agg_snark(id, degree, rng, real_chunk_hashes, real_and_padding_snarks) + .unwrap(); write_snark(&file_path, &snark); snark @@ -39,8 +41,6 @@ pub fn gen_comp_evm_proof( prover: &mut Prover, prev_snark: Snark, ) -> Proof { - // gupeng - set_var("VERIFY_CONFIG", format!("./configs/{id}.config")); set_var("COMPRESSION_CONFIG", format!("./configs/{id}.config")); let rng = gen_rng(); @@ -52,17 +52,21 @@ pub fn gen_comp_evm_proof( proof } -pub fn load_or_gen_chunk_snark( +pub fn load_or_gen_comp_snark( output_dir: &str, id: &str, + is_fresh: bool, + degree: u32, prover: &mut Prover, - witness_block: Block, + prev_snark: Snark, ) -> Snark { - let file_path = format!("{output_dir}/{id}_chunk_snark.json"); + set_var("COMPRESSION_CONFIG", format!("./configs/{id}.config")); + let file_path = format!("{output_dir}/{id}_snark.json"); load_snark(&file_path).unwrap().unwrap_or_else(|| { + let rng = gen_rng(); let snark = prover - .gen_chunk_snark::(&witness_block) + .gen_comp_snark(id, is_fresh, degree, rng, prev_snark) .unwrap(); write_snark(&file_path, &snark); @@ -70,23 +74,34 @@ pub fn load_or_gen_chunk_snark( }) } -pub fn load_or_gen_comp_snark( +pub fn load_or_gen_padding_chunk_snark( output_dir: &str, id: &str, - is_fresh: bool, - degree: u32, prover: &mut Prover, - prev_snark: Snark, + last_real_chunk_hash: &ChunkHash, ) -> Snark { - // gupeng - set_var("VERIFY_CONFIG", format!("./configs/{id}.config")); - set_var("COMPRESSION_CONFIG", format!("./configs/{id}.config")); - let file_path = format!("{output_dir}/{id}_snark.json"); + let file_path = format!("{output_dir}/{id}_chunk_snark.json"); load_snark(&file_path).unwrap().unwrap_or_else(|| { - let rng = gen_rng(); let snark = prover - .gen_comp_snark(id, is_fresh, degree, rng, prev_snark) + .gen_padding_chunk_snark(last_real_chunk_hash) + .unwrap(); + write_snark(&file_path, &snark); + + snark + }) +} +pub fn load_or_gen_real_chunk_snark( + output_dir: &str, + id: &str, + prover: &mut Prover, + witness_block: Block, +) -> Snark { + let file_path = format!("{output_dir}/{id}_chunk_snark.json"); + + load_snark(&file_path).unwrap().unwrap_or_else(|| { + let snark = prover + .gen_chunk_snark::(&witness_block) .unwrap(); write_snark(&file_path, &snark); diff --git a/prover/tests/aggregation_tests.rs b/prover/tests/aggregation_tests.rs index 92ca03397..4e0a1278e 100644 --- a/prover/tests/aggregation_tests.rs +++ b/prover/tests/aggregation_tests.rs @@ -1,4 +1,4 @@ -use aggregator::CompressionCircuit; +use aggregator::{CompressionCircuit, MAX_AGG_SNARKS}; use prover::{ aggregator::{Prover, Verifier}, config::{ @@ -6,14 +6,14 @@ use prover::{ }, test_util::{ aggregator::{ - gen_comp_evm_proof, load_or_gen_agg_snark, load_or_gen_chunk_snark, - load_or_gen_comp_snark, + gen_comp_evm_proof, load_or_gen_agg_snark, load_or_gen_comp_snark, + load_or_gen_padding_chunk_snark, load_or_gen_real_chunk_snark, }, load_block_traces_for_test, PARAMS_DIR, }, utils::{chunk_trace_to_witness_block, init_env_and_log}, }; -use std::{env::set_var, path::Path}; +use std::{env::set_var, iter::repeat, path::Path}; #[cfg(feature = "prove_verify")] #[test] @@ -38,19 +38,31 @@ fn test_agg_prove_verify() { log::info!("Got witness-blocks"); // Convert witness blocks to chunk hashes. - let chunk_hashes: Vec<_> = witness_blocks.iter().map(Into::into).collect(); - log::info!("Got chunk-hashes"); + let real_chunk_hashes: Vec<_> = witness_blocks.iter().map(Into::into).collect(); + log::info!("Got real-chunk-hashes"); let mut prover = Prover::from_params_dir(PARAMS_DIR, &*ALL_AGG_DEGREES); log::info!("Constructed prover"); - // Load or generate chunk snarks. - let chunk_snarks: Vec<_> = witness_blocks + // Load or generate real-chunk snarks. + let mut chunk_snarks: Vec<_> = witness_blocks .into_iter() .enumerate() - .map(|(i, block)| load_or_gen_chunk_snark(&output_dir, &i.to_string(), &mut prover, block)) + .map(|(i, block)| { + load_or_gen_real_chunk_snark(&output_dir, &format!("real{i}"), &mut prover, block) + }) .collect(); - log::info!("Got chunk-snarks"); + log::info!("Got real-chunk-snarks"); + + // Load or generate padding-chunk snark. + let padding_chunk_snark = load_or_gen_padding_chunk_snark( + &output_dir, + "padding", + &mut prover, + real_chunk_hashes.last().unwrap(), + ); + chunk_snarks.push(padding_chunk_snark); + log::info!("Got padding-chunk-snark"); // Load or generate compression wide snarks (layer-1). let layer1_snarks: Vec<_> = chunk_snarks @@ -69,7 +81,7 @@ fn test_agg_prove_verify() { log::info!("Got compression wide snarks (layer-1)"); // Load or generate compression thin snarks (layer-2). - let layer2_snarks: Vec<_> = layer1_snarks + let mut layer2_snarks: Vec<_> = layer1_snarks .into_iter() .map(|snark| { load_or_gen_comp_snark( @@ -84,13 +96,19 @@ fn test_agg_prove_verify() { .collect(); log::info!("Got compression thin snarks (layer-2)"); + // Extend to MAX_AGG_SNARKS by copying the last padding snark. + let padding_layer2_snarks = repeat(layer2_snarks.last().unwrap()) + .take(MAX_AGG_SNARKS - layer2_snarks.len()) + .collect(); + layer2_snarks.extend(padding_layer2_snarks); + // Load or generate aggregation snark (layer-3). let layer3_snark = load_or_gen_agg_snark( &output_dir, "agg_layer3", *AGG_LAYER3_DEGREE, &mut prover, - &chunk_hashes, + &real_chunk_hashes, &layer2_snarks, ); log::info!("Got aggregation snark (layer-3)"); diff --git a/prover/tests/compression_tests.rs b/prover/tests/compression_tests.rs index e3c007635..88d6a0351 100644 --- a/prover/tests/compression_tests.rs +++ b/prover/tests/compression_tests.rs @@ -5,7 +5,7 @@ use prover::{ config::{AGG_LAYER1_DEGREE, AGG_LAYER2_DEGREE, INNER_DEGREE}, io::serialize_vk, test_util::{ - aggregator::{gen_comp_evm_proof, load_or_gen_chunk_snark, load_or_gen_comp_snark}, + aggregator::{gen_comp_evm_proof, load_or_gen_comp_snark, load_or_gen_real_chunk_snark}, load_block_traces_for_test, PARAMS_DIR, }, utils::{chunk_trace_to_witness_block, init_env_and_log}, @@ -35,7 +35,7 @@ fn test_comp_prove_verify() { log::info!("Constructed prover"); // Load or generate chunk snark. - let chunk_snark = load_or_gen_chunk_snark(&output_dir, "comp", &mut prover, witness_block); + let chunk_snark = load_or_gen_real_chunk_snark(&output_dir, "comp", &mut prover, witness_block); log::info!("Got chunk-snark"); // Load or generate compression wide snark (layer-1). From c2d873fd10deef1dadc8ee92c6a3f1a8e506829a Mon Sep 17 00:00:00 2001 From: Steven Gu Date: Wed, 12 Jul 2023 12:18:42 +0800 Subject: [PATCH 06/16] Fix --- prover/tests/aggregation_tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prover/tests/aggregation_tests.rs b/prover/tests/aggregation_tests.rs index 4e0a1278e..9378029c2 100644 --- a/prover/tests/aggregation_tests.rs +++ b/prover/tests/aggregation_tests.rs @@ -97,7 +97,7 @@ fn test_agg_prove_verify() { log::info!("Got compression thin snarks (layer-2)"); // Extend to MAX_AGG_SNARKS by copying the last padding snark. - let padding_layer2_snarks = repeat(layer2_snarks.last().unwrap()) + let padding_layer2_snarks: Vec<_> = repeat(layer2_snarks.last().unwrap().clone()) .take(MAX_AGG_SNARKS - layer2_snarks.len()) .collect(); layer2_snarks.extend(padding_layer2_snarks); From 682c46d719ba7d0f27c0f7e8c2764081f8d13875 Mon Sep 17 00:00:00 2001 From: Steven Gu Date: Wed, 12 Jul 2023 14:59:00 +0800 Subject: [PATCH 07/16] Update --- ffi/src/prove.rs | 2 +- prover/src/aggregator/prover.rs | 6 +++++- prover/src/aggregator/prover/chunk.rs | 24 +++++++++++++++++++++++- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/ffi/src/prove.rs b/ffi/src/prove.rs index 801ab7e94..8b62b7e46 100644 --- a/ffi/src/prove.rs +++ b/ffi/src/prove.rs @@ -90,7 +90,7 @@ pub unsafe extern "C" fn create_agg_proof() -> *const c_char { let proof = AGG_PROVER .get_mut() .unwrap() - .gen_agg_proof(chunk_traces) + .gen_agg_proof(chunk_traces, None) .unwrap(); let proof_bytes = serde_json::to_vec(&proof).unwrap(); diff --git a/prover/src/aggregator/prover.rs b/prover/src/aggregator/prover.rs index b9a2136be..ab7476592 100644 --- a/prover/src/aggregator/prover.rs +++ b/prover/src/aggregator/prover.rs @@ -76,7 +76,11 @@ impl Prover { } } - pub fn gen_agg_proof(&mut self, chunk_traces: Vec>) -> Result { + pub fn gen_agg_proof( + &mut self, + chunk_traces: Vec>, + output_dir: Option<&str>, + ) -> Result { // Convert chunk traces to witness blocks. let witness_blocks = chunk_traces .into_iter() diff --git a/prover/src/aggregator/prover/chunk.rs b/prover/src/aggregator/prover/chunk.rs index 6f3d1af3d..773abb420 100644 --- a/prover/src/aggregator/prover/chunk.rs +++ b/prover/src/aggregator/prover/chunk.rs @@ -1,8 +1,9 @@ use super::Prover; use crate::{ config::INNER_DEGREE, + io::{load_snark, write_snark}, utils::{gen_rng, metric_of_witness_block}, - zkevm::circuit::TargetCircuit, + zkevm::circuit::{SuperCircuit, TargetCircuit}, }; use anyhow::Result; use halo2_proofs::halo2curves::bn256::Fr; @@ -26,4 +27,25 @@ impl Prover { Ok(snark) } + + pub fn load_or_gen_chunk_snark( + &mut self, + id: &str, + witness_block: Block, + output_dir: Option<&str>, + ) -> Result { + let file_path = format!("{}/{}_chunk_snark.json", output_dir.unwrap_or_default(), id); + + match output_dir.and_then(|_| load_snark(&file_path).ok().flatten()) { + Some(snark) => Ok(snark), + None => { + let res = self.gen_chunk_snark::(&witness_block); + if let (Some(_), Ok(snark)) = (output_dir, &res) { + write_snark(&file_path, &snark); + } + + res + } + } + } } From 2c227569c34bbfcf0de56539c64f05ec5ef3ea6d Mon Sep 17 00:00:00 2001 From: Steven Gu Date: Wed, 12 Jul 2023 15:47:40 +0800 Subject: [PATCH 08/16] Make aggregation-tests work. --- Cargo.lock | 22 ++++---- Cargo.toml | 2 +- prover/src/aggregator/prover.rs | 10 ++-- prover/src/aggregator/prover/chunk.rs | 2 +- prover/src/aggregator/prover/padding.rs | 4 +- prover/src/test_util/aggregator.rs | 75 +++++++++++++------------ prover/tests/aggregation_tests.rs | 48 ++++++++-------- prover/tests/compression_tests.rs | 6 +- 8 files changed, 88 insertions(+), 81 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3d831db03..b4b10f3c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,7 +23,7 @@ dependencies = [ [[package]] name = "aggregator" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d2579a143458d3f3e7c5f0c8aaa1a03537ef6e74" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d864ddf2c1914dec70475e565571deb58499e57d" dependencies = [ "ark-std", "env_logger 0.10.0", @@ -418,7 +418,7 @@ checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" [[package]] name = "bus-mapping" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d2579a143458d3f3e7c5f0c8aaa1a03537ef6e74" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d864ddf2c1914dec70475e565571deb58499e57d" dependencies = [ "eth-types", "ethers-core 0.17.0", @@ -1118,7 +1118,7 @@ dependencies = [ [[package]] name = "eth-types" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d2579a143458d3f3e7c5f0c8aaa1a03537ef6e74" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d864ddf2c1914dec70475e565571deb58499e57d" dependencies = [ "ethers-core 0.17.0", "ethers-signers", @@ -1384,7 +1384,7 @@ dependencies = [ [[package]] name = "external-tracer" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d2579a143458d3f3e7c5f0c8aaa1a03537ef6e74" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d864ddf2c1914dec70475e565571deb58499e57d" dependencies = [ "eth-types", "geth-utils", @@ -1613,7 +1613,7 @@ dependencies = [ [[package]] name = "gadgets" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d2579a143458d3f3e7c5f0c8aaa1a03537ef6e74" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d864ddf2c1914dec70475e565571deb58499e57d" dependencies = [ "digest 0.7.6", "eth-types", @@ -1653,7 +1653,7 @@ dependencies = [ [[package]] name = "geth-utils" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d2579a143458d3f3e7c5f0c8aaa1a03537ef6e74" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d864ddf2c1914dec70475e565571deb58499e57d" dependencies = [ "env_logger 0.9.3", "gobuild 0.1.0-alpha.2 (git+https://github.com/scroll-tech/gobuild.git)", @@ -1807,7 +1807,7 @@ dependencies = [ [[package]] name = "halo2_proofs" version = "0.2.0" -source = "git+https://github.com/scroll-tech/halo2.git?branch=feat/disable_selector_compression#031495f62172c144f392848043da2064261c13dc" +source = "git+https://github.com/scroll-tech/halo2.git?branch=develop#a00de9e0b5311647f0046ead97b96dede4e01945" dependencies = [ "ark-std", "blake2b_simd", @@ -2243,7 +2243,7 @@ dependencies = [ [[package]] name = "keccak256" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d2579a143458d3f3e7c5f0c8aaa1a03537ef6e74" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d864ddf2c1914dec70475e565571deb58499e57d" dependencies = [ "env_logger 0.9.3", "eth-types", @@ -2443,7 +2443,7 @@ dependencies = [ [[package]] name = "mock" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d2579a143458d3f3e7c5f0c8aaa1a03537ef6e74" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d864ddf2c1914dec70475e565571deb58499e57d" dependencies = [ "eth-types", "ethers-core 0.17.0", @@ -2458,7 +2458,7 @@ dependencies = [ [[package]] name = "mpt-zktrie" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d2579a143458d3f3e7c5f0c8aaa1a03537ef6e74" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d864ddf2c1914dec70475e565571deb58499e57d" dependencies = [ "bus-mapping", "eth-types", @@ -4683,7 +4683,7 @@ checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" [[package]] name = "zkevm-circuits" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d2579a143458d3f3e7c5f0c8aaa1a03537ef6e74" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d864ddf2c1914dec70475e565571deb58499e57d" dependencies = [ "array-init", "bus-mapping", diff --git a/Cargo.toml b/Cargo.toml index 73db3297b..d14d3648a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ members = [ ] [patch."https://github.com/privacy-scaling-explorations/halo2.git"] -halo2_proofs = { git = "https://github.com/scroll-tech/halo2.git", branch = "feat/disable_selector_compression" } +halo2_proofs = { git = "https://github.com/scroll-tech/halo2.git", branch = "develop" } [patch."https://github.com/privacy-scaling-explorations/poseidon.git"] poseidon = { git = "https://github.com/scroll-tech/poseidon.git", branch = "scroll-dev-0220" } [patch."https://github.com/privacy-scaling-explorations/halo2wrong.git"] diff --git a/prover/src/aggregator/prover.rs b/prover/src/aggregator/prover.rs index ab7476592..d8371a5c7 100644 --- a/prover/src/aggregator/prover.rs +++ b/prover/src/aggregator/prover.rs @@ -79,7 +79,7 @@ impl Prover { pub fn gen_agg_proof( &mut self, chunk_traces: Vec>, - output_dir: Option<&str>, + _output_dir: Option<&str>, ) -> Result { // Convert chunk traces to witness blocks. let witness_blocks = chunk_traces @@ -97,7 +97,7 @@ impl Prover { .collect::>>()?; // Generate compression wide snarks (layer-1). - set_var("VERIFY_CONFIG", "./configs/agg_layer1.config"); + set_var("COMPRESSION_CONFIG", "./configs/agg_layer1.config"); let layer1_snarks = chunk_snarks .into_iter() .map(|snark| { @@ -107,7 +107,7 @@ impl Prover { .collect::>>()?; // Generate compression thin snarks (layer-2). - set_var("VERIFY_CONFIG", "./configs/agg_layer2.config"); + set_var("COMPRESSION_CONFIG", "./configs/agg_layer2.config"); let layer2_snarks: Vec<_> = layer1_snarks .into_iter() .map(|snark| { @@ -117,7 +117,7 @@ impl Prover { .collect::>>()?; // Generate aggregation snark (layer-3). - set_var("VERIFY_CONFIG", "./configs/agg_layer3.config"); + set_var("AGGREGATION_CONFIG", "./configs/agg_layer3.config"); let rng = gen_rng(); let layer3_snark = self.gen_agg_snark( "agg_layer3", @@ -128,7 +128,7 @@ impl Prover { )?; // Generate final compression snarks (layer-4). - set_var("VERIFY_CONFIG", "./configs/agg_layer4.config"); + set_var("COMPRESSION_CONFIG", "./configs/agg_layer4.config"); let rng = gen_rng(); let layer4_snark = self.gen_comp_snark("agg_layer4", false, *AGG_LAYER4_DEGREE, rng, layer3_snark)?; diff --git a/prover/src/aggregator/prover/chunk.rs b/prover/src/aggregator/prover/chunk.rs index 773abb420..64b910057 100644 --- a/prover/src/aggregator/prover/chunk.rs +++ b/prover/src/aggregator/prover/chunk.rs @@ -41,7 +41,7 @@ impl Prover { None => { let res = self.gen_chunk_snark::(&witness_block); if let (Some(_), Ok(snark)) = (output_dir, &res) { - write_snark(&file_path, &snark); + write_snark(&file_path, snark); } res diff --git a/prover/src/aggregator/prover/padding.rs b/prover/src/aggregator/prover/padding.rs index 8c2c550b0..341ee4aa6 100644 --- a/prover/src/aggregator/prover/padding.rs +++ b/prover/src/aggregator/prover/padding.rs @@ -5,11 +5,11 @@ use anyhow::Result; use snark_verifier_sdk::{gen_snark_shplonk, Snark}; impl Prover { - pub fn gen_padding_chunk_snark(&mut self, last_real_chunk_hash: &ChunkHash) -> Result { + pub fn gen_padding_snark(&mut self, last_real_chunk_hash: &ChunkHash) -> Result { let chunk_hash = ChunkHash::dummy_chunk_hash(last_real_chunk_hash); let circuit = DummyChunkHashCircuit::new(chunk_hash); - let (params, pk) = self.params_and_pk("padding_chunk", &circuit, *INNER_DEGREE)?; + let (params, pk) = self.params_and_pk("padding", &circuit, *INNER_DEGREE)?; let snark = gen_snark_shplonk(params, pk, circuit, &mut gen_rng(), None::); Ok(snark) diff --git a/prover/src/test_util/aggregator.rs b/prover/src/test_util/aggregator.rs index 71abffe2a..d92e1f350 100644 --- a/prover/src/test_util/aggregator.rs +++ b/prover/src/test_util/aggregator.rs @@ -11,37 +11,16 @@ use snark_verifier_sdk::Snark; use std::{env::set_var, path::PathBuf}; use zkevm_circuits::evm_circuit::witness::Block; -pub fn load_or_gen_agg_snark( - output_dir: &str, - id: &str, - degree: u32, - prover: &mut Prover, - real_chunk_hashes: &[ChunkHash], - real_and_padding_snarks: &[Snark], -) -> Snark { - set_var("AGGREGATION_CONFIG", format!("./configs/{id}.config")); - let file_path = format!("{output_dir}/{id}_snark.json"); - - load_snark(&file_path).unwrap().unwrap_or_else(|| { - let rng = gen_rng(); - let snark = prover - .gen_agg_snark(id, degree, rng, real_chunk_hashes, real_and_padding_snarks) - .unwrap(); - write_snark(&file_path, &snark); - - snark - }) -} - pub fn gen_comp_evm_proof( output_dir: &str, + cfg: &str, id: &str, is_fresh: bool, degree: u32, prover: &mut Prover, prev_snark: Snark, ) -> Proof { - set_var("COMPRESSION_CONFIG", format!("./configs/{id}.config")); + set_var("COMPRESSION_CONFIG", format!("./configs/{cfg}.config")); let rng = gen_rng(); let proof = prover @@ -52,21 +31,22 @@ pub fn gen_comp_evm_proof( proof } -pub fn load_or_gen_comp_snark( +pub fn load_or_gen_agg_snark( output_dir: &str, + cfg: &str, id: &str, - is_fresh: bool, degree: u32, prover: &mut Prover, - prev_snark: Snark, + real_chunk_hashes: &[ChunkHash], + real_and_padding_snarks: &[Snark], ) -> Snark { - set_var("COMPRESSION_CONFIG", format!("./configs/{id}.config")); - let file_path = format!("{output_dir}/{id}_snark.json"); + set_var("AGGREGATION_CONFIG", format!("./configs/{cfg}.config")); + let file_path = format!("{output_dir}/agg_snark_{id}.json"); load_snark(&file_path).unwrap().unwrap_or_else(|| { let rng = gen_rng(); let snark = prover - .gen_comp_snark(id, is_fresh, degree, rng, prev_snark) + .gen_agg_snark(id, degree, rng, real_chunk_hashes, real_and_padding_snarks) .unwrap(); write_snark(&file_path, &snark); @@ -74,37 +54,58 @@ pub fn load_or_gen_comp_snark( }) } -pub fn load_or_gen_padding_chunk_snark( +pub fn load_or_gen_chunk_snark( output_dir: &str, id: &str, prover: &mut Prover, - last_real_chunk_hash: &ChunkHash, + witness_block: Block, ) -> Snark { - let file_path = format!("{output_dir}/{id}_chunk_snark.json"); + let file_path = format!("{output_dir}/chunk_snark_{id}.json"); load_snark(&file_path).unwrap().unwrap_or_else(|| { let snark = prover - .gen_padding_chunk_snark(last_real_chunk_hash) + .gen_chunk_snark::(&witness_block) .unwrap(); write_snark(&file_path, &snark); snark }) } -pub fn load_or_gen_real_chunk_snark( +pub fn load_or_gen_comp_snark( output_dir: &str, + cfg: &str, id: &str, + is_fresh: bool, + degree: u32, prover: &mut Prover, - witness_block: Block, + prev_snark: Snark, ) -> Snark { - let file_path = format!("{output_dir}/{id}_chunk_snark.json"); + set_var("COMPRESSION_CONFIG", format!("./configs/{cfg}.config")); + let file_path = format!("{output_dir}/comp_snark_{id}.json"); load_snark(&file_path).unwrap().unwrap_or_else(|| { + let rng = gen_rng(); let snark = prover - .gen_chunk_snark::(&witness_block) + .gen_comp_snark(id, is_fresh, degree, rng, prev_snark) .unwrap(); write_snark(&file_path, &snark); snark }) } + +pub fn load_or_gen_padding_snark( + output_dir: &str, + id: &str, + prover: &mut Prover, + last_real_chunk_hash: &ChunkHash, +) -> Snark { + let file_path = format!("{output_dir}/padding_snark_{id}.json"); + + load_snark(&file_path).unwrap().unwrap_or_else(|| { + let snark = prover.gen_padding_snark(last_real_chunk_hash).unwrap(); + write_snark(&file_path, &snark); + + snark + }) +} diff --git a/prover/tests/aggregation_tests.rs b/prover/tests/aggregation_tests.rs index 9378029c2..865018220 100644 --- a/prover/tests/aggregation_tests.rs +++ b/prover/tests/aggregation_tests.rs @@ -6,8 +6,8 @@ use prover::{ }, test_util::{ aggregator::{ - gen_comp_evm_proof, load_or_gen_agg_snark, load_or_gen_comp_snark, - load_or_gen_padding_chunk_snark, load_or_gen_real_chunk_snark, + gen_comp_evm_proof, load_or_gen_agg_snark, load_or_gen_chunk_snark, + load_or_gen_comp_snark, load_or_gen_padding_snark, }, load_block_traces_for_test, PARAMS_DIR, }, @@ -44,33 +44,23 @@ fn test_agg_prove_verify() { let mut prover = Prover::from_params_dir(PARAMS_DIR, &*ALL_AGG_DEGREES); log::info!("Constructed prover"); - // Load or generate real-chunk snarks. - let mut chunk_snarks: Vec<_> = witness_blocks + // Load or generate real chunk snarks. + let chunk_snarks: Vec<_> = witness_blocks .into_iter() .enumerate() - .map(|(i, block)| { - load_or_gen_real_chunk_snark(&output_dir, &format!("real{i}"), &mut prover, block) - }) + .map(|(i, block)| load_or_gen_chunk_snark(&output_dir, &i.to_string(), &mut prover, block)) .collect(); log::info!("Got real-chunk-snarks"); - // Load or generate padding-chunk snark. - let padding_chunk_snark = load_or_gen_padding_chunk_snark( - &output_dir, - "padding", - &mut prover, - real_chunk_hashes.last().unwrap(), - ); - chunk_snarks.push(padding_chunk_snark); - log::info!("Got padding-chunk-snark"); - // Load or generate compression wide snarks (layer-1). - let layer1_snarks: Vec<_> = chunk_snarks + let mut layer1_snarks: Vec<_> = chunk_snarks .into_iter() - .map(|snark| { + .enumerate() + .map(|(i, snark)| { load_or_gen_comp_snark( &output_dir, "agg_layer1", + &format!("layer1_{i}"), true, *AGG_LAYER1_DEGREE, &mut prover, @@ -80,13 +70,25 @@ fn test_agg_prove_verify() { .collect(); log::info!("Got compression wide snarks (layer-1)"); + // Load or generate layer-1 padding snark. + let layer1_padding_snark = load_or_gen_padding_snark( + &output_dir, + "layer1", + &mut prover, + real_chunk_hashes.last().unwrap(), + ); + layer1_snarks.push(layer1_padding_snark); + log::info!("Got layer1-padding-snark"); + // Load or generate compression thin snarks (layer-2). let mut layer2_snarks: Vec<_> = layer1_snarks .into_iter() - .map(|snark| { + .enumerate() + .map(|(i, snark)| { load_or_gen_comp_snark( &output_dir, "agg_layer2", + &format!("layer2_{i}"), false, *AGG_LAYER2_DEGREE, &mut prover, @@ -97,15 +99,16 @@ fn test_agg_prove_verify() { log::info!("Got compression thin snarks (layer-2)"); // Extend to MAX_AGG_SNARKS by copying the last padding snark. - let padding_layer2_snarks: Vec<_> = repeat(layer2_snarks.last().unwrap().clone()) + let layer2_padding_snarks: Vec<_> = repeat(layer2_snarks.last().unwrap().clone()) .take(MAX_AGG_SNARKS - layer2_snarks.len()) .collect(); - layer2_snarks.extend(padding_layer2_snarks); + layer2_snarks.extend(layer2_padding_snarks); // Load or generate aggregation snark (layer-3). let layer3_snark = load_or_gen_agg_snark( &output_dir, "agg_layer3", + "layer3_0", *AGG_LAYER3_DEGREE, &mut prover, &real_chunk_hashes, @@ -117,6 +120,7 @@ fn test_agg_prove_verify() { let proof = gen_comp_evm_proof( &output_dir, "agg_layer4", + "layer4_0", false, *AGG_LAYER4_DEGREE, &mut prover, diff --git a/prover/tests/compression_tests.rs b/prover/tests/compression_tests.rs index 88d6a0351..99cd8ed96 100644 --- a/prover/tests/compression_tests.rs +++ b/prover/tests/compression_tests.rs @@ -5,7 +5,7 @@ use prover::{ config::{AGG_LAYER1_DEGREE, AGG_LAYER2_DEGREE, INNER_DEGREE}, io::serialize_vk, test_util::{ - aggregator::{gen_comp_evm_proof, load_or_gen_comp_snark, load_or_gen_real_chunk_snark}, + aggregator::{gen_comp_evm_proof, load_or_gen_chunk_snark, load_or_gen_comp_snark}, load_block_traces_for_test, PARAMS_DIR, }, utils::{chunk_trace_to_witness_block, init_env_and_log}, @@ -35,13 +35,14 @@ fn test_comp_prove_verify() { log::info!("Constructed prover"); // Load or generate chunk snark. - let chunk_snark = load_or_gen_real_chunk_snark(&output_dir, "comp", &mut prover, witness_block); + let chunk_snark = load_or_gen_chunk_snark(&output_dir, "0", &mut prover, witness_block); log::info!("Got chunk-snark"); // Load or generate compression wide snark (layer-1). let layer1_snark = load_or_gen_comp_snark( &output_dir, "agg_layer1", + "layer1_0", true, *AGG_LAYER1_DEGREE, &mut prover, @@ -53,6 +54,7 @@ fn test_comp_prove_verify() { let proof = gen_comp_evm_proof( &output_dir, "agg_layer2", + "layer2_0", false, *AGG_LAYER2_DEGREE, &mut prover, From 092b17fe4a85db3a87fc0f654b369d10e97a4229 Mon Sep 17 00:00:00 2001 From: Steven Gu Date: Thu, 13 Jul 2023 09:36:30 +0800 Subject: [PATCH 09/16] Fix --- prover/tests/aggregation_tests.rs | 2 +- prover/tests/compression_tests.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/prover/tests/aggregation_tests.rs b/prover/tests/aggregation_tests.rs index 865018220..0a7af4ddc 100644 --- a/prover/tests/aggregation_tests.rs +++ b/prover/tests/aggregation_tests.rs @@ -130,7 +130,7 @@ fn test_agg_prove_verify() { // Construct verifier and EVM verify. let params = prover.params(*AGG_LAYER4_DEGREE).clone(); - let vk = prover.pk("agg_layer4").unwrap().get_vk().clone(); + let vk = prover.pk("layer4_0").unwrap().get_vk().clone(); let verifier = Verifier::new(params, Some(vk)); let yul_file_path = format!("{output_dir}/agg_verifier.yul"); verifier.evm_verify::(&proof, Some(Path::new(&yul_file_path))); diff --git a/prover/tests/compression_tests.rs b/prover/tests/compression_tests.rs index 99cd8ed96..3d6218db6 100644 --- a/prover/tests/compression_tests.rs +++ b/prover/tests/compression_tests.rs @@ -63,7 +63,7 @@ fn test_comp_prove_verify() { log::info!("Got compression EVM proof (layer-2)"); // Test vk deserialization. - let vk1 = prover.pk("agg_layer2").unwrap().get_vk().clone(); + let vk1 = prover.pk("layer2_0").unwrap().get_vk().clone(); let raw_vk1 = serialize_vk(&vk1); let mut vk2 = VerifyingKey::::read::<_, CompressionCircuit>( &mut Cursor::new(&raw_vk1), From fc94edf7f55a250b3a51eb298a21715857f8db6d Mon Sep 17 00:00:00 2001 From: Steven Gu Date: Thu, 13 Jul 2023 15:52:30 +0800 Subject: [PATCH 10/16] Refactor for FFI functions. --- Cargo.lock | 20 +-- ffi/src/prove.rs | 88 +++++----- ffi/src/verify.rs | 4 +- .../{agg_layer1.config => layer1.config} | 0 .../{agg_layer2.config => layer2.config} | 0 .../{agg_layer3.config => layer3.config} | 0 .../{agg_layer4.config => layer4.config} | 0 prover/src/aggregator.rs | 1 + prover/src/aggregator/prover.rs | 160 ++++++++++++------ prover/src/aggregator/prover/aggregation.rs | 37 ++++ prover/src/aggregator/prover/compression.rs | 35 +++- prover/src/aggregator/prover/evm.rs | 32 ++++ .../aggregator/prover/{chunk.rs => inner.rs} | 18 +- prover/src/aggregator/prover/padding.rs | 31 +++- prover/src/config.rs | 27 +-- prover/src/test_util.rs | 1 - prover/src/test_util/aggregator.rs | 111 ------------ prover/src/zkevm/prover.rs | 12 +- prover/src/zkevm/verifier.rs | 12 +- prover/tests/aggregation_tests.rs | 137 +++++++-------- prover/tests/compression_tests.rs | 71 ++++---- 21 files changed, 431 insertions(+), 366 deletions(-) rename prover/configs/{agg_layer1.config => layer1.config} (100%) rename prover/configs/{agg_layer2.config => layer2.config} (100%) rename prover/configs/{agg_layer3.config => layer3.config} (100%) rename prover/configs/{agg_layer4.config => layer4.config} (100%) create mode 100644 prover/src/aggregator/prover/evm.rs rename prover/src/aggregator/prover/{chunk.rs => inner.rs} (74%) delete mode 100644 prover/src/test_util/aggregator.rs diff --git a/Cargo.lock b/Cargo.lock index b4b10f3c4..b728030d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,7 +23,7 @@ dependencies = [ [[package]] name = "aggregator" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d864ddf2c1914dec70475e565571deb58499e57d" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#de2430030204e875f62ba0142d92ef794998a023" dependencies = [ "ark-std", "env_logger 0.10.0", @@ -418,7 +418,7 @@ checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" [[package]] name = "bus-mapping" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d864ddf2c1914dec70475e565571deb58499e57d" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#de2430030204e875f62ba0142d92ef794998a023" dependencies = [ "eth-types", "ethers-core 0.17.0", @@ -1118,7 +1118,7 @@ dependencies = [ [[package]] name = "eth-types" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d864ddf2c1914dec70475e565571deb58499e57d" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#de2430030204e875f62ba0142d92ef794998a023" dependencies = [ "ethers-core 0.17.0", "ethers-signers", @@ -1384,7 +1384,7 @@ dependencies = [ [[package]] name = "external-tracer" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d864ddf2c1914dec70475e565571deb58499e57d" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#de2430030204e875f62ba0142d92ef794998a023" dependencies = [ "eth-types", "geth-utils", @@ -1613,7 +1613,7 @@ dependencies = [ [[package]] name = "gadgets" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d864ddf2c1914dec70475e565571deb58499e57d" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#de2430030204e875f62ba0142d92ef794998a023" dependencies = [ "digest 0.7.6", "eth-types", @@ -1653,7 +1653,7 @@ dependencies = [ [[package]] name = "geth-utils" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d864ddf2c1914dec70475e565571deb58499e57d" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#de2430030204e875f62ba0142d92ef794998a023" dependencies = [ "env_logger 0.9.3", "gobuild 0.1.0-alpha.2 (git+https://github.com/scroll-tech/gobuild.git)", @@ -2243,7 +2243,7 @@ dependencies = [ [[package]] name = "keccak256" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d864ddf2c1914dec70475e565571deb58499e57d" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#de2430030204e875f62ba0142d92ef794998a023" dependencies = [ "env_logger 0.9.3", "eth-types", @@ -2443,7 +2443,7 @@ dependencies = [ [[package]] name = "mock" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d864ddf2c1914dec70475e565571deb58499e57d" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#de2430030204e875f62ba0142d92ef794998a023" dependencies = [ "eth-types", "ethers-core 0.17.0", @@ -2458,7 +2458,7 @@ dependencies = [ [[package]] name = "mpt-zktrie" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d864ddf2c1914dec70475e565571deb58499e57d" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#de2430030204e875f62ba0142d92ef794998a023" dependencies = [ "bus-mapping", "eth-types", @@ -4683,7 +4683,7 @@ checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" [[package]] name = "zkevm-circuits" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#d864ddf2c1914dec70475e565571deb58499e57d" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#de2430030204e875f62ba0142d92ef794998a023" dependencies = [ "array-init", "bus-mapping", diff --git a/ffi/src/prove.rs b/ffi/src/prove.rs index 8b62b7e46..9bf1cf7f0 100644 --- a/ffi/src/prove.rs +++ b/ffi/src/prove.rs @@ -1,21 +1,29 @@ use crate::utils::{c_char_to_str, c_char_to_vec, vec_to_c_char}; use libc::c_char; -use prover::{aggregator, config::ALL_AGG_DEGREES, utils::init_env_and_log, zkevm}; -use std::cell::OnceCell; +use once_cell::sync::Lazy; +use prover::{ + aggregator::{self, ChunkHash}, + config::AGG_DEGREES, + utils::init_env_and_log, + zkevm, Proof, +}; +use std::{cell::OnceCell, env}; use types::eth::BlockTrace; -static mut CHUNK_PROVER: OnceCell = OnceCell::new(); +static mut ZKEVM_PROVER: OnceCell = OnceCell::new(); static mut AGG_PROVER: OnceCell = OnceCell::new(); -static mut AGG_CHUNK_TRACES: OnceCell>> = OnceCell::new(); + +// Only used for debugging. +static OUTPUT_DIR: Lazy> = Lazy::new(|| env::var("PROVER_OUTPUT_DIR").ok()); /// # Safety #[no_mangle] -pub unsafe extern "C" fn init_chunk_prover(params_dir: *const c_char) { - init_env_and_log("ffi_chunk_prove"); +pub unsafe extern "C" fn init_zkevm_prover(params_dir: *const c_char) { + init_env_and_log("ffi_zkevm_prove"); let params_dir = c_char_to_str(params_dir); let prover = zkevm::Prover::from_params_dir(params_dir); - CHUNK_PROVER.set(prover).unwrap(); + ZKEVM_PROVER.set(prover).unwrap(); } /// # Safety @@ -25,72 +33,64 @@ pub unsafe extern "C" fn init_agg_prover(params_dir: *const c_char) { let params_dir = c_char_to_str(params_dir); - let prover = aggregator::Prover::from_params_dir(params_dir, &ALL_AGG_DEGREES); + let prover = aggregator::Prover::from_params_dir(params_dir, &AGG_DEGREES); AGG_PROVER.set(prover).unwrap(); } /// # Safety #[no_mangle] -pub unsafe extern "C" fn create_block_proof(trace_char: *const c_char) -> *const c_char { - let trace_vec = c_char_to_vec(trace_char); - let trace = serde_json::from_slice::(&trace_vec).unwrap(); - let proof = CHUNK_PROVER +pub unsafe extern "C" fn create_block_proof(block_trace: *const c_char) -> *const c_char { + let block_trace = c_char_to_vec(block_trace); + let block_trace = serde_json::from_slice::(&block_trace).unwrap(); + + let proof = ZKEVM_PROVER .get_mut() .unwrap() - .gen_chunk_proof(&[trace]) + .gen_chunk_proof(&[block_trace]) .unwrap(); + let proof_bytes = serde_json::to_vec(&proof).unwrap(); vec_to_c_char(proof_bytes) } /// # Safety #[no_mangle] -pub unsafe extern "C" fn create_chunk_proof(trace_char: *const c_char) -> *const c_char { - let trace_vec = c_char_to_vec(trace_char); - let traces = serde_json::from_slice::>(&trace_vec).unwrap(); - let proof = CHUNK_PROVER +pub unsafe extern "C" fn create_chunk_proof(block_traces: *const c_char) -> *const c_char { + let block_traces = c_char_to_vec(block_traces); + let block_traces = serde_json::from_slice::>(&block_traces).unwrap(); + + let proof = ZKEVM_PROVER .get_mut() .unwrap() - .gen_chunk_proof(traces.as_slice()) + .gen_chunk_proof(block_traces.as_slice()) .unwrap(); + let proof_bytes = serde_json::to_vec(&proof).unwrap(); vec_to_c_char(proof_bytes) } /// # Safety #[no_mangle] -pub unsafe extern "C" fn add_agg_chunk_trace(trace_char: *const c_char) { - let trace_vec = c_char_to_vec(trace_char); - let trace = serde_json::from_slice::>(&trace_vec).unwrap(); - - AGG_CHUNK_TRACES - .get_mut() - .or_else(|| { - AGG_CHUNK_TRACES.set(vec![]).unwrap(); - AGG_CHUNK_TRACES.get_mut() - }) - .unwrap() - .push(trace); -} +pub unsafe extern "C" fn create_agg_proof( + chunk_hashes: *const c_char, + chunk_proofs: *const c_char, +) -> *const c_char { + let chunk_hashes = c_char_to_vec(chunk_hashes); + let chunk_proofs = c_char_to_vec(chunk_proofs); -/// # Safety -#[no_mangle] -pub unsafe extern "C" fn clear_agg_chunk_traces() { - if let Some(chunk_traces) = AGG_CHUNK_TRACES.get_mut() { - chunk_traces.clear(); - } -} + let chunk_hashes = serde_json::from_slice::>(&chunk_hashes).unwrap(); + let chunk_proofs = serde_json::from_slice::>(&chunk_proofs).unwrap(); + assert_eq!(chunk_hashes.len(), chunk_proofs.len()); -/// # Safety -#[no_mangle] -pub unsafe extern "C" fn create_agg_proof() -> *const c_char { - // Consume the chunk traces (take and clear). - let chunk_traces = AGG_CHUNK_TRACES.take().unwrap(); + let chunks = chunk_hashes + .into_iter() + .zip(chunk_proofs.into_iter()) + .collect(); let proof = AGG_PROVER .get_mut() .unwrap() - .gen_agg_proof(chunk_traces, None) + .gen_agg_proof(chunks, OUTPUT_DIR.as_deref()) .unwrap(); let proof_bytes = serde_json::to_vec(&proof).unwrap(); diff --git a/ffi/src/verify.rs b/ffi/src/verify.rs index ab919dd8f..5b4467eb6 100644 --- a/ffi/src/verify.rs +++ b/ffi/src/verify.rs @@ -1,6 +1,6 @@ use crate::utils::{c_char_to_str, c_char_to_vec}; use libc::c_char; -use prover::{aggregator, config::AGG_LAYER4_DEGREE, utils::init_env_and_log, zkevm, Proof}; +use prover::{aggregator, config::LAYER4_DEGREE, utils::init_env_and_log, zkevm, Proof}; use std::{fs::File, io::Read}; static mut CHUNK_VERIFIER: Option<&zkevm::Verifier> = None; @@ -35,7 +35,7 @@ pub unsafe extern "C" fn init_agg_verifier(params_dir: *const c_char, vk_path: * let params_dir = c_char_to_str(params_dir); let verifier = Box::new(aggregator::Verifier::from_params_dir( params_dir, - *AGG_LAYER4_DEGREE, + *LAYER4_DEGREE, Some(vk), )); diff --git a/prover/configs/agg_layer1.config b/prover/configs/layer1.config similarity index 100% rename from prover/configs/agg_layer1.config rename to prover/configs/layer1.config diff --git a/prover/configs/agg_layer2.config b/prover/configs/layer2.config similarity index 100% rename from prover/configs/agg_layer2.config rename to prover/configs/layer2.config diff --git a/prover/configs/agg_layer3.config b/prover/configs/layer3.config similarity index 100% rename from prover/configs/agg_layer3.config rename to prover/configs/layer3.config diff --git a/prover/configs/agg_layer4.config b/prover/configs/layer4.config similarity index 100% rename from prover/configs/agg_layer4.config rename to prover/configs/layer4.config diff --git a/prover/src/aggregator.rs b/prover/src/aggregator.rs index ac0279ed5..ca8b5acb1 100644 --- a/prover/src/aggregator.rs +++ b/prover/src/aggregator.rs @@ -2,3 +2,4 @@ mod prover; mod verifier; pub use self::{prover::Prover, verifier::Verifier}; +pub use aggregator::ChunkHash; diff --git a/prover/src/aggregator/prover.rs b/prover/src/aggregator/prover.rs index d8371a5c7..81674b4cd 100644 --- a/prover/src/aggregator/prover.rs +++ b/prover/src/aggregator/prover.rs @@ -1,9 +1,9 @@ use crate::{ - config::{AGG_LAYER1_DEGREE, AGG_LAYER2_DEGREE, AGG_LAYER3_DEGREE, AGG_LAYER4_DEGREE}, - utils::{chunk_trace_to_witness_block, gen_rng, load_params, param_path_for_degree}, - zkevm::circuit::SuperCircuit, + config::{LAYER1_DEGREE, LAYER2_DEGREE, LAYER3_DEGREE, LAYER4_DEGREE}, + utils::{chunk_trace_to_witness_block, load_params, param_path_for_degree}, Proof, }; +use aggregator::{ChunkHash, MAX_AGG_SNARKS}; use anyhow::Result; use halo2_proofs::{ halo2curves::bn256::{Bn256, G1Affine}, @@ -12,14 +12,15 @@ use halo2_proofs::{ }; use std::{ collections::{BTreeMap, BTreeSet, HashMap}, - env::set_var, + iter::repeat, }; use types::eth::BlockTrace; mod aggregation; -mod chunk; mod common; mod compression; +mod evm; +mod inner; mod padding; #[derive(Debug)] @@ -78,61 +79,112 @@ impl Prover { pub fn gen_agg_proof( &mut self, - chunk_traces: Vec>, - _output_dir: Option<&str>, + chunks: Vec<(ChunkHash, Proof)>, + output_dir: Option<&str>, ) -> Result { - // Convert chunk traces to witness blocks. - let witness_blocks = chunk_traces - .into_iter() - .map(chunk_trace_to_witness_block) - .collect::>>()?; - - // Convert witness blocks to chunk hashes. - let real_chunk_hashes: Vec<_> = witness_blocks.iter().map(Into::into).collect(); - - // Generate chunk snarks. - let chunk_snarks = witness_blocks - .into_iter() - .map(|block| self.gen_chunk_snark::(&block)) - .collect::>>()?; + let real_chunk_count = chunks.len(); + assert!((1..=MAX_AGG_SNARKS).contains(&real_chunk_count)); + + let last_real_chunk_hash = chunks.last().unwrap().0; + let name = last_real_chunk_hash + .public_input_hash() + .to_low_u64_le() + .to_string(); + + // Load or generate padding snark (layer-1). + let layer1_padding_snark = + self.load_or_gen_padding_snark(&name, &last_real_chunk_hash, output_dir)?; + log::info!("Got padding snark (layer-1): {name}"); + + // Load or generate compression thin snark for padding (layer-2). + let layer2_padding_snark = self.load_or_gen_comp_snark( + &name, + "layer2", + false, + *LAYER2_DEGREE, + layer1_padding_snark, + output_dir, + )?; + log::info!("Got compression thin snark for padding (layer-2): {name}"); - // Generate compression wide snarks (layer-1). - set_var("COMPRESSION_CONFIG", "./configs/agg_layer1.config"); - let layer1_snarks = chunk_snarks - .into_iter() - .map(|snark| { - let rng = gen_rng(); - self.gen_comp_snark("agg_layer1", true, *AGG_LAYER1_DEGREE, rng, snark) - }) - .collect::>>()?; - - // Generate compression thin snarks (layer-2). - set_var("COMPRESSION_CONFIG", "./configs/agg_layer2.config"); - let layer2_snarks: Vec<_> = layer1_snarks + let (chunk_hashes, mut layer2_snarks): (Vec<_>, Vec<_>) = chunks .into_iter() - .map(|snark| { - let rng = gen_rng(); - self.gen_comp_snark("agg_layer2", false, *AGG_LAYER2_DEGREE, rng, snark) - }) - .collect::>>()?; - - // Generate aggregation snark (layer-3). - set_var("AGGREGATION_CONFIG", "./configs/agg_layer3.config"); - let rng = gen_rng(); - let layer3_snark = self.gen_agg_snark( - "agg_layer3", - *AGG_LAYER3_DEGREE, - rng, - &real_chunk_hashes, + .map(|chunk| (chunk.0, chunk.1.to_snark())) + .unzip(); + + // Extend to MAX_AGG_SNARKS by copying the padding snark. + layer2_snarks.extend(repeat(layer2_padding_snark).take(MAX_AGG_SNARKS - real_chunk_count)); + + // Load or generate aggregation snark (layer-3). + let layer3_snark = self.load_or_gen_agg_snark( + &name, + "layer3", + *LAYER3_DEGREE, + &chunk_hashes, &layer2_snarks, + output_dir, + )?; + log::info!("Got aggregation snark (layer-3): {name}"); + + // Load or generate final compression thin snark (layer-4). + let layer4_snark = self.load_or_gen_comp_snark( + &name, + "layer4", + false, + *LAYER4_DEGREE, + layer3_snark, + output_dir, )?; + log::info!("Got final compression thin snark (layer-4): {name}"); - // Generate final compression snarks (layer-4). - set_var("COMPRESSION_CONFIG", "./configs/agg_layer4.config"); - let rng = gen_rng(); - let layer4_snark = - self.gen_comp_snark("agg_layer4", false, *AGG_LAYER4_DEGREE, rng, layer3_snark)?; + let pk = self.pk("layer4").unwrap(); + Proof::from_snark(pk, &layer4_snark) + } + + pub fn gen_chunk_proof( + &mut self, + chunk_trace: Vec, + output_dir: Option<&str>, + ) -> Result { + assert!(!chunk_trace.is_empty()); + + let witness_block = chunk_trace_to_witness_block(chunk_trace)?; + log::info!("Got witness block"); + + let name = witness_block + .context + .first_or_default() + .number + .low_u64() + .to_string(); + + // Load or generate inner snark. + let inner_snark = self.load_or_gen_inner_snark(&name, witness_block, output_dir)?; + log::info!("Got inner snark: {name}"); + + // Load or generate compression wide snark (layer-1). + let layer1_snark = self.load_or_gen_comp_snark( + &name, + "layer1", + true, + *LAYER1_DEGREE, + inner_snark, + output_dir, + )?; + log::info!("Got compression wide snark (layer-1): {name}"); + + // Load or generate compression thin snark (layer-2). + let layer2_snark = self.load_or_gen_comp_snark( + &name, + "layer2", + false, + *LAYER2_DEGREE, + layer1_snark, + output_dir, + )?; + log::info!("Got compression thin snark (layer-2): {name}"); - Proof::from_snark(&self.pk_map["agg_layer4"], &layer4_snark) + let pk = self.pk("layer2").unwrap(); + Proof::from_snark(pk, &layer2_snark) } } diff --git a/prover/src/aggregator/prover/aggregation.rs b/prover/src/aggregator/prover/aggregation.rs index 4eb98630a..277d1fcac 100644 --- a/prover/src/aggregator/prover/aggregation.rs +++ b/prover/src/aggregator/prover/aggregation.rs @@ -1,8 +1,13 @@ use super::Prover; +use crate::{ + io::{load_snark, write_snark}, + utils::gen_rng, +}; use aggregator::{AggregationCircuit, BatchHash, ChunkHash}; use anyhow::Result; use rand::Rng; use snark_verifier_sdk::Snark; +use std::env::set_var; impl Prover { pub fn gen_agg_snark( @@ -24,4 +29,36 @@ impl Prover { self.gen_snark(id, degree, &mut rng, circuit) } + + pub fn load_or_gen_agg_snark( + &mut self, + name: &str, + id: &str, + degree: u32, + real_chunk_hashes: &[ChunkHash], + real_and_padding_snarks: &[Snark], + output_dir: Option<&str>, + ) -> Result { + let file_path = format!( + "{}/aggregation_snark_{}.json", + output_dir.unwrap_or_default(), + name + ); + + match output_dir.and_then(|_| load_snark(&file_path).ok().flatten()) { + Some(snark) => Ok(snark), + None => { + set_var("AGGREGATION_CONFIG", format!("./configs/{id}.config")); + + let rng = gen_rng(); + let result = + self.gen_agg_snark(id, degree, rng, real_chunk_hashes, real_and_padding_snarks); + if let (Some(_), Ok(snark)) = (output_dir, &result) { + write_snark(&file_path, snark); + } + + result + } + } + } } diff --git a/prover/src/aggregator/prover/compression.rs b/prover/src/aggregator/prover/compression.rs index 9a04df849..e1ef764af 100644 --- a/prover/src/aggregator/prover/compression.rs +++ b/prover/src/aggregator/prover/compression.rs @@ -1,9 +1,13 @@ use super::Prover; -use crate::Proof; +use crate::{ + io::{load_snark, write_snark}, + utils::gen_rng, +}; use aggregator::CompressionCircuit; use anyhow::{anyhow, Result}; use rand::Rng; use snark_verifier_sdk::Snark; +use std::env::set_var; impl Prover { pub fn gen_comp_snark( @@ -20,17 +24,34 @@ impl Prover { self.gen_snark(id, degree, &mut rng, circuit) } - pub fn gen_comp_evm_proof( + pub fn load_or_gen_comp_snark( &mut self, + name: &str, id: &str, is_fresh: bool, degree: u32, - mut rng: impl Rng + Send, prev_snark: Snark, - ) -> Result { - let circuit = CompressionCircuit::new(self.params(degree), prev_snark, is_fresh, &mut rng) - .map_err(|err| anyhow!("Failed to construct compression circuit: {err:?}"))?; + output_dir: Option<&str>, + ) -> Result { + let file_path = format!( + "{}/compression_snark_{}.json", + output_dir.unwrap_or_default(), + name + ); + + match output_dir.and_then(|_| load_snark(&file_path).ok().flatten()) { + Some(snark) => Ok(snark), + None => { + set_var("COMPRESSION_CONFIG", format!("./configs/{id}.config")); + + let rng = gen_rng(); + let result = self.gen_comp_snark(id, is_fresh, degree, rng, prev_snark); + if let (Some(_), Ok(snark)) = (output_dir, &result) { + write_snark(&file_path, snark); + } - self.gen_evm_proof(id, degree, &mut rng, circuit) + result + } + } } } diff --git a/prover/src/aggregator/prover/evm.rs b/prover/src/aggregator/prover/evm.rs new file mode 100644 index 000000000..ad83d6d5f --- /dev/null +++ b/prover/src/aggregator/prover/evm.rs @@ -0,0 +1,32 @@ +use super::Prover; +use crate::{utils::gen_rng, Proof}; +use aggregator::CompressionCircuit; +use anyhow::{anyhow, Result}; +use snark_verifier_sdk::Snark; +use std::{env::set_var, path::PathBuf}; + +impl Prover { + pub fn gen_comp_evm_proof( + &mut self, + name: &str, + id: &str, + is_fresh: bool, + degree: u32, + prev_snark: Snark, + output_dir: Option<&str>, + ) -> Result { + set_var("COMPRESSION_CONFIG", format!("./configs/{id}.config")); + + let mut rng = gen_rng(); + let circuit = CompressionCircuit::new(self.params(degree), prev_snark, is_fresh, &mut rng) + .map_err(|err| anyhow!("Failed to construct compression circuit: {err:?}"))?; + + let result = self.gen_evm_proof(id, degree, &mut rng, circuit); + + if let (Some(output_dir), Ok(proof)) = (output_dir, &result) { + proof.dump(&mut PathBuf::from(output_dir), name)?; + } + + result + } +} diff --git a/prover/src/aggregator/prover/chunk.rs b/prover/src/aggregator/prover/inner.rs similarity index 74% rename from prover/src/aggregator/prover/chunk.rs rename to prover/src/aggregator/prover/inner.rs index 64b910057..82ec76a8c 100644 --- a/prover/src/aggregator/prover/chunk.rs +++ b/prover/src/aggregator/prover/inner.rs @@ -11,7 +11,7 @@ use snark_verifier_sdk::{gen_snark_shplonk, Snark}; use zkevm_circuits::evm_circuit::witness::Block; impl Prover { - pub fn gen_chunk_snark( + pub fn gen_inner_snark( &mut self, witness_block: &Block, ) -> Result { @@ -28,23 +28,27 @@ impl Prover { Ok(snark) } - pub fn load_or_gen_chunk_snark( + pub fn load_or_gen_inner_snark( &mut self, - id: &str, + name: &str, witness_block: Block, output_dir: Option<&str>, ) -> Result { - let file_path = format!("{}/{}_chunk_snark.json", output_dir.unwrap_or_default(), id); + let file_path = format!( + "{}/inner_snark_{}.json", + output_dir.unwrap_or_default(), + name + ); match output_dir.and_then(|_| load_snark(&file_path).ok().flatten()) { Some(snark) => Ok(snark), None => { - let res = self.gen_chunk_snark::(&witness_block); - if let (Some(_), Ok(snark)) = (output_dir, &res) { + let result = self.gen_inner_snark::(&witness_block); + if let (Some(_), Ok(snark)) = (output_dir, &result) { write_snark(&file_path, snark); } - res + result } } } diff --git a/prover/src/aggregator/prover/padding.rs b/prover/src/aggregator/prover/padding.rs index 341ee4aa6..671b82385 100644 --- a/prover/src/aggregator/prover/padding.rs +++ b/prover/src/aggregator/prover/padding.rs @@ -1,5 +1,9 @@ use super::Prover; -use crate::{config::INNER_DEGREE, utils::gen_rng}; +use crate::{ + config::INNER_DEGREE, + io::{load_snark, write_snark}, + utils::gen_rng, +}; use aggregator::{ChunkHash, DummyChunkHashCircuit}; use anyhow::Result; use snark_verifier_sdk::{gen_snark_shplonk, Snark}; @@ -14,4 +18,29 @@ impl Prover { Ok(snark) } + + pub fn load_or_gen_padding_snark( + &mut self, + name: &str, + last_real_chunk_hash: &ChunkHash, + output_dir: Option<&str>, + ) -> Result { + let file_path = format!( + "{}/padding_snark_{}.json", + output_dir.unwrap_or_default(), + name + ); + + match output_dir.and_then(|_| load_snark(&file_path).ok().flatten()) { + Some(snark) => Ok(snark), + None => { + let result = self.gen_padding_snark(last_real_chunk_hash); + if let (Some(_), Ok(snark)) = (output_dir, &result) { + write_snark(&file_path, snark); + } + + result + } + } + } } diff --git a/prover/src/config.rs b/prover/src/config.rs index 2b26fbe7c..8cc23aa98 100644 --- a/prover/src/config.rs +++ b/prover/src/config.rs @@ -3,20 +3,23 @@ use once_cell::sync::Lazy; use std::collections::HashSet; pub static INNER_DEGREE: Lazy = Lazy::new(|| read_env_var("INNER_DEGREE", 20)); -pub static CHUNK_DEGREE: Lazy = Lazy::new(|| read_env_var("CHUNK_DEGREE", 25)); +pub static LAYER1_DEGREE: Lazy = Lazy::new(|| read_env_var("LAYER1_DEGREE", 25)); +pub static LAYER2_DEGREE: Lazy = Lazy::new(|| read_env_var("LAYER2_DEGREE", 25)); +pub static LAYER3_DEGREE: Lazy = Lazy::new(|| read_env_var("LAYER3_DEGREE", 25)); +pub static LAYER4_DEGREE: Lazy = Lazy::new(|| read_env_var("LAYER4_DEGREE", 25)); -pub static AGG_LAYER1_DEGREE: Lazy = Lazy::new(|| read_env_var("AGG_LAYER1_DEGREE", 25)); -pub static AGG_LAYER2_DEGREE: Lazy = Lazy::new(|| read_env_var("AGG_LAYER2_DEGREE", 25)); -pub static AGG_LAYER3_DEGREE: Lazy = Lazy::new(|| read_env_var("AGG_LAYER3_DEGREE", 25)); -pub static AGG_LAYER4_DEGREE: Lazy = Lazy::new(|| read_env_var("AGG_LAYER4_DEGREE", 25)); - -pub static ALL_AGG_DEGREES: Lazy> = Lazy::new(|| { +pub static ZKEVM_DEGREES: Lazy> = Lazy::new(|| { Vec::from_iter(HashSet::from([ *INNER_DEGREE, - *CHUNK_DEGREE, - *AGG_LAYER1_DEGREE, - *AGG_LAYER2_DEGREE, - *AGG_LAYER3_DEGREE, - *AGG_LAYER4_DEGREE, + *LAYER1_DEGREE, + *LAYER2_DEGREE, + ])) +}); + +pub static AGG_DEGREES: Lazy> = Lazy::new(|| { + Vec::from_iter(HashSet::from([ + *LAYER2_DEGREE, + *LAYER3_DEGREE, + *LAYER4_DEGREE, ])) }); diff --git a/prover/src/test_util.rs b/prover/src/test_util.rs index ec9c9f020..ddd8ab225 100644 --- a/prover/src/test_util.rs +++ b/prover/src/test_util.rs @@ -2,7 +2,6 @@ use crate::utils::{get_block_trace_from_file, read_env_var}; use glob::glob; use types::eth::BlockTrace; -pub mod aggregator; pub mod mock_plonk; pub const PARAMS_DIR: &str = "./test_params"; diff --git a/prover/src/test_util/aggregator.rs b/prover/src/test_util/aggregator.rs deleted file mode 100644 index d92e1f350..000000000 --- a/prover/src/test_util/aggregator.rs +++ /dev/null @@ -1,111 +0,0 @@ -use crate::{ - aggregator::Prover, - io::{load_snark, write_snark}, - utils::gen_rng, - zkevm::circuit::SuperCircuit, - Proof, -}; -use aggregator::ChunkHash; -use halo2_proofs::halo2curves::bn256::Fr; -use snark_verifier_sdk::Snark; -use std::{env::set_var, path::PathBuf}; -use zkevm_circuits::evm_circuit::witness::Block; - -pub fn gen_comp_evm_proof( - output_dir: &str, - cfg: &str, - id: &str, - is_fresh: bool, - degree: u32, - prover: &mut Prover, - prev_snark: Snark, -) -> Proof { - set_var("COMPRESSION_CONFIG", format!("./configs/{cfg}.config")); - - let rng = gen_rng(); - let proof = prover - .gen_comp_evm_proof(id, is_fresh, degree, rng, prev_snark) - .unwrap(); - proof.dump(&mut PathBuf::from(output_dir), id).unwrap(); - - proof -} - -pub fn load_or_gen_agg_snark( - output_dir: &str, - cfg: &str, - id: &str, - degree: u32, - prover: &mut Prover, - real_chunk_hashes: &[ChunkHash], - real_and_padding_snarks: &[Snark], -) -> Snark { - set_var("AGGREGATION_CONFIG", format!("./configs/{cfg}.config")); - let file_path = format!("{output_dir}/agg_snark_{id}.json"); - - load_snark(&file_path).unwrap().unwrap_or_else(|| { - let rng = gen_rng(); - let snark = prover - .gen_agg_snark(id, degree, rng, real_chunk_hashes, real_and_padding_snarks) - .unwrap(); - write_snark(&file_path, &snark); - - snark - }) -} - -pub fn load_or_gen_chunk_snark( - output_dir: &str, - id: &str, - prover: &mut Prover, - witness_block: Block, -) -> Snark { - let file_path = format!("{output_dir}/chunk_snark_{id}.json"); - - load_snark(&file_path).unwrap().unwrap_or_else(|| { - let snark = prover - .gen_chunk_snark::(&witness_block) - .unwrap(); - write_snark(&file_path, &snark); - - snark - }) -} -pub fn load_or_gen_comp_snark( - output_dir: &str, - cfg: &str, - id: &str, - is_fresh: bool, - degree: u32, - prover: &mut Prover, - prev_snark: Snark, -) -> Snark { - set_var("COMPRESSION_CONFIG", format!("./configs/{cfg}.config")); - let file_path = format!("{output_dir}/comp_snark_{id}.json"); - - load_snark(&file_path).unwrap().unwrap_or_else(|| { - let rng = gen_rng(); - let snark = prover - .gen_comp_snark(id, is_fresh, degree, rng, prev_snark) - .unwrap(); - write_snark(&file_path, &snark); - - snark - }) -} - -pub fn load_or_gen_padding_snark( - output_dir: &str, - id: &str, - prover: &mut Prover, - last_real_chunk_hash: &ChunkHash, -) -> Snark { - let file_path = format!("{output_dir}/padding_snark_{id}.json"); - - load_snark(&file_path).unwrap().unwrap_or_else(|| { - let snark = prover.gen_padding_snark(last_real_chunk_hash).unwrap(); - write_snark(&file_path, &snark); - - snark - }) -} diff --git a/prover/src/zkevm/prover.rs b/prover/src/zkevm/prover.rs index d0ac82e7a..c8c7150db 100644 --- a/prover/src/zkevm/prover.rs +++ b/prover/src/zkevm/prover.rs @@ -2,7 +2,7 @@ use super::circuit::{ block_traces_to_witness_block, check_batch_capacity, SuperCircuit, TargetCircuit, }; use crate::{ - config::{CHUNK_DEGREE, INNER_DEGREE}, + config::INNER_DEGREE, utils::{load_params, metric_of_witness_block, read_env_var, tick}, Proof, }; @@ -38,6 +38,8 @@ extern crate procfs; pub static OPT_MEM: Lazy = Lazy::new(|| read_env_var("OPT_MEM", false)); pub static MOCK_PROVE: Lazy = Lazy::new(|| read_env_var("MOCK_PROVE", false)); +const CHUNK_DEGREE: u32 = 25; + #[derive(Debug)] // This is the aggregation prover that takes in a list of traces, produces // a proof that can be verified on chain. @@ -53,13 +55,13 @@ pub struct Prover { impl Prover { pub fn from_params(inner_params: ParamsKZG, chunk_params: ParamsKZG) -> Self { assert!(inner_params.k() == *INNER_DEGREE); - assert!(chunk_params.k() == *CHUNK_DEGREE); + assert!(chunk_params.k() == CHUNK_DEGREE); // notice that `inner_k < chunk`_k which is not necessary the case in practice log::info!( "loaded parameters for degrees {} and {}", *INNER_DEGREE, - *CHUNK_DEGREE + CHUNK_DEGREE ); // this check can be skipped since the `params` is downsized? @@ -84,9 +86,9 @@ impl Prover { } pub fn from_params_dir(params_dir: &str) -> Self { - let chunk_params = load_params(params_dir, *CHUNK_DEGREE, None).unwrap(); + let chunk_params = load_params(params_dir, CHUNK_DEGREE, None).unwrap(); let inner_params = load_params(params_dir, *INNER_DEGREE, None).unwrap_or_else(|_| { - assert!(*CHUNK_DEGREE >= *INNER_DEGREE); + assert!(CHUNK_DEGREE >= *INNER_DEGREE); log::warn!( "Optimization: download params{} to params dir", *INNER_DEGREE diff --git a/prover/src/zkevm/verifier.rs b/prover/src/zkevm/verifier.rs index a49e9cc7f..dae1d5799 100644 --- a/prover/src/zkevm/verifier.rs +++ b/prover/src/zkevm/verifier.rs @@ -1,9 +1,5 @@ use super::circuit::TargetCircuit; -use crate::{ - config::{CHUNK_DEGREE, INNER_DEGREE}, - utils::load_params, - Proof, -}; +use crate::{config::INNER_DEGREE, utils::load_params, Proof}; use anyhow::{bail, Result}; use halo2_proofs::{ halo2curves::bn256::{Bn256, G1Affine}, @@ -21,6 +17,8 @@ use snark_verifier::system::halo2::transcript::evm::EvmTranscript; use snark_verifier_sdk::{verify_snark_shplonk, AggregationCircuit, Snark}; use std::{collections::HashMap, io::Cursor}; +const CHUNK_DEGREE: u32 = 25; + pub struct Verifier { inner_params: ParamsKZG, chunk_params: ParamsKZG, @@ -51,9 +49,9 @@ impl Verifier { } pub fn from_params_dir(params_dir: &str, chunk_vk: Option>) -> Self { - let chunk_params = load_params(params_dir, *CHUNK_DEGREE, None).unwrap(); + let chunk_params = load_params(params_dir, CHUNK_DEGREE, None).unwrap(); let inner_params = load_params(params_dir, *INNER_DEGREE, None).unwrap_or_else(|_| { - assert!(*CHUNK_DEGREE >= *INNER_DEGREE); + assert!(CHUNK_DEGREE >= *INNER_DEGREE); log::warn!( "Optimization: download params{} to params dir", *INNER_DEGREE diff --git a/prover/tests/aggregation_tests.rs b/prover/tests/aggregation_tests.rs index 0a7af4ddc..40acd7860 100644 --- a/prover/tests/aggregation_tests.rs +++ b/prover/tests/aggregation_tests.rs @@ -2,15 +2,9 @@ use aggregator::{CompressionCircuit, MAX_AGG_SNARKS}; use prover::{ aggregator::{Prover, Verifier}, config::{ - AGG_LAYER1_DEGREE, AGG_LAYER2_DEGREE, AGG_LAYER3_DEGREE, AGG_LAYER4_DEGREE, ALL_AGG_DEGREES, - }, - test_util::{ - aggregator::{ - gen_comp_evm_proof, load_or_gen_agg_snark, load_or_gen_chunk_snark, - load_or_gen_comp_snark, load_or_gen_padding_snark, - }, - load_block_traces_for_test, PARAMS_DIR, + AGG_DEGREES, LAYER1_DEGREE, LAYER2_DEGREE, LAYER3_DEGREE, LAYER4_DEGREE, ZKEVM_DEGREES, }, + test_util::{load_block_traces_for_test, PARAMS_DIR}, utils::{chunk_trace_to_witness_block, init_env_and_log}, }; use std::{env::set_var, iter::repeat, path::Path}; @@ -41,42 +35,49 @@ fn test_agg_prove_verify() { let real_chunk_hashes: Vec<_> = witness_blocks.iter().map(Into::into).collect(); log::info!("Got real-chunk-hashes"); - let mut prover = Prover::from_params_dir(PARAMS_DIR, &*ALL_AGG_DEGREES); - log::info!("Constructed prover"); + let mut zkevm_prover = Prover::from_params_dir(PARAMS_DIR, &*ZKEVM_DEGREES); + let mut agg_prover = Prover::from_params_dir(PARAMS_DIR, &*AGG_DEGREES); + log::info!("Constructed zkevm and aggregation provers"); - // Load or generate real chunk snarks. - let chunk_snarks: Vec<_> = witness_blocks + // Load or generate real inner snarks. + let inner_snarks: Vec<_> = witness_blocks .into_iter() .enumerate() - .map(|(i, block)| load_or_gen_chunk_snark(&output_dir, &i.to_string(), &mut prover, block)) + .map(|(i, witness_block)| { + zkevm_prover + .load_or_gen_inner_snark(&format!("layer0_{i}"), witness_block, Some(&output_dir)) + .unwrap() + }) .collect(); - log::info!("Got real-chunk-snarks"); + log::info!("Got real-inner-snarks"); // Load or generate compression wide snarks (layer-1). - let mut layer1_snarks: Vec<_> = chunk_snarks + let mut layer1_snarks: Vec<_> = inner_snarks .into_iter() .enumerate() .map(|(i, snark)| { - load_or_gen_comp_snark( - &output_dir, - "agg_layer1", - &format!("layer1_{i}"), - true, - *AGG_LAYER1_DEGREE, - &mut prover, - snark, - ) + zkevm_prover + .load_or_gen_comp_snark( + &format!("layer1_{i}"), + "layer1", + true, + *LAYER1_DEGREE, + snark, + Some(&output_dir), + ) + .unwrap() }) .collect(); - log::info!("Got compression wide snarks (layer-1)"); + log::info!("Got compression-wide-snarks (layer-1)"); // Load or generate layer-1 padding snark. - let layer1_padding_snark = load_or_gen_padding_snark( - &output_dir, - "layer1", - &mut prover, - real_chunk_hashes.last().unwrap(), - ); + let layer1_padding_snark = agg_prover + .load_or_gen_padding_snark( + "layer1", + real_chunk_hashes.last().unwrap(), + Some(&output_dir), + ) + .unwrap(); layer1_snarks.push(layer1_padding_snark); log::info!("Got layer1-padding-snark"); @@ -85,52 +86,54 @@ fn test_agg_prove_verify() { .into_iter() .enumerate() .map(|(i, snark)| { - load_or_gen_comp_snark( - &output_dir, - "agg_layer2", - &format!("layer2_{i}"), - false, - *AGG_LAYER2_DEGREE, - &mut prover, - snark, - ) + zkevm_prover + .load_or_gen_comp_snark( + &format!("layer2_{i}"), + "layer2", + false, + *LAYER2_DEGREE, + snark, + Some(&output_dir), + ) + .unwrap() }) .collect(); - log::info!("Got compression thin snarks (layer-2)"); + log::info!("Got compression-thin-snarks (layer-2)"); // Extend to MAX_AGG_SNARKS by copying the last padding snark. - let layer2_padding_snarks: Vec<_> = repeat(layer2_snarks.last().unwrap().clone()) - .take(MAX_AGG_SNARKS - layer2_snarks.len()) - .collect(); - layer2_snarks.extend(layer2_padding_snarks); + layer2_snarks.extend( + repeat(layer2_snarks.last().unwrap().clone()).take(MAX_AGG_SNARKS - layer2_snarks.len()), + ); // Load or generate aggregation snark (layer-3). - let layer3_snark = load_or_gen_agg_snark( - &output_dir, - "agg_layer3", - "layer3_0", - *AGG_LAYER3_DEGREE, - &mut prover, - &real_chunk_hashes, - &layer2_snarks, - ); - log::info!("Got aggregation snark (layer-3)"); + let layer3_snark = agg_prover + .load_or_gen_agg_snark( + "layer3_0", + "layer3", + *LAYER3_DEGREE, + &real_chunk_hashes, + &layer2_snarks, + Some(&output_dir), + ) + .unwrap(); + log::info!("Got aggregation-snark (layer-3)"); // Load or generate compression EVM proof (layer-4). - let proof = gen_comp_evm_proof( - &output_dir, - "agg_layer4", - "layer4_0", - false, - *AGG_LAYER4_DEGREE, - &mut prover, - layer3_snark, - ); - log::info!("Got compression EVM proof (layer-4)"); + let proof = agg_prover + .gen_comp_evm_proof( + "layer4_0", + "layer4", + false, + *LAYER4_DEGREE, + layer3_snark, + Some(&output_dir), + ) + .unwrap(); + log::info!("Got compression-EVM-proof (layer-4)"); // Construct verifier and EVM verify. - let params = prover.params(*AGG_LAYER4_DEGREE).clone(); - let vk = prover.pk("layer4_0").unwrap().get_vk().clone(); + let params = agg_prover.params(*LAYER4_DEGREE).clone(); + let vk = agg_prover.pk("layer4").unwrap().get_vk().clone(); let verifier = Verifier::new(params, Some(vk)); let yul_file_path = format!("{output_dir}/agg_verifier.yul"); verifier.evm_verify::(&proof, Some(Path::new(&yul_file_path))); diff --git a/prover/tests/compression_tests.rs b/prover/tests/compression_tests.rs index 3d6218db6..1a75cab97 100644 --- a/prover/tests/compression_tests.rs +++ b/prover/tests/compression_tests.rs @@ -2,16 +2,11 @@ use aggregator::CompressionCircuit; use halo2_proofs::{halo2curves::bn256::G1Affine, plonk::VerifyingKey, SerdeFormat}; use prover::{ aggregator::{Prover, Verifier}, - config::{AGG_LAYER1_DEGREE, AGG_LAYER2_DEGREE, INNER_DEGREE}, + config::{LAYER1_DEGREE, LAYER2_DEGREE, ZKEVM_DEGREES}, io::serialize_vk, - test_util::{ - aggregator::{gen_comp_evm_proof, load_or_gen_chunk_snark, load_or_gen_comp_snark}, - load_block_traces_for_test, PARAMS_DIR, - }, + test_util::{load_block_traces_for_test, PARAMS_DIR}, utils::{chunk_trace_to_witness_block, init_env_and_log}, }; -use snark_verifier::pcs::kzg::{Bdfg21, Kzg}; -use snark_verifier_sdk::{evm_verify, gen_evm_verifier, verify_snark_shplonk, CircuitExt}; use std::{io::Cursor, path::Path}; #[cfg(feature = "prove_verify")] @@ -28,42 +23,43 @@ fn test_comp_prove_verify() { let witness_block = chunk_trace_to_witness_block(chunk_trace).unwrap(); log::info!("Got witness-block"); - let mut prover = Prover::from_params_dir( - PARAMS_DIR, - &[*INNER_DEGREE, *AGG_LAYER1_DEGREE, *AGG_LAYER2_DEGREE], - ); - log::info!("Constructed prover"); + let mut zkevm_prover = Prover::from_params_dir(PARAMS_DIR, &*ZKEVM_DEGREES); + log::info!("Constructed zkevm-prover"); - // Load or generate chunk snark. - let chunk_snark = load_or_gen_chunk_snark(&output_dir, "0", &mut prover, witness_block); - log::info!("Got chunk-snark"); + // Load or generate inner snark. + let inner_snark = zkevm_prover + .load_or_gen_inner_snark("layer0", witness_block, Some(&output_dir)) + .unwrap(); + log::info!("Got inner-snark"); // Load or generate compression wide snark (layer-1). - let layer1_snark = load_or_gen_comp_snark( - &output_dir, - "agg_layer1", - "layer1_0", - true, - *AGG_LAYER1_DEGREE, - &mut prover, - chunk_snark, - ); - log::info!("Got compression wide snark (layer-1)"); + let layer1_snark = zkevm_prover + .load_or_gen_comp_snark( + "layer1_0", + "layer1", + true, + *LAYER1_DEGREE, + inner_snark, + Some(&output_dir), + ) + .unwrap(); + log::info!("Got compression-wide-snark (layer-1)"); // Load or generate compression EVM proof (layer-2). - let proof = gen_comp_evm_proof( - &output_dir, - "agg_layer2", - "layer2_0", - false, - *AGG_LAYER2_DEGREE, - &mut prover, - layer1_snark, - ); - log::info!("Got compression EVM proof (layer-2)"); + let proof = zkevm_prover + .gen_comp_evm_proof( + "layer2_0", + "layer2", + false, + *LAYER2_DEGREE, + layer1_snark, + Some(&output_dir), + ) + .unwrap(); + log::info!("Got compression-EVM-proof (layer-2)"); // Test vk deserialization. - let vk1 = prover.pk("layer2_0").unwrap().get_vk().clone(); + let vk1 = zkevm_prover.pk("layer2").unwrap().get_vk().clone(); let raw_vk1 = serialize_vk(&vk1); let mut vk2 = VerifyingKey::::read::<_, CompressionCircuit>( &mut Cursor::new(&raw_vk1), @@ -76,8 +72,7 @@ fn test_comp_prove_verify() { log::error!("test - vk2 = {:#?}", vk2); // Construct verifier and EVM verify. - let params = prover.params(*AGG_LAYER2_DEGREE).clone(); - // let vk = prover.pk("agg_layer2").unwrap().get_vk().clone(); + let params = zkevm_prover.params(*LAYER2_DEGREE).clone(); let verifier = Verifier::new(params, Some(vk2)); let yul_file_path = format!("{output_dir}/comp_verifier.yul"); verifier.evm_verify::(&proof, Some(Path::new(&yul_file_path))); From 8ab29feafab4f0e5ccb37a9f2f92bbcf374747e1 Mon Sep 17 00:00:00 2001 From: Steven Gu Date: Thu, 13 Jul 2023 16:15:40 +0800 Subject: [PATCH 11/16] Save YUL, BIN, publib-input and proof data in `evm_verify` function. --- Cargo.lock | 20 +++++------ prover/Cargo.toml | 12 +++---- prover/configs/layer4.config | 2 +- prover/src/aggregator/prover/aggregation.rs | 5 +-- prover/src/aggregator/verifier.rs | 37 +++++++++++++++++---- prover/tests/aggregation_tests.rs | 3 +- prover/tests/compression_tests.rs | 3 +- types/Cargo.toml | 2 +- 8 files changed, 54 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b728030d1..39a3fa547 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,7 +23,7 @@ dependencies = [ [[package]] name = "aggregator" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#de2430030204e875f62ba0142d92ef794998a023" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#919ab3e2f9eaa1bcc409156d00e20fa63734b804" dependencies = [ "ark-std", "env_logger 0.10.0", @@ -418,7 +418,7 @@ checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" [[package]] name = "bus-mapping" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#de2430030204e875f62ba0142d92ef794998a023" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#919ab3e2f9eaa1bcc409156d00e20fa63734b804" dependencies = [ "eth-types", "ethers-core 0.17.0", @@ -1118,7 +1118,7 @@ dependencies = [ [[package]] name = "eth-types" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#de2430030204e875f62ba0142d92ef794998a023" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#919ab3e2f9eaa1bcc409156d00e20fa63734b804" dependencies = [ "ethers-core 0.17.0", "ethers-signers", @@ -1384,7 +1384,7 @@ dependencies = [ [[package]] name = "external-tracer" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#de2430030204e875f62ba0142d92ef794998a023" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#919ab3e2f9eaa1bcc409156d00e20fa63734b804" dependencies = [ "eth-types", "geth-utils", @@ -1613,7 +1613,7 @@ dependencies = [ [[package]] name = "gadgets" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#de2430030204e875f62ba0142d92ef794998a023" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#919ab3e2f9eaa1bcc409156d00e20fa63734b804" dependencies = [ "digest 0.7.6", "eth-types", @@ -1653,7 +1653,7 @@ dependencies = [ [[package]] name = "geth-utils" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#de2430030204e875f62ba0142d92ef794998a023" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#919ab3e2f9eaa1bcc409156d00e20fa63734b804" dependencies = [ "env_logger 0.9.3", "gobuild 0.1.0-alpha.2 (git+https://github.com/scroll-tech/gobuild.git)", @@ -2243,7 +2243,7 @@ dependencies = [ [[package]] name = "keccak256" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#de2430030204e875f62ba0142d92ef794998a023" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#919ab3e2f9eaa1bcc409156d00e20fa63734b804" dependencies = [ "env_logger 0.9.3", "eth-types", @@ -2443,7 +2443,7 @@ dependencies = [ [[package]] name = "mock" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#de2430030204e875f62ba0142d92ef794998a023" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#919ab3e2f9eaa1bcc409156d00e20fa63734b804" dependencies = [ "eth-types", "ethers-core 0.17.0", @@ -2458,7 +2458,7 @@ dependencies = [ [[package]] name = "mpt-zktrie" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#de2430030204e875f62ba0142d92ef794998a023" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#919ab3e2f9eaa1bcc409156d00e20fa63734b804" dependencies = [ "bus-mapping", "eth-types", @@ -4683,7 +4683,7 @@ checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" [[package]] name = "zkevm-circuits" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg#de2430030204e875f62ba0142d92ef794998a023" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#919ab3e2f9eaa1bcc409156d00e20fa63734b804" dependencies = [ "array-init", "bus-mapping", diff --git a/prover/Cargo.toml b/prover/Cargo.toml index acd73705d..94eb90d88 100644 --- a/prover/Cargo.toml +++ b/prover/Cargo.toml @@ -8,12 +8,12 @@ edition = "2021" [dependencies] halo2_proofs = { git = "https://github.com/privacy-scaling-explorations/halo2.git", tag = "v2023_02_02" } -aggregator = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "integrate-dynamic-proof-agg" } -bus-mapping = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "integrate-dynamic-proof-agg" } -eth-types = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "integrate-dynamic-proof-agg" } -zkevm-circuits = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "integrate-dynamic-proof-agg", default-features = false, features = ["test","scroll","scroll-trace","shanghai"] } -mpt-zktrie = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "integrate-dynamic-proof-agg" } -mock = { git = "https://github.com/scroll-tech/zkevm-circuits", branch = "integrate-dynamic-proof-agg" } +aggregator = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "integrate-dynamic-proof-agg-new" } +bus-mapping = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "integrate-dynamic-proof-agg-new" } +eth-types = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "integrate-dynamic-proof-agg-new" } +zkevm-circuits = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "integrate-dynamic-proof-agg-new", default-features = false, features = ["test","scroll","scroll-trace","shanghai"] } +mpt-zktrie = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "integrate-dynamic-proof-agg-new" } +mock = { git = "https://github.com/scroll-tech/zkevm-circuits", branch = "integrate-dynamic-proof-agg-new" } snark-verifier = { git = "https://github.com/scroll-tech/snark-verifier", branch = "develop" } snark-verifier-sdk = { git = "https://github.com/scroll-tech/snark-verifier", branch = "develop" } diff --git a/prover/configs/layer4.config b/prover/configs/layer4.config index 83e53fe41..59864d22b 100644 --- a/prover/configs/layer4.config +++ b/prover/configs/layer4.config @@ -2,7 +2,7 @@ "strategy": "Simple", "degree": 25, "num_advice": [ - 1 + 2 ], "num_lookup_advice": [ 1 diff --git a/prover/src/aggregator/prover/aggregation.rs b/prover/src/aggregator/prover/aggregation.rs index 277d1fcac..e67edddd4 100644 --- a/prover/src/aggregator/prover/aggregation.rs +++ b/prover/src/aggregator/prover/aggregation.rs @@ -4,7 +4,7 @@ use crate::{ utils::gen_rng, }; use aggregator::{AggregationCircuit, BatchHash, ChunkHash}; -use anyhow::Result; +use anyhow::{anyhow, Result}; use rand::Rng; use snark_verifier_sdk::Snark; use std::env::set_var; @@ -25,7 +25,8 @@ impl Prover { real_and_padding_snarks, &mut rng, batch_hash, - ); + ) + .map_err(|err| anyhow!("Failed to construct aggregation circuit: {err:?}"))?; self.gen_snark(id, degree, &mut rng, circuit) } diff --git a/prover/src/aggregator/verifier.rs b/prover/src/aggregator/verifier.rs index 34b522355..eb6d53513 100644 --- a/prover/src/aggregator/verifier.rs +++ b/prover/src/aggregator/verifier.rs @@ -1,4 +1,4 @@ -use crate::{utils::load_params, Proof}; +use crate::{io::write_file, utils::load_params, Proof}; use aggregator::CompressionCircuit; use anyhow::Result; use halo2_proofs::{ @@ -7,9 +7,13 @@ use halo2_proofs::{ poly::kzg::commitment::ParamsKZG, SerdeFormat, }; -use snark_verifier::pcs::kzg::{Bdfg21, Kzg}; +use itertools::Itertools; +use snark_verifier::{ + pcs::kzg::{Bdfg21, Kzg}, + util::arithmetic::PrimeField, +}; use snark_verifier_sdk::{evm_verify, gen_evm_verifier, verify_snark_shplonk, CircuitExt}; -use std::{io::Cursor, path::Path}; +use std::{io::Cursor, path::PathBuf, str::FromStr}; #[derive(Debug)] pub struct Verifier { @@ -54,7 +58,7 @@ impl Verifier { } // Should panic if failed to verify. - pub fn evm_verify>(&self, proof: &Proof, yul_file_path: Option<&Path>) { + pub fn evm_verify>(&self, proof: &Proof, output_dir: &str) { let vk = match &self.vk { Some(vk) => vk, None => panic!("Aggregation verification key is missing"), @@ -62,13 +66,34 @@ impl Verifier { let num_instance = proof.num_instance().expect("Not a EVM proof").clone(); + let mut yul_file_path = PathBuf::from_str(&output_dir).unwrap(); + yul_file_path.push("evm_verifier.yul"); + + // Generate deployment code and dump YUL file. let deployment_code = gen_evm_verifier::>( &self.params, vk, num_instance, - yul_file_path, + Some(yul_file_path.as_path()), ); - evm_verify(deployment_code, proof.instances(), proof.proof().to_vec()); + // Dump bytecode. + let mut output_dir = PathBuf::from_str(&output_dir).unwrap(); + write_file(&mut output_dir, "evm_verifier.bin", &deployment_code); + + // Dump public input data. + let pi_data: Vec<_> = proof + .instances() + .iter() + .flatten() + .flat_map(|value| value.to_repr().as_ref().iter().rev().cloned().collect_vec()) + .collect(); + write_file(&mut output_dir, "evm_pi_data.data", &pi_data); + + // Dump proof. + let proof_data = proof.proof().to_vec(); + write_file(&mut output_dir, "evm_proof.data", &proof_data); + + evm_verify(deployment_code, proof.instances(), proof_data); } } diff --git a/prover/tests/aggregation_tests.rs b/prover/tests/aggregation_tests.rs index 40acd7860..66bf0b1bd 100644 --- a/prover/tests/aggregation_tests.rs +++ b/prover/tests/aggregation_tests.rs @@ -135,7 +135,6 @@ fn test_agg_prove_verify() { let params = agg_prover.params(*LAYER4_DEGREE).clone(); let vk = agg_prover.pk("layer4").unwrap().get_vk().clone(); let verifier = Verifier::new(params, Some(vk)); - let yul_file_path = format!("{output_dir}/agg_verifier.yul"); - verifier.evm_verify::(&proof, Some(Path::new(&yul_file_path))); + verifier.evm_verify::(&proof, &output_dir); log::info!("Finish EVM verify"); } diff --git a/prover/tests/compression_tests.rs b/prover/tests/compression_tests.rs index 1a75cab97..02786e9d7 100644 --- a/prover/tests/compression_tests.rs +++ b/prover/tests/compression_tests.rs @@ -74,7 +74,6 @@ fn test_comp_prove_verify() { // Construct verifier and EVM verify. let params = zkevm_prover.params(*LAYER2_DEGREE).clone(); let verifier = Verifier::new(params, Some(vk2)); - let yul_file_path = format!("{output_dir}/comp_verifier.yul"); - verifier.evm_verify::(&proof, Some(Path::new(&yul_file_path))); + verifier.evm_verify::(&proof, &output_dir); log::info!("Finish EVM verify"); } diff --git a/types/Cargo.toml b/types/Cargo.toml index 6e8fa53c3..b3cf84631 100644 --- a/types/Cargo.toml +++ b/types/Cargo.toml @@ -4,7 +4,7 @@ version = "0.4.0" edition = "2021" [dependencies] -eth-types = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "integrate-dynamic-proof-agg" } +eth-types = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "integrate-dynamic-proof-agg-new" } base64 = "0.13.0" blake2 = "0.10.3" ethers-core = "0.17.0" From d23c3e7892392c4164805abfad820f2949e9b08c Mon Sep 17 00:00:00 2001 From: Steven Gu Date: Mon, 17 Jul 2023 12:30:05 +0800 Subject: [PATCH 12/16] Re-organize FFI to aggregator and zkevm. --- ffi/src/aggregator.rs | 81 +++++++++++++++++++++++++++++++++++ ffi/src/lib.rs | 26 ++---------- ffi/src/prove.rs | 98 ------------------------------------------- ffi/src/utils.rs | 24 +++++++++++ ffi/src/verify.rs | 64 ---------------------------- ffi/src/zkevm.rs | 80 +++++++++++++++++++++++++++++++++++ 6 files changed, 188 insertions(+), 185 deletions(-) create mode 100644 ffi/src/aggregator.rs delete mode 100644 ffi/src/prove.rs create mode 100644 ffi/src/utils.rs delete mode 100644 ffi/src/verify.rs create mode 100644 ffi/src/zkevm.rs diff --git a/ffi/src/aggregator.rs b/ffi/src/aggregator.rs new file mode 100644 index 000000000..55544678e --- /dev/null +++ b/ffi/src/aggregator.rs @@ -0,0 +1,81 @@ +use crate::utils::{c_char_to_str, c_char_to_vec, vec_to_c_char, OUTPUT_DIR}; +use libc::c_char; +use prover::{ + aggregator::{ChunkHash, Prover, Verifier}, + config::{AGG_DEGREES, LAYER4_DEGREE}, + utils::init_env_and_log, + Proof, +}; +use std::{cell::OnceCell, fs::File, io::Read}; + +static mut AGG_PROVER: OnceCell = OnceCell::new(); +static mut AGG_VERIFIER: Option<&Verifier> = None; + +/// # Safety +#[no_mangle] +pub unsafe extern "C" fn init_agg_prover(params_dir: *const c_char) { + init_env_and_log("ffi_agg_prove"); + + let params_dir = c_char_to_str(params_dir); + + let prover = Prover::from_params_dir(params_dir, &AGG_DEGREES); + AGG_PROVER.set(prover).unwrap(); +} + +/// # Safety +#[no_mangle] +pub unsafe extern "C" fn init_agg_verifier(params_dir: *const c_char, vk_path: *const c_char) { + init_env_and_log("ffi_agg_verify"); + + let vk_path = c_char_to_str(vk_path); + let mut f = File::open(vk_path).unwrap(); + let mut vk = vec![]; + f.read_to_end(&mut vk).unwrap(); + + let params_dir = c_char_to_str(params_dir); + let verifier = Box::new(Verifier::from_params_dir( + params_dir, + *LAYER4_DEGREE, + Some(vk), + )); + + AGG_VERIFIER = Some(Box::leak(verifier)); +} + +/// # Safety +#[no_mangle] +pub unsafe extern "C" fn create_agg_proof( + chunk_hashes: *const c_char, + chunk_proofs: *const c_char, +) -> *const c_char { + let chunk_hashes = c_char_to_vec(chunk_hashes); + let chunk_proofs = c_char_to_vec(chunk_proofs); + + let chunk_hashes = serde_json::from_slice::>(&chunk_hashes).unwrap(); + let chunk_proofs = serde_json::from_slice::>(&chunk_proofs).unwrap(); + assert_eq!(chunk_hashes.len(), chunk_proofs.len()); + + let chunks = chunk_hashes + .into_iter() + .zip(chunk_proofs.into_iter()) + .collect(); + + let proof = AGG_PROVER + .get_mut() + .unwrap() + .gen_agg_proof(chunks, OUTPUT_DIR.as_deref()) + .unwrap(); + + let proof_bytes = serde_json::to_vec(&proof).unwrap(); + vec_to_c_char(proof_bytes) +} + +/// # Safety +#[no_mangle] +pub unsafe extern "C" fn verify_agg_proof(proof: *const c_char) -> c_char { + let proof = c_char_to_vec(proof); + let proof = serde_json::from_slice::(proof.as_slice()).unwrap(); + + let verified = AGG_VERIFIER.unwrap().verify_agg_proof(proof).is_ok(); + verified as c_char +} diff --git a/ffi/src/lib.rs b/ffi/src/lib.rs index c9a0070ad..3e27d48eb 100644 --- a/ffi/src/lib.rs +++ b/ffi/src/lib.rs @@ -1,25 +1,5 @@ #![feature(once_cell)] -pub mod prove; -pub mod verify; - -pub(crate) mod utils { - use std::{ - ffi::{CStr, CString}, - os::raw::c_char, - }; - - pub(crate) fn c_char_to_str(c: *const c_char) -> &'static str { - let cstr = unsafe { CStr::from_ptr(c) }; - cstr.to_str().unwrap() - } - - pub(crate) fn c_char_to_vec(c: *const c_char) -> Vec { - let cstr = unsafe { CStr::from_ptr(c) }; - cstr.to_bytes().to_vec() - } - - pub(crate) fn vec_to_c_char(bytes: Vec) -> *const c_char { - CString::new(bytes).unwrap().into_raw() - } -} +mod aggregator; +mod utils; +mod zkevm; diff --git a/ffi/src/prove.rs b/ffi/src/prove.rs deleted file mode 100644 index 9bf1cf7f0..000000000 --- a/ffi/src/prove.rs +++ /dev/null @@ -1,98 +0,0 @@ -use crate::utils::{c_char_to_str, c_char_to_vec, vec_to_c_char}; -use libc::c_char; -use once_cell::sync::Lazy; -use prover::{ - aggregator::{self, ChunkHash}, - config::AGG_DEGREES, - utils::init_env_and_log, - zkevm, Proof, -}; -use std::{cell::OnceCell, env}; -use types::eth::BlockTrace; - -static mut ZKEVM_PROVER: OnceCell = OnceCell::new(); -static mut AGG_PROVER: OnceCell = OnceCell::new(); - -// Only used for debugging. -static OUTPUT_DIR: Lazy> = Lazy::new(|| env::var("PROVER_OUTPUT_DIR").ok()); - -/// # Safety -#[no_mangle] -pub unsafe extern "C" fn init_zkevm_prover(params_dir: *const c_char) { - init_env_and_log("ffi_zkevm_prove"); - - let params_dir = c_char_to_str(params_dir); - let prover = zkevm::Prover::from_params_dir(params_dir); - ZKEVM_PROVER.set(prover).unwrap(); -} - -/// # Safety -#[no_mangle] -pub unsafe extern "C" fn init_agg_prover(params_dir: *const c_char) { - init_env_and_log("ffi_agg_prove"); - - let params_dir = c_char_to_str(params_dir); - - let prover = aggregator::Prover::from_params_dir(params_dir, &AGG_DEGREES); - AGG_PROVER.set(prover).unwrap(); -} - -/// # Safety -#[no_mangle] -pub unsafe extern "C" fn create_block_proof(block_trace: *const c_char) -> *const c_char { - let block_trace = c_char_to_vec(block_trace); - let block_trace = serde_json::from_slice::(&block_trace).unwrap(); - - let proof = ZKEVM_PROVER - .get_mut() - .unwrap() - .gen_chunk_proof(&[block_trace]) - .unwrap(); - - let proof_bytes = serde_json::to_vec(&proof).unwrap(); - vec_to_c_char(proof_bytes) -} - -/// # Safety -#[no_mangle] -pub unsafe extern "C" fn create_chunk_proof(block_traces: *const c_char) -> *const c_char { - let block_traces = c_char_to_vec(block_traces); - let block_traces = serde_json::from_slice::>(&block_traces).unwrap(); - - let proof = ZKEVM_PROVER - .get_mut() - .unwrap() - .gen_chunk_proof(block_traces.as_slice()) - .unwrap(); - - let proof_bytes = serde_json::to_vec(&proof).unwrap(); - vec_to_c_char(proof_bytes) -} - -/// # Safety -#[no_mangle] -pub unsafe extern "C" fn create_agg_proof( - chunk_hashes: *const c_char, - chunk_proofs: *const c_char, -) -> *const c_char { - let chunk_hashes = c_char_to_vec(chunk_hashes); - let chunk_proofs = c_char_to_vec(chunk_proofs); - - let chunk_hashes = serde_json::from_slice::>(&chunk_hashes).unwrap(); - let chunk_proofs = serde_json::from_slice::>(&chunk_proofs).unwrap(); - assert_eq!(chunk_hashes.len(), chunk_proofs.len()); - - let chunks = chunk_hashes - .into_iter() - .zip(chunk_proofs.into_iter()) - .collect(); - - let proof = AGG_PROVER - .get_mut() - .unwrap() - .gen_agg_proof(chunks, OUTPUT_DIR.as_deref()) - .unwrap(); - - let proof_bytes = serde_json::to_vec(&proof).unwrap(); - vec_to_c_char(proof_bytes) -} diff --git a/ffi/src/utils.rs b/ffi/src/utils.rs new file mode 100644 index 000000000..2017dd34c --- /dev/null +++ b/ffi/src/utils.rs @@ -0,0 +1,24 @@ +use once_cell::sync::Lazy; +use std::{ + env, + ffi::{CStr, CString}, + os::raw::c_char, +}; + +// Only used for debugging. +pub(crate) static OUTPUT_DIR: Lazy> = + Lazy::new(|| env::var("PROVER_OUTPUT_DIR").ok()); + +pub(crate) fn c_char_to_str(c: *const c_char) -> &'static str { + let cstr = unsafe { CStr::from_ptr(c) }; + cstr.to_str().unwrap() +} + +pub(crate) fn c_char_to_vec(c: *const c_char) -> Vec { + let cstr = unsafe { CStr::from_ptr(c) }; + cstr.to_bytes().to_vec() +} + +pub(crate) fn vec_to_c_char(bytes: Vec) -> *const c_char { + CString::new(bytes).unwrap().into_raw() +} diff --git a/ffi/src/verify.rs b/ffi/src/verify.rs deleted file mode 100644 index 5b4467eb6..000000000 --- a/ffi/src/verify.rs +++ /dev/null @@ -1,64 +0,0 @@ -use crate::utils::{c_char_to_str, c_char_to_vec}; -use libc::c_char; -use prover::{aggregator, config::LAYER4_DEGREE, utils::init_env_and_log, zkevm, Proof}; -use std::{fs::File, io::Read}; - -static mut CHUNK_VERIFIER: Option<&zkevm::Verifier> = None; -static mut AGG_VERIFIER: Option<&aggregator::Verifier> = None; - -/// # Safety -#[no_mangle] -pub unsafe extern "C" fn init_chunk_verifier(params_dir: *const c_char, vk_path: *const c_char) { - init_env_and_log("ffi_chunk_verify"); - - let vk_path = c_char_to_str(vk_path); - let mut f = File::open(vk_path).unwrap(); - let mut vk = vec![]; - f.read_to_end(&mut vk).unwrap(); - - let params_dir = c_char_to_str(params_dir); - let verifier = Box::new(zkevm::Verifier::from_params_dir(params_dir, Some(vk))); - - CHUNK_VERIFIER = Some(Box::leak(verifier)); -} - -/// # Safety -#[no_mangle] -pub unsafe extern "C" fn init_agg_verifier(params_dir: *const c_char, vk_path: *const c_char) { - init_env_and_log("ffi_agg_verify"); - - let vk_path = c_char_to_str(vk_path); - let mut f = File::open(vk_path).unwrap(); - let mut vk = vec![]; - f.read_to_end(&mut vk).unwrap(); - - let params_dir = c_char_to_str(params_dir); - let verifier = Box::new(aggregator::Verifier::from_params_dir( - params_dir, - *LAYER4_DEGREE, - Some(vk), - )); - - AGG_VERIFIER = Some(Box::leak(verifier)); -} - -/// # Safety -#[no_mangle] -pub unsafe extern "C" fn verify_chunk_proof(proof: *const c_char) -> c_char { - let proof_vec = c_char_to_vec(proof); - let proof = serde_json::from_slice::(proof_vec.as_slice()).unwrap(); - let verified = CHUNK_VERIFIER.unwrap().verify_chunk_proof(proof).is_ok(); - - verified as c_char -} - -/// # Safety -#[no_mangle] -pub unsafe extern "C" fn verify_agg_proof(proof: *const c_char) -> c_char { - let proof_vec = c_char_to_vec(proof); - let proof = serde_json::from_slice::(proof_vec.as_slice()).unwrap(); - - let verified = AGG_VERIFIER.unwrap().verify_agg_proof(proof).is_ok(); - - verified as c_char -} diff --git a/ffi/src/zkevm.rs b/ffi/src/zkevm.rs new file mode 100644 index 000000000..8adb135ce --- /dev/null +++ b/ffi/src/zkevm.rs @@ -0,0 +1,80 @@ +use crate::utils::{c_char_to_str, c_char_to_vec, vec_to_c_char}; +use libc::c_char; +use prover::{ + utils::init_env_and_log, + zkevm::{Prover, Verifier}, + Proof, +}; +use std::{cell::OnceCell, fs::File, io::Read}; +use types::eth::BlockTrace; + +static mut ZKEVM_PROVER: OnceCell = OnceCell::new(); +static mut ZKEVM_VERIFIER: Option<&Verifier> = None; + +/// # Safety +#[no_mangle] +pub unsafe extern "C" fn init_zkevm_prover(params_dir: *const c_char) { + init_env_and_log("ffi_zkevm_prove"); + + let params_dir = c_char_to_str(params_dir); + let prover = Prover::from_params_dir(params_dir); + ZKEVM_PROVER.set(prover).unwrap(); +} + +/// # Safety +#[no_mangle] +pub unsafe extern "C" fn init_zkevm_verifier(params_dir: *const c_char, vk_path: *const c_char) { + init_env_and_log("ffi_zkevm_verify"); + + let vk_path = c_char_to_str(vk_path); + let mut f = File::open(vk_path).unwrap(); + let mut vk = vec![]; + f.read_to_end(&mut vk).unwrap(); + + let params_dir = c_char_to_str(params_dir); + let verifier = Box::new(Verifier::from_params_dir(params_dir, Some(vk))); + + ZKEVM_VERIFIER = Some(Box::leak(verifier)); +} + +/// # Safety +#[no_mangle] +pub unsafe extern "C" fn create_block_proof(block_trace: *const c_char) -> *const c_char { + let block_trace = c_char_to_vec(block_trace); + let block_trace = serde_json::from_slice::(&block_trace).unwrap(); + + let proof = ZKEVM_PROVER + .get_mut() + .unwrap() + .gen_chunk_proof(&[block_trace]) + .unwrap(); + + let proof_bytes = serde_json::to_vec(&proof).unwrap(); + vec_to_c_char(proof_bytes) +} + +/// # Safety +#[no_mangle] +pub unsafe extern "C" fn create_chunk_proof(block_traces: *const c_char) -> *const c_char { + let block_traces = c_char_to_vec(block_traces); + let block_traces = serde_json::from_slice::>(&block_traces).unwrap(); + + let proof = ZKEVM_PROVER + .get_mut() + .unwrap() + .gen_chunk_proof(block_traces.as_slice()) + .unwrap(); + + let proof_bytes = serde_json::to_vec(&proof).unwrap(); + vec_to_c_char(proof_bytes) +} + +/// # Safety +#[no_mangle] +pub unsafe extern "C" fn verify_chunk_proof(proof: *const c_char) -> c_char { + let proof = c_char_to_vec(proof); + let proof = serde_json::from_slice::(proof.as_slice()).unwrap(); + + let verified = ZKEVM_VERIFIER.unwrap().verify_chunk_proof(proof).is_ok(); + verified as c_char +} From 201addb0a7d3027fc10e50a7cd03ee0a20664491 Mon Sep 17 00:00:00 2001 From: Steven Gu Date: Mon, 17 Jul 2023 17:24:18 +0800 Subject: [PATCH 13/16] Extract to `common` prover and verifier. --- Cargo.lock | 22 +-- prover/src/aggregator/prover.rs | 133 +++--------------- prover/src/aggregator/verifier.rs | 82 ++--------- prover/src/common.rs | 5 + prover/src/common/prover.rs | 122 ++++++++++++++++ .../prover/aggregation.rs | 0 .../prover/compression.rs | 0 .../src/{aggregator => common}/prover/evm.rs | 0 .../{aggregator => common}/prover/inner.rs | 0 .../{aggregator => common}/prover/padding.rs | 0 .../common.rs => common/prover/utils.rs} | 0 prover/src/common/verifier.rs | 42 ++++++ prover/src/common/verifier/evm.rs | 52 +++++++ prover/src/common/verifier/utils.rs | 16 +++ prover/src/lib.rs | 2 + prover/tests/aggregation_tests.rs | 29 ++-- prover/tests/compression_tests.rs | 20 +-- 17 files changed, 307 insertions(+), 218 deletions(-) create mode 100644 prover/src/common.rs create mode 100644 prover/src/common/prover.rs rename prover/src/{aggregator => common}/prover/aggregation.rs (100%) rename prover/src/{aggregator => common}/prover/compression.rs (100%) rename prover/src/{aggregator => common}/prover/evm.rs (100%) rename prover/src/{aggregator => common}/prover/inner.rs (100%) rename prover/src/{aggregator => common}/prover/padding.rs (100%) rename prover/src/{aggregator/prover/common.rs => common/prover/utils.rs} (100%) create mode 100644 prover/src/common/verifier.rs create mode 100644 prover/src/common/verifier/evm.rs create mode 100644 prover/src/common/verifier/utils.rs diff --git a/Cargo.lock b/Cargo.lock index 39a3fa547..ada2bc432 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,7 +23,7 @@ dependencies = [ [[package]] name = "aggregator" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#919ab3e2f9eaa1bcc409156d00e20fa63734b804" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#9d3ca9f47f913735b9404a364cb0066114f20a49" dependencies = [ "ark-std", "env_logger 0.10.0", @@ -418,7 +418,7 @@ checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" [[package]] name = "bus-mapping" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#919ab3e2f9eaa1bcc409156d00e20fa63734b804" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#9d3ca9f47f913735b9404a364cb0066114f20a49" dependencies = [ "eth-types", "ethers-core 0.17.0", @@ -1118,7 +1118,7 @@ dependencies = [ [[package]] name = "eth-types" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#919ab3e2f9eaa1bcc409156d00e20fa63734b804" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#9d3ca9f47f913735b9404a364cb0066114f20a49" dependencies = [ "ethers-core 0.17.0", "ethers-signers", @@ -1384,7 +1384,7 @@ dependencies = [ [[package]] name = "external-tracer" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#919ab3e2f9eaa1bcc409156d00e20fa63734b804" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#9d3ca9f47f913735b9404a364cb0066114f20a49" dependencies = [ "eth-types", "geth-utils", @@ -1613,7 +1613,7 @@ dependencies = [ [[package]] name = "gadgets" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#919ab3e2f9eaa1bcc409156d00e20fa63734b804" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#9d3ca9f47f913735b9404a364cb0066114f20a49" dependencies = [ "digest 0.7.6", "eth-types", @@ -1653,7 +1653,7 @@ dependencies = [ [[package]] name = "geth-utils" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#919ab3e2f9eaa1bcc409156d00e20fa63734b804" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#9d3ca9f47f913735b9404a364cb0066114f20a49" dependencies = [ "env_logger 0.9.3", "gobuild 0.1.0-alpha.2 (git+https://github.com/scroll-tech/gobuild.git)", @@ -1785,7 +1785,7 @@ dependencies = [ [[package]] name = "halo2-mpt-circuits" version = "0.1.0" -source = "git+https://github.com/scroll-tech/mpt-circuit.git?branch=v0.4#9d129125bd792e906c30e56386424bc3ab5920ba" +source = "git+https://github.com/scroll-tech/mpt-circuit.git?branch=v0.4#2e216c2d30d35a459f933a58b8002fe817abb1b4" dependencies = [ "ethers-core 0.17.0", "halo2_proofs", @@ -2243,7 +2243,7 @@ dependencies = [ [[package]] name = "keccak256" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#919ab3e2f9eaa1bcc409156d00e20fa63734b804" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#9d3ca9f47f913735b9404a364cb0066114f20a49" dependencies = [ "env_logger 0.9.3", "eth-types", @@ -2443,7 +2443,7 @@ dependencies = [ [[package]] name = "mock" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#919ab3e2f9eaa1bcc409156d00e20fa63734b804" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#9d3ca9f47f913735b9404a364cb0066114f20a49" dependencies = [ "eth-types", "ethers-core 0.17.0", @@ -2458,7 +2458,7 @@ dependencies = [ [[package]] name = "mpt-zktrie" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#919ab3e2f9eaa1bcc409156d00e20fa63734b804" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#9d3ca9f47f913735b9404a364cb0066114f20a49" dependencies = [ "bus-mapping", "eth-types", @@ -4683,7 +4683,7 @@ checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" [[package]] name = "zkevm-circuits" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#919ab3e2f9eaa1bcc409156d00e20fa63734b804" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#9d3ca9f47f913735b9404a364cb0066114f20a49" dependencies = [ "array-init", "bus-mapping", diff --git a/prover/src/aggregator/prover.rs b/prover/src/aggregator/prover.rs index 81674b4cd..13dffcc11 100644 --- a/prover/src/aggregator/prover.rs +++ b/prover/src/aggregator/prover.rs @@ -1,80 +1,31 @@ use crate::{ - config::{LAYER1_DEGREE, LAYER2_DEGREE, LAYER3_DEGREE, LAYER4_DEGREE}, - utils::{chunk_trace_to_witness_block, load_params, param_path_for_degree}, + common, + config::{LAYER2_DEGREE, LAYER3_DEGREE, LAYER4_DEGREE}, Proof, }; use aggregator::{ChunkHash, MAX_AGG_SNARKS}; use anyhow::Result; -use halo2_proofs::{ - halo2curves::bn256::{Bn256, G1Affine}, - plonk::ProvingKey, - poly::{commitment::Params, kzg::commitment::ParamsKZG}, -}; -use std::{ - collections::{BTreeMap, BTreeSet, HashMap}, - iter::repeat, -}; -use types::eth::BlockTrace; - -mod aggregation; -mod common; -mod compression; -mod evm; -mod inner; -mod padding; +use halo2_proofs::{halo2curves::bn256::Bn256, poly::kzg::commitment::ParamsKZG}; +use std::{collections::BTreeMap, iter::repeat}; #[derive(Debug)] pub struct Prover { - // degree -> params (use BTreeMap to find proper degree for params downsize) - params_map: BTreeMap>, - // Cached id -> pk - pk_map: HashMap>, + inner: common::Prover, +} + +impl From for Prover { + fn from(inner: common::Prover) -> Self { + Self { inner } + } } impl Prover { pub fn from_params(params_map: BTreeMap>) -> Self { - Self { - params_map, - pk_map: HashMap::new(), - } + common::Prover::from_params(params_map).into() } pub fn from_params_dir(params_dir: &str, degrees: &[u32]) -> Self { - let degrees = BTreeSet::from_iter(degrees); - let max_degree = **degrees.last().unwrap(); - - // Downsize params if any params of degree doesn't exist. - let mut params_map = BTreeMap::new(); - for d in BTreeSet::from_iter(degrees).into_iter().rev() { - let params = match load_params(params_dir, *d, None) { - Ok(params) => params, - Err(_) => { - let params: &ParamsKZG<_> = params_map - .first_key_value() - .unwrap_or_else(|| { - panic!( - "File `{}` must exist", - param_path_for_degree(params_dir, max_degree) - ) - }) - .1; - - let mut params: ParamsKZG<_> = params.clone(); - params.downsize(*d); - - log::warn!("Optimization: download params{d} to params dir",); - - params - } - }; - - params_map.insert(*d, params); - } - - Self { - params_map, - pk_map: HashMap::new(), - } + common::Prover::from_params_dir(params_dir, degrees).into() } pub fn gen_agg_proof( @@ -93,11 +44,12 @@ impl Prover { // Load or generate padding snark (layer-1). let layer1_padding_snark = - self.load_or_gen_padding_snark(&name, &last_real_chunk_hash, output_dir)?; + self.inner + .load_or_gen_padding_snark(&name, &last_real_chunk_hash, output_dir)?; log::info!("Got padding snark (layer-1): {name}"); // Load or generate compression thin snark for padding (layer-2). - let layer2_padding_snark = self.load_or_gen_comp_snark( + let layer2_padding_snark = self.inner.load_or_gen_comp_snark( &name, "layer2", false, @@ -116,7 +68,7 @@ impl Prover { layer2_snarks.extend(repeat(layer2_padding_snark).take(MAX_AGG_SNARKS - real_chunk_count)); // Load or generate aggregation snark (layer-3). - let layer3_snark = self.load_or_gen_agg_snark( + let layer3_snark = self.inner.load_or_gen_agg_snark( &name, "layer3", *LAYER3_DEGREE, @@ -127,7 +79,7 @@ impl Prover { log::info!("Got aggregation snark (layer-3): {name}"); // Load or generate final compression thin snark (layer-4). - let layer4_snark = self.load_or_gen_comp_snark( + let layer4_snark = self.inner.load_or_gen_comp_snark( &name, "layer4", false, @@ -137,54 +89,7 @@ impl Prover { )?; log::info!("Got final compression thin snark (layer-4): {name}"); - let pk = self.pk("layer4").unwrap(); + let pk = self.inner.pk("layer4").unwrap(); Proof::from_snark(pk, &layer4_snark) } - - pub fn gen_chunk_proof( - &mut self, - chunk_trace: Vec, - output_dir: Option<&str>, - ) -> Result { - assert!(!chunk_trace.is_empty()); - - let witness_block = chunk_trace_to_witness_block(chunk_trace)?; - log::info!("Got witness block"); - - let name = witness_block - .context - .first_or_default() - .number - .low_u64() - .to_string(); - - // Load or generate inner snark. - let inner_snark = self.load_or_gen_inner_snark(&name, witness_block, output_dir)?; - log::info!("Got inner snark: {name}"); - - // Load or generate compression wide snark (layer-1). - let layer1_snark = self.load_or_gen_comp_snark( - &name, - "layer1", - true, - *LAYER1_DEGREE, - inner_snark, - output_dir, - )?; - log::info!("Got compression wide snark (layer-1): {name}"); - - // Load or generate compression thin snark (layer-2). - let layer2_snark = self.load_or_gen_comp_snark( - &name, - "layer2", - false, - *LAYER2_DEGREE, - layer1_snark, - output_dir, - )?; - log::info!("Got compression thin snark (layer-2): {name}"); - - let pk = self.pk("layer2").unwrap(); - Proof::from_snark(pk, &layer2_snark) - } } diff --git a/prover/src/aggregator/verifier.rs b/prover/src/aggregator/verifier.rs index eb6d53513..35b8a4bc1 100644 --- a/prover/src/aggregator/verifier.rs +++ b/prover/src/aggregator/verifier.rs @@ -1,99 +1,47 @@ -use crate::{io::write_file, utils::load_params, Proof}; +use crate::{common, Proof}; use aggregator::CompressionCircuit; use anyhow::Result; use halo2_proofs::{ - halo2curves::bn256::{Bn256, Fr, G1Affine}, + halo2curves::bn256::{Bn256, G1Affine}, plonk::VerifyingKey, poly::kzg::commitment::ParamsKZG, - SerdeFormat, }; -use itertools::Itertools; -use snark_verifier::{ - pcs::kzg::{Bdfg21, Kzg}, - util::arithmetic::PrimeField, -}; -use snark_verifier_sdk::{evm_verify, gen_evm_verifier, verify_snark_shplonk, CircuitExt}; -use std::{io::Cursor, path::PathBuf, str::FromStr}; +use snark_verifier_sdk::verify_snark_shplonk; #[derive(Debug)] pub struct Verifier { - params: ParamsKZG, - vk: Option>, + inner: common::Verifier, +} + +impl From for Verifier { + fn from(inner: common::Verifier) -> Self { + Self { inner } + } } impl Verifier { pub fn new(params: ParamsKZG, vk: Option>) -> Self { - Self { params, vk } + common::Verifier::new(params, vk).into() } pub fn from_params(params: ParamsKZG, raw_vk: Option>) -> Self { - let vk = raw_vk.as_ref().map(|k| { - VerifyingKey::::read::<_, CompressionCircuit>( - &mut Cursor::new(&k), - SerdeFormat::Processed, - ) - .unwrap() - }); - - Self { params, vk } + common::Verifier::from_params(params, raw_vk).into() } pub fn from_params_dir(params_dir: &str, degree: u32, vk: Option>) -> Self { - let params = load_params(params_dir, degree, None).unwrap(); - - Self::from_params(params, vk) + common::Verifier::from_params_dir(params_dir, degree, vk).into() } pub fn verify_agg_proof(&self, proof: Proof) -> Result { - let vk = match &self.vk { + let vk = match self.inner.vk() { Some(vk) => vk, None => panic!("Aggregation verification key is missing"), }; Ok(verify_snark_shplonk::( - &self.params, + self.inner.params(), proof.to_snark(), vk, )) } - - // Should panic if failed to verify. - pub fn evm_verify>(&self, proof: &Proof, output_dir: &str) { - let vk = match &self.vk { - Some(vk) => vk, - None => panic!("Aggregation verification key is missing"), - }; - - let num_instance = proof.num_instance().expect("Not a EVM proof").clone(); - - let mut yul_file_path = PathBuf::from_str(&output_dir).unwrap(); - yul_file_path.push("evm_verifier.yul"); - - // Generate deployment code and dump YUL file. - let deployment_code = gen_evm_verifier::>( - &self.params, - vk, - num_instance, - Some(yul_file_path.as_path()), - ); - - // Dump bytecode. - let mut output_dir = PathBuf::from_str(&output_dir).unwrap(); - write_file(&mut output_dir, "evm_verifier.bin", &deployment_code); - - // Dump public input data. - let pi_data: Vec<_> = proof - .instances() - .iter() - .flatten() - .flat_map(|value| value.to_repr().as_ref().iter().rev().cloned().collect_vec()) - .collect(); - write_file(&mut output_dir, "evm_pi_data.data", &pi_data); - - // Dump proof. - let proof_data = proof.proof().to_vec(); - write_file(&mut output_dir, "evm_proof.data", &proof_data); - - evm_verify(deployment_code, proof.instances(), proof_data); - } } diff --git a/prover/src/common.rs b/prover/src/common.rs new file mode 100644 index 000000000..ca8b5acb1 --- /dev/null +++ b/prover/src/common.rs @@ -0,0 +1,5 @@ +mod prover; +mod verifier; + +pub use self::{prover::Prover, verifier::Verifier}; +pub use aggregator::ChunkHash; diff --git a/prover/src/common/prover.rs b/prover/src/common/prover.rs new file mode 100644 index 000000000..859ac24d2 --- /dev/null +++ b/prover/src/common/prover.rs @@ -0,0 +1,122 @@ +use crate::{ + config::{LAYER1_DEGREE, LAYER2_DEGREE}, + utils::{chunk_trace_to_witness_block, load_params, param_path_for_degree}, + Proof, +}; +use anyhow::Result; +use halo2_proofs::{ + halo2curves::bn256::{Bn256, G1Affine}, + plonk::ProvingKey, + poly::{commitment::Params, kzg::commitment::ParamsKZG}, +}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use types::eth::BlockTrace; + +mod aggregation; +mod compression; +mod evm; +mod inner; +mod padding; +mod utils; + +#[derive(Debug)] +pub struct Prover { + // degree -> params (use BTreeMap to find proper degree for params downsize) + params_map: BTreeMap>, + // Cached id -> pk + pk_map: HashMap>, +} + +impl Prover { + pub fn from_params(params_map: BTreeMap>) -> Self { + Self { + params_map, + pk_map: HashMap::new(), + } + } + + pub fn from_params_dir(params_dir: &str, degrees: &[u32]) -> Self { + let degrees = BTreeSet::from_iter(degrees); + let max_degree = **degrees.last().unwrap(); + + // Downsize params if any params of degree doesn't exist. + let mut params_map = BTreeMap::new(); + for d in BTreeSet::from_iter(degrees).into_iter().rev() { + let params = match load_params(params_dir, *d, None) { + Ok(params) => params, + Err(_) => { + let params: &ParamsKZG<_> = params_map + .first_key_value() + .unwrap_or_else(|| { + panic!( + "File `{}` must exist", + param_path_for_degree(params_dir, max_degree) + ) + }) + .1; + + let mut params: ParamsKZG<_> = params.clone(); + params.downsize(*d); + + log::warn!("Optimization: download params{d} to params dir",); + + params + } + }; + + params_map.insert(*d, params); + } + + Self { + params_map, + pk_map: HashMap::new(), + } + } + + pub fn gen_chunk_proof( + &mut self, + chunk_trace: Vec, + output_dir: Option<&str>, + ) -> Result { + assert!(!chunk_trace.is_empty()); + + let witness_block = chunk_trace_to_witness_block(chunk_trace)?; + log::info!("Got witness block"); + + let name = witness_block + .context + .first_or_default() + .number + .low_u64() + .to_string(); + + // Load or generate inner snark. + let inner_snark = self.load_or_gen_inner_snark(&name, witness_block, output_dir)?; + log::info!("Got inner snark: {name}"); + + // Load or generate compression wide snark (layer-1). + let layer1_snark = self.load_or_gen_comp_snark( + &name, + "layer1", + true, + *LAYER1_DEGREE, + inner_snark, + output_dir, + )?; + log::info!("Got compression wide snark (layer-1): {name}"); + + // Load or generate compression thin snark (layer-2). + let layer2_snark = self.load_or_gen_comp_snark( + &name, + "layer2", + false, + *LAYER2_DEGREE, + layer1_snark, + output_dir, + )?; + log::info!("Got compression thin snark (layer-2): {name}"); + + let pk = self.pk("layer2").unwrap(); + Proof::from_snark(pk, &layer2_snark) + } +} diff --git a/prover/src/aggregator/prover/aggregation.rs b/prover/src/common/prover/aggregation.rs similarity index 100% rename from prover/src/aggregator/prover/aggregation.rs rename to prover/src/common/prover/aggregation.rs diff --git a/prover/src/aggregator/prover/compression.rs b/prover/src/common/prover/compression.rs similarity index 100% rename from prover/src/aggregator/prover/compression.rs rename to prover/src/common/prover/compression.rs diff --git a/prover/src/aggregator/prover/evm.rs b/prover/src/common/prover/evm.rs similarity index 100% rename from prover/src/aggregator/prover/evm.rs rename to prover/src/common/prover/evm.rs diff --git a/prover/src/aggregator/prover/inner.rs b/prover/src/common/prover/inner.rs similarity index 100% rename from prover/src/aggregator/prover/inner.rs rename to prover/src/common/prover/inner.rs diff --git a/prover/src/aggregator/prover/padding.rs b/prover/src/common/prover/padding.rs similarity index 100% rename from prover/src/aggregator/prover/padding.rs rename to prover/src/common/prover/padding.rs diff --git a/prover/src/aggregator/prover/common.rs b/prover/src/common/prover/utils.rs similarity index 100% rename from prover/src/aggregator/prover/common.rs rename to prover/src/common/prover/utils.rs diff --git a/prover/src/common/verifier.rs b/prover/src/common/verifier.rs new file mode 100644 index 000000000..d2eeea3ee --- /dev/null +++ b/prover/src/common/verifier.rs @@ -0,0 +1,42 @@ +use crate::utils::load_params; +use aggregator::CompressionCircuit; +use halo2_proofs::{ + halo2curves::bn256::{Bn256, G1Affine}, + plonk::VerifyingKey, + poly::kzg::commitment::ParamsKZG, + SerdeFormat, +}; +use std::io::Cursor; + +mod evm; +mod utils; + +#[derive(Debug)] +pub struct Verifier { + params: ParamsKZG, + vk: Option>, +} + +impl Verifier { + pub fn new(params: ParamsKZG, vk: Option>) -> Self { + Self { params, vk } + } + + pub fn from_params(params: ParamsKZG, raw_vk: Option>) -> Self { + let vk = raw_vk.as_ref().map(|k| { + VerifyingKey::::read::<_, CompressionCircuit>( + &mut Cursor::new(&k), + SerdeFormat::Processed, + ) + .unwrap() + }); + + Self { params, vk } + } + + pub fn from_params_dir(params_dir: &str, degree: u32, vk: Option>) -> Self { + let params = load_params(params_dir, degree, None).unwrap(); + + Self::from_params(params, vk) + } +} diff --git a/prover/src/common/verifier/evm.rs b/prover/src/common/verifier/evm.rs new file mode 100644 index 000000000..8f467b079 --- /dev/null +++ b/prover/src/common/verifier/evm.rs @@ -0,0 +1,52 @@ +use super::Verifier; +use crate::{io::write_file, Proof}; +use halo2_proofs::halo2curves::bn256::{Bn256, Fr}; +use itertools::Itertools; +use snark_verifier::{ + pcs::kzg::{Bdfg21, Kzg}, + util::arithmetic::PrimeField, +}; +use snark_verifier_sdk::{evm_verify, gen_evm_verifier, CircuitExt}; +use std::{path::PathBuf, str::FromStr}; + +impl Verifier { + // Should panic if failed to verify. + pub fn evm_verify>(&self, proof: &Proof, output_dir: &str) { + let vk = match &self.vk { + Some(vk) => vk, + None => panic!("Aggregation verification key is missing"), + }; + + let num_instance = proof.num_instance().expect("Not a EVM proof").clone(); + + let mut yul_file_path = PathBuf::from_str(&output_dir).unwrap(); + yul_file_path.push("evm_verifier.yul"); + + // Generate deployment code and dump YUL file. + let deployment_code = gen_evm_verifier::>( + &self.params, + vk, + num_instance, + Some(yul_file_path.as_path()), + ); + + // Dump bytecode. + let mut output_dir = PathBuf::from_str(&output_dir).unwrap(); + write_file(&mut output_dir, "evm_verifier.bin", &deployment_code); + + // Dump public input data. + let pi_data: Vec<_> = proof + .instances() + .iter() + .flatten() + .flat_map(|value| value.to_repr().as_ref().iter().rev().cloned().collect_vec()) + .collect(); + write_file(&mut output_dir, "evm_pi_data.data", &pi_data); + + // Dump proof. + let proof_data = proof.proof().to_vec(); + write_file(&mut output_dir, "evm_proof.data", &proof_data); + + evm_verify(deployment_code, proof.instances(), proof_data); + } +} diff --git a/prover/src/common/verifier/utils.rs b/prover/src/common/verifier/utils.rs new file mode 100644 index 000000000..07fc8f57b --- /dev/null +++ b/prover/src/common/verifier/utils.rs @@ -0,0 +1,16 @@ +use super::Verifier; +use halo2_proofs::{ + halo2curves::bn256::{Bn256, G1Affine}, + plonk::VerifyingKey, + poly::kzg::commitment::ParamsKZG, +}; + +impl Verifier { + pub fn params(&self) -> &ParamsKZG { + &self.params + } + + pub fn vk(&self) -> Option<&VerifyingKey> { + self.vk.as_ref() + } +} diff --git a/prover/src/lib.rs b/prover/src/lib.rs index 7c1191499..2efd3356d 100644 --- a/prover/src/lib.rs +++ b/prover/src/lib.rs @@ -1,4 +1,5 @@ pub mod aggregator; +pub mod common; pub mod config; mod evm_verifier; pub mod io; @@ -7,6 +8,7 @@ pub mod test_util; pub mod utils; pub mod zkevm; +pub use common::ChunkHash; pub use evm_verifier::EvmVerifier; pub use proof::Proof; diff --git a/prover/tests/aggregation_tests.rs b/prover/tests/aggregation_tests.rs index 66bf0b1bd..af69a0f8f 100644 --- a/prover/tests/aggregation_tests.rs +++ b/prover/tests/aggregation_tests.rs @@ -1,13 +1,11 @@ use aggregator::{CompressionCircuit, MAX_AGG_SNARKS}; use prover::{ - aggregator::{Prover, Verifier}, - config::{ - AGG_DEGREES, LAYER1_DEGREE, LAYER2_DEGREE, LAYER3_DEGREE, LAYER4_DEGREE, ZKEVM_DEGREES, - }, + common::{Prover, Verifier}, + config::{AGG_DEGREES, LAYER1_DEGREE, LAYER2_DEGREE, LAYER3_DEGREE, LAYER4_DEGREE}, test_util::{load_block_traces_for_test, PARAMS_DIR}, utils::{chunk_trace_to_witness_block, init_env_and_log}, }; -use std::{env::set_var, iter::repeat, path::Path}; +use std::{env::set_var, iter::repeat}; #[cfg(feature = "prove_verify")] #[test] @@ -35,16 +33,15 @@ fn test_agg_prove_verify() { let real_chunk_hashes: Vec<_> = witness_blocks.iter().map(Into::into).collect(); log::info!("Got real-chunk-hashes"); - let mut zkevm_prover = Prover::from_params_dir(PARAMS_DIR, &*ZKEVM_DEGREES); - let mut agg_prover = Prover::from_params_dir(PARAMS_DIR, &*AGG_DEGREES); - log::info!("Constructed zkevm and aggregation provers"); + let mut prover = Prover::from_params_dir(PARAMS_DIR, &*AGG_DEGREES); + log::info!("Constructed prover"); // Load or generate real inner snarks. let inner_snarks: Vec<_> = witness_blocks .into_iter() .enumerate() .map(|(i, witness_block)| { - zkevm_prover + prover .load_or_gen_inner_snark(&format!("layer0_{i}"), witness_block, Some(&output_dir)) .unwrap() }) @@ -56,7 +53,7 @@ fn test_agg_prove_verify() { .into_iter() .enumerate() .map(|(i, snark)| { - zkevm_prover + prover .load_or_gen_comp_snark( &format!("layer1_{i}"), "layer1", @@ -71,7 +68,7 @@ fn test_agg_prove_verify() { log::info!("Got compression-wide-snarks (layer-1)"); // Load or generate layer-1 padding snark. - let layer1_padding_snark = agg_prover + let layer1_padding_snark = prover .load_or_gen_padding_snark( "layer1", real_chunk_hashes.last().unwrap(), @@ -86,7 +83,7 @@ fn test_agg_prove_verify() { .into_iter() .enumerate() .map(|(i, snark)| { - zkevm_prover + prover .load_or_gen_comp_snark( &format!("layer2_{i}"), "layer2", @@ -106,7 +103,7 @@ fn test_agg_prove_verify() { ); // Load or generate aggregation snark (layer-3). - let layer3_snark = agg_prover + let layer3_snark = prover .load_or_gen_agg_snark( "layer3_0", "layer3", @@ -119,7 +116,7 @@ fn test_agg_prove_verify() { log::info!("Got aggregation-snark (layer-3)"); // Load or generate compression EVM proof (layer-4). - let proof = agg_prover + let proof = prover .gen_comp_evm_proof( "layer4_0", "layer4", @@ -132,8 +129,8 @@ fn test_agg_prove_verify() { log::info!("Got compression-EVM-proof (layer-4)"); // Construct verifier and EVM verify. - let params = agg_prover.params(*LAYER4_DEGREE).clone(); - let vk = agg_prover.pk("layer4").unwrap().get_vk().clone(); + let params = prover.params(*LAYER4_DEGREE).clone(); + let vk = prover.pk("layer4").unwrap().get_vk().clone(); let verifier = Verifier::new(params, Some(vk)); verifier.evm_verify::(&proof, &output_dir); log::info!("Finish EVM verify"); diff --git a/prover/tests/compression_tests.rs b/prover/tests/compression_tests.rs index 02786e9d7..bbd11668f 100644 --- a/prover/tests/compression_tests.rs +++ b/prover/tests/compression_tests.rs @@ -1,13 +1,13 @@ use aggregator::CompressionCircuit; use halo2_proofs::{halo2curves::bn256::G1Affine, plonk::VerifyingKey, SerdeFormat}; use prover::{ - aggregator::{Prover, Verifier}, + common::{Prover, Verifier}, config::{LAYER1_DEGREE, LAYER2_DEGREE, ZKEVM_DEGREES}, io::serialize_vk, test_util::{load_block_traces_for_test, PARAMS_DIR}, utils::{chunk_trace_to_witness_block, init_env_and_log}, }; -use std::{io::Cursor, path::Path}; +use std::io::Cursor; #[cfg(feature = "prove_verify")] #[test] @@ -23,17 +23,17 @@ fn test_comp_prove_verify() { let witness_block = chunk_trace_to_witness_block(chunk_trace).unwrap(); log::info!("Got witness-block"); - let mut zkevm_prover = Prover::from_params_dir(PARAMS_DIR, &*ZKEVM_DEGREES); - log::info!("Constructed zkevm-prover"); + let mut prover = Prover::from_params_dir(PARAMS_DIR, &*ZKEVM_DEGREES); + log::info!("Constructed prover"); // Load or generate inner snark. - let inner_snark = zkevm_prover + let inner_snark = prover .load_or_gen_inner_snark("layer0", witness_block, Some(&output_dir)) .unwrap(); log::info!("Got inner-snark"); // Load or generate compression wide snark (layer-1). - let layer1_snark = zkevm_prover + let layer1_snark = prover .load_or_gen_comp_snark( "layer1_0", "layer1", @@ -46,7 +46,7 @@ fn test_comp_prove_verify() { log::info!("Got compression-wide-snark (layer-1)"); // Load or generate compression EVM proof (layer-2). - let proof = zkevm_prover + let proof = prover .gen_comp_evm_proof( "layer2_0", "layer2", @@ -59,9 +59,9 @@ fn test_comp_prove_verify() { log::info!("Got compression-EVM-proof (layer-2)"); // Test vk deserialization. - let vk1 = zkevm_prover.pk("layer2").unwrap().get_vk().clone(); + let vk1 = prover.pk("layer2").unwrap().get_vk().clone(); let raw_vk1 = serialize_vk(&vk1); - let mut vk2 = VerifyingKey::::read::<_, CompressionCircuit>( + let vk2 = VerifyingKey::::read::<_, CompressionCircuit>( &mut Cursor::new(&raw_vk1), SerdeFormat::Processed, ) @@ -72,7 +72,7 @@ fn test_comp_prove_verify() { log::error!("test - vk2 = {:#?}", vk2); // Construct verifier and EVM verify. - let params = zkevm_prover.params(*LAYER2_DEGREE).clone(); + let params = prover.params(*LAYER2_DEGREE).clone(); let verifier = Verifier::new(params, Some(vk2)); verifier.evm_verify::(&proof, &output_dir); log::info!("Finish EVM verify"); From dfa6def6d5cff1d9c4e278c4d7bf2c438ad19af4 Mon Sep 17 00:00:00 2001 From: Steven Gu Date: Tue, 18 Jul 2023 08:28:03 +0800 Subject: [PATCH 14/16] Update zkevm prover and verifier. --- Makefile | 3 - bin/src/prove.rs | 2 +- ffi/src/aggregator.rs | 9 +- ffi/src/zkevm.rs | 6 +- prover/src/aggregator/prover.rs | 6 +- prover/src/aggregator/verifier.rs | 18 +- prover/src/common/prover.rs | 55 +----- prover/src/common/verifier.rs | 19 +- prover/src/common/verifier/evm.rs | 6 +- prover/src/zkevm.rs | 2 +- prover/src/zkevm/prover.rs | 280 +++++++----------------------- prover/src/zkevm/prover/evm.rs | 21 --- prover/src/zkevm/verifier.rs | 120 +++---------- prover/tests/chunk_tests.rs | 98 ----------- prover/tests/integration.rs | 61 +++---- 15 files changed, 148 insertions(+), 558 deletions(-) delete mode 100644 prover/src/zkevm/prover/evm.rs delete mode 100644 prover/tests/chunk_tests.rs diff --git a/Makefile b/Makefile index 5eb57cb9c..1aa5f06e2 100644 --- a/Makefile +++ b/Makefile @@ -39,9 +39,6 @@ test-inner-prove: @cargo test --features prove_verify --release test_inner_prove_verify test-chunk-prove: - @cargo test --features prove_verify --release test_chunk_prove_verify - -test-comp-prove: @cargo test --features prove_verify --release test_comp_prove_verify test-agg-prove: diff --git a/bin/src/prove.rs b/bin/src/prove.rs index f5d32422b..32dca7ca6 100644 --- a/bin/src/prove.rs +++ b/bin/src/prove.rs @@ -47,7 +47,7 @@ fn main() { let now = Instant::now(); let chunk_proof = prover - .gen_chunk_proof(traces.as_slice()) + .gen_chunk_proof(traces, None) .expect("cannot generate chunk proof"); info!( "finish generating chunk proof, elapsed: {:?}", diff --git a/ffi/src/aggregator.rs b/ffi/src/aggregator.rs index 55544678e..a65e7943c 100644 --- a/ffi/src/aggregator.rs +++ b/ffi/src/aggregator.rs @@ -2,7 +2,6 @@ use crate::utils::{c_char_to_str, c_char_to_vec, vec_to_c_char, OUTPUT_DIR}; use libc::c_char; use prover::{ aggregator::{ChunkHash, Prover, Verifier}, - config::{AGG_DEGREES, LAYER4_DEGREE}, utils::init_env_and_log, Proof, }; @@ -18,7 +17,7 @@ pub unsafe extern "C" fn init_agg_prover(params_dir: *const c_char) { let params_dir = c_char_to_str(params_dir); - let prover = Prover::from_params_dir(params_dir, &AGG_DEGREES); + let prover = Prover::from_params_dir(params_dir); AGG_PROVER.set(prover).unwrap(); } @@ -33,11 +32,7 @@ pub unsafe extern "C" fn init_agg_verifier(params_dir: *const c_char, vk_path: * f.read_to_end(&mut vk).unwrap(); let params_dir = c_char_to_str(params_dir); - let verifier = Box::new(Verifier::from_params_dir( - params_dir, - *LAYER4_DEGREE, - Some(vk), - )); + let verifier = Box::new(Verifier::from_params_dir(params_dir, Some(vk))); AGG_VERIFIER = Some(Box::leak(verifier)); } diff --git a/ffi/src/zkevm.rs b/ffi/src/zkevm.rs index 8adb135ce..dcd7e6c46 100644 --- a/ffi/src/zkevm.rs +++ b/ffi/src/zkevm.rs @@ -1,4 +1,4 @@ -use crate::utils::{c_char_to_str, c_char_to_vec, vec_to_c_char}; +use crate::utils::{c_char_to_str, c_char_to_vec, vec_to_c_char, OUTPUT_DIR}; use libc::c_char; use prover::{ utils::init_env_and_log, @@ -46,7 +46,7 @@ pub unsafe extern "C" fn create_block_proof(block_trace: *const c_char) -> *cons let proof = ZKEVM_PROVER .get_mut() .unwrap() - .gen_chunk_proof(&[block_trace]) + .gen_chunk_proof(vec![block_trace], OUTPUT_DIR.as_deref()) .unwrap(); let proof_bytes = serde_json::to_vec(&proof).unwrap(); @@ -62,7 +62,7 @@ pub unsafe extern "C" fn create_chunk_proof(block_traces: *const c_char) -> *con let proof = ZKEVM_PROVER .get_mut() .unwrap() - .gen_chunk_proof(block_traces.as_slice()) + .gen_chunk_proof(block_traces, OUTPUT_DIR.as_deref()) .unwrap(); let proof_bytes = serde_json::to_vec(&proof).unwrap(); diff --git a/prover/src/aggregator/prover.rs b/prover/src/aggregator/prover.rs index 13dffcc11..07a10068c 100644 --- a/prover/src/aggregator/prover.rs +++ b/prover/src/aggregator/prover.rs @@ -1,6 +1,6 @@ use crate::{ common, - config::{LAYER2_DEGREE, LAYER3_DEGREE, LAYER4_DEGREE}, + config::{AGG_DEGREES, LAYER2_DEGREE, LAYER3_DEGREE, LAYER4_DEGREE}, Proof, }; use aggregator::{ChunkHash, MAX_AGG_SNARKS}; @@ -24,8 +24,8 @@ impl Prover { common::Prover::from_params(params_map).into() } - pub fn from_params_dir(params_dir: &str, degrees: &[u32]) -> Self { - common::Prover::from_params_dir(params_dir, degrees).into() + pub fn from_params_dir(params_dir: &str) -> Self { + common::Prover::from_params_dir(params_dir, &AGG_DEGREES).into() } pub fn gen_agg_proof( diff --git a/prover/src/aggregator/verifier.rs b/prover/src/aggregator/verifier.rs index 35b8a4bc1..a7928eb11 100644 --- a/prover/src/aggregator/verifier.rs +++ b/prover/src/aggregator/verifier.rs @@ -1,4 +1,4 @@ -use crate::{common, Proof}; +use crate::{common, config::LAYER4_DEGREE, Proof}; use aggregator::CompressionCircuit; use anyhow::Result; use halo2_proofs::{ @@ -6,7 +6,6 @@ use halo2_proofs::{ plonk::VerifyingKey, poly::kzg::commitment::ParamsKZG, }; -use snark_verifier_sdk::verify_snark_shplonk; #[derive(Debug)] pub struct Verifier { @@ -28,20 +27,11 @@ impl Verifier { common::Verifier::from_params(params, raw_vk).into() } - pub fn from_params_dir(params_dir: &str, degree: u32, vk: Option>) -> Self { - common::Verifier::from_params_dir(params_dir, degree, vk).into() + pub fn from_params_dir(params_dir: &str, vk: Option>) -> Self { + common::Verifier::from_params_dir(params_dir, *LAYER4_DEGREE, vk).into() } pub fn verify_agg_proof(&self, proof: Proof) -> Result { - let vk = match self.inner.vk() { - Some(vk) => vk, - None => panic!("Aggregation verification key is missing"), - }; - - Ok(verify_snark_shplonk::( - self.inner.params(), - proof.to_snark(), - vk, - )) + self.inner.verify_proof::(proof) } } diff --git a/prover/src/common/prover.rs b/prover/src/common/prover.rs index 859ac24d2..0223d8252 100644 --- a/prover/src/common/prover.rs +++ b/prover/src/common/prover.rs @@ -1,16 +1,10 @@ -use crate::{ - config::{LAYER1_DEGREE, LAYER2_DEGREE}, - utils::{chunk_trace_to_witness_block, load_params, param_path_for_degree}, - Proof, -}; -use anyhow::Result; +use crate::utils::{load_params, param_path_for_degree}; use halo2_proofs::{ halo2curves::bn256::{Bn256, G1Affine}, plonk::ProvingKey, poly::{commitment::Params, kzg::commitment::ParamsKZG}, }; use std::collections::{BTreeMap, BTreeSet, HashMap}; -use types::eth::BlockTrace; mod aggregation; mod compression; @@ -72,51 +66,4 @@ impl Prover { pk_map: HashMap::new(), } } - - pub fn gen_chunk_proof( - &mut self, - chunk_trace: Vec, - output_dir: Option<&str>, - ) -> Result { - assert!(!chunk_trace.is_empty()); - - let witness_block = chunk_trace_to_witness_block(chunk_trace)?; - log::info!("Got witness block"); - - let name = witness_block - .context - .first_or_default() - .number - .low_u64() - .to_string(); - - // Load or generate inner snark. - let inner_snark = self.load_or_gen_inner_snark(&name, witness_block, output_dir)?; - log::info!("Got inner snark: {name}"); - - // Load or generate compression wide snark (layer-1). - let layer1_snark = self.load_or_gen_comp_snark( - &name, - "layer1", - true, - *LAYER1_DEGREE, - inner_snark, - output_dir, - )?; - log::info!("Got compression wide snark (layer-1): {name}"); - - // Load or generate compression thin snark (layer-2). - let layer2_snark = self.load_or_gen_comp_snark( - &name, - "layer2", - false, - *LAYER2_DEGREE, - layer1_snark, - output_dir, - )?; - log::info!("Got compression thin snark (layer-2): {name}"); - - let pk = self.pk("layer2").unwrap(); - Proof::from_snark(pk, &layer2_snark) - } } diff --git a/prover/src/common/verifier.rs b/prover/src/common/verifier.rs index d2eeea3ee..607651e29 100644 --- a/prover/src/common/verifier.rs +++ b/prover/src/common/verifier.rs @@ -1,11 +1,13 @@ -use crate::utils::load_params; +use crate::{utils::load_params, Proof}; use aggregator::CompressionCircuit; +use anyhow::Result; use halo2_proofs::{ - halo2curves::bn256::{Bn256, G1Affine}, + halo2curves::bn256::{Bn256, Fr, G1Affine}, plonk::VerifyingKey, poly::kzg::commitment::ParamsKZG, SerdeFormat, }; +use snark_verifier_sdk::{verify_snark_shplonk, CircuitExt}; use std::io::Cursor; mod evm; @@ -39,4 +41,17 @@ impl Verifier { Self::from_params(params, vk) } + + pub fn verify_proof>(&self, proof: Proof) -> Result { + let vk = match &self.vk { + Some(vk) => vk, + None => panic!("Verification key is missing"), + }; + + Ok(verify_snark_shplonk::( + &self.params, + proof.to_snark(), + vk, + )) + } } diff --git a/prover/src/common/verifier/evm.rs b/prover/src/common/verifier/evm.rs index 8f467b079..4527093e8 100644 --- a/prover/src/common/verifier/evm.rs +++ b/prover/src/common/verifier/evm.rs @@ -14,12 +14,12 @@ impl Verifier { pub fn evm_verify>(&self, proof: &Proof, output_dir: &str) { let vk = match &self.vk { Some(vk) => vk, - None => panic!("Aggregation verification key is missing"), + None => panic!("Verification key is missing"), }; let num_instance = proof.num_instance().expect("Not a EVM proof").clone(); - let mut yul_file_path = PathBuf::from_str(&output_dir).unwrap(); + let mut yul_file_path = PathBuf::from_str(output_dir).unwrap(); yul_file_path.push("evm_verifier.yul"); // Generate deployment code and dump YUL file. @@ -31,7 +31,7 @@ impl Verifier { ); // Dump bytecode. - let mut output_dir = PathBuf::from_str(&output_dir).unwrap(); + let mut output_dir = PathBuf::from_str(output_dir).unwrap(); write_file(&mut output_dir, "evm_verifier.bin", &deployment_code); // Dump public input data. diff --git a/prover/src/zkevm.rs b/prover/src/zkevm.rs index 90ddf0720..d72933653 100644 --- a/prover/src/zkevm.rs +++ b/prover/src/zkevm.rs @@ -3,6 +3,6 @@ pub mod circuit; mod prover; mod verifier; -pub use self::prover::Prover; pub use capacity_checker::CircuitCapacityChecker; +pub use prover::Prover; pub use verifier::Verifier; diff --git a/prover/src/zkevm/prover.rs b/prover/src/zkevm/prover.rs index c8c7150db..ef9809040 100644 --- a/prover/src/zkevm/prover.rs +++ b/prover/src/zkevm/prover.rs @@ -1,238 +1,82 @@ -use super::circuit::{ - block_traces_to_witness_block, check_batch_capacity, SuperCircuit, TargetCircuit, -}; use crate::{ - config::INNER_DEGREE, - utils::{load_params, metric_of_witness_block, read_env_var, tick}, + common, + config::{LAYER1_DEGREE, LAYER2_DEGREE, ZKEVM_DEGREES}, + utils::chunk_trace_to_witness_block, Proof, }; -use anyhow::{bail, Result}; -use halo2_proofs::poly::{ - commitment::{Params, ParamsProver}, - kzg::commitment::ParamsVerifierKZG, -}; -use log::info; -use once_cell::sync::Lazy; -use rand::SeedableRng; -use rand_xorshift::XorShiftRng; -use std::collections::HashMap; +use anyhow::Result; +use halo2_proofs::{halo2curves::bn256::Bn256, poly::kzg::commitment::ParamsKZG}; +use std::collections::BTreeMap; use types::eth::BlockTrace; -use halo2_proofs::{ - dev::MockProver, - halo2curves::bn256::{Bn256, Fr, G1Affine}, - plonk::{keygen_pk2, ProvingKey}, - poly::kzg::commitment::ParamsKZG, -}; -use snark_verifier_sdk::{ - gen_evm_proof_shplonk, gen_pk, gen_snark_shplonk, AggregationCircuit, CircuitExt, Snark, -}; - -mod evm; mod mock; -#[cfg(target_os = "linux")] -extern crate procfs; - -#[allow(dead_code)] -pub static OPT_MEM: Lazy = Lazy::new(|| read_env_var("OPT_MEM", false)); -pub static MOCK_PROVE: Lazy = Lazy::new(|| read_env_var("MOCK_PROVE", false)); - -const CHUNK_DEGREE: u32 = 25; - #[derive(Debug)] -// This is the aggregation prover that takes in a list of traces, produces -// a proof that can be verified on chain. pub struct Prover { - pub inner_params: ParamsKZG, - pub chunk_params: ParamsKZG, - /// We may have a list of public keys for different inner circuits. - /// Those keys are stored as a hash map, and keyed by a `name` String. - pub inner_pks: HashMap>, - pub chunk_pk: Option>, + inner: common::Prover, } -impl Prover { - pub fn from_params(inner_params: ParamsKZG, chunk_params: ParamsKZG) -> Self { - assert!(inner_params.k() == *INNER_DEGREE); - assert!(chunk_params.k() == CHUNK_DEGREE); - - // notice that `inner_k < chunk`_k which is not necessary the case in practice - log::info!( - "loaded parameters for degrees {} and {}", - *INNER_DEGREE, - CHUNK_DEGREE - ); - - // this check can be skipped since the `params` is downsized? - { - let target_params_verifier: &ParamsVerifierKZG = inner_params.verifier_params(); - let agg_params_verifier: &ParamsVerifierKZG = chunk_params.verifier_params(); - log::info!( - "params g2 {:?} s_g2 {:?}", - target_params_verifier.g2(), - target_params_verifier.s_g2() - ); - debug_assert_eq!(target_params_verifier.s_g2(), agg_params_verifier.s_g2()); - debug_assert_eq!(target_params_verifier.g2(), agg_params_verifier.g2()); - } - - Self { - inner_params, - chunk_params, - inner_pks: Default::default(), - chunk_pk: None, - } - } - - pub fn from_params_dir(params_dir: &str) -> Self { - let chunk_params = load_params(params_dir, CHUNK_DEGREE, None).unwrap(); - let inner_params = load_params(params_dir, *INNER_DEGREE, None).unwrap_or_else(|_| { - assert!(CHUNK_DEGREE >= *INNER_DEGREE); - log::warn!( - "Optimization: download params{} to params dir", - *INNER_DEGREE - ); - - let mut new_params = chunk_params.clone(); - new_params.downsize(*INNER_DEGREE); - new_params - }); - - Self::from_params(inner_params, chunk_params) +impl From for Prover { + fn from(inner: common::Prover) -> Self { + Self { inner } } +} - // Generate the chunk proof given the chunk trace using Poseidon hash for challenges. - // The returned proof is expected to be verified by only rust verifier not solidity verifier. - pub fn gen_chunk_proof(&mut self, chunk_trace: &[BlockTrace]) -> Result { - let inner_snark = self.gen_inner_snark::(chunk_trace)?; - // Compress the inner snark using the aggregation proof. - self.gen_agg_proof(vec![inner_snark]) +impl Prover { + pub fn from_params(params_map: BTreeMap>) -> Self { + common::Prover::from_params(params_map).into() } - // Generate the chunk proof given the chunk trace using Keccak hash for challenges. - // The returned proof can be efficiently verified by solidity verifier. - pub fn gen_chunk_evm_proof(&mut self, chunk_trace: &[BlockTrace]) -> Result { - let inner_snark = self.gen_inner_snark::(chunk_trace)?; - // Compress the inner snark using the aggregation proof. - self.gen_agg_evm_proof(vec![inner_snark]) + pub fn from_params_dir(params_dir: &str) -> Self { + common::Prover::from_params_dir(params_dir, &ZKEVM_DEGREES).into() } - // Generate the snark of the inner circuit - pub fn gen_inner_snark( + pub fn gen_chunk_proof( &mut self, - chunk_trace: &[BlockTrace], - ) -> Result { - if chunk_trace.is_empty() { - bail!("Empty chunk trace"); - } - - let mut block_traces = chunk_trace.to_vec(); - - let (circuit, instance) = { - // will return early if the check finds out the trace exceeds the circuit capacity - check_batch_capacity(&mut block_traces)?; - - let witness_block = block_traces_to_witness_block(&block_traces)?; - log::info!( - "proving the chunk: {:?}", - metric_of_witness_block(&witness_block) - ); - - C::from_witness_block(&witness_block)? - }; - - // generate the proof for the inner circuit - info!( - "Create {} proof of block {} ... block {}, batch len {}", - C::name(), - chunk_trace.first().unwrap().header.hash.unwrap(), - chunk_trace.last().unwrap().header.hash.unwrap(), - chunk_trace.len() - ); - - let seed = [0u8; 16]; - let mut rng = XorShiftRng::from_seed(seed); - - if *MOCK_PROVE { - log::info!("mock prove {} start", C::name()); - let prover = MockProver::::run(*INNER_DEGREE, &circuit, instance)?; - if let Err(errs) = prover.verify_par() { - log::error!("err num: {}", errs.len()); - for err in &errs { - log::error!("{}", err); - } - bail!("{:#?}", errs); - } - log::info!("mock prove {} done", C::name()); - } - - if !self.inner_pks.contains_key(&C::name()) { - self.gen_inner_pk::(&C::dummy_inner_circuit()); - } - let pk = &self.inner_pks[&C::name()]; - - // Generate the SNARK proof for the inner circuit - let snark_proof = - gen_snark_shplonk(&self.inner_params, pk, circuit, &mut rng, None::); - Ok(snark_proof) - } - - // Generate the aggregation proof given the proofs of inner circuit - pub fn gen_agg_proof(&mut self, snarks: Vec) -> Result { - // build the aggregation circuit inputs from the inner circuit outputs - let seed = [0u8; 16]; - let mut rng = XorShiftRng::from_seed(seed); - - let agg_circuit = AggregationCircuit::new(&self.chunk_params, snarks, &mut rng); - let chunk_pk = self - .chunk_pk - .get_or_insert_with(|| gen_pk(&self.chunk_params, &agg_circuit, None)); - - let agg_proof = gen_snark_shplonk( - &self.chunk_params, - chunk_pk, - agg_circuit, - &mut rng, - None::, - ); - - Proof::from_snark(chunk_pk, &agg_proof) - } - - // Generate the aggregation evm proof given the proofs of inner circuit - pub fn gen_agg_evm_proof(&mut self, snarks: Vec) -> Result { - // build the aggregation circuit inputs from the inner circuit outputs - let seed = [0u8; 16]; - let mut rng = XorShiftRng::from_seed(seed); - - let agg_circuit = AggregationCircuit::new(&self.chunk_params, snarks, &mut rng); - let chunk_pk = self - .chunk_pk - .get_or_insert_with(|| gen_pk(&self.chunk_params, &agg_circuit, None)); - - let agg_proof = gen_evm_proof_shplonk( - &self.chunk_params, - chunk_pk, - agg_circuit.clone(), - agg_circuit.instances(), - &mut rng, - ); - - Proof::new( - chunk_pk, - agg_proof, - &agg_circuit.instances(), - Some(agg_circuit.num_instance()), - ) - } - - // Initiates the public key for a given inner circuit. - pub(crate) fn gen_inner_pk(&mut self, circuit: &::Inner) { - tick(&format!("before init pk of {}", C::name())); - let pk = keygen_pk2(&self.inner_params, circuit) - .unwrap_or_else(|e| panic!("failed to generate {} pk: {:?}", C::name(), e)); - self.inner_pks.insert(C::name(), pk); - tick(&format!("after init pk of {}", C::name())); + chunk_trace: Vec, + output_dir: Option<&str>, + ) -> Result { + assert!(!chunk_trace.is_empty()); + + let witness_block = chunk_trace_to_witness_block(chunk_trace)?; + log::info!("Got witness block"); + + let name = witness_block + .context + .first_or_default() + .number + .low_u64() + .to_string(); + + // Load or generate inner snark. + let inner_snark = self + .inner + .load_or_gen_inner_snark(&name, witness_block, output_dir)?; + log::info!("Got inner snark: {name}"); + + // Load or generate compression wide snark (layer-1). + let layer1_snark = self.inner.load_or_gen_comp_snark( + &name, + "layer1", + true, + *LAYER1_DEGREE, + inner_snark, + output_dir, + )?; + log::info!("Got compression wide snark (layer-1): {name}"); + + // Load or generate compression thin snark (layer-2). + let layer2_snark = self.inner.load_or_gen_comp_snark( + &name, + "layer2", + false, + *LAYER2_DEGREE, + layer1_snark, + output_dir, + )?; + log::info!("Got compression thin snark (layer-2): {name}"); + + let pk = self.inner.pk("layer2").unwrap(); + Proof::from_snark(pk, &layer2_snark) } } diff --git a/prover/src/zkevm/prover/evm.rs b/prover/src/zkevm/prover/evm.rs deleted file mode 100644 index 1ec64d6a3..000000000 --- a/prover/src/zkevm/prover/evm.rs +++ /dev/null @@ -1,21 +0,0 @@ -use super::Prover; -use halo2_proofs::{halo2curves::bn256::G1Affine, plonk::VerifyingKey}; -use snark_verifier_sdk::{gen_evm_verifier_shplonk, AggregationCircuit, CircuitExt}; -use std::path::Path; - -impl Prover { - /// Generate the EVM bytecode for plonk verifier. - pub fn create_evm_verifier_bytecode( - &self, - agg_circuit: &AggregationCircuit, - agg_vk: &VerifyingKey, - path: Option<&Path>, - ) -> Vec { - gen_evm_verifier_shplonk::( - &self.chunk_params, - agg_vk, - agg_circuit.num_instance(), - path, - ) - } -} diff --git a/prover/src/zkevm/verifier.rs b/prover/src/zkevm/verifier.rs index dae1d5799..f30258072 100644 --- a/prover/src/zkevm/verifier.rs +++ b/prover/src/zkevm/verifier.rs @@ -1,117 +1,37 @@ -use super::circuit::TargetCircuit; -use crate::{config::INNER_DEGREE, utils::load_params, Proof}; -use anyhow::{bail, Result}; +use crate::{common, config::LAYER2_DEGREE, Proof}; +use aggregator::CompressionCircuit; +use anyhow::Result; use halo2_proofs::{ halo2curves::bn256::{Bn256, G1Affine}, - plonk::{keygen_vk, verify_proof, VerifyingKey}, - poly::{ - commitment::{Params, ParamsProver}, - kzg::{commitment::ParamsKZG, multiopen::VerifierSHPLONK, strategy::AccumulatorStrategy}, - VerificationStrategy, - }, - transcript::TranscriptReadBuffer, - SerdeFormat, + plonk::VerifyingKey, + poly::kzg::commitment::ParamsKZG, }; -use itertools::Itertools; -use snark_verifier::system::halo2::transcript::evm::EvmTranscript; -use snark_verifier_sdk::{verify_snark_shplonk, AggregationCircuit, Snark}; -use std::{collections::HashMap, io::Cursor}; - -const CHUNK_DEGREE: u32 = 25; +#[derive(Debug)] pub struct Verifier { - inner_params: ParamsKZG, - chunk_params: ParamsKZG, - chunk_vk: Option>, - inner_vks: HashMap>, + inner: common::Verifier, } -impl Verifier { - pub fn from_params( - inner_params: ParamsKZG, - chunk_params: ParamsKZG, - raw_chunk_vk: Option>, - ) -> Self { - let chunk_vk = raw_chunk_vk.as_ref().map(|k| { - VerifyingKey::::read::<_, AggregationCircuit>( - &mut Cursor::new(&k), - SerdeFormat::Processed, - ) - .unwrap() - }); - - Self { - inner_params, - chunk_params, - chunk_vk, - inner_vks: Default::default(), - } +impl From for Verifier { + fn from(inner: common::Verifier) -> Self { + Self { inner } } +} - pub fn from_params_dir(params_dir: &str, chunk_vk: Option>) -> Self { - let chunk_params = load_params(params_dir, CHUNK_DEGREE, None).unwrap(); - let inner_params = load_params(params_dir, *INNER_DEGREE, None).unwrap_or_else(|_| { - assert!(CHUNK_DEGREE >= *INNER_DEGREE); - log::warn!( - "Optimization: download params{} to params dir", - *INNER_DEGREE - ); - - let mut new_params = chunk_params.clone(); - new_params.downsize(*INNER_DEGREE); - new_params - }); - - Self::from_params(inner_params, chunk_params, chunk_vk) +impl Verifier { + pub fn new(params: ParamsKZG, vk: Option>) -> Self { + common::Verifier::new(params, vk).into() } - pub fn verify_chunk_proof(&self, proof: Proof) -> Result { - let chunk_vk = match &self.chunk_vk { - Some(vk) => vk, - None => panic!("Chunk verification key is missing"), - }; - - Ok(verify_snark_shplonk::( - &self.chunk_params, - proof.to_snark(), - chunk_vk, - )) + pub fn from_params(params: ParamsKZG, raw_vk: Option>) -> Self { + common::Verifier::from_params(params, raw_vk).into() } - pub fn verify_chunk_evm_proof(&self, proof: Proof) -> Result { - let chunk_vk = match &self.chunk_vk { - Some(vk) => vk, - None => panic!("Chunk verification key is missing"), - }; - - let mut transcript = TranscriptReadBuffer::<_, G1Affine, _>::init(proof.proof()); - - // Deserialize instances - let instances = proof.instances(); - let instances = instances.iter().map(|ins| ins.as_slice()).collect_vec(); - - Ok(VerificationStrategy::<_, VerifierSHPLONK>::finalize( - verify_proof::<_, VerifierSHPLONK, _, EvmTranscript<_, _, _, _>, _>( - &self.chunk_params, - chunk_vk, - AccumulatorStrategy::new(&self.inner_params), - &[instances.as_slice()], - &mut transcript, - )?, - )) + pub fn from_params_dir(params_dir: &str, vk: Option>) -> Self { + common::Verifier::from_params_dir(params_dir, *LAYER2_DEGREE, vk).into() } - pub fn verify_inner_proof(&mut self, snark: &Snark) -> Result<()> { - let verifier_params = self.inner_params.verifier_params(); - let vk = self.inner_vks.entry(C::name()).or_insert_with(|| { - let circuit = C::dummy_inner_circuit(); - keygen_vk(&self.inner_params, &circuit) - .unwrap_or_else(|_| panic!("Failed to generate {} vk", C::name())) - }); - if verify_snark_shplonk::(verifier_params, snark.clone(), vk) { - Ok(()) - } else { - bail!("Snark verification failed".to_string()) - } + pub fn verify_chunk_proof(&self, proof: Proof) -> Result { + self.inner.verify_proof::(proof) } } diff --git a/prover/tests/chunk_tests.rs b/prover/tests/chunk_tests.rs deleted file mode 100644 index 952e0f6c4..000000000 --- a/prover/tests/chunk_tests.rs +++ /dev/null @@ -1,98 +0,0 @@ -use prover::{ - io::{load_snark, write_file, write_snark}, - test_util::{load_block_traces_for_test, PARAMS_DIR}, - utils::init_env_and_log, - zkevm::{ - circuit::{SuperCircuit, TargetCircuit}, - Prover, - }, - EvmVerifier, -}; -use rand::SeedableRng; -use rand_xorshift::XorShiftRng; -use snark_verifier_sdk::{AggregationCircuit, CircuitExt}; -use std::{ - path::{Path, PathBuf}, - str::FromStr, -}; - -#[cfg(feature = "prove_verify")] -#[test] -fn test_chunk_prove_verify() { - std::env::set_var("VERIFY_CONFIG", "./configs/verify_circuit.config"); - - let output_dir = init_env_and_log("chunk_tests"); - let mut output_path = PathBuf::from_str(&output_dir).unwrap(); - log::info!("Inited ENV and created output-dir {output_dir}"); - - let block_traces = load_block_traces_for_test().1; - log::info!("Loaded block-traces"); - - // ==================================================== - // A whole aggregation procedure takes the following steps - // 1. instantiation the parameters and the prover - // 2. read inner circuit proofs (a.k.a. SNARKs) from previous dumped file or - // convert block traces into - // 3. build an aggregation circuit proof - // 4. generate bytecode for evm to verify aggregation circuit proof - // 5. validate the proof with evm bytecode - // ==================================================== - // - // 1. instantiation the parameters and the prover - // - - let mut prover = Prover::from_params_dir(PARAMS_DIR); - log::info!("build prover"); - - // - // 2. read inner circuit proofs (a.k.a. SNARKs) from previous dumped file or - // convert block traces into - // - let inner_snark_file_path = format!("{}/{}_snark.json", output_dir, SuperCircuit::name()); - let inner_snark = load_snark(&inner_snark_file_path) - .unwrap() - .unwrap_or_else(|| { - let snark = prover - .gen_inner_snark::(block_traces.as_slice()) - .unwrap(); - - // Dump inner circuit proof. - write_snark(&inner_snark_file_path, &snark); - - snark - }); - log::info!("got super circuit proof"); - - // sanity check: the inner proof is correct - - // 3. build an aggregation circuit proof - let agg_circuit = AggregationCircuit::new( - &prover.chunk_params, - vec![inner_snark.clone()], - XorShiftRng::from_seed([0u8; 16]), - ); - - let chunk_proof = prover.gen_agg_evm_proof(vec![inner_snark]).unwrap(); - - // Dump aggregation proof, vk and instance. - chunk_proof.dump(&mut output_path, &"chunk").unwrap(); - - log::info!("finished aggregation generation"); - - // 4. generate bytecode for evm to verify aggregation circuit proof - let agg_vk = prover.chunk_pk.as_ref().unwrap().get_vk(); - - // Create bytecode and dump yul-code. - let yul_file_path = format!("{}/verifier.yul", output_dir); - let deployment_code = - prover.create_evm_verifier_bytecode(&agg_circuit, agg_vk, Some(Path::new(&yul_file_path))); - - // Dump bytecode. - write_file(&mut output_path, "verifier.bin", &deployment_code); - - log::info!("finished byte code generation"); - - // 5. validate the proof with evm bytecode - EvmVerifier::new(deployment_code).verify(agg_circuit.instances(), chunk_proof.proof().to_vec()); - log::info!("end to end test completed"); -} diff --git a/prover/tests/integration.rs b/prover/tests/integration.rs index 060293bba..9dd9d8de1 100644 --- a/prover/tests/integration.rs +++ b/prover/tests/integration.rs @@ -1,4 +1,3 @@ -use chrono::Utc; use halo2_proofs::{plonk::keygen_vk, SerdeFormat}; use prover::{ config::INNER_DEGREE, @@ -206,33 +205,35 @@ fn test_vk_same() { } fn test_target_circuit_prove_verify() { - use std::time::Instant; - - init_env_and_log("integration"); - - let (_, block_traces) = load_block_traces_for_test(); - - log::info!("start generating {} snark", C::name()); - let now = Instant::now(); - let mut prover = Prover::from_params_dir(PARAMS_DIR); - log::info!("build prover"); - let snark = prover - .gen_inner_snark::(block_traces.as_slice()) - .unwrap(); - log::info!("finish generating snark, elapsed: {:?}", now.elapsed()); - - let output_file = format!( - "/tmp/{}_{}.json", - C::name(), - Utc::now().format("%Y%m%d_%H%M%S") - ); - let mut fd = std::fs::File::create(&output_file).unwrap(); - serde_json::to_writer_pretty(&mut fd, &snark).unwrap(); - log::info!("write snark to {}", output_file); - - log::info!("start verifying snark"); - let now = Instant::now(); - let mut verifier = Verifier::from_params_dir(PARAMS_DIR, None); - assert!(verifier.verify_inner_proof::(&snark).is_ok()); - log::info!("finish verifying snark, elapsed: {:?}", now.elapsed()); + /* TODO + use std::time::Instant; + + init_env_and_log("integration"); + + let (_, block_traces) = load_block_traces_for_test(); + + log::info!("start generating {} snark", C::name()); + let now = Instant::now(); + let mut prover = Prover::from_params_dir(PARAMS_DIR); + log::info!("build prover"); + let snark = prover + .gen_inner_snark::(block_traces.as_slice()) + .unwrap(); + log::info!("finish generating snark, elapsed: {:?}", now.elapsed()); + + let output_file = format!( + "/tmp/{}_{}.json", + C::name(), + Utc::now().format("%Y%m%d_%H%M%S") + ); + let mut fd = std::fs::File::create(&output_file).unwrap(); + serde_json::to_writer_pretty(&mut fd, &snark).unwrap(); + log::info!("write snark to {}", output_file); + + log::info!("start verifying snark"); + let now = Instant::now(); + let mut verifier = Verifier::from_params_dir(PARAMS_DIR, None); + assert!(verifier.verify_inner_proof::(&snark).is_ok()); + log::info!("finish verifying snark, elapsed: {:?}", now.elapsed()); + */ } From 5b01a205f6efc5b90656b85e0b3dc5193f3e5740 Mon Sep 17 00:00:00 2001 From: Steven Gu Date: Tue, 18 Jul 2023 21:52:26 +0800 Subject: [PATCH 15/16] Clean up code. --- Makefile | 2 +- README.md | 2 -- bin/src/verify.rs | 4 +-- ffi/src/aggregator.rs | 8 ++--- ffi/src/zkevm.rs | 4 +-- prover/src/aggregator.rs | 1 - prover/src/aggregator/verifier.rs | 9 +++-- prover/src/common/prover/evm.rs | 20 ++++++++++- prover/src/common/prover/inner.rs | 7 ++-- prover/src/common/prover/padding.rs | 12 +++++-- prover/src/common/prover/utils.rs | 19 +---------- prover/src/common/verifier.rs | 34 ++++++------------- prover/src/common/verifier/evm.rs | 7 +--- prover/src/common/verifier/utils.rs | 4 +-- prover/src/config.rs | 2 +- prover/src/proof.rs | 8 ++++- prover/src/zkevm/verifier.rs | 9 +++-- prover/tests/aggregation_tests.rs | 4 +-- .../{compression_tests.rs => chunk_tests.rs} | 21 ++---------- prover/tests/integration.rs | 4 +-- 20 files changed, 80 insertions(+), 101 deletions(-) rename prover/tests/{compression_tests.rs => chunk_tests.rs} (74%) diff --git a/Makefile b/Makefile index 1aa5f06e2..15d3119f9 100644 --- a/Makefile +++ b/Makefile @@ -39,7 +39,7 @@ test-inner-prove: @cargo test --features prove_verify --release test_inner_prove_verify test-chunk-prove: - @cargo test --features prove_verify --release test_comp_prove_verify + @cargo test --features prove_verify --release test_chunk_prove_verify test-agg-prove: @cargo test --features prove_verify --release test_agg_prove_verify diff --git a/README.md b/README.md index 4eb43a466..e99892ae7 100644 --- a/README.md +++ b/README.md @@ -28,8 +28,6 @@ make download-setup -e degree=DEGREE params_dir=PARAMS_DIR `make test-chunk-prove` and `make test-agg-prove` are the main testing entries for multi-level circuit constraint system of scroll-prover. Developers could understand how the system works by reading the codes of these tests. -Besides it, `make test-inner-prove` could be used to test the first-level circuit, and `make-comp-prove` could be used to test two-layers compression circuits. - ### Binaries This repository is designed to be used as a Rust crate, rather than a standalone running process. However, you can still use the following command to run binaries locally. diff --git a/bin/src/verify.rs b/bin/src/verify.rs index 159879371..d842ca332 100644 --- a/bin/src/verify.rs +++ b/bin/src/verify.rs @@ -20,12 +20,12 @@ fn main() { let args = Args::parse(); let chunk_vk = read_from_file(&args.vk_path); - let v = Verifier::from_params_dir(&args.params_path, Some(chunk_vk)); + let v = Verifier::from_params_dir(&args.params_path, &chunk_vk); let proof_path = PathBuf::from("proof_data").join("chunk_full_proof.json"); let proof_vec = read_from_file(&proof_path.to_string_lossy()); let proof = serde_json::from_slice::(proof_vec.as_slice()).unwrap(); - let verified = v.verify_chunk_proof(proof).is_ok(); + let verified = v.verify_chunk_proof(proof); info!("verify agg proof: {}", verified) } diff --git a/ffi/src/aggregator.rs b/ffi/src/aggregator.rs index a65e7943c..6add25150 100644 --- a/ffi/src/aggregator.rs +++ b/ffi/src/aggregator.rs @@ -1,9 +1,9 @@ use crate::utils::{c_char_to_str, c_char_to_vec, vec_to_c_char, OUTPUT_DIR}; use libc::c_char; use prover::{ - aggregator::{ChunkHash, Prover, Verifier}, + aggregator::{Prover, Verifier}, utils::init_env_and_log, - Proof, + ChunkHash, Proof, }; use std::{cell::OnceCell, fs::File, io::Read}; @@ -32,7 +32,7 @@ pub unsafe extern "C" fn init_agg_verifier(params_dir: *const c_char, vk_path: * f.read_to_end(&mut vk).unwrap(); let params_dir = c_char_to_str(params_dir); - let verifier = Box::new(Verifier::from_params_dir(params_dir, Some(vk))); + let verifier = Box::new(Verifier::from_params_dir(params_dir, &vk)); AGG_VERIFIER = Some(Box::leak(verifier)); } @@ -71,6 +71,6 @@ pub unsafe extern "C" fn verify_agg_proof(proof: *const c_char) -> c_char { let proof = c_char_to_vec(proof); let proof = serde_json::from_slice::(proof.as_slice()).unwrap(); - let verified = AGG_VERIFIER.unwrap().verify_agg_proof(proof).is_ok(); + let verified = AGG_VERIFIER.unwrap().verify_agg_proof(proof); verified as c_char } diff --git a/ffi/src/zkevm.rs b/ffi/src/zkevm.rs index dcd7e6c46..cadf87ed4 100644 --- a/ffi/src/zkevm.rs +++ b/ffi/src/zkevm.rs @@ -32,7 +32,7 @@ pub unsafe extern "C" fn init_zkevm_verifier(params_dir: *const c_char, vk_path: f.read_to_end(&mut vk).unwrap(); let params_dir = c_char_to_str(params_dir); - let verifier = Box::new(Verifier::from_params_dir(params_dir, Some(vk))); + let verifier = Box::new(Verifier::from_params_dir(params_dir, &vk)); ZKEVM_VERIFIER = Some(Box::leak(verifier)); } @@ -75,6 +75,6 @@ pub unsafe extern "C" fn verify_chunk_proof(proof: *const c_char) -> c_char { let proof = c_char_to_vec(proof); let proof = serde_json::from_slice::(proof.as_slice()).unwrap(); - let verified = ZKEVM_VERIFIER.unwrap().verify_chunk_proof(proof).is_ok(); + let verified = ZKEVM_VERIFIER.unwrap().verify_chunk_proof(proof); verified as c_char } diff --git a/prover/src/aggregator.rs b/prover/src/aggregator.rs index ca8b5acb1..ac0279ed5 100644 --- a/prover/src/aggregator.rs +++ b/prover/src/aggregator.rs @@ -2,4 +2,3 @@ mod prover; mod verifier; pub use self::{prover::Prover, verifier::Verifier}; -pub use aggregator::ChunkHash; diff --git a/prover/src/aggregator/verifier.rs b/prover/src/aggregator/verifier.rs index a7928eb11..fb2a53935 100644 --- a/prover/src/aggregator/verifier.rs +++ b/prover/src/aggregator/verifier.rs @@ -1,6 +1,5 @@ use crate::{common, config::LAYER4_DEGREE, Proof}; use aggregator::CompressionCircuit; -use anyhow::Result; use halo2_proofs::{ halo2curves::bn256::{Bn256, G1Affine}, plonk::VerifyingKey, @@ -19,19 +18,19 @@ impl From for Verifier { } impl Verifier { - pub fn new(params: ParamsKZG, vk: Option>) -> Self { + pub fn new(params: ParamsKZG, vk: VerifyingKey) -> Self { common::Verifier::new(params, vk).into() } - pub fn from_params(params: ParamsKZG, raw_vk: Option>) -> Self { + pub fn from_params(params: ParamsKZG, raw_vk: &[u8]) -> Self { common::Verifier::from_params(params, raw_vk).into() } - pub fn from_params_dir(params_dir: &str, vk: Option>) -> Self { + pub fn from_params_dir(params_dir: &str, vk: &[u8]) -> Self { common::Verifier::from_params_dir(params_dir, *LAYER4_DEGREE, vk).into() } - pub fn verify_agg_proof(&self, proof: Proof) -> Result { + pub fn verify_agg_proof(&self, proof: Proof) -> bool { self.inner.verify_proof::(proof) } } diff --git a/prover/src/common/prover/evm.rs b/prover/src/common/prover/evm.rs index ad83d6d5f..00120a4f9 100644 --- a/prover/src/common/prover/evm.rs +++ b/prover/src/common/prover/evm.rs @@ -2,7 +2,9 @@ use super::Prover; use crate::{utils::gen_rng, Proof}; use aggregator::CompressionCircuit; use anyhow::{anyhow, Result}; -use snark_verifier_sdk::Snark; +use halo2_proofs::halo2curves::bn256::Fr; +use rand::Rng; +use snark_verifier_sdk::{gen_evm_proof_shplonk, CircuitExt, Snark}; use std::{env::set_var, path::PathBuf}; impl Prover { @@ -29,4 +31,20 @@ impl Prover { result } + + fn gen_evm_proof>( + &mut self, + id: &str, + degree: u32, + rng: &mut (impl Rng + Send), + circuit: C, + ) -> Result { + let (params, pk) = self.params_and_pk(id, &circuit, degree)?; + + let instances = circuit.instances(); + let num_instance = circuit.num_instance(); + let proof = gen_evm_proof_shplonk(params, pk, circuit, instances.clone(), rng); + + Proof::new(pk, proof, &instances, Some(num_instance)) + } } diff --git a/prover/src/common/prover/inner.rs b/prover/src/common/prover/inner.rs index 82ec76a8c..c797fdfd3 100644 --- a/prover/src/common/prover/inner.rs +++ b/prover/src/common/prover/inner.rs @@ -7,12 +7,14 @@ use crate::{ }; use anyhow::Result; use halo2_proofs::halo2curves::bn256::Fr; +use rand::Rng; use snark_verifier_sdk::{gen_snark_shplonk, Snark}; use zkevm_circuits::evm_circuit::witness::Block; impl Prover { pub fn gen_inner_snark( &mut self, + mut rng: impl Rng + Send, witness_block: &Block, ) -> Result { log::info!( @@ -23,7 +25,7 @@ impl Prover { let (circuit, _instance) = C::from_witness_block(witness_block)?; let (params, pk) = self.params_and_pk(&C::name(), &C::dummy_inner_circuit(), *INNER_DEGREE)?; - let snark = gen_snark_shplonk(params, pk, circuit, &mut gen_rng(), None::); + let snark = gen_snark_shplonk(params, pk, circuit, &mut rng, None::); Ok(snark) } @@ -43,7 +45,8 @@ impl Prover { match output_dir.and_then(|_| load_snark(&file_path).ok().flatten()) { Some(snark) => Ok(snark), None => { - let result = self.gen_inner_snark::(&witness_block); + let rng = gen_rng(); + let result = self.gen_inner_snark::(rng, &witness_block); if let (Some(_), Ok(snark)) = (output_dir, &result) { write_snark(&file_path, snark); } diff --git a/prover/src/common/prover/padding.rs b/prover/src/common/prover/padding.rs index 671b82385..a3240c435 100644 --- a/prover/src/common/prover/padding.rs +++ b/prover/src/common/prover/padding.rs @@ -6,15 +6,20 @@ use crate::{ }; use aggregator::{ChunkHash, DummyChunkHashCircuit}; use anyhow::Result; +use rand::Rng; use snark_verifier_sdk::{gen_snark_shplonk, Snark}; impl Prover { - pub fn gen_padding_snark(&mut self, last_real_chunk_hash: &ChunkHash) -> Result { + pub fn gen_padding_snark( + &mut self, + mut rng: impl Rng + Send, + last_real_chunk_hash: &ChunkHash, + ) -> Result { let chunk_hash = ChunkHash::dummy_chunk_hash(last_real_chunk_hash); let circuit = DummyChunkHashCircuit::new(chunk_hash); let (params, pk) = self.params_and_pk("padding", &circuit, *INNER_DEGREE)?; - let snark = gen_snark_shplonk(params, pk, circuit, &mut gen_rng(), None::); + let snark = gen_snark_shplonk(params, pk, circuit, &mut rng, None::); Ok(snark) } @@ -34,7 +39,8 @@ impl Prover { match output_dir.and_then(|_| load_snark(&file_path).ok().flatten()) { Some(snark) => Ok(snark), None => { - let result = self.gen_padding_snark(last_real_chunk_hash); + let rng = gen_rng(); + let result = self.gen_padding_snark(rng, last_real_chunk_hash); if let (Some(_), Ok(snark)) = (output_dir, &result) { write_snark(&file_path, snark); } diff --git a/prover/src/common/prover/utils.rs b/prover/src/common/prover/utils.rs index 59ef777bd..9f530c7a9 100644 --- a/prover/src/common/prover/utils.rs +++ b/prover/src/common/prover/utils.rs @@ -1,5 +1,4 @@ use super::Prover; -use crate::Proof; use anyhow::Result; use halo2_proofs::{ halo2curves::bn256::{Bn256, Fr, G1Affine}, @@ -7,7 +6,7 @@ use halo2_proofs::{ poly::{commitment::Params, kzg::commitment::ParamsKZG}, }; use rand::Rng; -use snark_verifier_sdk::{gen_evm_proof_shplonk, gen_snark_shplonk, CircuitExt, Snark}; +use snark_verifier_sdk::{gen_snark_shplonk, CircuitExt, Snark}; impl Prover { pub fn gen_snark>( @@ -22,22 +21,6 @@ impl Prover { Ok(gen_snark_shplonk(params, pk, circuit, rng, None::)) } - pub fn gen_evm_proof>( - &mut self, - id: &str, - degree: u32, - rng: &mut (impl Rng + Send), - circuit: C, - ) -> Result { - let (params, pk) = self.params_and_pk(id, &circuit, degree)?; - - let instances = circuit.instances(); - let num_instance = circuit.num_instance(); - let proof = gen_evm_proof_shplonk(params, pk, circuit, instances.clone(), rng); - - Proof::new(pk, proof, &instances, Some(num_instance)) - } - pub fn params(&mut self, degree: u32) -> &ParamsKZG { if self.params_map.contains_key(°ree) { return &self.params_map[°ree]; diff --git a/prover/src/common/verifier.rs b/prover/src/common/verifier.rs index 607651e29..65dbadd8b 100644 --- a/prover/src/common/verifier.rs +++ b/prover/src/common/verifier.rs @@ -1,6 +1,5 @@ use crate::{utils::load_params, Proof}; use aggregator::CompressionCircuit; -use anyhow::Result; use halo2_proofs::{ halo2curves::bn256::{Bn256, Fr, G1Affine}, plonk::VerifyingKey, @@ -16,42 +15,31 @@ mod utils; #[derive(Debug)] pub struct Verifier { params: ParamsKZG, - vk: Option>, + vk: VerifyingKey, } impl Verifier { - pub fn new(params: ParamsKZG, vk: Option>) -> Self { + pub fn new(params: ParamsKZG, vk: VerifyingKey) -> Self { Self { params, vk } } - pub fn from_params(params: ParamsKZG, raw_vk: Option>) -> Self { - let vk = raw_vk.as_ref().map(|k| { - VerifyingKey::::read::<_, CompressionCircuit>( - &mut Cursor::new(&k), - SerdeFormat::Processed, - ) - .unwrap() - }); + pub fn from_params(params: ParamsKZG, raw_vk: &[u8]) -> Self { + let vk = VerifyingKey::::read::<_, CompressionCircuit>( + &mut Cursor::new(raw_vk), + SerdeFormat::Processed, + ) + .unwrap(); Self { params, vk } } - pub fn from_params_dir(params_dir: &str, degree: u32, vk: Option>) -> Self { + pub fn from_params_dir(params_dir: &str, degree: u32, vk: &[u8]) -> Self { let params = load_params(params_dir, degree, None).unwrap(); Self::from_params(params, vk) } - pub fn verify_proof>(&self, proof: Proof) -> Result { - let vk = match &self.vk { - Some(vk) => vk, - None => panic!("Verification key is missing"), - }; - - Ok(verify_snark_shplonk::( - &self.params, - proof.to_snark(), - vk, - )) + pub fn verify_proof>(&self, proof: Proof) -> bool { + verify_snark_shplonk::(&self.params, proof.to_snark(), &self.vk) } } diff --git a/prover/src/common/verifier/evm.rs b/prover/src/common/verifier/evm.rs index 4527093e8..18dbba1ca 100644 --- a/prover/src/common/verifier/evm.rs +++ b/prover/src/common/verifier/evm.rs @@ -12,11 +12,6 @@ use std::{path::PathBuf, str::FromStr}; impl Verifier { // Should panic if failed to verify. pub fn evm_verify>(&self, proof: &Proof, output_dir: &str) { - let vk = match &self.vk { - Some(vk) => vk, - None => panic!("Verification key is missing"), - }; - let num_instance = proof.num_instance().expect("Not a EVM proof").clone(); let mut yul_file_path = PathBuf::from_str(output_dir).unwrap(); @@ -25,7 +20,7 @@ impl Verifier { // Generate deployment code and dump YUL file. let deployment_code = gen_evm_verifier::>( &self.params, - vk, + &self.vk, num_instance, Some(yul_file_path.as_path()), ); diff --git a/prover/src/common/verifier/utils.rs b/prover/src/common/verifier/utils.rs index 07fc8f57b..60dfa75f4 100644 --- a/prover/src/common/verifier/utils.rs +++ b/prover/src/common/verifier/utils.rs @@ -10,7 +10,7 @@ impl Verifier { &self.params } - pub fn vk(&self) -> Option<&VerifyingKey> { - self.vk.as_ref() + pub fn vk(&self) -> &VerifyingKey { + &self.vk } } diff --git a/prover/src/config.rs b/prover/src/config.rs index 8cc23aa98..92b4e3f1d 100644 --- a/prover/src/config.rs +++ b/prover/src/config.rs @@ -18,7 +18,7 @@ pub static ZKEVM_DEGREES: Lazy> = Lazy::new(|| { pub static AGG_DEGREES: Lazy> = Lazy::new(|| { Vec::from_iter(HashSet::from([ - *LAYER2_DEGREE, + *LAYER2_DEGREE, // For padding *LAYER3_DEGREE, *LAYER4_DEGREE, ])) diff --git a/prover/src/proof.rs b/prover/src/proof.rs index 5ef39567f..bffb670dc 100644 --- a/prover/src/proof.rs +++ b/prover/src/proof.rs @@ -2,7 +2,7 @@ use crate::io::{deserialize_fr_matrix, serialize_fr_matrix, serialize_vk, write_ use anyhow::Result; use halo2_proofs::{ halo2curves::bn256::{Fr, G1Affine}, - plonk::ProvingKey, + plonk::{Circuit, ProvingKey, VerifyingKey}, SerdeFormat, }; use serde_derive::{Deserialize, Serialize}; @@ -16,6 +16,7 @@ use snark_verifier::{ use snark_verifier_sdk::Snark; use std::{ fs::File, + io::Cursor, path::{Path, PathBuf}, }; use types::base64; @@ -104,6 +105,11 @@ impl Proof { &self.proof } + pub fn vk>(&self) -> VerifyingKey { + VerifyingKey::::read::<_, C>(&mut Cursor::new(&self.vk), SerdeFormat::Processed) + .unwrap() + } + pub fn instances(&self) -> Vec> { let buf: Vec>> = serde_json::from_reader(self.instances.as_slice()).unwrap(); diff --git a/prover/src/zkevm/verifier.rs b/prover/src/zkevm/verifier.rs index f30258072..4cd024ff9 100644 --- a/prover/src/zkevm/verifier.rs +++ b/prover/src/zkevm/verifier.rs @@ -1,6 +1,5 @@ use crate::{common, config::LAYER2_DEGREE, Proof}; use aggregator::CompressionCircuit; -use anyhow::Result; use halo2_proofs::{ halo2curves::bn256::{Bn256, G1Affine}, plonk::VerifyingKey, @@ -19,19 +18,19 @@ impl From for Verifier { } impl Verifier { - pub fn new(params: ParamsKZG, vk: Option>) -> Self { + pub fn new(params: ParamsKZG, vk: VerifyingKey) -> Self { common::Verifier::new(params, vk).into() } - pub fn from_params(params: ParamsKZG, raw_vk: Option>) -> Self { + pub fn from_params(params: ParamsKZG, raw_vk: &[u8]) -> Self { common::Verifier::from_params(params, raw_vk).into() } - pub fn from_params_dir(params_dir: &str, vk: Option>) -> Self { + pub fn from_params_dir(params_dir: &str, vk: &[u8]) -> Self { common::Verifier::from_params_dir(params_dir, *LAYER2_DEGREE, vk).into() } - pub fn verify_chunk_proof(&self, proof: Proof) -> Result { + pub fn verify_chunk_proof(&self, proof: Proof) -> bool { self.inner.verify_proof::(proof) } } diff --git a/prover/tests/aggregation_tests.rs b/prover/tests/aggregation_tests.rs index af69a0f8f..71097bb02 100644 --- a/prover/tests/aggregation_tests.rs +++ b/prover/tests/aggregation_tests.rs @@ -130,8 +130,8 @@ fn test_agg_prove_verify() { // Construct verifier and EVM verify. let params = prover.params(*LAYER4_DEGREE).clone(); - let vk = prover.pk("layer4").unwrap().get_vk().clone(); - let verifier = Verifier::new(params, Some(vk)); + let vk = proof.vk::(); + let verifier = Verifier::new(params, vk); verifier.evm_verify::(&proof, &output_dir); log::info!("Finish EVM verify"); } diff --git a/prover/tests/compression_tests.rs b/prover/tests/chunk_tests.rs similarity index 74% rename from prover/tests/compression_tests.rs rename to prover/tests/chunk_tests.rs index bbd11668f..58e74ae7c 100644 --- a/prover/tests/compression_tests.rs +++ b/prover/tests/chunk_tests.rs @@ -1,17 +1,14 @@ use aggregator::CompressionCircuit; -use halo2_proofs::{halo2curves::bn256::G1Affine, plonk::VerifyingKey, SerdeFormat}; use prover::{ common::{Prover, Verifier}, config::{LAYER1_DEGREE, LAYER2_DEGREE, ZKEVM_DEGREES}, - io::serialize_vk, test_util::{load_block_traces_for_test, PARAMS_DIR}, utils::{chunk_trace_to_witness_block, init_env_and_log}, }; -use std::io::Cursor; #[cfg(feature = "prove_verify")] #[test] -fn test_comp_prove_verify() { +fn test_chunk_prove_verify() { // Init, load block traces and construct prover. let output_dir = init_env_and_log("comp_tests"); @@ -58,22 +55,10 @@ fn test_comp_prove_verify() { .unwrap(); log::info!("Got compression-EVM-proof (layer-2)"); - // Test vk deserialization. - let vk1 = prover.pk("layer2").unwrap().get_vk().clone(); - let raw_vk1 = serialize_vk(&vk1); - let vk2 = VerifyingKey::::read::<_, CompressionCircuit>( - &mut Cursor::new(&raw_vk1), - SerdeFormat::Processed, - ) - .unwrap(); - let raw_vk2 = serialize_vk(&vk2); - assert_eq!(raw_vk1, raw_vk2); - log::error!("test - vk1 = {:#?}", vk1); - log::error!("test - vk2 = {:#?}", vk2); - // Construct verifier and EVM verify. let params = prover.params(*LAYER2_DEGREE).clone(); - let verifier = Verifier::new(params, Some(vk2)); + let vk = proof.vk::(); + let verifier = Verifier::new(params, vk); verifier.evm_verify::(&proof, &output_dir); log::info!("Finish EVM verify"); } diff --git a/prover/tests/integration.rs b/prover/tests/integration.rs index a15b114c4..0cabe0028 100644 --- a/prover/tests/integration.rs +++ b/prover/tests/integration.rs @@ -6,7 +6,7 @@ use prover::{ utils::{get_block_trace_from_file, init_env_and_log, load_params}, zkevm::{ circuit::{SuperCircuit, TargetCircuit}, - CircuitCapacityChecker, Prover, Verifier, + CircuitCapacityChecker, Prover, }, }; @@ -202,7 +202,7 @@ fn test_vk_same() { } fn test_target_circuit_prove_verify() { - /* TODO + /* TODO: verify inner proof. use std::time::Instant; init_env_and_log("integration"); From 534195206b0b72798b1b4aea1f4bdc49df24712b Mon Sep 17 00:00:00 2001 From: Steven Gu Date: Wed, 19 Jul 2023 13:46:48 +0800 Subject: [PATCH 16/16] Add `MockProver` for all circuits. --- Cargo.lock | 20 ++++++++++---------- prover/src/aggregator/prover.rs | 11 +++++++---- prover/src/common/prover.rs | 1 + prover/src/common/prover/evm.rs | 2 ++ prover/src/common/prover/inner.rs | 9 +++++++-- prover/src/common/prover/mock.rs | 23 +++++++++++++++++++++++ prover/src/common/prover/padding.rs | 12 +++++------- prover/src/common/prover/utils.rs | 2 ++ prover/src/config.rs | 4 +++- prover/tests/aggregation_tests.rs | 1 + 10 files changed, 61 insertions(+), 24 deletions(-) create mode 100644 prover/src/common/prover/mock.rs diff --git a/Cargo.lock b/Cargo.lock index 4d1e1518e..5529596b4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,7 +23,7 @@ dependencies = [ [[package]] name = "aggregator" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#0dbf46f811fc2337df809272da6b8df2c64dd9de" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#496cca2ac06eae9b28e14ea377cef1d64eb4059a" dependencies = [ "ark-std", "env_logger 0.10.0", @@ -418,7 +418,7 @@ checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" [[package]] name = "bus-mapping" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#0dbf46f811fc2337df809272da6b8df2c64dd9de" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#496cca2ac06eae9b28e14ea377cef1d64eb4059a" dependencies = [ "eth-types", "ethers-core 0.17.0", @@ -1118,7 +1118,7 @@ dependencies = [ [[package]] name = "eth-types" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#0dbf46f811fc2337df809272da6b8df2c64dd9de" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#496cca2ac06eae9b28e14ea377cef1d64eb4059a" dependencies = [ "ethers-core 0.17.0", "ethers-signers", @@ -1384,7 +1384,7 @@ dependencies = [ [[package]] name = "external-tracer" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#0dbf46f811fc2337df809272da6b8df2c64dd9de" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#496cca2ac06eae9b28e14ea377cef1d64eb4059a" dependencies = [ "eth-types", "geth-utils", @@ -1613,7 +1613,7 @@ dependencies = [ [[package]] name = "gadgets" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#0dbf46f811fc2337df809272da6b8df2c64dd9de" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#496cca2ac06eae9b28e14ea377cef1d64eb4059a" dependencies = [ "digest 0.7.6", "eth-types", @@ -1653,7 +1653,7 @@ dependencies = [ [[package]] name = "geth-utils" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#0dbf46f811fc2337df809272da6b8df2c64dd9de" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#496cca2ac06eae9b28e14ea377cef1d64eb4059a" dependencies = [ "env_logger 0.9.3", "gobuild 0.1.0-alpha.2 (git+https://github.com/scroll-tech/gobuild.git)", @@ -2243,7 +2243,7 @@ dependencies = [ [[package]] name = "keccak256" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#0dbf46f811fc2337df809272da6b8df2c64dd9de" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#496cca2ac06eae9b28e14ea377cef1d64eb4059a" dependencies = [ "env_logger 0.9.3", "eth-types", @@ -2443,7 +2443,7 @@ dependencies = [ [[package]] name = "mock" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#0dbf46f811fc2337df809272da6b8df2c64dd9de" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#496cca2ac06eae9b28e14ea377cef1d64eb4059a" dependencies = [ "eth-types", "ethers-core 0.17.0", @@ -2458,7 +2458,7 @@ dependencies = [ [[package]] name = "mpt-zktrie" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#0dbf46f811fc2337df809272da6b8df2c64dd9de" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#496cca2ac06eae9b28e14ea377cef1d64eb4059a" dependencies = [ "bus-mapping", "eth-types", @@ -4683,7 +4683,7 @@ checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" [[package]] name = "zkevm-circuits" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#0dbf46f811fc2337df809272da6b8df2c64dd9de" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=integrate-dynamic-proof-agg-new#496cca2ac06eae9b28e14ea377cef1d64eb4059a" dependencies = [ "array-init", "bus-mapping", diff --git a/prover/src/aggregator/prover.rs b/prover/src/aggregator/prover.rs index 07a10068c..f94d370d0 100644 --- a/prover/src/aggregator/prover.rs +++ b/prover/src/aggregator/prover.rs @@ -1,6 +1,6 @@ use crate::{ common, - config::{AGG_DEGREES, LAYER2_DEGREE, LAYER3_DEGREE, LAYER4_DEGREE}, + config::{AGG_DEGREES, LAYER1_DEGREE, LAYER2_DEGREE, LAYER3_DEGREE, LAYER4_DEGREE}, Proof, }; use aggregator::{ChunkHash, MAX_AGG_SNARKS}; @@ -43,9 +43,12 @@ impl Prover { .to_string(); // Load or generate padding snark (layer-1). - let layer1_padding_snark = - self.inner - .load_or_gen_padding_snark(&name, &last_real_chunk_hash, output_dir)?; + let layer1_padding_snark = self.inner.load_or_gen_padding_snark( + &name, + *LAYER1_DEGREE, + &last_real_chunk_hash, + output_dir, + )?; log::info!("Got padding snark (layer-1): {name}"); // Load or generate compression thin snark for padding (layer-2). diff --git a/prover/src/common/prover.rs b/prover/src/common/prover.rs index 0223d8252..7c9d0cef4 100644 --- a/prover/src/common/prover.rs +++ b/prover/src/common/prover.rs @@ -10,6 +10,7 @@ mod aggregation; mod compression; mod evm; mod inner; +mod mock; mod padding; mod utils; diff --git a/prover/src/common/prover/evm.rs b/prover/src/common/prover/evm.rs index 00120a4f9..90f911fb7 100644 --- a/prover/src/common/prover/evm.rs +++ b/prover/src/common/prover/evm.rs @@ -39,6 +39,8 @@ impl Prover { rng: &mut (impl Rng + Send), circuit: C, ) -> Result { + Self::assert_if_mock_prover(id, degree, &circuit); + let (params, pk) = self.params_and_pk(id, &circuit, degree)?; let instances = circuit.instances(); diff --git a/prover/src/common/prover/inner.rs b/prover/src/common/prover/inner.rs index c797fdfd3..6dc5b38ea 100644 --- a/prover/src/common/prover/inner.rs +++ b/prover/src/common/prover/inner.rs @@ -22,9 +22,14 @@ impl Prover { metric_of_witness_block(witness_block) ); + let id = C::name(); + let degree = *INNER_DEGREE; + let (circuit, _instance) = C::from_witness_block(witness_block)?; - let (params, pk) = - self.params_and_pk(&C::name(), &C::dummy_inner_circuit(), *INNER_DEGREE)?; + + Self::assert_if_mock_prover(&id, degree, &circuit); + + let (params, pk) = self.params_and_pk(&id, &C::dummy_inner_circuit(), degree)?; let snark = gen_snark_shplonk(params, pk, circuit, &mut rng, None::); Ok(snark) diff --git a/prover/src/common/prover/mock.rs b/prover/src/common/prover/mock.rs new file mode 100644 index 000000000..e6beab5fb --- /dev/null +++ b/prover/src/common/prover/mock.rs @@ -0,0 +1,23 @@ +use super::Prover; +use crate::utils::read_env_var; +use halo2_proofs::{dev::MockProver, halo2curves::bn256::Fr}; +use once_cell::sync::Lazy; +use snark_verifier_sdk::CircuitExt; + +pub static MOCK_PROVE: Lazy = Lazy::new(|| read_env_var("MOCK_PROVE", false)); + +impl Prover { + pub fn assert_if_mock_prover>(id: &str, degree: u32, circuit: &C) { + if !*MOCK_PROVE { + return; + } + + log::info!("Mock prove for {id} - BEGIN"); + + let instances = circuit.instances(); + let mock_prover = MockProver::::run(degree, circuit, instances).unwrap(); + mock_prover.assert_satisfied_par(); + + log::info!("Mock prove for {id} - END"); + } +} diff --git a/prover/src/common/prover/padding.rs b/prover/src/common/prover/padding.rs index a3240c435..a42649960 100644 --- a/prover/src/common/prover/padding.rs +++ b/prover/src/common/prover/padding.rs @@ -1,32 +1,30 @@ use super::Prover; use crate::{ - config::INNER_DEGREE, io::{load_snark, write_snark}, utils::gen_rng, }; use aggregator::{ChunkHash, DummyChunkHashCircuit}; use anyhow::Result; use rand::Rng; -use snark_verifier_sdk::{gen_snark_shplonk, Snark}; +use snark_verifier_sdk::Snark; impl Prover { pub fn gen_padding_snark( &mut self, + degree: u32, mut rng: impl Rng + Send, last_real_chunk_hash: &ChunkHash, ) -> Result { let chunk_hash = ChunkHash::dummy_chunk_hash(last_real_chunk_hash); let circuit = DummyChunkHashCircuit::new(chunk_hash); - let (params, pk) = self.params_and_pk("padding", &circuit, *INNER_DEGREE)?; - let snark = gen_snark_shplonk(params, pk, circuit, &mut rng, None::); - - Ok(snark) + self.gen_snark("padding", degree, &mut rng, circuit) } pub fn load_or_gen_padding_snark( &mut self, name: &str, + degree: u32, last_real_chunk_hash: &ChunkHash, output_dir: Option<&str>, ) -> Result { @@ -40,7 +38,7 @@ impl Prover { Some(snark) => Ok(snark), None => { let rng = gen_rng(); - let result = self.gen_padding_snark(rng, last_real_chunk_hash); + let result = self.gen_padding_snark(degree, rng, last_real_chunk_hash); if let (Some(_), Ok(snark)) = (output_dir, &result) { write_snark(&file_path, snark); } diff --git a/prover/src/common/prover/utils.rs b/prover/src/common/prover/utils.rs index 9f530c7a9..37125138c 100644 --- a/prover/src/common/prover/utils.rs +++ b/prover/src/common/prover/utils.rs @@ -16,6 +16,8 @@ impl Prover { rng: &mut (impl Rng + Send), circuit: C, ) -> Result { + Self::assert_if_mock_prover(id, degree, &circuit); + let (params, pk) = self.params_and_pk(id, &circuit, degree)?; Ok(gen_snark_shplonk(params, pk, circuit, rng, None::)) diff --git a/prover/src/config.rs b/prover/src/config.rs index 92b4e3f1d..b891ed2fe 100644 --- a/prover/src/config.rs +++ b/prover/src/config.rs @@ -18,7 +18,9 @@ pub static ZKEVM_DEGREES: Lazy> = Lazy::new(|| { pub static AGG_DEGREES: Lazy> = Lazy::new(|| { Vec::from_iter(HashSet::from([ - *LAYER2_DEGREE, // For padding + // TODO: optimize to decrease degree for padding. + *LAYER1_DEGREE, // For layer-1 padding snark generation + *LAYER2_DEGREE, // For layer-2 padding snark generation *LAYER3_DEGREE, *LAYER4_DEGREE, ])) diff --git a/prover/tests/aggregation_tests.rs b/prover/tests/aggregation_tests.rs index 71097bb02..28db52e4f 100644 --- a/prover/tests/aggregation_tests.rs +++ b/prover/tests/aggregation_tests.rs @@ -71,6 +71,7 @@ fn test_agg_prove_verify() { let layer1_padding_snark = prover .load_or_gen_padding_snark( "layer1", + *LAYER1_DEGREE, real_chunk_hashes.last().unwrap(), Some(&output_dir), )