feat: Move prover data to /home/popzxc/workspace/current/zksync-era/p…

…rover/data (#2778)

⚠️ this change is non-destructive globally (e.g. no changes in
deployments/configs are needed), but for local deployment if you have
setup keys locally, you need to move them to the new folder:

```
cd $ZKSYNC_HOME
mv prover/crates/bin/vk_setup_data_generator_server_fri/data/setup* prover/data/keys
rmdir prover/crates/bin/vk_setup_data_generator_server_fri/data
```

## What

- Moves prover data (`data`/`historical_data`) from
`prover/crates/bin/vk_setup_data_generator_server_fri` to `prover/data`
- Updates all the relevant paths
- Adds some minimal documentation to `prover/data/README.md`

## Why

- More intuitive
- Simplifies refactoring of the workspace: no need to worry about layout
of data when renaming crates/changing structure/etc.

.dockerignore

@@ -39,7 +39,7 @@ contracts/.git
 !etc/multivm_bootloaders
 !cargo
 !bellman-cuda
-!prover/crates/bin/vk_setup_data_generator_server_fri/data/
+!prover/data/
 !.github/release-please/manifest.json
 
 !etc/env/file_based

.gitignore

@@ -110,6 +110,7 @@ hyperchain-*.yml
 
 # Prover keys that should not be commited
 prover/crates/bin/vk_setup_data_generator_server_fri/data/setup_*
+prover/data/keys/setup_*
 
 # Zk Toolbox
 chains/era/configs/*

core/lib/env_config/src/fri_prover.rs

@@ -32,7 +32,7 @@ mod tests {
 
  fn expected_config() -> FriProverConfig {
  FriProverConfig {
- setup_data_path: "vk_setup_data_generator_server_fri/data".to_string(),
+ setup_data_path: "prover/data/keys".to_string(),
  prometheus_port: 3315,
  max_attempts: 10,
  generation_timeout_in_secs: 300,
@@ -68,7 +68,7 @@ mod tests {
  fn from_env() {
  let mut lock = MUTEX.lock();
  let config = r#"
- FRI_PROVER_SETUP_DATA_PATH="vk_setup_data_generator_server_fri/data"
+ FRI_PROVER_SETUP_DATA_PATH="prover/data/keys"
  FRI_PROVER_PROMETHEUS_PORT="3315"
  FRI_PROVER_MAX_ATTEMPTS="10"
  FRI_PROVER_GENERATION_TIMEOUT_IN_SECS="300"

docker/proof-fri-gpu-compressor/Dockerfile

@@ -37,7 +37,7 @@ FROM nvidia/cuda:12.2.0-runtime-ubuntu22.04
 RUN apt-get update && apt-get install -y curl libpq5 ca-certificates && rm -rf /var/lib/apt/lists/*
 
 # copy VK required for proof wrapping
-COPY prover/crates/bin/vk_setup_data_generator_server_fri/data/ /prover/crates/bin/vk_setup_data_generator_server_fri/data/
+COPY prover/data/keys/ /prover/data/keys
 
 COPY setup_2\^24.key /setup_2\^24.key
 

docker/prover-fri-gateway/Dockerfile

@@ -11,7 +11,7 @@ FROM debian:bookworm-slim
 RUN apt-get update && apt-get install -y curl libpq5 ca-certificates && rm -rf /var/lib/apt/lists/*
 
 # copy VK required for proof wrapping
-COPY prover/crates/bin/vk_setup_data_generator_server_fri/data/ /prover/crates/bin/vk_setup_data_generator_server_fri/data/
+COPY prover/data/keys/ /prover/data/keys/
 
 COPY --from=builder /usr/src/zksync/prover/target/release/zksync_prover_fri_gateway /usr/bin/
 

docker/prover-gpu-fri-gar/Dockerfile

@@ -9,7 +9,7 @@ COPY *.bin /
 RUN apt-get update && apt-get install -y libpq5 ca-certificates openssl && rm -rf /var/lib/apt/lists/*
 
 # copy finalization hints required for assembly generation
-COPY --from=prover prover/crates/bin/vk_setup_data_generator_server_fri/data/ /prover/crates/bin/vk_setup_data_generator_server_fri/data/
+COPY --from=prover prover/data/keys/ /prover/data/keys/
 COPY --from=prover /usr/bin/zksync_prover_fri /usr/bin/
 
 ENTRYPOINT ["zksync_prover_fri"]

docker/prover-gpu-fri/Dockerfile

@@ -31,7 +31,7 @@ FROM nvidia/cuda:12.2.0-runtime-ubuntu22.04
 RUN apt-get update && apt-get install -y curl libpq5 ca-certificates && rm -rf /var/lib/apt/lists/*
 
 # copy finalization hints required for assembly generation
-COPY prover/crates/bin/vk_setup_data_generator_server_fri/data/ /prover/crates/bin/vk_setup_data_generator_server_fri/data/
+COPY prover/data/keys/ /prover/data/keys/
 
 COPY --from=builder /usr/src/zksync/prover/target/release/zksync_prover_fri /usr/bin/
 

docker/witness-generator/Dockerfile

@@ -13,7 +13,7 @@ FROM debian:bookworm-slim
 
 RUN apt-get update && apt-get install -y curl libpq5 ca-certificates && rm -rf /var/lib/apt/lists/*
 
-COPY prover/crates/bin/vk_setup_data_generator_server_fri/data/ /prover/crates/bin/vk_setup_data_generator_server_fri/data/
+COPY prover/data/keys/ /prover/data/keys/
 
 COPY --from=builder /usr/src/zksync/prover/target/release/zksync_witness_generator /usr/bin/
 

docker/witness-vector-generator/Dockerfile

@@ -12,7 +12,7 @@ FROM debian:bookworm-slim
 RUN apt-get update && apt-get install -y curl libpq5 ca-certificates && rm -rf /var/lib/apt/lists/*
 
 # copy finalization hints required for witness vector generation
-COPY prover/crates/bin/vk_setup_data_generator_server_fri/data/ /prover/crates/bin/vk_setup_data_generator_server_fri/data/
+COPY prover/data/keys/ /prover/crates/bin/vk_setup_data_generator_server_fri/data/
 
 COPY --from=builder /usr/src/zksync/prover/target/release/zksync_witness_vector_generator /usr/bin/
 

docs/guides/advanced/15_prover_keys.md

@@ -118,9 +118,9 @@ friendly hash function (currently Poseidon2).
 [recursive_circuit_list]:
  https://github.com/matter-labs/era-zkevm_test_harness/blob/3cd647aa57fc2e1180bab53f7a3b61ec47502a46/circuit_definitions/src/circuit_definitions/recursion_layer/mod.rs#L29
 [verification_key_list]:
- https://github.com/matter-labs/zksync-era/tree/boojum-integration/prover/vk_setup_data_generator_server_fri/data
+ https://github.com/matter-labs/zksync-era/tree/6d18061df4a18803d3c6377305ef711ce60317e1/prover/data/keys
 [env_variables_for_hash]:
- https://github.com/matter-labs/zksync-era/blob/boojum-integration/etc/env/base/contracts.toml#L44
+ https://github.com/matter-labs/zksync-era/blob/6d18061df4a18803d3c6377305ef711ce60317e1/etc/env/base/contracts.toml#L61
 [prover_setup_data]:
  https://github.com/matter-labs/zksync-era/blob/d2ca29bf20b4ec2d9ec9e327b4ba6b281d9793de/prover/vk_setup_data_generator_server_fri/src/lib.rs#L61
 [verifier_computation]:

etc/env/base/fri_prover.toml

@@ -1,5 +1,5 @@
 [fri_prover]
-setup_data_path = "crates/bin/vk_setup_data_generator_server_fri/data"
+setup_data_path = "data/keys"
 prometheus_port = 3315
 max_attempts = 10
 generation_timeout_in_secs = 600

etc/env/file_based/general.yaml

@@ -140,7 +140,7 @@ prover:
  file_backed:
  file_backed_base_path: artifacts
  max_retries: 10
- setup_data_path: crates/bin/vk_setup_data_generator_server_fri/data
+ setup_data_path: data/keys
  prometheus_port: 3315
  max_attempts: 10
  generation_timeout_in_secs: 600

infrastructure/zk/src/prover_setup.ts

@@ -30,8 +30,7 @@ export async function setupProver(proverType: ProverType) {
  } else {
  env.modify(
  'FRI_PROVER_SETUP_DATA_PATH',
- `${process.env.ZKSYNC_HOME}/etc/hyperchains/prover-keys/${process.env.ZKSYNC_ENV}/${
- proverType === ProverType.GPU ? 'gpu' : 'cpu'
+ `${process.env.ZKSYNC_HOME}/etc/hyperchains/prover-keys/${process.env.ZKSYNC_ENV}/${proverType === ProverType.GPU ? 'gpu' : 'cpu'
  }/`,
  process.env.ENV_FILE!
  );
@@ -98,8 +97,7 @@ async function setupProverKeys(proverType: ProverType) {
 
  env.modify(
  'FRI_PROVER_SETUP_DATA_PATH',
- `${process.env.ZKSYNC_HOME}/etc/hyperchains/prover-keys/${process.env.ZKSYNC_ENV}/${
- proverType === ProverType.GPU ? 'gpu' : 'cpu'
+ `${process.env.ZKSYNC_HOME}/etc/hyperchains/prover-keys/${process.env.ZKSYNC_ENV}/${proverType === ProverType.GPU ? 'gpu' : 'cpu'
  }/`,
  process.env.ENV_FILE!
  );
@@ -204,7 +202,7 @@ async function downloadDefaultSetupKeys(proverType: ProverType, region: string)
  );
 
  await utils.spawn(
- `cp -r ${process.env.ZKSYNC_HOME}/prover/vk_setup_data_generator_server_fri/data/* ${process.env.ZKSYNC_HOME}/etc/hyperchains/prover-keys/${currentEnv}/${proverType}/`
+ `cp -r ${process.env.ZKSYNC_HOME}/prover/data/keys/* ${process.env.ZKSYNC_HOME}/etc/hyperchains/prover-keys/${currentEnv}/${proverType}/`
  );
 }
 

prover/crates/bin/vk_setup_data_generator_server_fri/src/keystore.rs

@@ -47,16 +47,14 @@ pub struct Keystore {
 fn get_base_path() -> PathBuf {
  let path = core_workspace_dir_or_current_dir();
 
- let new_path = path.join("prover/crates/bin/vk_setup_data_generator_server_fri/data");
+ let new_path = path.join("data/keys");
  if new_path.exists() {
  return new_path;
  }
 
  let mut components = path.components();
  components.next_back().unwrap();
- components
- .as_path()
- .join("prover/crates/bin/vk_setup_data_generator_server_fri/data")
+ components.as_path().join("data/keys")
 }
 
 impl Default for Keystore {

prover/crates/bin/vk_setup_data_generator_server_fri/src/utils.rs

@@ -132,7 +132,7 @@ mod tests {
  #[test]
  fn test_keyhash_generation() {
  let mut path_to_input = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap());
- path_to_input.push("historical_data");
+ path_to_input.push("../../../data/historical_data");
 
  for entry in std::fs::read_dir(path_to_input.clone()).unwrap().flatten() {
  if entry.metadata().unwrap().is_dir() {

prover/data/README.md

@@ -0,0 +1,23 @@
+# Prover data directory
+
+This directory contains the data required to run provers.
+
+Currently, it has the following sub-directories:
+
+- [keys](./keys/): Data required for proof generation. This data is mapped to a single protocol version.
+- [historical_data](./historical_data/) Descriptors for the protocol versions used in the past.
+
+## Keys directory
+
+`keys` directory is used by various components in the prover subsystem, and it generally can contain two kinds of data:
+
+- Small static files, like commitments, finalization hints, or verification keys.
+- Big generated blobs, like setup keys.
+
+Small static files are committed to the repository. Big files are expected to be downloaded or generated on demand. Two
+important notices as of Sep 2024:
+
+- Path to setup keys can be overridden via configuration.
+- Proof compressor requires an universal setup file, named, for example, `setup_2^24.bin` or `setup_2^26.bin`. It's
+ handled separately from the rest of the keys, e.g. it has separate configuration variables, and can naturally occur in
+ the `$ZKSYNC_HOME/keys/setup` during development.

zk_toolbox/crates/zk_inception/src/commands/prover/generate_sk.rs

@@ -18,8 +18,8 @@ pub(crate) async fn run(shell: &Shell) -> anyhow::Result<()> {
  shell,
  "cargo run --features gpu --release --bin key_generator -- 
  generate-sk-gpu all --recompute-if-missing
- --setup-path=crates/bin/vk_setup_data_generator_server_fri/data
- --path={link_to_prover}/crates/bin/vk_setup_data_generator_server_fri/data"
+ --setup-path=data/keys
+ --path={link_to_prover}/data/keys"
  ));
  cmd.run()?;
  spinner.finish();

zk_toolbox/crates/zk_supervisor/src/commands/prover_version.rs

@@ -27,8 +27,7 @@ async fn get_protocol_version(shell: &Shell, link_to_prover: &Path) -> anyhow::R
 }
 
 async fn get_snark_wrapper(link_to_prover: &Path) -> anyhow::Result<String> {
- let path =
- link_to_prover.join("crates/bin/vk_setup_data_generator_server_fri/data/commitments.json");
+ let path = link_to_prover.join("data/keys/commitments.json");
  let file = fs::File::open(path).expect("Could not find commitments file in zksync-era");
  let json: serde_json::Value =
  serde_json::from_reader(file).expect("Could not parse commitments.json");