Merge branch 'develop' of github.com:0xPolygonZero/zk_evm into arpit/704

0xPolygonZero · Nov 11, 2024 · 00ae12e · 00ae12e
2 parents 8db8683 + 4b4a9df
commit 00ae12e
Show file tree

Hide file tree

Showing 70 changed files with 95,446 additions and 323 deletions.
diff --git a/.cargo/config.toml b/.cargo/config.toml
@@ -2,3 +2,6 @@
 # https://github.com/rust-lang/rust/pull/124129
 # https://github.com/dtolnay/linkme/pull/88
 rustflags = ["-Z", "linker-features=-lld"]
+
+[alias]
+xtask = ["run", "--package=xtask", "--"]
diff --git a/.github/actions/rust/action.yml b/.github/actions/rust/action.yml
@@ -1,10 +1,12 @@
+---  # Common CI steps
+
 name: "Rust"
 description: |
   Common steps for CI
-  
   See <https://docs.github.com/en/actions/sharing-automations/creating-actions/creating-a-composite-action>
+
 runs:
-  using: composite  
+  using: composite
   steps:
     - uses: actions-rust-lang/setup-rust-toolchain@v1
     - uses: Swatinem/rust-cache@v2

diff --git a/.github/labeler.yml b/.github/labeler.yml
@@ -1,27 +1,29 @@
+---  # Github labeler configuration file
+
 # Add 'crate: mpt_trie' label to any changes within 'mpt_trie' folder.
 'crate: mpt_trie':
-- changed-files:
-  - any-glob-to-any-file: mpt_trie/**
+  - changed-files:
+      - any-glob-to-any-file: mpt_trie/**
 
 # Add 'crate: evm_arithmetization' label to any changes within 'evm_arithmetization' folder.
 'crate: evm_arithmetization':
-- changed-files:
-  - any-glob-to-any-file: evm_arithmetization/**
+  - changed-files:
+      - any-glob-to-any-file: evm_arithmetization/**
 
 # Add 'crate: trace_decoder' label to any changes within 'trace_decoder' folder.
 'crate: trace_decoder':
-- changed-files:
-  - any-glob-to-any-file: trace_decoder/**
+  - changed-files:
+      - any-glob-to-any-file: trace_decoder/**
 
 # Add 'crate: zero_bin' label to any changes within 'zero' folder.
 'crate: zero_bin':
-- changed-files:
-  - any-glob-to-any-file: zero/**
+  - changed-files:
+      - any-glob-to-any-file: zero/**
 
 # Add 'specs' label to any changes within 'docs' or `book` folder.
 'specs':
-- changed-files:
-  - any-glob-to-any-file: ['docs/**', 'book/**']
+  - changed-files:
+      - any-glob-to-any-file: ['docs/**', 'book/**']
 
 # Add 'crate: common' label to any changes within 'common' folder.
 'crate: common':

diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml
@@ -1,14 +1,17 @@
+---   # Rust cargo audit workflow
+
 name: Security audit
 on:
   push:
-    paths: 
+    paths:
       - '**/Cargo.toml'
+
 jobs:
   security_audit:
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: rustsec/[email protected]
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -0,0 +1,64 @@
+---   # Proof generation benchmarking workflow
+
+name: Benchmark proving
+
+on:
+  schedule:
+    - cron: "0 4 * * *"
+  workflow_dispatch:
+    branches:
+      - "**"
+
+env:
+  BENCHMARK_WITNESS: artifacts/witness_b1000_b1019.json
+
+jobs:
+  benchmark_proving:
+    name: Benchmark proving for representative blocks
+    runs-on: zero-reg
+    timeout-minutes: 300
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+
+      - name: Build the project
+        run: |
+          RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' cargo build --release
+          sudo sysctl kernel.perf_event_paranoid=0
+
+      - name: Download previous results
+        uses: dawidd6/action-download-artifact@v6
+        with:
+          workflow: benchmark.yml
+          workflow_conclusion: success
+          name: proving_benchmark
+          path: ./
+          if_no_artifact_found: ignore
+
+      - name: Run the script
+        run: |
+          echo "Benchmarking proving with $BENCHMARK_WITNESS"
+          ./scripts/benchmark_input.sh $BENCHMARK_WITNESS | tee benchmark_output.log
+          MEASURED_PROVING_TIME_SEC=`cat benchmark_output.log | grep 'Proving duration:' | tail -1 | awk '{ print $3}'`
+          PERF_TIME=`cat output.log | grep "seconds time elapsed" |  tail -1 | awk '{ print ($1)}'`
+          PERF_USER_TIME=`cat output.log | grep "seconds user" |  tail -1 | awk '{ print ($1)}'`
+          PERF_SYS_TIME=`cat output.log | grep "seconds sys" |  tail -1 | awk '{ print ($1)}'`
+          echo "RUN_ID=${{ github.run_id }} MEASURED_PROVING_TIME_SEC=$MEASURED_PROVING_TIME_SEC \
+            PERF_TIME=$PERF_TIME PERF_USER_TIME=$PERF_USER_TIME \
+            PERF_SYS_TIME=$PERF_SYS_TIME FILE=$BENCHMARK_WITNESS"
+          printf '%12s     %-12s     %-24s    %-20s    %-20s    %-20s    %-s\n' \
+            `date --utc +%y-%m-%d-%H:%M:%S` ${{ github.run_id }} \
+            $MEASURED_PROVING_TIME_SEC $PERF_TIME $PERF_USER_TIME \
+            $PERF_SYS_TIME $BENCHMARK_WITNESS >> proving_benchmark_results.txt
+
+      - name: Upload new results
+        uses: actions/upload-artifact@v4
+        with:
+          name: proving_benchmark
+          path: |
+            ./proving_benchmark_results.txt
+            ./output.log
+          retention-days: 90
+          overwrite: true
diff --git a/.github/workflows/book.yml b/.github/workflows/book.yml
@@ -1,3 +1,5 @@
+---   # MD Book generation and deployment workflow
+
 name: zkEVM mdbook
 
 on:
@@ -12,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions-rs/toolchain@v1
         with:
           toolchain: nightly
@@ -47,7 +49,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: build
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Download built book
         uses: actions/download-artifact@v3

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -1,3 +1,5 @@
+---   # Workflow with multiple jobs to test different parts of the project
+
 name: Continuous Integration
 
 on:
@@ -22,7 +24,6 @@ jobs:
     name: Test mpt_trie
     runs-on: ubuntu-latest
     timeout-minutes: 30
-    if: ${{ ! contains(toJSON(github.event.commits.*.message), '[skip-ci]') }}
     steps:
       - name: Checkout sources
         uses: actions/checkout@v4
@@ -37,7 +38,11 @@ jobs:
       - name: Test in mpt_trie subdirectory
         run: cargo test --manifest-path mpt_trie/Cargo.toml
         env:
-          RUSTFLAGS: -Copt-level=3 -Cdebug-assertions -Coverflow-checks=y -Cdebuginfo=0
+          RUSTFLAGS: >
+            -Copt-level=3
+            -Cdebug-assertions
+            -Coverflow-checks=y
+            -Cdebuginfo=0
           RUST_LOG: 1
           CARGO_INCREMENTAL: 1
           RUST_BACKTRACE: 1
@@ -50,8 +55,6 @@ jobs:
       RUST_LOG: info
       CARGO_INCREMENTAL: 1
       RUST_BACKTRACE: 1
-
-    if: ${{ ! contains(toJSON(github.event.commits.*.message), '[skip-ci]') }}
     steps:
       - name: Checkout sources
         uses: actions/checkout@v4
@@ -63,7 +66,7 @@ jobs:
         with:
           cache-on-failure: true
 
-      - name: build # build separately so test logs are actually nice
+      - name: build  # build separately so test logs are actually nice
         run: cargo build --tests --manifest-path trace_decoder/Cargo.toml
 
       - name: test
@@ -73,7 +76,6 @@ jobs:
     name: Test evm_arithmetization
     runs-on: ubuntu-latest
     timeout-minutes: 20
-    if: ${{ ! contains(toJSON(github.event.commits.*.message), '[skip-ci]') }}
     steps:
       - name: Checkout sources
         uses: actions/checkout@v4
@@ -93,11 +95,30 @@ jobs:
           CARGO_INCREMENTAL: 1
           RUST_BACKTRACE: 1
 
+  run_ignored_tests:
+    name: Slow evm_arithmetization tests in release mode
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v4
+
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+
+      - name: Set up rust cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          cache-on-failure: true
+
+      - name: Run specific ignored tests in release mode
+        run: |
+          cargo test --release \
+          --test empty_tables --test erc721 --test two_to_one_block -- --ignored
+
   test_zero_bin:
     name: Test zero_bin
     runs-on: ubuntu-latest
     timeout-minutes: 10
-    if: ${{ ! contains(toJSON(github.event.commits.*.message), '[skip-ci]') }}
     steps:
       - name: Checkout sources
         uses: actions/checkout@v4
@@ -112,7 +133,6 @@ jobs:
       - name: Test in zero_bin subdirectory
         run: |
           cargo test --manifest-path zero/Cargo.toml
-
         env:
           RUSTFLAGS: -Copt-level=3 -Cdebug-assertions -Coverflow-checks=y -Cdebuginfo=0
           RUST_LOG: 1
@@ -123,7 +143,6 @@ jobs:
     name: Test zk_evm_proc_macro
     runs-on: ubuntu-latest
     timeout-minutes: 5
-    if: ${{ ! contains(toJSON(github.event.commits.*.message), '[skip-ci]') }}
     steps:
       - name: Checkout sources
         uses: actions/checkout@v4
@@ -149,7 +168,7 @@ jobs:
     timeout-minutes: 15
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Run the script
         run: ./scripts/prove_stdio.sh artifacts/witness_b19807080.json
@@ -160,7 +179,7 @@ jobs:
     timeout-minutes: 5
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Run the script
         run: ./scripts/prove_stdio.sh artifacts/witness_b19807080.json test_only
@@ -171,7 +190,7 @@ jobs:
     timeout-minutes: 10
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Run the script
-        run: ./scripts/prove_stdio.sh artifacts/witness_b3_b6.json
+        run: ./scripts/prove_stdio.sh artifacts/witness_b3_b6.json "" use_test_config
diff --git a/.github/workflows/docker_build.yml b/.github/workflows/docker_build.yml
@@ -1,3 +1,5 @@
+---   # Workflow for building and running docker images as regression test
+
 name: Docker Build & Run
 
 on:

diff --git a/.github/workflows/docker_build_push.yml b/.github/workflows/docker_build_push.yml
@@ -1,3 +1,5 @@
+---   # Workflow for building and deploying docker images
+
 name: Docker Build & Push
 
 on:

diff --git a/.github/workflows/jerigon-native.yml b/.github/workflows/jerigon-native.yml
@@ -1,3 +1,5 @@
+---   # Run and populate blockchain with transactions and generate proofs using native tracer
+
 name: Jerigon Integration
 
 on:
@@ -19,7 +21,6 @@ jobs:
     name: Native tracer proof generation
     runs-on: zero-ci
     timeout-minutes: 30
-    if: "! contains(toJSON(github.event.commits.*.message), '[skip-ci]')"
     steps:
       - name: Checkout sources
         uses: actions/checkout@v4
@@ -52,16 +53,17 @@ jobs:
         run: |
           echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
           sudo apt update
-          sudo apt install kurtosis-cli
+          sudo apt install kurtosis-cli=1.3.1
 
-      #It is much easier to use cast tool in scripts so install foundry
+      # It is much easier to use cast tool in scripts so install foundry
       - name: Install Foundry
         uses: foundry-rs/foundry-toolchain@v1
 
       - name: Run cancun test network
         run: |
           docker pull ghcr.io/0xpolygonzero/erigon:feat-zero
-          kurtosis run --enclave cancun-testnet github.com/ethpandaops/[email protected] --args-file jerigon-test-network/network_params.yml
+          kurtosis run --enclave cancun-testnet github.com/ethpandaops/[email protected] \
+            --args-file jerigon-test-network/network_params.yml
 
       - name: Generate blocks with transactions
         run: |

diff --git a/.github/workflows/jerigon-zero.yml b/.github/workflows/jerigon-zero.yml
@@ -1,3 +1,5 @@
+---   # Run and populate blockchain with transactions and generate proofs using zero tracer
+
 name: Jerigon Integration
 
 on:
@@ -19,7 +21,6 @@ jobs:
     name: Zero tracer proof generation
     runs-on: zero-ci
     timeout-minutes: 30
-    if: "! contains(toJSON(github.event.commits.*.message), '[skip-ci]')"
     steps:
       - name: Checkout sources
         uses: actions/checkout@v4
@@ -52,16 +53,17 @@ jobs:
         run: |
           echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
           sudo apt update
-          sudo apt install kurtosis-cli
+          sudo apt install kurtosis-cli=1.3.1
 
-      #It is much easier to use cast tool in scripts so install foundry
+      # It is much easier to use cast tool in scripts so install foundry
       - name: Install Foundry
         uses: foundry-rs/foundry-toolchain@v1
 
       - name: Run cancun test network
         run: |
           docker pull ghcr.io/0xpolygonzero/erigon:feat-zero
-          kurtosis run --enclave cancun-testnet github.com/ethpandaops/[email protected] --args-file jerigon-test-network/network_params.yml
+          kurtosis run --enclave cancun-testnet github.com/ethpandaops/[email protected] \
+            --args-file jerigon-test-network/network_params.yml
 
       - name: Generate blocks with transactions
         run: |