From 2844be3479bd82d9df90a8711e80c46c3f8374c4 Mon Sep 17 00:00:00 2001 From: syntrust Date: Fri, 19 Jul 2024 12:37:14 +0800 Subject: [PATCH 01/27] split init and start --- init.sh | 152 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ run.sh | 124 ++++++++------------------------------------- 2 files changed, 174 insertions(+), 102 deletions(-) create mode 100755 init.sh diff --git a/init.sh b/init.sh new file mode 100755 index 00000000..5d5918f0 --- /dev/null +++ b/init.sh @@ -0,0 +1,152 @@ +#!/bin/bash + +# usage 1 (use snarkjs, will check Node.js version and snarkjs installation): +# env ES_NODE_STORAGE_MINER= ./init.sh +# usage 2 (use go-rapisnark): +# env ES_NODE_STORAGE_MINER= ./init.sh --miner.zk-prover-impl 2 + +# Note: currently only zk prover mode is supported + +if [ -z "$ES_NODE_STORAGE_MINER" ]; then + echo "Please provide 'ES_NODE_STORAGE_MINER' as an environment variable" + exit 1 +fi + +if [ ${#ES_NODE_STORAGE_MINER} -ne 42 ] || case $ES_NODE_STORAGE_MINER in 0x*) false;; *) true;; esac; then + echo "Error: ES_NODE_STORAGE_MINER should be prefixed with '0x' and have a total length of 42" + exit 1 +fi + + +# ZK prover implementation, 1: snarkjs, 2: go-rapidsnark. Default: 1 +zkp_impl=1 +# ZK prover mode, 1: one proof per sample, 2: one proof for multiple samples. Default: 2 +zkp_mode=2 + + +while [[ "$#" -gt 0 ]]; do + case "$1" in + --miner.zk-prover-impl) + if [ -n "$2" ]; then + zkp_impl="$2" + shift + else + echo "Error: Missing value for --miner.zk-prover-impl option." + exit 1 + fi + ;; + --miner.zk-prover-mode) + if [ -n "$2" ]; then + zkp_mode="$2" + shift + else + echo "Error: Missing value for --miner.zk-prover-mode option." + exit 1 + fi + ;; + *) + echo "Error: Unknown option: $1" + exit 1 + ;; + esac + shift +done + +echo "zkp_impl is set to $zkp_impl" + +if [ -n "$zkp_mode" ] && [ "$zkp_mode" != 1 ] && [ "$zkp_mode" != 2 ]; then + echo "Error: zk prover mode can only be 1 or 2." + exit 1 +fi + +# download zkey if not yet +zkey_name="blob_poseidon2.zkey" +zkey_size=560301223 +zkey_url="https://es-node-zkey.s3.us-west-1.amazonaws.com/blob_poseidon2_testnet1.zkey" +if [ "$zkp_mode" = 1 ]; then + zkey_name="blob_poseidon.zkey" + zkey_size=280151245 + zkey_url="https://drive.usercontent.google.com/download?id=1ZLfhYeCXMnbk6wUiBADRAn1mZ8MI_zg-&export=download&confirm=t&uuid=16ddcd58-2498-4d65-8931-934df3d0065c" +fi +zkey_file="./build/bin/snark_lib/$zkey_name" +if [ ! -e ${zkey_file} ] || [ $(wc -c < ${zkey_file}) -ne ${zkey_size} ]; then + echo "Start downloading ${zkey_file}..." + curl $zkey_url -o ${zkey_file} + if [ ! -e ${zkey_file} ]; then + echo "Error: The zkey file was not downloaded. Please try again." + exit 1 + fi + if [ $(wc -c < ${zkey_file}) -ne ${zkey_size} ]; then + echo "Error: The zkey file was not downloaded correctly. You can check the file content for more information." + exit 1 + fi +else + echo "√ ${zkey_file} already exists." +fi + +if [ -n "$zkp_impl" ] && [ "$zkp_impl" != 1 ] && [ "$zkp_impl" != 2 ]; then + echo "Error: miner.zk-prover-impl can only be 1 or 2" + exit 1 +fi + + +if [ "$zkp_impl" = 1 ]; then + + if ! [ -x "$(command -v node)" ]; then + echo 'Error: Node.js is not installed.' + exit 1 + fi + + # check node js version + node_version=$(node -v) + major_version=$(echo $node_version | cut -d'v' -f2 | cut -d'.' -f1) + if [ "$major_version" -lt 16 ]; then + echo "Error: Node.js version is too old: $node_version; must be 16 and above." + exit 1 + else + echo "√ Node.js version is compatible." + fi + + # install snarkjs if not + if ! [ "$(command -v snarkjs)" ]; then + echo "snarkjs not found, start installing..." + snarkjs_install=$(npm install -g snarkjs 2>&1) + if [ $? -eq 0 ]; then + echo "√ snarkjs installed successfully." + else + echo "Error: snarkjs install failed with the following error:" + echo "$snarkjs_install" + exit 1 + fi + else + echo "√ snarkjs is already installed." + fi + +fi + +executable="./build/bin/es-node" +echo "============================" +$executable --version +echo "============================" + +data_dir="./es-data" +storage_file_0="$data_dir/shard-0.dat" + +es_node_init="$executable init --shard_index 0 \ + --datadir $data_dir \ + --l1.rpc http://88.99.30.186:8545 \ + --storage.l1contract 0x804C520d3c084C805E37A35E90057Ac32831F96f \ + --storage.miner $ES_NODE_STORAGE_MINER \ +$@" + +# create data file for shard 0 if not yet +if [ ! -e $storage_file_0 ]; then + if $es_node_init ; then + echo "Initialized ${storage_file_0} successfully" + else + echo "Error: failed to initialize ${storage_file_0}" + exit 1 + fi +else + echo "Warning: storage file ${storage_file_0} already exists, skip initialization." +fi \ No newline at end of file diff --git a/run.sh b/run.sh index 375279dc..661a817b 100755 --- a/run.sh +++ b/run.sh @@ -1,9 +1,12 @@ #!/bin/bash -# usage: +# usage 1: # env ES_NODE_STORAGE_MINER= ES_NODE_SIGNER_PRIVATE_KEY= ./run.sh -# for one zk proof per sample (if the storage contract supports): -# env ES_NODE_STORAGE_MINER= ES_NODE_SIGNER_PRIVATE_KEY= ./run.sh --miner.zk-prover-mode 1 --l1.rpc --l1.beacon +# usage 2 (overriding rpc urls): +# env ES_NODE_STORAGE_MINER= ES_NODE_SIGNER_PRIVATE_KEY= ./run.sh --l1.rpc --l1.beacon + +# Note: currently only zk prover mode is supported +zkey_name="blob_poseidon2.zkey" if [ -z "$ES_NODE_STORAGE_MINER" ]; then echo "Please provide 'ES_NODE_STORAGE_MINER' as an environment variable" @@ -25,119 +28,36 @@ if [ ${#ES_NODE_SIGNER_PRIVATE_KEY} -ne 64 ]; then exit 1 fi -if ! [ -x "$(command -v node)" ]; then - echo 'Error: Node.js is not installed.' - exit 1 -fi - -# check node js version -node_version=$(node -v) -major_version=$(echo $node_version | cut -d'v' -f2 | cut -d'.' -f1) - -if [ "$major_version" -lt 16 ]; then - echo "Error: Node.js version is too old." - exit 1 -fi - -# install snarkjs if not -if ! [ "$(command -v snarkjs)" ]; then - echo "snarkjs not found, start installing..." - snarkjs_install=$(npm install -g snarkjs 2>&1) - if [ $? -eq 0 ]; then - echo "snarkjs installed successfully." - else - echo "Error: snarkjs install failed with the following error:" - echo "$snarkjs_install" - exit 1 - fi -fi - -# ZK prover mode, 1: one proof per sample, 2: one proof for multiple samples. -zkp_mode=2 -i=1 -while [ $i -le $# ]; do - if [ "${!i}" = "--miner.zk-prover-mode" ]; then - j=$((i+1)) - zkp_mode="${!j}" - break - else - if echo "${!i}" | grep -qE -- "--miner\.zk-prover-mode=([0-9]+)"; then - zkp_mode=$(echo "${!i}" | sed -E 's/.*=([0-9]+)/\1/') - break - fi - fi - i=$((i+1)) -done - -if [ "$zkp_mode" != 1 ] && [ "$zkp_mode" != 2 ]; then - echo "Error: zk prover mode can only be 1 or 2." - exit 1 -fi +executable="./build/bin/es-node" +echo "============================" +$executable --version +echo "============================" -echo "zk prover mode is $zkp_mode" +data_dir="./es-data" +storage_file_0="$data_dir/shard-0.dat" -# download zkey if not yet -zkey_name="blob_poseidon2.zkey" -zkey_size=560301223 -zkey_url="https://es-node-zkey.s3.us-west-1.amazonaws.com/blob_poseidon2_testnet1.zkey" -if [ "$zkp_mode" = 1 ]; then - zkey_name="blob_poseidon.zkey" - zkey_size=280151245 - zkey_url="https://drive.usercontent.google.com/download?id=1ZLfhYeCXMnbk6wUiBADRAn1mZ8MI_zg-&export=download&confirm=t&uuid=16ddcd58-2498-4d65-8931-934df3d0065c" -fi -zkey_file="./build/bin/snarkjs/$zkey_name" -if [ ! -e ${zkey_file} ] || [ $(wc -c < ${zkey_file}) -ne ${zkey_size} ]; then - echo "Start downloading ${zkey_file}..." - curl $zkey_url -o ${zkey_file} - if [ ! -e ${zkey_file} ]; then - echo "Error: The zkey file was not downloaded. Please try again." - exit 1 - fi - if [ $(wc -c < ${zkey_file}) -ne ${zkey_size} ]; then - echo "Error: The zkey file was not downloaded correctly. You can check the file content for more information." +if [ ! -e $storage_file_0 ]; then + echo "Error: storage file not found: ${storage_file_0}. Please run 'init.sh' first." exit 1 - fi fi -executable="./build/bin/es-node" -data_dir="./es-data" -storage_file_0="$data_dir/shard-0.dat" - -common_flags=" --datadir $data_dir \ - --l1.rpc http://88.99.30.186:8545 \ +start_flags=" --network devnet \ + --datadir $data_dir \ --storage.l1contract 0x804C520d3c084C805E37A35E90057Ac32831F96f \ - --storage.miner $ES_NODE_STORAGE_MINER \ - " - -# init shard 0 -es_node_init="init --shard_index 0" - -# start node -# TODO remove --network -es_node_start=" --network devnet \ - --miner.enabled \ - --miner.zkey $zkey_name \ --storage.files $storage_file_0 \ - --signer.private-key $ES_NODE_SIGNER_PRIVATE_KEY \ + --storage.miner $ES_NODE_STORAGE_MINER \ + --l1.rpc http://88.99.30.186:8545 \ --l1.beacon http://88.99.30.186:3500 \ --l1.beacon-based-time 1706684472 \ --l1.beacon-based-slot 4245906 \ + --signer.private-key $ES_NODE_SIGNER_PRIVATE_KEY \ + --miner.enabled \ + --miner.zkey $zkey_name \ --download.thread 32 \ --state.upload.url http://metrics.ethstorage.io:8080 \ --p2p.listen.udp 30305 \ --p2p.sync.concurrency 32 \ --p2p.bootnodes enr:-Li4QF3vBkkDQYNLHlVjW5NcEpXAsfNtE1lUVb_LgUQ_Ot2afS8jbDfnYQBDABJud_5Hd1hX_1cNeGVU6Tem06WDlfaGAY1e3vNvimV0aHN0b3JhZ2XbAYDY15SATFINPAhMgF43o16QBXrDKDH5b8GAgmlkgnY0gmlwhEFtP5qJc2VjcDI1NmsxoQK8XODtSv0IsrhBxZmTZBZEoLssb7bTX0YOVl6S0yLxuYN0Y3CCJAaDdWRwgnZh \ $@" - -# create data file for shard 0 if not yet -if [ ! -e $storage_file_0 ]; then - if $executable $es_node_init $common_flags ; then - echo "Initialized ${storage_file_0} successfully" - else - echo "Error: failed to initialize ${storage_file_0}" - exit 1 - fi -fi -# start es-node -exec $executable $common_flags $es_node_start +exec $executable $start_flags \ No newline at end of file From 06cb8e7f20350e10f1f45408952a700d80d0f957 Mon Sep 17 00:00:00 2001 From: syntrust Date: Fri, 19 Jul 2024 17:47:30 +0800 Subject: [PATCH 02/27] support flags override --- init.sh | 75 ++++++++++++++++++++++++++++++--------------------------- 1 file changed, 40 insertions(+), 35 deletions(-) diff --git a/init.sh b/init.sh index 5d5918f0..52e0eaef 100755 --- a/init.sh +++ b/init.sh @@ -16,43 +16,44 @@ if [ ${#ES_NODE_STORAGE_MINER} -ne 42 ] || case $ES_NODE_STORAGE_MINER in 0x*) f echo "Error: ES_NODE_STORAGE_MINER should be prefixed with '0x' and have a total length of 42" exit 1 fi - - # ZK prover implementation, 1: snarkjs, 2: go-rapidsnark. Default: 1 zkp_impl=1 # ZK prover mode, 1: one proof per sample, 2: one proof for multiple samples. Default: 2 zkp_mode=2 +#!/bin/bash - -while [[ "$#" -gt 0 ]]; do - case "$1" in - --miner.zk-prover-impl) - if [ -n "$2" ]; then - zkp_impl="$2" - shift - else - echo "Error: Missing value for --miner.zk-prover-impl option." - exit 1 - fi - ;; - --miner.zk-prover-mode) - if [ -n "$2" ]; then - zkp_mode="$2" - shift - else - echo "Error: Missing value for --miner.zk-prover-mode option." - exit 1 - fi - ;; - *) - echo "Error: Unknown option: $1" - exit 1 - ;; - esac - shift -done - -echo "zkp_impl is set to $zkp_impl" +process_zkprover_options() { + local args=("$@") + + for ((i=0; i < ${#args[@]}; i++)); do + case "${args[$i]}" in + --miner.zk-prover-impl) + if [ -n "${args[$((i+1))]}" ]; then + zkp_impl="${args[$((i+1))]}" + unset 'args[i]' + unset 'args[i+1]' + else + echo "Error: Missing value for --miner.zk-prover-impl option." + exit 1 + fi + ;; + --miner.zk-prover-mode) + if [ -n "${args[$((i+1))]}" ]; then + zkp_mode="${args[$((i+1))]}" + unset 'args[i]' + unset 'args[i+1]' + else + echo "Error: Missing value for --miner.zk-prover-mode option." + exit 1 + fi + ;; + esac + done + + echo "${args[@]}" +} + +remaining_args=$(process_zkprover_options "$@") if [ -n "$zkp_mode" ] && [ "$zkp_mode" != 1 ] && [ "$zkp_mode" != 2 ]; then echo "Error: zk prover mode can only be 1 or 2." @@ -84,6 +85,7 @@ else echo "√ ${zkey_file} already exists." fi +echo "zkp_impl is set to $zkp_impl" if [ -n "$zkp_impl" ] && [ "$zkp_impl" != 1 ] && [ "$zkp_impl" != 2 ]; then echo "Error: miner.zk-prover-impl can only be 1 or 2" exit 1 @@ -125,9 +127,9 @@ if [ "$zkp_impl" = 1 ]; then fi executable="./build/bin/es-node" -echo "============================" +echo "========== build info ==================" $executable --version -echo "============================" +echo "========================================" data_dir="./es-data" storage_file_0="$data_dir/shard-0.dat" @@ -137,7 +139,10 @@ es_node_init="$executable init --shard_index 0 \ --l1.rpc http://88.99.30.186:8545 \ --storage.l1contract 0x804C520d3c084C805E37A35E90057Ac32831F96f \ --storage.miner $ES_NODE_STORAGE_MINER \ -$@" +$remaining_args" + + +echo "$es_node_init" # create data file for shard 0 if not yet if [ ! -e $storage_file_0 ]; then From 04b205af15af97396e79c9266d915c9448335c39 Mon Sep 17 00:00:00 2001 From: syntrust Date: Mon, 22 Jul 2024 10:18:13 +0800 Subject: [PATCH 03/27] fix docker --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index bdaf72b0..181184fa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,8 +10,9 @@ COPY --from=builder /es-node/build/ /es-node/build/ RUN apk add --no-cache bash curl libstdc++ gcompat libgomp # Entrypoint +COPY --from=builder /es-node/init.sh /es-node/ COPY --from=builder /es-node/run.sh /es-node/ -RUN chmod +x /es-node/run.sh +RUN chmod +x /es-node/init.sh /es-node/run.sh WORKDIR /es-node EXPOSE 9545 9222 30305/udp From bd56ee3d4cb1e3ad7115416ad9e710b4a894a8dd Mon Sep 17 00:00:00 2001 From: syntrust Date: Mon, 22 Jul 2024 10:30:18 +0800 Subject: [PATCH 04/27] fix build --- .github/workflows/publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 440fd92a..6986fbf3 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -44,6 +44,7 @@ jobs: run: | mkdir -p ${{ env.BIN_DIR }}/snarkbuild cp -r ethstorage/prover/snark_lib ${{ env.BIN_DIR }} + cp init.sh ${{ env.BUILD_DIR }} cp run.sh ${{ env.BUILD_DIR }} - name: Build From 30e1438df050ae079f6ec7a42566c9f58521ae42 Mon Sep 17 00:00:00 2001 From: syntrust Date: Mon, 22 Jul 2024 16:29:10 +0800 Subject: [PATCH 05/27] fix flag read --- init.sh | 46 ++++++++++++++-------------------------------- 1 file changed, 14 insertions(+), 32 deletions(-) diff --git a/init.sh b/init.sh index 52e0eaef..5b83d0f5 100755 --- a/init.sh +++ b/init.sh @@ -22,38 +22,20 @@ zkp_impl=1 zkp_mode=2 #!/bin/bash -process_zkprover_options() { - local args=("$@") - - for ((i=0; i < ${#args[@]}; i++)); do - case "${args[$i]}" in - --miner.zk-prover-impl) - if [ -n "${args[$((i+1))]}" ]; then - zkp_impl="${args[$((i+1))]}" - unset 'args[i]' - unset 'args[i+1]' - else - echo "Error: Missing value for --miner.zk-prover-impl option." - exit 1 - fi - ;; - --miner.zk-prover-mode) - if [ -n "${args[$((i+1))]}" ]; then - zkp_mode="${args[$((i+1))]}" - unset 'args[i]' - unset 'args[i+1]' - else - echo "Error: Missing value for --miner.zk-prover-mode option." - exit 1 - fi - ;; - esac - done - - echo "${args[@]}" -} - -remaining_args=$(process_zkprover_options "$@") +remaining_args="" + +while [ $# -gt 0 ]; do + if [[ $1 == --miner.zk-prover-impl ]]; then + zkp_impl=$2 + shift 2 + elif [[ $1 == --miner.zk-prover-mode ]]; then + zkp_mode=$2 + shift 2 + else + remaining_args="$remaining_args $1" + shift + fi +done if [ -n "$zkp_mode" ] && [ "$zkp_mode" != 1 ] && [ "$zkp_mode" != 2 ]; then echo "Error: zk prover mode can only be 1 or 2." From 86d74dfc12407e1b2e767e51857eb85873a2b7d9 Mon Sep 17 00:00:00 2001 From: syntrust Date: Mon, 22 Jul 2024 18:16:11 +0800 Subject: [PATCH 06/27] use zkey with path --- ethstorage/miner/cli.go | 16 ++++++++++++---- ethstorage/prover/kzg_poseidon_prover.go | 7 +++---- ethstorage/prover/zk_prover.go | 8 ++++---- ethstorage/prover/zk_prover_go.go | 4 ++-- init.sh | 13 +++++++------ run.sh | 4 ++-- 6 files changed, 30 insertions(+), 22 deletions(-) diff --git a/ethstorage/miner/cli.go b/ethstorage/miner/cli.go index ecd56ff8..0180fe4a 100644 --- a/ethstorage/miner/cli.go +++ b/ethstorage/miner/cli.go @@ -55,7 +55,7 @@ func CLIFlags(envPrefix string) []cli.Flag { }, cli.StringFlag{ Name: ZKeyFileNameFlagName, - Usage: "zkey file name which should be put in the snark_lib folder", + Usage: "zkey file name with path", Value: DefaultConfig.ZKeyFileName, EnvVar: rollup.PrefixEnvVar(envPrefix, "ZKEY_FILE"), }, @@ -118,14 +118,22 @@ func (c CLIConfig) ToMinerConfig() (Config, error) { } zkWorkingDir = dir } + zkFile := c.ZKeyFileName + if !filepath.IsAbs(zkFile) { + dir, err := filepath.Abs(zkFile) + if err != nil { + return Config{}, fmt.Errorf("check ZKeyFileName error: %v", err) + } + zkFile = dir + } cfg := DefaultConfig cfg.ZKWorkingDir = zkWorkingDir + cfg.ZKeyFileName = zkFile + cfg.ZKProverMode = c.ZKProverMode + cfg.ZKProverImpl = c.ZKProverImpl cfg.GasPrice = c.GasPrice cfg.PriorityGasPrice = c.PriorityGasPrice cfg.MinimumProfit = c.MinimumProfit - cfg.ZKeyFileName = c.ZKeyFileName - cfg.ZKProverMode = c.ZKProverMode - cfg.ZKProverImpl = c.ZKProverImpl cfg.ThreadsPerShard = c.ThreadsPerShard return cfg, nil } diff --git a/ethstorage/prover/kzg_poseidon_prover.go b/ethstorage/prover/kzg_poseidon_prover.go index e2dfb744..3a9b1ee4 100644 --- a/ethstorage/prover/kzg_poseidon_prover.go +++ b/ethstorage/prover/kzg_poseidon_prover.go @@ -40,18 +40,17 @@ type KZGPoseidonProver struct { // Prover that can be used directly by miner to prove both KZG and Poseidon hash // workingDir specifies the working directory of the command relative to the caller. -// zkeyFileName specifies the zkey file name to generate snark proof +// zkeyFile specifies the zkey file with path to generate snark proof // zkProverMode specifies the mode of the zk prover, 1 for per sample, 2 for samples // zkProverImpl specifies the implementation of the snark prover, 1 for snarkjs, 2 for go-rapidsnark // lg specifies the logger to log the info // returns a prover that can generate a combined KZG + zk proof -func NewKZGPoseidonProver(workingDir, zkeyFileName string, zkProverMode, zkProverImpl uint64, lg log.Logger) KZGPoseidonProver { +func NewKZGPoseidonProver(workingDir, zkeyFile string, zkProverMode, zkProverImpl uint64, lg log.Logger) KZGPoseidonProver { // check dependencies when es-node starts libDir := filepath.Join(workingDir, SnarkLib) if _, err := os.Stat(libDir); errors.Is(err, os.ErrNotExist) { lg.Crit("Init ZK prover failed", "error", "snark lib does not exist", "dir", libDir) } - zkeyFile := filepath.Join(libDir, zkeyFileName) if _, err := os.Stat(zkeyFile); errors.Is(err, os.ErrNotExist) { lg.Crit("Init ZK prover failed", "error", "zkey does not exist", "dir", zkeyFile) } @@ -71,7 +70,7 @@ func NewKZGPoseidonProver(workingDir, zkeyFileName string, zkProverMode, zkProve zkProverMode: zkProverMode, zkProverImpl: zkProverImpl, libDir: libDir, - zkey: zkeyFileName, + zkey: zkeyFile, wasm: wasmName, lg: lg, } diff --git a/ethstorage/prover/zk_prover.go b/ethstorage/prover/zk_prover.go index 0e8c2565..348f7941 100644 --- a/ethstorage/prover/zk_prover.go +++ b/ethstorage/prover/zk_prover.go @@ -27,14 +27,14 @@ const ( ) type ZKProver struct { - dir, zkeyName, wasmName string + dir, zkeyFile, wasmName string lg log.Logger } -func NewZKProver(workingDir, zkeyName, wasmName string, lg log.Logger) *ZKProver { +func NewZKProver(workingDir, zkey, wasmName string, lg log.Logger) *ZKProver { return &ZKProver{ dir: workingDir, - zkeyName: zkeyName, + zkeyFile: zkey, wasmName: wasmName, lg: lg, } @@ -146,7 +146,7 @@ func (p *ZKProver) prove(dir string, pubInputs []byte) ([]byte, string, error) { proofFile := filepath.Join(buildDir, proofName) publicFile := filepath.Join(buildDir, publicName) cmd = exec.Command("snarkjs", "groth16", "prove", - filepath.Join(libDir, p.zkeyName), + p.zkeyFile, wtnsFile, proofFile, publicFile, diff --git a/ethstorage/prover/zk_prover_go.go b/ethstorage/prover/zk_prover_go.go index e665bd67..ce90ae1b 100644 --- a/ethstorage/prover/zk_prover_go.go +++ b/ethstorage/prover/zk_prover_go.go @@ -24,7 +24,7 @@ type ZKProverGo struct { lg log.Logger } -func NewZKProverGo(libDir, zkeyName, wasmName string, lg log.Logger) (*ZKProverGo, error) { +func NewZKProverGo(libDir, zkeyFile, wasmName string, lg log.Logger) (*ZKProverGo, error) { wasmBytes, err := os.ReadFile(filepath.Join(libDir, wasmName)) if err != nil { lg.Error("Read wasm file failed", "error", err) @@ -35,7 +35,7 @@ func NewZKProverGo(libDir, zkeyName, wasmName string, lg log.Logger) (*ZKProverG lg.Error("Create witness calculator failed", "error", err) return nil, err } - zkey, err := os.ReadFile(filepath.Join(libDir, zkeyName)) + zkey, err := os.ReadFile(zkeyFile) if err != nil { lg.Error("Read zkey file failed", "error", err) return nil, err diff --git a/init.sh b/init.sh index 5b83d0f5..a0ddfe93 100755 --- a/init.sh +++ b/init.sh @@ -16,6 +16,12 @@ if [ ${#ES_NODE_STORAGE_MINER} -ne 42 ] || case $ES_NODE_STORAGE_MINER in 0x*) f echo "Error: ES_NODE_STORAGE_MINER should be prefixed with '0x' and have a total length of 42" exit 1 fi + +executable="./build/bin/es-node" +echo "========== build info ==================" +$executable --version +echo "========================================" + # ZK prover implementation, 1: snarkjs, 2: go-rapidsnark. Default: 1 zkp_impl=1 # ZK prover mode, 1: one proof per sample, 2: one proof for multiple samples. Default: 2 @@ -51,7 +57,7 @@ if [ "$zkp_mode" = 1 ]; then zkey_size=280151245 zkey_url="https://drive.usercontent.google.com/download?id=1ZLfhYeCXMnbk6wUiBADRAn1mZ8MI_zg-&export=download&confirm=t&uuid=16ddcd58-2498-4d65-8931-934df3d0065c" fi -zkey_file="./build/bin/snark_lib/$zkey_name" +zkey_file="./build/bin/snark_lib/zkey/$zkey_name" if [ ! -e ${zkey_file} ] || [ $(wc -c < ${zkey_file}) -ne ${zkey_size} ]; then echo "Start downloading ${zkey_file}..." curl $zkey_url -o ${zkey_file} @@ -108,11 +114,6 @@ if [ "$zkp_impl" = 1 ]; then fi -executable="./build/bin/es-node" -echo "========== build info ==================" -$executable --version -echo "========================================" - data_dir="./es-data" storage_file_0="$data_dir/shard-0.dat" diff --git a/run.sh b/run.sh index 661a817b..97a93ddc 100755 --- a/run.sh +++ b/run.sh @@ -6,7 +6,7 @@ # env ES_NODE_STORAGE_MINER= ES_NODE_SIGNER_PRIVATE_KEY= ./run.sh --l1.rpc --l1.beacon # Note: currently only zk prover mode is supported -zkey_name="blob_poseidon2.zkey" +zkey_file="./build/bin/snark_lib/zkey/blob_poseidon2.zkey" if [ -z "$ES_NODE_STORAGE_MINER" ]; then echo "Please provide 'ES_NODE_STORAGE_MINER' as an environment variable" @@ -52,7 +52,7 @@ start_flags=" --network devnet \ --l1.beacon-based-slot 4245906 \ --signer.private-key $ES_NODE_SIGNER_PRIVATE_KEY \ --miner.enabled \ - --miner.zkey $zkey_name \ + --miner.zkey $zkey_file \ --download.thread 32 \ --state.upload.url http://metrics.ethstorage.io:8080 \ --p2p.listen.udp 30305 \ From 6126e26fa168da47e2e2a86c5360356037e617e1 Mon Sep 17 00:00:00 2001 From: syntrust Date: Tue, 23 Jul 2024 10:19:08 +0800 Subject: [PATCH 07/27] minor --- init.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/init.sh b/init.sh index a0ddfe93..66d8438e 100755 --- a/init.sh +++ b/init.sh @@ -124,9 +124,6 @@ es_node_init="$executable init --shard_index 0 \ --storage.miner $ES_NODE_STORAGE_MINER \ $remaining_args" - -echo "$es_node_init" - # create data file for shard 0 if not yet if [ ! -e $storage_file_0 ]; then if $es_node_init ; then From 2c2058e6445f69a5ec41bb3de541455469755dd1 Mon Sep 17 00:00:00 2001 From: syntrust Date: Thu, 25 Jul 2024 16:00:21 +0800 Subject: [PATCH 08/27] fix config --- ethstorage/miner/cli.go | 2 +- ethstorage/miner/config.go | 3 ++- ethstorage/miner/miner_test.go | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ethstorage/miner/cli.go b/ethstorage/miner/cli.go index 0180fe4a..66afa493 100644 --- a/ethstorage/miner/cli.go +++ b/ethstorage/miner/cli.go @@ -61,7 +61,7 @@ func CLIFlags(envPrefix string) []cli.Flag { }, cli.StringFlag{ Name: ZKWorkingDirFlagName, - Usage: "Path to the snark_lib folder", + Usage: "Path to the snark library folder", Value: DefaultConfig.ZKWorkingDir, EnvVar: rollup.PrefixEnvVar(envPrefix, "ZK_WORKING_DIR"), }, diff --git a/ethstorage/miner/config.go b/ethstorage/miner/config.go index f159c66c..3ff37136 100644 --- a/ethstorage/miner/config.go +++ b/ethstorage/miner/config.go @@ -9,6 +9,7 @@ import ( "runtime" "github.com/ethereum/go-ethereum/common" + "github.com/ethstorage/go-ethstorage/ethstorage/prover" "github.com/ethstorage/go-ethstorage/ethstorage/signer" ) @@ -48,7 +49,7 @@ var DefaultConfig = Config{ GasPrice: nil, PriorityGasPrice: nil, - ZKeyFileName: "blob_poseidon2.zkey", + ZKeyFileName: filepath.Join("build", "bin", prover.SnarkLib, "blob_poseidon2.zkey"), ZKWorkingDir: filepath.Join("build", "bin"), ZKProverMode: 2, ZKProverImpl: 1, diff --git a/ethstorage/miner/miner_test.go b/ethstorage/miner/miner_test.go index 9fe16090..a5ebe3d2 100644 --- a/ethstorage/miner/miner_test.go +++ b/ethstorage/miner/miner_test.go @@ -77,7 +77,7 @@ func newMiner(t *testing.T, storageMgr *es.StorageManager, client *eth.PollingCl } defer os.Remove(zkey) } - pvr := prover.NewKZGPoseidonProver(zkWorkingDir, defaultConfig.ZKeyFileName, defaultConfig.ZKProverMode, defaultConfig.ZKProverImpl, lg) + pvr := prover.NewKZGPoseidonProver(zkWorkingDir, zkey, defaultConfig.ZKProverMode, defaultConfig.ZKProverImpl, lg) fd := new(event.Feed) db := rawdb.NewMemoryDatabase() br := blobs.NewBlobReader(downloader.NewBlobMemCache(), storageMgr, lg) From e31dff7e16f052ac517edb14084b6e013141601b Mon Sep 17 00:00:00 2001 From: syntrust Date: Thu, 25 Jul 2024 17:59:12 +0800 Subject: [PATCH 09/27] update zkey path --- Makefile | 1 + ethstorage/miner/cli.go | 4 ++-- ethstorage/miner/config.go | 4 ++-- ethstorage/miner/miner_test.go | 37 +++++++++++++---------------- ethstorage/node/node.go | 2 +- init.sh | 2 +- integration_tests/node_mine_test.go | 4 ++-- 7 files changed, 25 insertions(+), 29 deletions(-) diff --git a/Makefile b/Makefile index c8f4dc28..04a1f607 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,7 @@ LDFLAGS := -ldflags "$(LDFLAGSSTRING)" es-node: build cp -r ethstorage/prover/snark_lib build/bin + mkdir -p build/bin/snark_lib/zkey mkdir -p build/bin/snarkbuild build: diff --git a/ethstorage/miner/cli.go b/ethstorage/miner/cli.go index 66afa493..684ed257 100644 --- a/ethstorage/miner/cli.go +++ b/ethstorage/miner/cli.go @@ -56,7 +56,7 @@ func CLIFlags(envPrefix string) []cli.Flag { cli.StringFlag{ Name: ZKeyFileNameFlagName, Usage: "zkey file name with path", - Value: DefaultConfig.ZKeyFileName, + Value: DefaultConfig.ZKeyFile, EnvVar: rollup.PrefixEnvVar(envPrefix, "ZKEY_FILE"), }, cli.StringFlag{ @@ -128,7 +128,7 @@ func (c CLIConfig) ToMinerConfig() (Config, error) { } cfg := DefaultConfig cfg.ZKWorkingDir = zkWorkingDir - cfg.ZKeyFileName = zkFile + cfg.ZKeyFile = zkFile cfg.ZKProverMode = c.ZKProverMode cfg.ZKProverImpl = c.ZKProverImpl cfg.GasPrice = c.GasPrice diff --git a/ethstorage/miner/config.go b/ethstorage/miner/config.go index 3ff37136..b55efca0 100644 --- a/ethstorage/miner/config.go +++ b/ethstorage/miner/config.go @@ -30,7 +30,7 @@ type Config struct { // cli GasPrice *big.Int PriorityGasPrice *big.Int - ZKeyFileName string + ZKeyFile string ZKWorkingDir string ZKProverMode uint64 ZKProverImpl uint64 @@ -49,7 +49,7 @@ var DefaultConfig = Config{ GasPrice: nil, PriorityGasPrice: nil, - ZKeyFileName: filepath.Join("build", "bin", prover.SnarkLib, "blob_poseidon2.zkey"), + ZKeyFile: filepath.Join("build", "bin", prover.SnarkLib, "zkey", "blob_poseidon2.zkey"), ZKWorkingDir: filepath.Join("build", "bin"), ZKProverMode: 2, ZKProverImpl: 1, diff --git a/ethstorage/miner/miner_test.go b/ethstorage/miner/miner_test.go index a5ebe3d2..c9eaefa6 100644 --- a/ethstorage/miner/miner_test.go +++ b/ethstorage/miner/miner_test.go @@ -6,7 +6,6 @@ package miner import ( - "math/big" "os" "path/filepath" "runtime/debug" @@ -14,7 +13,6 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/event" es "github.com/ethstorage/go-ethstorage/ethstorage" @@ -38,7 +36,6 @@ var ( kvSize uint64 = 1 << kvSizeBits kvEntries uint64 = 1 << kvEntriesBits shardID = uint64(0) - value = hexutil.EncodeUint64(10000000000000) lg = esLog.NewLogger(esLog.DefaultCLIConfig()) ) @@ -54,34 +51,32 @@ func initStorageManager(t *testing.T, client *eth.PollingClient) *es.StorageMana } func newMiner(t *testing.T, storageMgr *es.StorageManager, client *eth.PollingClient) *Miner { - defaultConfig := &Config{ - RandomChecks: 2, - NonceLimit: 1048576, - MinimumDiff: new(big.Int).SetUint64(5000000), - Cutoff: new(big.Int).SetUint64(60), - DiffAdjDivisor: new(big.Int).SetUint64(1024), - GasPrice: nil, - PriorityGasPrice: new(big.Int).SetUint64(10), - ThreadsPerShard: 1, - ZKProverMode: 2, - ZKProverImpl: 1, - ZKeyFileName: "blob_poseidon2.zkey", - } - l1api := NewL1MiningAPI(client, nil, lg) + testConfig := &DefaultConfig + zkWorkingDir, _ := filepath.Abs("../prover") - zkey := filepath.Join(zkWorkingDir, prover.SnarkLib, defaultConfig.ZKeyFileName) + testConfig.ZKWorkingDir = zkWorkingDir + dir := filepath.Join(zkWorkingDir, prover.SnarkLib, "zkey") + zkey := filepath.Join(dir, "blob_poseidon2.zkey") + testConfig.ZKeyFile = zkey + if _, err := os.Stat(dir); os.IsNotExist(err) { + err := os.Mkdir(dir, 0755) + if err != nil { + t.Fatalf("Mkdir failed %v", err) + } + } if _, err := os.Stat(zkey); os.IsNotExist(err) { _, err := os.Create(zkey) if err != nil { t.Fatalf("Create failed %v", err) } - defer os.Remove(zkey) + defer os.RemoveAll(dir) } - pvr := prover.NewKZGPoseidonProver(zkWorkingDir, zkey, defaultConfig.ZKProverMode, defaultConfig.ZKProverImpl, lg) + pvr := prover.NewKZGPoseidonProver(zkWorkingDir, zkey, testConfig.ZKProverMode, testConfig.ZKProverImpl, lg) fd := new(event.Feed) db := rawdb.NewMemoryDatabase() br := blobs.NewBlobReader(downloader.NewBlobMemCache(), storageMgr, lg) - miner := New(defaultConfig, db, storageMgr, l1api, br, &pvr, fd, lg) + l1api := NewL1MiningAPI(client, nil, lg) + miner := New(testConfig, db, storageMgr, l1api, br, &pvr, fd, lg) return miner } diff --git a/ethstorage/node/node.go b/ethstorage/node/node.go index 8f13c216..f1e76a1d 100644 --- a/ethstorage/node/node.go +++ b/ethstorage/node/node.go @@ -300,7 +300,7 @@ func (n *EsNode) initMiner(ctx context.Context, cfg *Config) error { l1api := miner.NewL1MiningAPI(n.l1Source, n.randaoSource, n.log) pvr := prover.NewKZGPoseidonProver( cfg.Mining.ZKWorkingDir, - cfg.Mining.ZKeyFileName, + cfg.Mining.ZKeyFile, cfg.Mining.ZKProverMode, cfg.Mining.ZKProverImpl, n.log, diff --git a/init.sh b/init.sh index 66d8438e..3fed9d1b 100755 --- a/init.sh +++ b/init.sh @@ -127,7 +127,7 @@ $remaining_args" # create data file for shard 0 if not yet if [ ! -e $storage_file_0 ]; then if $es_node_init ; then - echo "Initialized ${storage_file_0} successfully" + echo "√ Initialized ${storage_file_0} successfully" else echo "Error: failed to initialize ${storage_file_0}" exit 1 diff --git a/integration_tests/node_mine_test.go b/integration_tests/node_mine_test.go index ed2c24fc..df0030e4 100644 --- a/integration_tests/node_mine_test.go +++ b/integration_tests/node_mine_test.go @@ -78,7 +78,7 @@ func TestMining(t *testing.T) { l1api := miner.NewL1MiningAPI(pClient, nil, lg) pvr := prover.NewKZGPoseidonProver( miningConfig.ZKWorkingDir, - miningConfig.ZKeyFileName, + miningConfig.ZKeyFile, miningConfig.ZKProverMode, miningConfig.ZKProverImpl, lg, @@ -415,7 +415,7 @@ func initMiningConfig(t *testing.T, l1Contract common.Address, client *eth.Polli } miningConfig.PrepaidAmount = new(big.Int).SetBytes(result) - miningConfig.ZKeyFileName = zkey2Name + miningConfig.ZKeyFile = zkey2Name proverPath, _ := filepath.Abs(prPath) miningConfig.ZKWorkingDir = proverPath miningConfig.ZKProverMode = 2 From 16c171049427a5ec3e8f9a3da1620d04918b3ac3 Mon Sep 17 00:00:00 2001 From: syntrust Date: Thu, 25 Jul 2024 18:43:40 +0800 Subject: [PATCH 10/27] fix zkey path in git --- .github/workflows/publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 6986fbf3..9b7bfea6 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -43,6 +43,7 @@ jobs: - name: Create package run: | mkdir -p ${{ env.BIN_DIR }}/snarkbuild + mkdir -p ${{ env.BIN_DIR }}/snark_lib/zkey cp -r ethstorage/prover/snark_lib ${{ env.BIN_DIR }} cp init.sh ${{ env.BUILD_DIR }} cp run.sh ${{ env.BUILD_DIR }} From 8910db67e0f9028dff4ffab8ed5c1acaefaf1ba2 Mon Sep 17 00:00:00 2001 From: syntrust Date: Wed, 31 Jul 2024 16:43:17 +0800 Subject: [PATCH 11/27] fix compose --- docker-compose.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 9bd31192..ebca2fdd 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -8,9 +8,10 @@ services: - '30305:30305/udp' volumes: - ./es-data:/es-node/es-data + - ./es-node/build/bin/snark_lib/zkey:/es-node/build/bin/snark_lib/zkey environment: - ES_NODE_STORAGE_MINER=${ES_NODE_STORAGE_MINER} - ES_NODE_SIGNER_PRIVATE_KEY=${ES_NODE_SIGNER_PRIVATE_KEY} - command: ["bash", "-c", "/es-node/run.sh --miner.zk-prover-impl=2"] + command: ["bash", "-c", "/es-node/run.sh"] container_name: es tty: true From 3bba285f656c40b6e6ea24b98a757081be961e72 Mon Sep 17 00:00:00 2001 From: syntrust Date: Wed, 31 Jul 2024 17:33:25 +0800 Subject: [PATCH 12/27] fix zkey path --- docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index ebca2fdd..e0d76c12 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -8,7 +8,7 @@ services: - '30305:30305/udp' volumes: - ./es-data:/es-node/es-data - - ./es-node/build/bin/snark_lib/zkey:/es-node/build/bin/snark_lib/zkey + - ./build/bin/snark_lib/zkey:/es-node/build/bin/snark_lib/zkey environment: - ES_NODE_STORAGE_MINER=${ES_NODE_STORAGE_MINER} - ES_NODE_SIGNER_PRIVATE_KEY=${ES_NODE_SIGNER_PRIVATE_KEY} From 71317a7a7e896232be0562ccd0c02ab46a907641 Mon Sep 17 00:00:00 2001 From: syntrust Date: Wed, 31 Jul 2024 17:33:57 +0800 Subject: [PATCH 13/27] default to --miner.zk-prover-impl 2 --- init.sh | 15 +++++++-------- run-docker.sh | 30 +++++++++++++++++++----------- run.sh | 3 ++- 3 files changed, 28 insertions(+), 20 deletions(-) diff --git a/init.sh b/init.sh index 3fed9d1b..bd04304e 100755 --- a/init.sh +++ b/init.sh @@ -1,11 +1,9 @@ #!/bin/bash -# usage 1 (use snarkjs, will check Node.js version and snarkjs installation): +# usage 1 (use go-rapisnark): # env ES_NODE_STORAGE_MINER= ./init.sh -# usage 2 (use go-rapisnark): -# env ES_NODE_STORAGE_MINER= ./init.sh --miner.zk-prover-impl 2 - -# Note: currently only zk prover mode is supported +# usage 2 (use snarkjs, will check Node.js version and snarkjs installation): +# env ES_NODE_STORAGE_MINER= ./init.sh --miner.zk-prover-impl 1 if [ -z "$ES_NODE_STORAGE_MINER" ]; then echo "Please provide 'ES_NODE_STORAGE_MINER' as an environment variable" @@ -22,9 +20,10 @@ echo "========== build info ==================" $executable --version echo "========================================" -# ZK prover implementation, 1: snarkjs, 2: go-rapidsnark. Default: 1 -zkp_impl=1 -# ZK prover mode, 1: one proof per sample, 2: one proof for multiple samples. Default: 2 +# ZK prover implementation, 1: snarkjs, 2: go-rapidsnark. +zkp_impl=2 +# ZK prover mode, 1: one proof per sample, 2: one proof for multiple samples. +# Note: currently only zk prover mode 2 is supported zkp_mode=2 #!/bin/bash diff --git a/run-docker.sh b/run-docker.sh index dc948435..1b656c79 100755 --- a/run-docker.sh +++ b/run-docker.sh @@ -40,18 +40,26 @@ else sudo docker build -t $image_name . echo "image $image_name built" fi + + sudo docker run --rm \ + -v ./es-data:/es-node/es-data \ + -v ./build/bin/snark_lib/zkey:/es-node/build/bin/snark_lib/zkey \ + -e ES_NODE_STORAGE_MINER=$ES_NODE_STORAGE_MINER \ + --entrypoint /es-node/init.sh \ + $image_name + # run container in the background - sudo docker run --name $container_name \ - -v ./es-data:/es-node/es-data \ - -e ES_NODE_STORAGE_MINER=$ES_NODE_STORAGE_MINER \ - -e ES_NODE_SIGNER_PRIVATE_KEY=$ES_NODE_SIGNER_PRIVATE_KEY \ - -p 9545:9545 \ - -p 9222:9222 \ - -p 30305:30305/udp \ - -d \ - --entrypoint /es-node/run.sh \ - $image_name \ - --miner.zk-prover-impl 2 + sudo docker run --name $container_name \ + -v ./es-data:/es-node/es-data \ + -v ./build/bin/snark_lib/zkey:/es-node/build/bin/snark_lib/zkey \ + -e ES_NODE_STORAGE_MINER=$ES_NODE_STORAGE_MINER \ + -e ES_NODE_SIGNER_PRIVATE_KEY=$ES_NODE_SIGNER_PRIVATE_KEY \ + -p 9545:9545 \ + -p 9222:9222 \ + -p 30305:30305/udp \ + -d \ + --entrypoint /es-node/run.sh \ + $image_name echo "container $container_name started" fi fi diff --git a/run.sh b/run.sh index 97a93ddc..8265dddf 100755 --- a/run.sh +++ b/run.sh @@ -5,7 +5,7 @@ # usage 2 (overriding rpc urls): # env ES_NODE_STORAGE_MINER= ES_NODE_SIGNER_PRIVATE_KEY= ./run.sh --l1.rpc --l1.beacon -# Note: currently only zk prover mode is supported +# Note: currently only zk prover mode 2 is supported zkey_file="./build/bin/snark_lib/zkey/blob_poseidon2.zkey" if [ -z "$ES_NODE_STORAGE_MINER" ]; then @@ -53,6 +53,7 @@ start_flags=" --network devnet \ --signer.private-key $ES_NODE_SIGNER_PRIVATE_KEY \ --miner.enabled \ --miner.zkey $zkey_file \ + --miner.zk-prover-impl 2 \ --download.thread 32 \ --state.upload.url http://metrics.ethstorage.io:8080 \ --p2p.listen.udp 30305 \ From 5d3573296e861647ad5997d4c4d998b1ac364c5d Mon Sep 17 00:00:00 2001 From: syntrust Date: Wed, 31 Jul 2024 18:54:42 +0800 Subject: [PATCH 14/27] l2 --- init-l2.sh | 136 ++++++++++++++++++++++++++++++++++++++++++++ run-l2.sh | 161 ++++++++--------------------------------------------- run.sh | 4 +- 3 files changed, 161 insertions(+), 140 deletions(-) create mode 100755 init-l2.sh diff --git a/init-l2.sh b/init-l2.sh new file mode 100755 index 00000000..a16810f5 --- /dev/null +++ b/init-l2.sh @@ -0,0 +1,136 @@ +#!/bin/bash + +# usage 1 (use go-rapisnark): +# env ES_NODE_STORAGE_MINER= ./init.sh +# usage 2 (use snarkjs, will check Node.js version and snarkjs installation): +# env ES_NODE_STORAGE_MINER= ./init.sh --miner.zk-prover-impl 1 + +if [ -z "$ES_NODE_STORAGE_MINER" ]; then + echo "Please provide 'ES_NODE_STORAGE_MINER' as an environment variable" + exit 1 +fi + +if [ ${#ES_NODE_STORAGE_MINER} -ne 42 ] || case $ES_NODE_STORAGE_MINER in 0x*) false;; *) true;; esac; then + echo "Error: ES_NODE_STORAGE_MINER should be prefixed with '0x' and have a total length of 42" + exit 1 +fi + +executable="./build/bin/es-node" +echo "========== build info ==================" +$executable --version +echo "========================================" + +# ZK prover implementation, 1: snarkjs, 2: go-rapidsnark. +zkp_impl=2 +# ZK prover mode, 1: one proof per sample, 2: one proof for multiple samples. +# Note: currently only zk prover mode 2 is supported +zkp_mode=2 +#!/bin/bash + +remaining_args="" + +while [ $# -gt 0 ]; do + if [[ $1 == --miner.zk-prover-impl ]]; then + zkp_impl=$2 + shift 2 + elif [[ $1 == --miner.zk-prover-mode ]]; then + zkp_mode=$2 + shift 2 + else + remaining_args="$remaining_args $1" + shift + fi +done + +if [ -n "$zkp_mode" ] && [ "$zkp_mode" != 1 ] && [ "$zkp_mode" != 2 ]; then + echo "Error: zk prover mode can only be 1 or 2." + exit 1 +fi + +# download zkey if not yet +zkey_name="blob_poseidon2.zkey" +zkey_size=560301223 +zkey_url="https://es-node-zkey.s3.us-west-1.amazonaws.com/blob_poseidon2_testnet1.zkey" +if [ "$zkp_mode" = 1 ]; then + zkey_name="blob_poseidon.zkey" + zkey_size=280151245 + zkey_url="https://drive.usercontent.google.com/download?id=1ZLfhYeCXMnbk6wUiBADRAn1mZ8MI_zg-&export=download&confirm=t&uuid=16ddcd58-2498-4d65-8931-934df3d0065c" +fi +zkey_file="./build/bin/snark_lib/zkey/$zkey_name" +if [ ! -e ${zkey_file} ] || [ $(wc -c < ${zkey_file}) -ne ${zkey_size} ]; then + echo "Start downloading ${zkey_file}..." + curl $zkey_url -o ${zkey_file} + if [ ! -e ${zkey_file} ]; then + echo "Error: The zkey file was not downloaded. Please try again." + exit 1 + fi + if [ $(wc -c < ${zkey_file}) -ne ${zkey_size} ]; then + echo "Error: The zkey file was not downloaded correctly. You can check the file content for more information." + exit 1 + fi +else + echo "√ ${zkey_file} already exists." +fi + +echo "zkp_impl is set to $zkp_impl" +if [ -n "$zkp_impl" ] && [ "$zkp_impl" != 1 ] && [ "$zkp_impl" != 2 ]; then + echo "Error: miner.zk-prover-impl can only be 1 or 2" + exit 1 +fi + + +if [ "$zkp_impl" = 1 ]; then + + if ! [ -x "$(command -v node)" ]; then + echo 'Error: Node.js is not installed.' + exit 1 + fi + + # check node js version + node_version=$(node -v) + major_version=$(echo $node_version | cut -d'v' -f2 | cut -d'.' -f1) + if [ "$major_version" -lt 16 ]; then + echo "Error: Node.js version is too old: $node_version; must be 16 and above." + exit 1 + else + echo "√ Node.js version is compatible." + fi + + # install snarkjs if not + if ! [ "$(command -v snarkjs)" ]; then + echo "snarkjs not found, start installing..." + snarkjs_install=$(npm install -g snarkjs 2>&1) + if [ $? -eq 0 ]; then + echo "√ snarkjs installed successfully." + else + echo "Error: snarkjs install failed with the following error:" + echo "$snarkjs_install" + exit 1 + fi + else + echo "√ snarkjs is already installed." + fi + +fi + +data_dir="./es-data" +storage_file_0="$data_dir/shard-0.dat" + +es_node_init="$executable init --shard_index 0 \ + --datadir $data_dir \ + --l1.rpc http://65.109.20.29:8545 \ + --storage.l1contract 0x64003adbdf3014f7E38FC6BE752EB047b95da89A \ + --storage.miner $ES_NODE_STORAGE_MINER \ +$remaining_args" + +# create data file for shard 0 if not yet +if [ ! -e $storage_file_0 ]; then + if $es_node_init ; then + echo "√ Initialized ${storage_file_0} successfully" + else + echo "Error: failed to initialize ${storage_file_0}" + exit 1 + fi +else + echo "Warning: storage file ${storage_file_0} already exists, skip initialization." +fi \ No newline at end of file diff --git a/run-l2.sh b/run-l2.sh index 0b8ab4e4..a958a7ee 100755 --- a/run-l2.sh +++ b/run-l2.sh @@ -1,9 +1,10 @@ #!/bin/bash # usage: -# env ES_NODE_STORAGE_MINER= ES_NODE_SIGNER_PRIVATE_KEY= ./run.sh -# for one zk proof per sample (if the storage contract supports): -# env ES_NODE_STORAGE_MINER= ES_NODE_SIGNER_PRIVATE_KEY= ./run.sh --miner.zk-prover-mode 1 --l1.rpc --l1.beacon +# env ES_NODE_STORAGE_MINER= ES_NODE_SIGNER_PRIVATE_KEY= ./run-l2.sh + +# Note: currently only zk prover mode 2 is supported +zkey_file="./build/bin/snark_lib/zkey/blob_poseidon2.zkey" if [ -z "$ES_NODE_STORAGE_MINER" ]; then echo "Please provide 'ES_NODE_STORAGE_MINER' as an environment variable" @@ -25,152 +26,36 @@ if [ ${#ES_NODE_SIGNER_PRIVATE_KEY} -ne 64 ]; then exit 1 fi -# ZK prover mode, 1: one proof per sample, 2: one proof for multiple samples. -zkp_mode= -i=1 -while [ $i -le $# ]; do - if [ "${!i}" = "--miner.zk-prover-mode" ]; then - j=$((i+1)) - zkp_mode="${!j}" - break - else - if echo "${!i}" | grep -qE -- "--miner\.zk-prover-mode=([0-9]+)"; then - zkp_mode=$(echo "${!i}" | sed -E 's/.*=([0-9]+)/\1/') - break - fi - fi - i=$((i+1)) -done - -if [ -n "$zkp_mode" ] && [ "$zkp_mode" != 1 ] && [ "$zkp_mode" != 2 ]; then - echo "Error: zk prover mode can only be 1 or 2." - exit 1 -fi +executable="./build/bin/es-node" +echo "========== build info ==================" +$executable --version +echo "========================================" -if [ -n "$zkp_mode" ]; then - echo "The zk prover mode has been overridden to $zkp_mode" -fi +data_dir="./es-data" +storage_file_0="$data_dir/shard-0.dat" -# download zkey if not yet -zkey_name="blob_poseidon2.zkey" -zkey_size=560301223 -zkey_url="https://es-node-zkey.s3.us-west-1.amazonaws.com/blob_poseidon2_testnet1.zkey" -if [ "$zkp_mode" = 1 ]; then - zkey_name="blob_poseidon.zkey" - zkey_size=280151245 - zkey_url="https://drive.usercontent.google.com/download?id=1ZLfhYeCXMnbk6wUiBADRAn1mZ8MI_zg-&export=download&confirm=t&uuid=16ddcd58-2498-4d65-8931-934df3d0065c" -fi -zkey_file="./build/bin/snark_lib/$zkey_name" -if [ ! -e ${zkey_file} ] || [ $(wc -c < ${zkey_file}) -ne ${zkey_size} ]; then - echo "Start downloading ${zkey_file}..." - curl $zkey_url -o ${zkey_file} - if [ ! -e ${zkey_file} ]; then - echo "Error: The zkey file was not downloaded. Please try again." - exit 1 - fi - if [ $(wc -c < ${zkey_file}) -ne ${zkey_size} ]; then - echo "Error: The zkey file was not downloaded correctly. You can check the file content for more information." +if [ ! -e $storage_file_0 ]; then + echo "Error: storage file not found: ${storage_file_0}. Please run 'init.sh' first." exit 1 - fi fi - -# ZK prover implementation, 1: snarkjs, 2: go-rapidsnark. -zkp_impl= -i=1 -while [ $i -le $# ]; do - if [ "${!i}" = "--miner.zk-prover-impl" ]; then - j=$((i+1)) - zkp_impl="${!j}" - break - else - if echo "${!i}" | grep -qE -- "--miner\.zk-prover-impl=([0-9]+)"; then - zkp_impl=$(echo "${!i}" | sed -E 's/.*=([0-9]+)/\1/') - break - fi - fi - i=$((i+1)) -done - - -if [ -n "$zkp_impl" ] && [ "$zkp_impl" != 1 ] && [ "$zkp_impl" != 2 ]; then - echo "miner.zk-prover-impl can only be 1 or 2" - exit 1 -fi - -if [ -n "$zkp_impl" ]; then - echo "The zk prover implementation has been overridden to $zkp_impl" -fi - -if [ "$zkp_impl" = 1 ]; then - - if ! [ -x "$(command -v node)" ]; then - echo 'Error: Node.js is not installed.' - exit 1 - fi - - # check node js version - node_version=$(node -v) - major_version=$(echo $node_version | cut -d'v' -f2 | cut -d'.' -f1) - - if [ "$major_version" -lt 16 ]; then - echo "Error: Node.js version is too old." - exit 1 - fi - - # install snarkjs if not - if ! [ "$(command -v snarkjs)" ]; then - echo "snarkjs not found, start installing..." - snarkjs_install=$(npm install -g snarkjs 2>&1) - if [ $? -eq 0 ]; then - echo "snarkjs installed successfully." - else - echo "Error: snarkjs install failed with the following error:" - echo "$snarkjs_install" - exit 1 - fi - fi - -fi - -executable="./build/bin/es-node" -data_dir="./es-data" -storage_file_0="$data_dir/shard-0.dat" - -common_flags=" --datadir $data_dir \ - --l1.rpc http://65.109.20.29:8545 \ +start_flags=" --network devnet \ + --datadir $data_dir \ --storage.l1contract 0x64003adbdf3014f7E38FC6BE752EB047b95da89A \ - --storage.miner $ES_NODE_STORAGE_MINER \ - " - -# init shard 0 -es_node_init="init --shard_index 0" - -# start node -# TODO remove --network -es_node_start=" --network devnet \ - --miner.enabled \ - --miner.zkey $zkey_name \ --storage.files $storage_file_0 \ - --signer.private-key $ES_NODE_SIGNER_PRIVATE_KEY \ + --storage.miner $ES_NODE_STORAGE_MINER \ + --l1.rpc http://65.109.20.29:8545 \ + --l1.block_time 2 \ --da.url http://65.109.20.29:8888 \ --randao.url http://88.99.30.186:8545 \ - --l1.block_time 2 \ + --signer.private-key $ES_NODE_SIGNER_PRIVATE_KEY \ + --miner.enabled \ + --miner.zkey $zkey_file \ + --miner.zk-prover-impl 2 \ --download.thread 32 \ --p2p.listen.udp 30305 \ --p2p.sync.concurrency 32 \ --p2p.bootnodes enr:-Li4QA-fcxSHHu68uzHxsGnR8Q8lnvPir8L3cb5-RSq5fvU7cmxukZinZ9N-XRcnvWauQl6KK2tnlD3RZTwOxI4KgIaGAZC-hjTfimV0aHN0b3JhZ2XbAYDY15RkADrb3zAU9-OPxr51LrBHuV2omsGAgmlkgnY0gmlwhEFtcySJc2VjcDI1NmsxoQNY8raIsHIGPniQ738UiNmIvifax5L6R51YLPoflGzix4N0Y3CCJAaDdWRwgnZh \ -$@" - -# create data file for shard 0 if not yet -if [ ! -e $storage_file_0 ]; then - if $executable $es_node_init $common_flags ; then - echo "Initialized ${storage_file_0} successfully" - else - echo "Error: failed to initialize ${storage_file_0}" - exit 1 - fi -fi + $@" -# start es-node -exec $executable $common_flags $es_node_start +exec $executable $start_flags \ No newline at end of file diff --git a/run.sh b/run.sh index 8265dddf..f0f67c88 100755 --- a/run.sh +++ b/run.sh @@ -29,9 +29,9 @@ if [ ${#ES_NODE_SIGNER_PRIVATE_KEY} -ne 64 ]; then fi executable="./build/bin/es-node" -echo "============================" +echo "========== build info ==================" $executable --version -echo "============================" +echo "========================================" data_dir="./es-data" storage_file_0="$data_dir/shard-0.dat" From cce92e11e4d874886cca8da6d7aa1ee3b97a79be Mon Sep 17 00:00:00 2001 From: syntrust Date: Wed, 31 Jul 2024 19:17:52 +0800 Subject: [PATCH 15/27] add init condition --- run-docker.sh | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/run-docker.sh b/run-docker.sh index 1b656c79..80bdbbed 100755 --- a/run-docker.sh +++ b/run-docker.sh @@ -41,12 +41,15 @@ else echo "image $image_name built" fi - sudo docker run --rm \ - -v ./es-data:/es-node/es-data \ - -v ./build/bin/snark_lib/zkey:/es-node/build/bin/snark_lib/zkey \ - -e ES_NODE_STORAGE_MINER=$ES_NODE_STORAGE_MINER \ - --entrypoint /es-node/init.sh \ - $image_name + # check if data folder exist + if [ ! -d "./es-data" ]; then + sudo docker run --rm \ + -v ./es-data:/es-node/es-data \ + -v ./build/bin/snark_lib/zkey:/es-node/build/bin/snark_lib/zkey \ + -e ES_NODE_STORAGE_MINER=$ES_NODE_STORAGE_MINER \ + --entrypoint /es-node/init.sh \ + $image_name + fi # run container in the background sudo docker run --name $container_name \ From 51440d3f956bc5f96e3a7629c9e2beafa2659dd2 Mon Sep 17 00:00:00 2001 From: syntrust Date: Tue, 6 Aug 2024 19:16:37 +0800 Subject: [PATCH 16/27] zk impl 1 --- Dockerfile | 3 ++- init-l2.sh | 2 +- init.sh | 2 +- run-docker.sh | 2 -- run-l2.sh | 3 +-- run.sh | 2 +- 6 files changed, 6 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 181184fa..fff8291b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,8 @@ RUN make # Pull ES node into a second stage deploy alpine container FROM alpine:latest COPY --from=builder /es-node/build/ /es-node/build/ -RUN apk add --no-cache bash curl libstdc++ gcompat libgomp +RUN apk add --no-cache bash curl libstdc++ gcompat libgomp nodejs npm +RUN npm install -g snarkjs # Entrypoint COPY --from=builder /es-node/init.sh /es-node/ diff --git a/init-l2.sh b/init-l2.sh index a16810f5..4ff73c40 100755 --- a/init-l2.sh +++ b/init-l2.sh @@ -21,7 +21,7 @@ $executable --version echo "========================================" # ZK prover implementation, 1: snarkjs, 2: go-rapidsnark. -zkp_impl=2 +zkp_impl=1 # ZK prover mode, 1: one proof per sample, 2: one proof for multiple samples. # Note: currently only zk prover mode 2 is supported zkp_mode=2 diff --git a/init.sh b/init.sh index bd04304e..77f4e675 100755 --- a/init.sh +++ b/init.sh @@ -21,7 +21,7 @@ $executable --version echo "========================================" # ZK prover implementation, 1: snarkjs, 2: go-rapidsnark. -zkp_impl=2 +zkp_impl=1 # ZK prover mode, 1: one proof per sample, 2: one proof for multiple samples. # Note: currently only zk prover mode 2 is supported zkp_mode=2 diff --git a/run-docker.sh b/run-docker.sh index 80bdbbed..1be7e074 100755 --- a/run-docker.sh +++ b/run-docker.sh @@ -33,7 +33,6 @@ else # start container if exist if sudo docker ps -a --format "{{.Names}}" | grep -q "^$container_name$"; then sudo docker start $container_name - echo "container $container_name started" else # build an image if not exist if ! sudo docker images --format "{{.Repository}}" | grep -q "^$image_name$"; then @@ -63,6 +62,5 @@ else -d \ --entrypoint /es-node/run.sh \ $image_name - echo "container $container_name started" fi fi diff --git a/run-l2.sh b/run-l2.sh index a958a7ee..82aab464 100755 --- a/run-l2.sh +++ b/run-l2.sh @@ -3,7 +3,6 @@ # usage: # env ES_NODE_STORAGE_MINER= ES_NODE_SIGNER_PRIVATE_KEY= ./run-l2.sh -# Note: currently only zk prover mode 2 is supported zkey_file="./build/bin/snark_lib/zkey/blob_poseidon2.zkey" if [ -z "$ES_NODE_STORAGE_MINER" ]; then @@ -51,7 +50,7 @@ start_flags=" --network devnet \ --signer.private-key $ES_NODE_SIGNER_PRIVATE_KEY \ --miner.enabled \ --miner.zkey $zkey_file \ - --miner.zk-prover-impl 2 \ + --miner.zk-prover-impl 1 \ --download.thread 32 \ --p2p.listen.udp 30305 \ --p2p.sync.concurrency 32 \ diff --git a/run.sh b/run.sh index f0f67c88..6620de48 100755 --- a/run.sh +++ b/run.sh @@ -53,7 +53,7 @@ start_flags=" --network devnet \ --signer.private-key $ES_NODE_SIGNER_PRIVATE_KEY \ --miner.enabled \ --miner.zkey $zkey_file \ - --miner.zk-prover-impl 2 \ + --miner.zk-prover-impl 1 \ --download.thread 32 \ --state.upload.url http://metrics.ethstorage.io:8080 \ --p2p.listen.udp 30305 \ From 5b4c4f7634733e9ce90f28b7889f1c3aca823f44 Mon Sep 17 00:00:00 2001 From: syntrust Date: Wed, 7 Aug 2024 10:23:36 +0800 Subject: [PATCH 17/27] minor --- ethstorage/miner/cli.go | 6 +++--- init-l2.sh | 1 - init.sh | 1 - 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/ethstorage/miner/cli.go b/ethstorage/miner/cli.go index 684ed257..89d62611 100644 --- a/ethstorage/miner/cli.go +++ b/ethstorage/miner/cli.go @@ -92,7 +92,7 @@ type CLIConfig struct { GasPrice *big.Int PriorityGasPrice *big.Int MinimumProfit *big.Int - ZKeyFileName string + ZKeyFile string ZKWorkingDir string ZKProverMode uint64 ZKProverImpl uint64 @@ -118,7 +118,7 @@ func (c CLIConfig) ToMinerConfig() (Config, error) { } zkWorkingDir = dir } - zkFile := c.ZKeyFileName + zkFile := c.ZKeyFile if !filepath.IsAbs(zkFile) { dir, err := filepath.Abs(zkFile) if err != nil { @@ -144,7 +144,7 @@ func ReadCLIConfig(ctx *cli.Context) CLIConfig { GasPrice: types.GlobalBig(ctx, GasPriceFlagName), PriorityGasPrice: types.GlobalBig(ctx, PriorityGasPriceFlagName), MinimumProfit: types.GlobalBig(ctx, MinimumProfitFlagName), - ZKeyFileName: ctx.GlobalString(ZKeyFileNameFlagName), + ZKeyFile: ctx.GlobalString(ZKeyFileNameFlagName), ZKWorkingDir: ctx.GlobalString(ZKWorkingDirFlagName), ZKProverMode: ctx.GlobalUint64(ZKProverModeFlagName), ZKProverImpl: ctx.GlobalUint64(ZKProverImplFlagName), diff --git a/init-l2.sh b/init-l2.sh index 4ff73c40..7a33939d 100755 --- a/init-l2.sh +++ b/init-l2.sh @@ -72,7 +72,6 @@ else echo "√ ${zkey_file} already exists." fi -echo "zkp_impl is set to $zkp_impl" if [ -n "$zkp_impl" ] && [ "$zkp_impl" != 1 ] && [ "$zkp_impl" != 2 ]; then echo "Error: miner.zk-prover-impl can only be 1 or 2" exit 1 diff --git a/init.sh b/init.sh index 77f4e675..efa413fd 100755 --- a/init.sh +++ b/init.sh @@ -72,7 +72,6 @@ else echo "√ ${zkey_file} already exists." fi -echo "zkp_impl is set to $zkp_impl" if [ -n "$zkp_impl" ] && [ "$zkp_impl" != 1 ] && [ "$zkp_impl" != 2 ]; then echo "Error: miner.zk-prover-impl can only be 1 or 2" exit 1 From 5518f2b60c343797a5697f53d9f0b027da4d970e Mon Sep 17 00:00:00 2001 From: syntrust Date: Wed, 7 Aug 2024 17:27:59 +0800 Subject: [PATCH 18/27] check miner address --- cmd/es-node/config.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cmd/es-node/config.go b/cmd/es-node/config.go index 9781b8b2..005e31c1 100644 --- a/cmd/es-node/config.go +++ b/cmd/es-node/config.go @@ -67,7 +67,7 @@ func NewConfig(ctx *cli.Context, log log.Logger) (*node.Config, error) { } dlConfig := NewDownloaderConfig(ctx) - minerConfig, err := NewMinerConfig(ctx, client, storageConfig.L1Contract) + minerConfig, err := NewMinerConfig(ctx, client, storageConfig.L1Contract, storageConfig.Miner) if err != nil { return nil, fmt.Errorf("failed to load miner config: %w", err) } @@ -124,11 +124,14 @@ func NewConfig(ctx *cli.Context, log log.Logger) (*node.Config, error) { return cfg, nil } -func NewMinerConfig(ctx *cli.Context, client *ethclient.Client, l1Contract common.Address) (*miner.Config, error) { +func NewMinerConfig(ctx *cli.Context, client *ethclient.Client, l1Contract, minerAddr common.Address) (*miner.Config, error) { cliConfig := miner.ReadCLIConfig(ctx) if !cliConfig.Enabled { return nil, nil } + if minerAddr == (common.Address{}) { + return nil, fmt.Errorf("miner address cannot be empty") + } log.Debug("Read mining config from cli", "config", fmt.Sprintf("%+v", cliConfig)) err := cliConfig.Check() if err != nil { @@ -243,6 +246,7 @@ func NewRollupConfig(ctx *cli.Context) (*rollup.EsConfig, error) { func NewStorageConfig(ctx *cli.Context, client *ethclient.Client) (*storage.StorageConfig, error) { l1Contract := common.HexToAddress(ctx.GlobalString(flags.StorageL1Contract.Name)) miner := common.HexToAddress(ctx.GlobalString(flags.StorageMiner.Name)) + log.Info("Loaded storage config", "l1Contract", l1Contract, "miner", miner) storageCfg, err := initStorageConfig(context.Background(), client, l1Contract, miner) if err != nil { log.Error("Failed to load storage config from contract", "error", err) From 3e3bc641251a3eadce729330d56e2b99d82399a1 Mon Sep 17 00:00:00 2001 From: syntrust Date: Wed, 7 Aug 2024 19:21:52 +0800 Subject: [PATCH 19/27] simplify scritps --- cmd/es-node/main.go | 8 +-- cmd/es-node/utils.go | 5 +- init-l2-rpc.sh | 3 + init-l2.sh | 133 ++----------------------------------------- init-rpc.sh | 3 + init.sh | 15 +---- 6 files changed, 18 insertions(+), 149 deletions(-) create mode 100755 init-l2-rpc.sh create mode 100755 init-rpc.sh diff --git a/cmd/es-node/main.go b/cmd/es-node/main.go index 101d09ed..05f0fec3 100644 --- a/cmd/es-node/main.go +++ b/cmd/es-node/main.go @@ -181,13 +181,13 @@ func EsNodeInit(ctx *cli.Context) error { } log := eslog.NewLogger(logCfg) log.Info("Will create data files for storage node") - l1Rpc := readRequiredFlag(ctx, flags.L1NodeAddr.Name) - contract := readRequiredFlag(ctx, flags.StorageL1Contract.Name) + l1Rpc := readRequiredFlag(ctx, flags.L1NodeAddr) + contract := readRequiredFlag(ctx, flags.StorageL1Contract) if !common.IsHexAddress(contract) { return fmt.Errorf("invalid contract address %s", contract) } - datadir := readRequiredFlag(ctx, flags.DataDir.Name) + datadir := readRequiredFlag(ctx, flags.DataDir) encodingType := ethstorage.ENCODE_BLOB_POSEIDON miner := "0x" if ctx.IsSet(encodingTypeFlagName) { @@ -198,7 +198,7 @@ func EsNodeInit(ctx *cli.Context) error { } } if encodingType != ethstorage.NO_ENCODE { - miner = readRequiredFlag(ctx, flags.StorageMiner.Name) + miner = readRequiredFlag(ctx, flags.StorageMiner) if !common.IsHexAddress(miner) { return fmt.Errorf("invalid miner address %s", miner) } diff --git a/cmd/es-node/utils.go b/cmd/es-node/utils.go index e47c29e7..43ff06e9 100644 --- a/cmd/es-node/utils.go +++ b/cmd/es-node/utils.go @@ -190,9 +190,10 @@ func sortBigIntSlice(slice []*big.Int) []int { return indices } -func readRequiredFlag(ctx *cli.Context, name string) string { +func readRequiredFlag(ctx *cli.Context, flag cli.StringFlag) string { + name := flag.GetName() if !ctx.IsSet(name) { - log.Crit("Flag is required", "flag", name) + log.Crit("Flag or environment variable is required", "flag", name, "envVar", flag.EnvVar) } value := ctx.String(name) log.Info("Read flag", "name", name, "value", value) diff --git a/init-l2-rpc.sh b/init-l2-rpc.sh new file mode 100755 index 00000000..0c5bb92f --- /dev/null +++ b/init-l2-rpc.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +./init-l2.sh --encoding_type=0 \ No newline at end of file diff --git a/init-l2.sh b/init-l2.sh index 7a33939d..9aa41b42 100755 --- a/init-l2.sh +++ b/init-l2.sh @@ -1,135 +1,10 @@ #!/bin/bash -# usage 1 (use go-rapisnark): -# env ES_NODE_STORAGE_MINER= ./init.sh -# usage 2 (use snarkjs, will check Node.js version and snarkjs installation): -# env ES_NODE_STORAGE_MINER= ./init.sh --miner.zk-prover-impl 1 +# usage: +# env ES_NODE_STORAGE_MINER= ./init-l2.sh -if [ -z "$ES_NODE_STORAGE_MINER" ]; then - echo "Please provide 'ES_NODE_STORAGE_MINER' as an environment variable" - exit 1 -fi -if [ ${#ES_NODE_STORAGE_MINER} -ne 42 ] || case $ES_NODE_STORAGE_MINER in 0x*) false;; *) true;; esac; then - echo "Error: ES_NODE_STORAGE_MINER should be prefixed with '0x' and have a total length of 42" - exit 1 -fi - -executable="./build/bin/es-node" -echo "========== build info ==================" -$executable --version -echo "========================================" - -# ZK prover implementation, 1: snarkjs, 2: go-rapidsnark. -zkp_impl=1 -# ZK prover mode, 1: one proof per sample, 2: one proof for multiple samples. -# Note: currently only zk prover mode 2 is supported -zkp_mode=2 -#!/bin/bash - -remaining_args="" - -while [ $# -gt 0 ]; do - if [[ $1 == --miner.zk-prover-impl ]]; then - zkp_impl=$2 - shift 2 - elif [[ $1 == --miner.zk-prover-mode ]]; then - zkp_mode=$2 - shift 2 - else - remaining_args="$remaining_args $1" - shift - fi -done - -if [ -n "$zkp_mode" ] && [ "$zkp_mode" != 1 ] && [ "$zkp_mode" != 2 ]; then - echo "Error: zk prover mode can only be 1 or 2." - exit 1 -fi - -# download zkey if not yet -zkey_name="blob_poseidon2.zkey" -zkey_size=560301223 -zkey_url="https://es-node-zkey.s3.us-west-1.amazonaws.com/blob_poseidon2_testnet1.zkey" -if [ "$zkp_mode" = 1 ]; then - zkey_name="blob_poseidon.zkey" - zkey_size=280151245 - zkey_url="https://drive.usercontent.google.com/download?id=1ZLfhYeCXMnbk6wUiBADRAn1mZ8MI_zg-&export=download&confirm=t&uuid=16ddcd58-2498-4d65-8931-934df3d0065c" -fi -zkey_file="./build/bin/snark_lib/zkey/$zkey_name" -if [ ! -e ${zkey_file} ] || [ $(wc -c < ${zkey_file}) -ne ${zkey_size} ]; then - echo "Start downloading ${zkey_file}..." - curl $zkey_url -o ${zkey_file} - if [ ! -e ${zkey_file} ]; then - echo "Error: The zkey file was not downloaded. Please try again." - exit 1 - fi - if [ $(wc -c < ${zkey_file}) -ne ${zkey_size} ]; then - echo "Error: The zkey file was not downloaded correctly. You can check the file content for more information." - exit 1 - fi -else - echo "√ ${zkey_file} already exists." -fi - -if [ -n "$zkp_impl" ] && [ "$zkp_impl" != 1 ] && [ "$zkp_impl" != 2 ]; then - echo "Error: miner.zk-prover-impl can only be 1 or 2" - exit 1 -fi - - -if [ "$zkp_impl" = 1 ]; then - - if ! [ -x "$(command -v node)" ]; then - echo 'Error: Node.js is not installed.' - exit 1 - fi - - # check node js version - node_version=$(node -v) - major_version=$(echo $node_version | cut -d'v' -f2 | cut -d'.' -f1) - if [ "$major_version" -lt 16 ]; then - echo "Error: Node.js version is too old: $node_version; must be 16 and above." - exit 1 - else - echo "√ Node.js version is compatible." - fi - - # install snarkjs if not - if ! [ "$(command -v snarkjs)" ]; then - echo "snarkjs not found, start installing..." - snarkjs_install=$(npm install -g snarkjs 2>&1) - if [ $? -eq 0 ]; then - echo "√ snarkjs installed successfully." - else - echo "Error: snarkjs install failed with the following error:" - echo "$snarkjs_install" - exit 1 - fi - else - echo "√ snarkjs is already installed." - fi - -fi - -data_dir="./es-data" -storage_file_0="$data_dir/shard-0.dat" - -es_node_init="$executable init --shard_index 0 \ - --datadir $data_dir \ +./init.sh \ --l1.rpc http://65.109.20.29:8545 \ --storage.l1contract 0x64003adbdf3014f7E38FC6BE752EB047b95da89A \ - --storage.miner $ES_NODE_STORAGE_MINER \ -$remaining_args" - -# create data file for shard 0 if not yet -if [ ! -e $storage_file_0 ]; then - if $es_node_init ; then - echo "√ Initialized ${storage_file_0} successfully" - else - echo "Error: failed to initialize ${storage_file_0}" - exit 1 - fi -else - echo "Warning: storage file ${storage_file_0} already exists, skip initialization." -fi \ No newline at end of file +$@ \ No newline at end of file diff --git a/init-rpc.sh b/init-rpc.sh new file mode 100755 index 00000000..44e56bd1 --- /dev/null +++ b/init-rpc.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +./init.sh --encoding_type=0 \ No newline at end of file diff --git a/init.sh b/init.sh index efa413fd..717a7b45 100755 --- a/init.sh +++ b/init.sh @@ -1,19 +1,7 @@ #!/bin/bash -# usage 1 (use go-rapisnark): +# usage: # env ES_NODE_STORAGE_MINER= ./init.sh -# usage 2 (use snarkjs, will check Node.js version and snarkjs installation): -# env ES_NODE_STORAGE_MINER= ./init.sh --miner.zk-prover-impl 1 - -if [ -z "$ES_NODE_STORAGE_MINER" ]; then - echo "Please provide 'ES_NODE_STORAGE_MINER' as an environment variable" - exit 1 -fi - -if [ ${#ES_NODE_STORAGE_MINER} -ne 42 ] || case $ES_NODE_STORAGE_MINER in 0x*) false;; *) true;; esac; then - echo "Error: ES_NODE_STORAGE_MINER should be prefixed with '0x' and have a total length of 42" - exit 1 -fi executable="./build/bin/es-node" echo "========== build info ==================" @@ -119,7 +107,6 @@ es_node_init="$executable init --shard_index 0 \ --datadir $data_dir \ --l1.rpc http://88.99.30.186:8545 \ --storage.l1contract 0x804C520d3c084C805E37A35E90057Ac32831F96f \ - --storage.miner $ES_NODE_STORAGE_MINER \ $remaining_args" # create data file for shard 0 if not yet From a32e7b32a1cd51f89113fa8a24629ef499c07032 Mon Sep 17 00:00:00 2001 From: syntrust Date: Thu, 8 Aug 2024 10:16:37 +0800 Subject: [PATCH 20/27] simplify run.sh --- cmd/es-node/config.go | 1 + run-l2-rpc.sh | 34 ++-------------------------------- run-l2.sh | 27 --------------------------- run-rpc.sh | 36 +++--------------------------------- run.sh | 28 ---------------------------- 5 files changed, 6 insertions(+), 120 deletions(-) diff --git a/cmd/es-node/config.go b/cmd/es-node/config.go index 005e31c1..55068c65 100644 --- a/cmd/es-node/config.go +++ b/cmd/es-node/config.go @@ -127,6 +127,7 @@ func NewConfig(ctx *cli.Context, log log.Logger) (*node.Config, error) { func NewMinerConfig(ctx *cli.Context, client *ethclient.Client, l1Contract, minerAddr common.Address) (*miner.Config, error) { cliConfig := miner.ReadCLIConfig(ctx) if !cliConfig.Enabled { + log.Info("Miner is not enabled.") return nil, nil } if minerAddr == (common.Address{}) { diff --git a/run-l2-rpc.sh b/run-l2-rpc.sh index f8f94b3c..cb827d57 100755 --- a/run-l2-rpc.sh +++ b/run-l2-rpc.sh @@ -1,36 +1,6 @@ #!/bin/sh -# usage: -# ./run-l2-rpc.sh - -executable="./build/bin/es-node" -data_dir="./es-data" -storage_file_0="$data_dir/shard-0.dat" - -common_flags=" --datadir $data_dir \ - --l1.rpc http://65.109.20.29:8545 \ - --storage.l1contract 0x64003adbdf3014f7E38FC6BE752EB047b95da89A \ -" -# init shard 0 -es_node_init="init --shard_index 0" - -# start node -# TODO remove --network -es_node_start=" --network devnet \ - --storage.files $storage_file_0 \ - --da.url http://65.109.20.29:8888 \ - --l1.block_time 2 \ - --download.thread 32 \ +./run-l2.sh \ --rpc.addr 0.0.0.0 \ - --p2p.listen.udp 30305 \ - --p2p.sync.concurrency 32 \ + --miner.enabled=false \ --p2p.bootnodes enr:-Li4QA-fcxSHHu68uzHxsGnR8Q8lnvPir8L3cb5-RSq5fvU7cmxukZinZ9N-XRcnvWauQl6KK2tnlD3RZTwOxI4KgIaGAZC-hjTfimV0aHN0b3JhZ2XbAYDY15RkADrb3zAU9-OPxr51LrBHuV2omsGAgmlkgnY0gmlwhEFtcySJc2VjcDI1NmsxoQNY8raIsHIGPniQ738UiNmIvifax5L6R51YLPoflGzix4N0Y3CCJAaDdWRwgnZh \ -" -# create data file for shard 0 if not yet -if [ ! -e $storage_file_0 ]; then - $executable $es_node_init $common_flags --encoding_type 0 - echo "initialized ${storage_file_0}" -fi - -# start es-node -exec $executable $es_node_start $common_flags diff --git a/run-l2.sh b/run-l2.sh index 82aab464..a11b13dc 100755 --- a/run-l2.sh +++ b/run-l2.sh @@ -5,26 +5,6 @@ zkey_file="./build/bin/snark_lib/zkey/blob_poseidon2.zkey" -if [ -z "$ES_NODE_STORAGE_MINER" ]; then - echo "Please provide 'ES_NODE_STORAGE_MINER' as an environment variable" - exit 1 -fi - -if [ ${#ES_NODE_STORAGE_MINER} -ne 42 ] || case $ES_NODE_STORAGE_MINER in 0x*) false;; *) true;; esac; then - echo "Error: ES_NODE_STORAGE_MINER should be prefixed with '0x' and have a total length of 42" - exit 1 -fi - -if [ -z "$ES_NODE_SIGNER_PRIVATE_KEY" ]; then - echo "Please provide 'ES_NODE_SIGNER_PRIVATE_KEY' as an environment variable" - exit 1 -fi - -if [ ${#ES_NODE_SIGNER_PRIVATE_KEY} -ne 64 ]; then - echo "Error: ES_NODE_SIGNER_PRIVATE_KEY should have a length of 64" - exit 1 -fi - executable="./build/bin/es-node" echo "========== build info ==================" $executable --version @@ -33,21 +13,14 @@ echo "========================================" data_dir="./es-data" storage_file_0="$data_dir/shard-0.dat" -if [ ! -e $storage_file_0 ]; then - echo "Error: storage file not found: ${storage_file_0}. Please run 'init.sh' first." - exit 1 -fi - start_flags=" --network devnet \ --datadir $data_dir \ --storage.l1contract 0x64003adbdf3014f7E38FC6BE752EB047b95da89A \ --storage.files $storage_file_0 \ - --storage.miner $ES_NODE_STORAGE_MINER \ --l1.rpc http://65.109.20.29:8545 \ --l1.block_time 2 \ --da.url http://65.109.20.29:8888 \ --randao.url http://88.99.30.186:8545 \ - --signer.private-key $ES_NODE_SIGNER_PRIVATE_KEY \ --miner.enabled \ --miner.zkey $zkey_file \ --miner.zk-prover-impl 1 \ diff --git a/run-rpc.sh b/run-rpc.sh index 2b92f882..c28508a2 100755 --- a/run-rpc.sh +++ b/run-rpc.sh @@ -1,36 +1,6 @@ #!/bin/sh -# usage: -# ./run_rpc.sh - -executable="./build/bin/es-node" -data_dir="./es-data" -storage_file_0="$data_dir/shard-0.dat" - -common_flags=" --datadir $data_dir \ - --l1.rpc http://88.99.30.186:8545 \ - --storage.l1contract 0x804C520d3c084C805E37A35E90057Ac32831F96f \ -" -# init shard 0 -es_node_init="init --shard_index 0" - -# start node -# TODO remove --network -es_node_start=" --network devnet \ - --storage.files $storage_file_0 \ - --l1.beacon http://88.99.30.186:3500 \ - --l1.beacon-based-time 1706684472 \ - --l1.beacon-based-slot 4245906 \ +./run.sh \ --rpc.addr 0.0.0.0 \ - --p2p.listen.udp 30305 \ - --p2p.sync.concurrency 32 \ - --p2p.bootnodes enr:-Li4QFpDtIlnf02Bli8jnZEkVAFyWkOOtaUZL7yKp3ySKmhGNiqRSe4AuUcFip3F4o_YLh30HJUg2UlcmIxx5W-fsK2GAY1eoPcdimV0aHN0b3JhZ2XbAYDY15SATFINPAhMgF43o16QBXrDKDH5b8GAgmlkgnY0gmlwhEFtMpGJc2VjcDI1NmsxoQL0mXwUXANkLHIAjN23dPfnOOhu-jhFUN13jcjHWeIP04N0Y3CCJAaDdWRwgnZh \ -" -# create data file for shard 0 if not yet -if [ ! -e $storage_file_0 ]; then - $executable $es_node_init $common_flags --encoding_type 0 - echo "initialized ${storage_file_0}" -fi - -# start es-node -exec $executable $es_node_start $common_flags + --miner.enabled=false \ + --p2p.bootnodes enr:-Li4QFpDtIlnf02Bli8jnZEkVAFyWkOOtaUZL7yKp3ySKmhGNiqRSe4AuUcFip3F4o_YLh30HJUg2UlcmIxx5W-fsK2GAY1eoPcdimV0aHN0b3JhZ2XbAYDY15SATFINPAhMgF43o16QBXrDKDH5b8GAgmlkgnY0gmlwhEFtMpGJc2VjcDI1NmsxoQL0mXwUXANkLHIAjN23dPfnOOhu-jhFUN13jcjHWeIP04N0Y3CCJAaDdWRwgnZh diff --git a/run.sh b/run.sh index 6620de48..dfd14074 100755 --- a/run.sh +++ b/run.sh @@ -5,29 +5,8 @@ # usage 2 (overriding rpc urls): # env ES_NODE_STORAGE_MINER= ES_NODE_SIGNER_PRIVATE_KEY= ./run.sh --l1.rpc --l1.beacon -# Note: currently only zk prover mode 2 is supported zkey_file="./build/bin/snark_lib/zkey/blob_poseidon2.zkey" -if [ -z "$ES_NODE_STORAGE_MINER" ]; then - echo "Please provide 'ES_NODE_STORAGE_MINER' as an environment variable" - exit 1 -fi - -if [ ${#ES_NODE_STORAGE_MINER} -ne 42 ] || case $ES_NODE_STORAGE_MINER in 0x*) false;; *) true;; esac; then - echo "Error: ES_NODE_STORAGE_MINER should be prefixed with '0x' and have a total length of 42" - exit 1 -fi - -if [ -z "$ES_NODE_SIGNER_PRIVATE_KEY" ]; then - echo "Please provide 'ES_NODE_SIGNER_PRIVATE_KEY' as an environment variable" - exit 1 -fi - -if [ ${#ES_NODE_SIGNER_PRIVATE_KEY} -ne 64 ]; then - echo "Error: ES_NODE_SIGNER_PRIVATE_KEY should have a length of 64" - exit 1 -fi - executable="./build/bin/es-node" echo "========== build info ==================" $executable --version @@ -36,21 +15,14 @@ echo "========================================" data_dir="./es-data" storage_file_0="$data_dir/shard-0.dat" -if [ ! -e $storage_file_0 ]; then - echo "Error: storage file not found: ${storage_file_0}. Please run 'init.sh' first." - exit 1 -fi - start_flags=" --network devnet \ --datadir $data_dir \ --storage.l1contract 0x804C520d3c084C805E37A35E90057Ac32831F96f \ --storage.files $storage_file_0 \ - --storage.miner $ES_NODE_STORAGE_MINER \ --l1.rpc http://88.99.30.186:8545 \ --l1.beacon http://88.99.30.186:3500 \ --l1.beacon-based-time 1706684472 \ --l1.beacon-based-slot 4245906 \ - --signer.private-key $ES_NODE_SIGNER_PRIVATE_KEY \ --miner.enabled \ --miner.zkey $zkey_file \ --miner.zk-prover-impl 1 \ From a3dde17dce8e62fa25756558395e88da79874f5c Mon Sep 17 00:00:00 2001 From: syntrust Date: Thu, 8 Aug 2024 10:59:52 +0800 Subject: [PATCH 21/27] minor --- run-l2-rpc.sh | 3 +-- run-rpc.sh | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/run-l2-rpc.sh b/run-l2-rpc.sh index cb827d57..0aee92ac 100755 --- a/run-l2-rpc.sh +++ b/run-l2-rpc.sh @@ -2,5 +2,4 @@ ./run-l2.sh \ --rpc.addr 0.0.0.0 \ - --miner.enabled=false \ - --p2p.bootnodes enr:-Li4QA-fcxSHHu68uzHxsGnR8Q8lnvPir8L3cb5-RSq5fvU7cmxukZinZ9N-XRcnvWauQl6KK2tnlD3RZTwOxI4KgIaGAZC-hjTfimV0aHN0b3JhZ2XbAYDY15RkADrb3zAU9-OPxr51LrBHuV2omsGAgmlkgnY0gmlwhEFtcySJc2VjcDI1NmsxoQNY8raIsHIGPniQ738UiNmIvifax5L6R51YLPoflGzix4N0Y3CCJAaDdWRwgnZh \ + --miner.enabled=false \ No newline at end of file diff --git a/run-rpc.sh b/run-rpc.sh index c28508a2..7deb74f2 100755 --- a/run-rpc.sh +++ b/run-rpc.sh @@ -2,5 +2,4 @@ ./run.sh \ --rpc.addr 0.0.0.0 \ - --miner.enabled=false \ - --p2p.bootnodes enr:-Li4QFpDtIlnf02Bli8jnZEkVAFyWkOOtaUZL7yKp3ySKmhGNiqRSe4AuUcFip3F4o_YLh30HJUg2UlcmIxx5W-fsK2GAY1eoPcdimV0aHN0b3JhZ2XbAYDY15SATFINPAhMgF43o16QBXrDKDH5b8GAgmlkgnY0gmlwhEFtMpGJc2VjcDI1NmsxoQL0mXwUXANkLHIAjN23dPfnOOhu-jhFUN13jcjHWeIP04N0Y3CCJAaDdWRwgnZh + --miner.enabled=false \ No newline at end of file From 60c0bd4ab3212cc36eeb6fdf3405a6f3eade796b Mon Sep 17 00:00:00 2001 From: syntrust Date: Thu, 8 Aug 2024 11:31:41 +0800 Subject: [PATCH 22/27] simplify --- run-docker.sh | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/run-docker.sh b/run-docker.sh index 1be7e074..14ff8a8c 100755 --- a/run-docker.sh +++ b/run-docker.sh @@ -3,26 +3,6 @@ # usage: # env ES_NODE_STORAGE_MINER= ES_NODE_SIGNER_PRIVATE_KEY= ./run-docker.sh -if [ -z "$ES_NODE_STORAGE_MINER" ]; then - echo "Please provide 'ES_NODE_STORAGE_MINER' as an environment variable" - exit 1 -fi - -if [ ${#ES_NODE_STORAGE_MINER} -ne 42 ] || case $ES_NODE_STORAGE_MINER in 0x*) false;; *) true;; esac; then - echo "Error: ES_NODE_STORAGE_MINER should be prefixed with '0x' and have a total length of 42" - exit 1 -fi - -if [ -z "$ES_NODE_SIGNER_PRIVATE_KEY" ]; then - echo "Please provide 'ES_NODE_SIGNER_PRIVATE_KEY' as an environment variable" - exit 1 -fi - -if [ ${#ES_NODE_SIGNER_PRIVATE_KEY} -ne 64 ]; then - echo "Error: ES_NODE_SIGNER_PRIVATE_KEY should have a length of 64" - exit 1 -fi - container_name="es" image_name="es-node" From 2240219c43215f911c91c34bda05c5a74c697c61 Mon Sep 17 00:00:00 2001 From: syntrust Date: Tue, 13 Aug 2024 12:27:13 +0800 Subject: [PATCH 23/27] support run-l2 options override run.sh --- ethstorage/node/node.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ethstorage/node/node.go b/ethstorage/node/node.go index 51bafe07..ff4e8c68 100644 --- a/ethstorage/node/node.go +++ b/ethstorage/node/node.go @@ -160,10 +160,12 @@ func (n *EsNode) initL1(ctx context.Context, cfg *Config) error { } n.l1Source = client - if cfg.L1.L1BeaconURL != "" { - n.l1Beacon = eth.NewBeaconClient(cfg.L1.L1BeaconURL, cfg.L1.L1BeaconBasedTime, cfg.L1.L1BeaconBasedSlot, cfg.L1.L1BeaconSlotTime) - } else if cfg.L1.DAURL != "" { + if cfg.L1.DAURL != "" { n.daClient = eth.NewDAClient(cfg.L1.DAURL) + n.log.Info("Using DA URL", "url", cfg.L1.DAURL) + } else if cfg.L1.L1BeaconURL != "" { + n.l1Beacon = eth.NewBeaconClient(cfg.L1.L1BeaconURL, cfg.L1.L1BeaconBasedTime, cfg.L1.L1BeaconBasedSlot, cfg.L1.L1BeaconSlotTime) + n.log.Info("Using L1 Beacon URL", "url", cfg.L1.L1BeaconURL) } else { return fmt.Errorf("no L1 beacon or DA URL provided") } @@ -173,6 +175,7 @@ func (n *EsNode) initL1(ctx context.Context, cfg *Config) error { return fmt.Errorf("failed to create randao source: %w", err) } n.randaoSource = rc + n.log.Info("Using randao source", "url", cfg.RandaoSourceURL) } return nil } From 3d62b1088293935e734467f50af04cb2e1f899ae Mon Sep 17 00:00:00 2001 From: syntrust Date: Tue, 13 Aug 2024 12:27:32 +0800 Subject: [PATCH 24/27] support run-l2 options override run.sh --- run-l2.sh | 27 ++------------------------- 1 file changed, 2 insertions(+), 25 deletions(-) diff --git a/run-l2.sh b/run-l2.sh index a11b13dc..fa04e87f 100755 --- a/run-l2.sh +++ b/run-l2.sh @@ -1,33 +1,10 @@ #!/bin/bash -# usage: -# env ES_NODE_STORAGE_MINER= ES_NODE_SIGNER_PRIVATE_KEY= ./run-l2.sh - -zkey_file="./build/bin/snark_lib/zkey/blob_poseidon2.zkey" - -executable="./build/bin/es-node" -echo "========== build info ==================" -$executable --version -echo "========================================" - -data_dir="./es-data" -storage_file_0="$data_dir/shard-0.dat" - -start_flags=" --network devnet \ - --datadir $data_dir \ +./run.sh \ --storage.l1contract 0x64003adbdf3014f7E38FC6BE752EB047b95da89A \ - --storage.files $storage_file_0 \ --l1.rpc http://65.109.20.29:8545 \ --l1.block_time 2 \ --da.url http://65.109.20.29:8888 \ --randao.url http://88.99.30.186:8545 \ - --miner.enabled \ - --miner.zkey $zkey_file \ - --miner.zk-prover-impl 1 \ - --download.thread 32 \ - --p2p.listen.udp 30305 \ - --p2p.sync.concurrency 32 \ --p2p.bootnodes enr:-Li4QA-fcxSHHu68uzHxsGnR8Q8lnvPir8L3cb5-RSq5fvU7cmxukZinZ9N-XRcnvWauQl6KK2tnlD3RZTwOxI4KgIaGAZC-hjTfimV0aHN0b3JhZ2XbAYDY15RkADrb3zAU9-OPxr51LrBHuV2omsGAgmlkgnY0gmlwhEFtcySJc2VjcDI1NmsxoQNY8raIsHIGPniQ738UiNmIvifax5L6R51YLPoflGzix4N0Y3CCJAaDdWRwgnZh \ - $@" - -exec $executable $start_flags \ No newline at end of file + $@ \ No newline at end of file From 0e66d5584f3a08dabe42970b91f89e5b0c699dc1 Mon Sep 17 00:00:00 2001 From: syntrust Date: Tue, 13 Aug 2024 12:28:13 +0800 Subject: [PATCH 25/27] minor --- init.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/init.sh b/init.sh index 717a7b45..d38f267e 100755 --- a/init.sh +++ b/init.sh @@ -13,7 +13,6 @@ zkp_impl=1 # ZK prover mode, 1: one proof per sample, 2: one proof for multiple samples. # Note: currently only zk prover mode 2 is supported zkp_mode=2 -#!/bin/bash remaining_args="" From e70ef18761584f1c3fe48e7e6774d55e2974e059 Mon Sep 17 00:00:00 2001 From: syntrust Date: Tue, 13 Aug 2024 12:28:31 +0800 Subject: [PATCH 26/27] docs --- run.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/run.sh b/run.sh index dfd14074..b77ace1c 100755 --- a/run.sh +++ b/run.sh @@ -1,10 +1,14 @@ #!/bin/bash -# usage 1: +# usage example 1: # env ES_NODE_STORAGE_MINER= ES_NODE_SIGNER_PRIVATE_KEY= ./run.sh -# usage 2 (overriding rpc urls): +# usage example 2 (overriding rpc urls): # env ES_NODE_STORAGE_MINER= ES_NODE_SIGNER_PRIVATE_KEY= ./run.sh --l1.rpc --l1.beacon +# usage example 3 (overriding zk options, make sure to use the same configuration when running both init.sh and run.sh): +# env ES_NODE_STORAGE_MINER= ES_NODE_SIGNER_PRIVATE_KEY= ./run.sh --miner.zk-prover-impl 2 --miner.zk-prover-mode 1 +# The following is the default zkey file path downloaded by `init.sh`, which is compatible with zk mode 2. +# You can override the zkey file by using the `--miner.zkey` flag. Just ensure that the provided zkey file is compatible with the zkey mode. zkey_file="./build/bin/snark_lib/zkey/blob_poseidon2.zkey" executable="./build/bin/es-node" @@ -25,7 +29,6 @@ start_flags=" --network devnet \ --l1.beacon-based-slot 4245906 \ --miner.enabled \ --miner.zkey $zkey_file \ - --miner.zk-prover-impl 1 \ --download.thread 32 \ --state.upload.url http://metrics.ethstorage.io:8080 \ --p2p.listen.udp 30305 \ From a6c7f99820ca78fc42e445722d7b67d51195da98 Mon Sep 17 00:00:00 2001 From: syntrust Date: Tue, 13 Aug 2024 17:56:19 +0800 Subject: [PATCH 27/27] no need --miner.zk-working-dir --- ethstorage/miner/cli.go | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/ethstorage/miner/cli.go b/ethstorage/miner/cli.go index 89d62611..ad739fae 100644 --- a/ethstorage/miner/cli.go +++ b/ethstorage/miner/cli.go @@ -20,7 +20,6 @@ const ( GasPriceFlagName = "miner.gas-price" PriorityGasPriceFlagName = "miner.priority-gas-price" ZKeyFileNameFlagName = "miner.zkey" - ZKWorkingDirFlagName = "miner.zk-working-dir" ZKProverModeFlagName = "miner.zk-prover-mode" ZKProverImplFlagName = "miner.zk-prover-impl" ThreadsPerShardFlagName = "miner.threads-per-shard" @@ -59,12 +58,6 @@ func CLIFlags(envPrefix string) []cli.Flag { Value: DefaultConfig.ZKeyFile, EnvVar: rollup.PrefixEnvVar(envPrefix, "ZKEY_FILE"), }, - cli.StringFlag{ - Name: ZKWorkingDirFlagName, - Usage: "Path to the snark library folder", - Value: DefaultConfig.ZKWorkingDir, - EnvVar: rollup.PrefixEnvVar(envPrefix, "ZK_WORKING_DIR"), - }, cli.Uint64Flag{ Name: ZKProverModeFlagName, Usage: "ZK prover mode, 1: one proof per sample, 2: one proof for multiple samples", @@ -145,7 +138,7 @@ func ReadCLIConfig(ctx *cli.Context) CLIConfig { PriorityGasPrice: types.GlobalBig(ctx, PriorityGasPriceFlagName), MinimumProfit: types.GlobalBig(ctx, MinimumProfitFlagName), ZKeyFile: ctx.GlobalString(ZKeyFileNameFlagName), - ZKWorkingDir: ctx.GlobalString(ZKWorkingDirFlagName), + ZKWorkingDir: DefaultConfig.ZKWorkingDir, ZKProverMode: ctx.GlobalUint64(ZKProverModeFlagName), ZKProverImpl: ctx.GlobalUint64(ZKProverImplFlagName), ThreadsPerShard: ctx.GlobalUint64(ThreadsPerShardFlagName),