Move init data file functionality out of run script

.github/workflows/publish.yml

@@ -43,7 +43,9 @@ jobs:
       - name: Create package
         run: |
           mkdir -p ${{ env.BIN_DIR }}/snarkbuild
+          mkdir -p ${{ env.BIN_DIR }}/snark_lib/zkey
           cp -r ethstorage/prover/snark_lib ${{ env.BIN_DIR }}
+          cp init.sh ${{ env.BUILD_DIR }}
           cp run.sh ${{ env.BUILD_DIR }}
 
       - name: Build

Dockerfile

@@ -7,11 +7,13 @@ RUN make
 # Pull ES node into a second stage deploy alpine container
 FROM alpine:latest
 COPY --from=builder /es-node/build/ /es-node/build/
-RUN apk add --no-cache bash curl libstdc++ gcompat libgomp
+RUN apk add --no-cache bash curl libstdc++ gcompat libgomp nodejs npm
+RUN npm install -g snarkjs
 
 # Entrypoint
+COPY --from=builder /es-node/init.sh /es-node/
 COPY --from=builder /es-node/run.sh /es-node/
-RUN chmod +x /es-node/run.sh
+RUN chmod +x /es-node/init.sh /es-node/run.sh
 WORKDIR /es-node
 
 EXPOSE 9545 9222 30305/udp

Makefile

@@ -10,6 +10,7 @@ LDFLAGS := -ldflags "$(LDFLAGSSTRING)"
 
 es-node: build
 	cp -r ethstorage/prover/snark_lib build/bin
+	mkdir -p build/bin/snark_lib/zkey
 	mkdir -p build/bin/snarkbuild
 
 build:

docker-compose.yaml

@@ -8,9 +8,10 @@ services:
       - '30305:30305/udp'
     volumes:
       - ./es-data:/es-node/es-data
+      - ./build/bin/snark_lib/zkey:/es-node/build/bin/snark_lib/zkey
     environment:
       - ES_NODE_STORAGE_MINER=${ES_NODE_STORAGE_MINER}
       - ES_NODE_SIGNER_PRIVATE_KEY=${ES_NODE_SIGNER_PRIVATE_KEY}
-    command:  ["bash", "-c", "/es-node/run.sh --miner.zk-prover-impl=2"]
+    command:  ["bash", "-c", "/es-node/run.sh"]
     container_name: es
     tty: true

ethstorage/miner/cli.go

@@ -55,13 +55,13 @@ func CLIFlags(envPrefix string) []cli.Flag {
 		},
 		cli.StringFlag{
 			Name:   ZKeyFileNameFlagName,
-			Usage:  "zkey file name which should be put in the snark_lib folder",
-			Value:  DefaultConfig.ZKeyFileName,
+			Usage:  "zkey file name with path",
+			Value:  DefaultConfig.ZKeyFile,
 			EnvVar: rollup.PrefixEnvVar(envPrefix, "ZKEY_FILE"),
 		},
 		cli.StringFlag{
 			Name:   ZKWorkingDirFlagName,
-			Usage:  "Path to the snark_lib folder",
+			Usage:  "Path to the snark library folder",
 			Value:  DefaultConfig.ZKWorkingDir,
 			EnvVar: rollup.PrefixEnvVar(envPrefix, "ZK_WORKING_DIR"),
 		},
@@ -92,7 +92,7 @@ type CLIConfig struct {
 	GasPrice         *big.Int
 	PriorityGasPrice *big.Int
 	MinimumProfit    *big.Int
-	ZKeyFileName     string
+	ZKeyFile         string
 	ZKWorkingDir     string
 	ZKProverMode     uint64
 	ZKProverImpl     uint64
@@ -118,14 +118,22 @@ func (c CLIConfig) ToMinerConfig() (Config, error) {
 		}
 		zkWorkingDir = dir
 	}
+	zkFile := c.ZKeyFile
+	if !filepath.IsAbs(zkFile) {
+		dir, err := filepath.Abs(zkFile)
+		if err != nil {
+			return Config{}, fmt.Errorf("check ZKeyFileName error: %v", err)
+		}
+		zkFile = dir
+	}
 	cfg := DefaultConfig
 	cfg.ZKWorkingDir = zkWorkingDir
+	cfg.ZKeyFile = zkFile
+	cfg.ZKProverMode = c.ZKProverMode
+	cfg.ZKProverImpl = c.ZKProverImpl
 	cfg.GasPrice = c.GasPrice
 	cfg.PriorityGasPrice = c.PriorityGasPrice
 	cfg.MinimumProfit = c.MinimumProfit
-	cfg.ZKeyFileName = c.ZKeyFileName
-	cfg.ZKProverMode = c.ZKProverMode
-	cfg.ZKProverImpl = c.ZKProverImpl
 	cfg.ThreadsPerShard = c.ThreadsPerShard
 	return cfg, nil
 }
@@ -136,7 +144,7 @@ func ReadCLIConfig(ctx *cli.Context) CLIConfig {
 		GasPrice:         types.GlobalBig(ctx, GasPriceFlagName),
 		PriorityGasPrice: types.GlobalBig(ctx, PriorityGasPriceFlagName),
 		MinimumProfit:    types.GlobalBig(ctx, MinimumProfitFlagName),
-		ZKeyFileName:     ctx.GlobalString(ZKeyFileNameFlagName),
+		ZKeyFile:         ctx.GlobalString(ZKeyFileNameFlagName),
 		ZKWorkingDir:     ctx.GlobalString(ZKWorkingDirFlagName),
 		ZKProverMode:     ctx.GlobalUint64(ZKProverModeFlagName),
 		ZKProverImpl:     ctx.GlobalUint64(ZKProverImplFlagName),

ethstorage/miner/config.go

@@ -9,6 +9,7 @@ import (
 	"runtime"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethstorage/go-ethstorage/ethstorage/prover"
 	"github.com/ethstorage/go-ethstorage/ethstorage/signer"
 )
 
@@ -29,7 +30,7 @@ type Config struct {
 	// cli
 	GasPrice         *big.Int
 	PriorityGasPrice *big.Int
-	ZKeyFileName     string
+	ZKeyFile         string
 	ZKWorkingDir     string
 	ZKProverMode     uint64
 	ZKProverImpl     uint64
@@ -48,7 +49,7 @@ var DefaultConfig = Config{
 
 	GasPrice:         nil,
 	PriorityGasPrice: nil,
-	ZKeyFileName:     "blob_poseidon2.zkey",
+	ZKeyFile:         filepath.Join("build", "bin", prover.SnarkLib, "zkey", "blob_poseidon2.zkey"),
 	ZKWorkingDir:     filepath.Join("build", "bin"),
 	ZKProverMode:     2,
 	ZKProverImpl:     1,

ethstorage/miner/miner_test.go

@@ -6,15 +6,13 @@
 package miner
 
 import (
-	"math/big"
 	"os"
 	"path/filepath"
 	"runtime/debug"
 	"testing"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/event"
 	es "github.com/ethstorage/go-ethstorage/ethstorage"
@@ -38,7 +36,6 @@ var (
 	kvSize       uint64 = 1 << kvSizeBits
 	kvEntries    uint64 = 1 << kvEntriesBits
 	shardID             = uint64(0)
-	value               = hexutil.EncodeUint64(10000000000000)
 	lg                  = esLog.NewLogger(esLog.DefaultCLIConfig())
 )
 
@@ -54,34 +51,32 @@ func initStorageManager(t *testing.T, client *eth.PollingClient) *es.StorageMana
 }
 
 func newMiner(t *testing.T, storageMgr *es.StorageManager, client *eth.PollingClient) *Miner {
-	defaultConfig := &Config{
-		RandomChecks:     2,
-		NonceLimit:       1048576,
-		MinimumDiff:      new(big.Int).SetUint64(5000000),
-		Cutoff:           new(big.Int).SetUint64(60),
-		DiffAdjDivisor:   new(big.Int).SetUint64(1024),
-		GasPrice:         nil,
-		PriorityGasPrice: new(big.Int).SetUint64(10),
-		ThreadsPerShard:  1,
-		ZKProverMode:     2,
-		ZKProverImpl:     1,
-		ZKeyFileName:     "blob_poseidon2.zkey",
-	}
-	l1api := NewL1MiningAPI(client, nil, lg)
+	testConfig := &DefaultConfig
+
 	zkWorkingDir, _ := filepath.Abs("../prover")
-	zkey := filepath.Join(zkWorkingDir, prover.SnarkLib, defaultConfig.ZKeyFileName)
+	testConfig.ZKWorkingDir = zkWorkingDir
+	dir := filepath.Join(zkWorkingDir, prover.SnarkLib, "zkey")
+	zkey := filepath.Join(dir, "blob_poseidon2.zkey")
+	testConfig.ZKeyFile = zkey
+	if _, err := os.Stat(dir); os.IsNotExist(err) {
+		err := os.Mkdir(dir, 0755)
+		if err != nil {
+			t.Fatalf("Mkdir failed %v", err)
+		}
+	}
 	if _, err := os.Stat(zkey); os.IsNotExist(err) {
 		_, err := os.Create(zkey)
 		if err != nil {
 			t.Fatalf("Create failed %v", err)
 		}
-		defer os.Remove(zkey)
+		defer os.RemoveAll(dir)
 	}
-	pvr := prover.NewKZGPoseidonProver(zkWorkingDir, defaultConfig.ZKeyFileName, defaultConfig.ZKProverMode, defaultConfig.ZKProverImpl, lg)
+	pvr := prover.NewKZGPoseidonProver(zkWorkingDir, zkey, testConfig.ZKProverMode, testConfig.ZKProverImpl, lg)
 	fd := new(event.Feed)
 	db := rawdb.NewMemoryDatabase()
 	br := blobs.NewBlobReader(downloader.NewBlobMemCache(), storageMgr, lg)
-	miner := New(defaultConfig, db, storageMgr, l1api, br, &pvr, fd, lg)
+	l1api := NewL1MiningAPI(client, nil, lg)
+	miner := New(testConfig, db, storageMgr, l1api, br, &pvr, fd, lg)
 	return miner
 }
 

ethstorage/node/node.go

@@ -300,7 +300,7 @@ func (n *EsNode) initMiner(ctx context.Context, cfg *Config) error {
 	l1api := miner.NewL1MiningAPI(n.l1Source, n.randaoSource, n.log)
 	pvr := prover.NewKZGPoseidonProver(
 		cfg.Mining.ZKWorkingDir,
-		cfg.Mining.ZKeyFileName,
+		cfg.Mining.ZKeyFile,
 		cfg.Mining.ZKProverMode,
 		cfg.Mining.ZKProverImpl,
 		n.log,

ethstorage/prover/kzg_poseidon_prover.go

@@ -40,18 +40,17 @@ type KZGPoseidonProver struct {
 
 // Prover that can be used directly by miner to prove both KZG and Poseidon hash
 // workingDir specifies the working directory of the command relative to the caller.
-// zkeyFileName specifies the zkey file name to generate snark proof
+// zkeyFile specifies the zkey file with path to generate snark proof
 // zkProverMode specifies the mode of the zk prover, 1 for per sample, 2 for samples
 // zkProverImpl specifies the implementation of the snark prover, 1 for snarkjs, 2 for go-rapidsnark
 // lg specifies the logger to log the info
 // returns a prover that can generate a combined KZG + zk proof
-func NewKZGPoseidonProver(workingDir, zkeyFileName string, zkProverMode, zkProverImpl uint64, lg log.Logger) KZGPoseidonProver {
+func NewKZGPoseidonProver(workingDir, zkeyFile string, zkProverMode, zkProverImpl uint64, lg log.Logger) KZGPoseidonProver {
 	// check dependencies when es-node starts
 	libDir := filepath.Join(workingDir, SnarkLib)
 	if _, err := os.Stat(libDir); errors.Is(err, os.ErrNotExist) {
 		lg.Crit("Init ZK prover failed", "error", "snark lib does not exist", "dir", libDir)
 	}
-	zkeyFile := filepath.Join(libDir, zkeyFileName)
 	if _, err := os.Stat(zkeyFile); errors.Is(err, os.ErrNotExist) {
 		lg.Crit("Init ZK prover failed", "error", "zkey does not exist", "dir", zkeyFile)
 	}
@@ -71,7 +70,7 @@ func NewKZGPoseidonProver(workingDir, zkeyFileName string, zkProverMode, zkProve
 		zkProverMode: zkProverMode,
 		zkProverImpl: zkProverImpl,
 		libDir:       libDir,
-		zkey:         zkeyFileName,
+		zkey:         zkeyFile,
 		wasm:         wasmName,
 		lg:           lg,
 	}

ethstorage/prover/zk_prover.go

@@ -27,14 +27,14 @@ const (
 )
 
 type ZKProver struct {
-	dir, zkeyName, wasmName string
+	dir, zkeyFile, wasmName string
 	lg                      log.Logger
 }
 
-func NewZKProver(workingDir, zkeyName, wasmName string, lg log.Logger) *ZKProver {
+func NewZKProver(workingDir, zkey, wasmName string, lg log.Logger) *ZKProver {
 	return &ZKProver{
 		dir:      workingDir,
-		zkeyName: zkeyName,
+		zkeyFile: zkey,
 		wasmName: wasmName,
 		lg:       lg,
 	}
@@ -146,7 +146,7 @@ func (p *ZKProver) prove(dir string, pubInputs []byte) ([]byte, string, error) {
 	proofFile := filepath.Join(buildDir, proofName)
 	publicFile := filepath.Join(buildDir, publicName)
 	cmd = exec.Command("snarkjs", "groth16", "prove",
-		filepath.Join(libDir, p.zkeyName),
+		p.zkeyFile,
 		wtnsFile,
 		proofFile,
 		publicFile,

ethstorage/prover/zk_prover_go.go

@@ -24,7 +24,7 @@ type ZKProverGo struct {
 	lg   log.Logger
 }
 
-func NewZKProverGo(libDir, zkeyName, wasmName string, lg log.Logger) (*ZKProverGo, error) {
+func NewZKProverGo(libDir, zkeyFile, wasmName string, lg log.Logger) (*ZKProverGo, error) {
 	wasmBytes, err := os.ReadFile(filepath.Join(libDir, wasmName))
 	if err != nil {
 		lg.Error("Read wasm file failed", "error", err)
@@ -35,7 +35,7 @@ func NewZKProverGo(libDir, zkeyName, wasmName string, lg log.Logger) (*ZKProverG
 		lg.Error("Create witness calculator failed", "error", err)
 		return nil, err
 	}
-	zkey, err := os.ReadFile(filepath.Join(libDir, zkeyName))
+	zkey, err := os.ReadFile(zkeyFile)
 	if err != nil {
 		lg.Error("Read zkey file failed", "error", err)
 		return nil, err