Merge branch 'feat/upgrade4' into feat-coordinator-enhance-batch-proo…

…f-sanity-check

Makefile

@@ -1,6 +1,6 @@
 .PHONY: fmt dev_docker build_test_docker run_test_docker clean update
 
-L2GETH_TAG=scroll-v5.3.0
+L2GETH_TAG=scroll-v5.5.1
 
 help: ## Display this help message
 	@grep -h \

common/forks/forks.go

@@ -5,6 +5,7 @@ import (
 	"math/big"
 	"sort"
 
+	"github.com/scroll-tech/da-codec/encoding"
 	"github.com/scroll-tech/go-ethereum/params"
 )
 
@@ -63,17 +64,6 @@ func CollectSortedForkHeights(config *params.ChainConfig) ([]uint64, map[uint64]
 	return forkHeights, forkHeightsMap, forkNameHeightMap
 }
 
-// BlocksUntilFork returns the number of blocks until the next fork
-// returns 0 if there is no fork scheduled for the future
-func BlocksUntilFork(blockHeight uint64, forkHeights []uint64) uint64 {
-	for _, forkHeight := range forkHeights {
-		if forkHeight > blockHeight {
-			return forkHeight - blockHeight
-		}
-	}
-	return 0
-}
-
 // BlockRange returns the block range of the hard fork
 // Need ensure the forkHeights is incremental
 func BlockRange(currentForkHeight uint64, forkHeights []uint64) (from, to uint64) {
@@ -87,3 +77,45 @@ func BlockRange(currentForkHeight uint64, forkHeights []uint64) (from, to uint64
 	}
 	return
 }
+
+// GetHardforkName returns the name of the hardfork active at the given block height and timestamp.
+// It checks the chain configuration to determine which hardfork is active.
+func GetHardforkName(config *params.ChainConfig, blockHeight, blockTimestamp uint64) string {
+	if !config.IsBernoulli(new(big.Int).SetUint64(blockHeight)) {
+		return "homestead"
+	} else if !config.IsCurie(new(big.Int).SetUint64(blockHeight)) {
+		return "bernoulli"
+	} else if !config.IsDarwin(blockTimestamp) {
+		return "curie"
+	} else {
+		return "darwin"
+	}
+}
+
+// GetCodecVersion returns the encoding codec version for the given block height and timestamp.
+// It determines the appropriate codec version based on the active hardfork.
+func GetCodecVersion(config *params.ChainConfig, blockHeight, blockTimestamp uint64) encoding.CodecVersion {
+	if !config.IsBernoulli(new(big.Int).SetUint64(blockHeight)) {
+		return encoding.CodecV0
+	} else if !config.IsCurie(new(big.Int).SetUint64(blockHeight)) {
+		return encoding.CodecV1
+	} else if !config.IsDarwin(blockTimestamp) {
+		return encoding.CodecV2
+	} else {
+		return encoding.CodecV3
+	}
+}
+
+// GetMaxChunksPerBatch returns the maximum number of chunks allowed per batch for the given block height and timestamp.
+// This value may change depending on the active hardfork.
+func GetMaxChunksPerBatch(config *params.ChainConfig, blockHeight, blockTimestamp uint64) uint64 {
+	if !config.IsBernoulli(new(big.Int).SetUint64(blockHeight)) {
+		return 15
+	} else if !config.IsCurie(new(big.Int).SetUint64(blockHeight)) {
+		return 15
+	} else if !config.IsDarwin(blockTimestamp) {
+		return 45
+	} else {
+		return 45
+	}
+}

common/forks/forks_test.go

@@ -33,46 +33,6 @@ func TestCollectSortedForkBlocks(t *testing.T) {
 	}, n)
 }
 
-func TestBlocksUntilFork(t *testing.T) {
-	tests := map[string]struct {
-		block    uint64
-		forks    []uint64
-		expected uint64
-	}{
-		"NoFork": {
-			block:    44,
-			forks:    []uint64{},
-			expected: 0,
-		},
-		"BeforeFork": {
-			block:    0,
-			forks:    []uint64{1, 5},
-			expected: 1,
-		},
-		"OnFork": {
-			block:    1,
-			forks:    []uint64{1, 5},
-			expected: 4,
-		},
-		"OnLastFork": {
-			block:    5,
-			forks:    []uint64{1, 5},
-			expected: 0,
-		},
-		"AfterFork": {
-			block:    5,
-			forks:    []uint64{1, 5},
-			expected: 0,
-		},
-	}
-
-	for name, test := range tests {
-		t.Run(name, func(t *testing.T) {
-			require.Equal(t, test.expected, BlocksUntilFork(test.block, test.forks))
-		})
-	}
-}
-
 func TestBlockRange(t *testing.T) {
 	tests := []struct {
 		name         string

common/go.mod

@@ -13,7 +13,8 @@ require (
 	github.com/modern-go/reflect2 v1.0.2
 	github.com/orcaman/concurrent-map v1.0.0
 	github.com/prometheus/client_golang v1.19.0
-	github.com/scroll-tech/go-ethereum v1.10.14-0.20240426041101-a860446ebaea
+	github.com/scroll-tech/da-codec v0.0.0-20240626090813-e197995302f3
+	github.com/scroll-tech/go-ethereum v1.10.14-0.20240626125436-418bc6f728b6
 	github.com/stretchr/testify v1.9.0
 	github.com/testcontainers/testcontainers-go v0.30.0
 	github.com/testcontainers/testcontainers-go/modules/compose v0.30.0
@@ -182,7 +183,7 @@ require (
 	github.com/rjeczalik/notify v0.9.1 // indirect
 	github.com/rs/cors v1.7.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/scroll-tech/zktrie v0.8.2 // indirect
+	github.com/scroll-tech/zktrie v0.8.4 // indirect
 	github.com/secure-systems-lab/go-securesystemslib v0.4.0 // indirect
 	github.com/serialx/hashring v0.0.0-20190422032157-8b2912629002 // indirect
 	github.com/shibumi/go-pathspec v1.3.0 // indirect

common/go.sum

@@ -633,10 +633,12 @@ github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
 github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/scroll-tech/go-ethereum v1.10.14-0.20240426041101-a860446ebaea h1:CH1WXWrpEpLaP3N+bFs2a1xdE0+lRm1AuJQb5YvE6Ls=
-github.com/scroll-tech/go-ethereum v1.10.14-0.20240426041101-a860446ebaea/go.mod h1:i4VBgWoaW/y0D8MmQb7hSOulyw1dKhuiSFAbznwivCA=
-github.com/scroll-tech/zktrie v0.8.2 h1:UMuIfA+jdgWMLmTgTL64Emo+zzMOdcnH0+eYdDcshxQ=
-github.com/scroll-tech/zktrie v0.8.2/go.mod h1:XvNo7vAk8yxNyTjBDj5WIiFzYW4bx/gJ78+NK6Zn6Uk=
+github.com/scroll-tech/da-codec v0.0.0-20240626090813-e197995302f3 h1:aDnxfgflaWcRnRb7qCQdeJs980s9UDhAvE9QVqjxxJ0=
+github.com/scroll-tech/da-codec v0.0.0-20240626090813-e197995302f3/go.mod h1:D6XEESeNVJkQJlv3eK+FyR+ufPkgVQbJzERylQi53Bs=
+github.com/scroll-tech/go-ethereum v1.10.14-0.20240626125436-418bc6f728b6 h1:Q8YyvrcPIcXQwE4ucm4bqmPh6TP6IB1GUTXripf2WyQ=
+github.com/scroll-tech/go-ethereum v1.10.14-0.20240626125436-418bc6f728b6/go.mod h1:byf/mZ8jLYUCnUePTicjJWn+RvKdxDn7buS6glTnMwQ=
+github.com/scroll-tech/zktrie v0.8.4 h1:UagmnZ4Z3ITCk+aUq9NQZJNAwnWl4gSxsLb2Nl7IgRE=
+github.com/scroll-tech/zktrie v0.8.4/go.mod h1:XvNo7vAk8yxNyTjBDj5WIiFzYW4bx/gJ78+NK6Zn6Uk=
 github.com/secure-systems-lab/go-securesystemslib v0.4.0 h1:b23VGrQhTA8cN2CbBw7/FulN9fTtqYUdS5+Oxzt+DUE=
 github.com/secure-systems-lab/go-securesystemslib v0.4.0/go.mod h1:FGBZgq2tXWICsxWQW1msNf49F0Pf2Op5Htayx335Qbs=
 github.com/serialx/hashring v0.0.0-20190422032157-8b2912629002 h1:ka9QPuQg2u4LGipiZGsgkg3rJCo4iIUCy75FddM0GRQ=

common/types/db.go

@@ -188,6 +188,31 @@ func (s ChunkProofsStatus) String() string {
 	}
 }
 
+// BatchProofsStatus describes the proving status of batches that belong to a bundle.
+type BatchProofsStatus int
+
+const (
+	// BatchProofsStatusUndefined represents an undefined batch proofs status
+	BatchProofsStatusUndefined BatchProofsStatus = iota
+
+	// BatchProofsStatusPending means that some batches that belong to this bundle have not been proven
+	BatchProofsStatusPending
+
+	// BatchProofsStatusReady means that all batches that belong to this bundle have been proven
+	BatchProofsStatusReady
+)
+
+func (s BatchProofsStatus) String() string {
+	switch s {
+	case BatchProofsStatusPending:
+		return "BatchProofsStatusPending"
+	case BatchProofsStatusReady:
+		return "BatchProofsStatusReady"
+	default:
+		return fmt.Sprintf("Undefined BatchProofsStatus (%d)", int32(s))
+	}
+}
+
 // RollupStatus block_batch rollup_status (pending, committing, committed, commit_failed, finalizing, finalized, finalize_skipped, finalize_failed)
 type RollupStatus int
 

common/types/message/message.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 
+	"github.com/scroll-tech/da-codec/encoding/codecv3"
 	"github.com/scroll-tech/go-ethereum/common"
 	"github.com/scroll-tech/go-ethereum/common/hexutil"
 	"github.com/scroll-tech/go-ethereum/crypto"
@@ -44,6 +45,8 @@ func (r ProofType) String() string {
 		return "proof type chunk"
 	case ProofTypeBatch:
 		return "proof type batch"
+	case ProofTypeBundle:
+		return "proof type bundle"
 	default:
 		return fmt.Sprintf("illegal proof type: %d", r)
 	}
@@ -52,10 +55,12 @@ func (r ProofType) String() string {
 const (
 	// ProofTypeUndefined is an unknown proof type
 	ProofTypeUndefined ProofType = iota
-	// ProofTypeChunk is default prover, it only generates zk proof from traces.
+	// ProofTypeChunk generates a proof for a ZkEvm chunk, where the inputs are the execution traces for blocks contained in the chunk. ProofTypeChunk is the default proof type.
 	ProofTypeChunk
-	// ProofTypeBatch generates zk proof from other zk proofs and aggregate them into one proof.
+	// ProofTypeBatch generates a single proof, aggregating one or more proofs of the type ProofTypeChunk.
 	ProofTypeBatch
+	// ProofTypeBundle generates a single proof by recursing over more than one proofs of the type ProofTypeBatch.
+	ProofTypeBundle
 )
 
 // GenerateToken generates token
@@ -132,11 +137,12 @@ func (a *ProofMsg) PublicKey() (string, error) {
 
 // TaskMsg is a wrapper type around db ProveTask type.
 type TaskMsg struct {
-	UUID            string           `json:"uuid"`
-	ID              string           `json:"id"`
-	Type            ProofType        `json:"type,omitempty"`
-	BatchTaskDetail *BatchTaskDetail `json:"batch_task_detail,omitempty"`
-	ChunkTaskDetail *ChunkTaskDetail `json:"chunk_task_detail,omitempty"`
+	UUID             string            `json:"uuid"`
+	ID               string            `json:"id"`
+	Type             ProofType         `json:"type,omitempty"`
+	ChunkTaskDetail  *ChunkTaskDetail  `json:"chunk_task_detail,omitempty"`
+	BatchTaskDetail  *BatchTaskDetail  `json:"batch_task_detail,omitempty"`
+	BundleTaskDetail *BundleTaskDetail `json:"bundle_task_detail,omitempty"`
 }
 
 // ChunkTaskDetail is a type containing ChunkTask detail.
@@ -146,19 +152,34 @@ type ChunkTaskDetail struct {
 
 // BatchTaskDetail is a type containing BatchTask detail.
 type BatchTaskDetail struct {
-	ChunkInfos  []*ChunkInfo  `json:"chunk_infos"`
-	ChunkProofs []*ChunkProof `json:"chunk_proofs"`
+	ChunkInfos      []*ChunkInfo     `json:"chunk_infos"`
+	ChunkProofs     []*ChunkProof    `json:"chunk_proofs"`
+	ParentStateRoot common.Hash      `json:"parent_state_root"`
+	ParentBatchHash common.Hash      `json:"parent_batch_hash"`
+	BatchHeader     *codecv3.DABatch `json:"batch_header"`
+}
+
+// BundleTaskDetail consists of all the information required to describe the task to generate a proof for a bundle of batches.
+type BundleTaskDetail struct {
+	ChainID             uint64        `json:"chain_id"`
+	FinalizedBatchHash  common.Hash   `json:"finalized_batch_hash"`
+	FinalizedStateRoot  common.Hash   `json:"finalized_state_root"`
+	PendingBatchHash    common.Hash   `json:"pending_batch_hash"`
+	PendingStateRoot    common.Hash   `json:"pending_state_root"`
+	PendingWithdrawRoot common.Hash   `json:"pending_withdraw_root"`
+	BatchProofs         []*BatchProof `json:"batch_proofs"`
 }
 
 // ProofDetail is the message received from provers that contains zk proof, the status of
 // the proof generation succeeded, and an error message if proof generation failed.
 type ProofDetail struct {
-	ID         string      `json:"id"`
-	Type       ProofType   `json:"type,omitempty"`
-	Status     RespStatus  `json:"status"`
-	ChunkProof *ChunkProof `json:"chunk_proof,omitempty"`
-	BatchProof *BatchProof `json:"batch_proof,omitempty"`
-	Error      string      `json:"error,omitempty"`
+	ID          string       `json:"id"`
+	Type        ProofType    `json:"type,omitempty"`
+	Status      RespStatus   `json:"status"`
+	ChunkProof  *ChunkProof  `json:"chunk_proof,omitempty"`
+	BatchProof  *BatchProof  `json:"batch_proof,omitempty"`
+	BundleProof *BundleProof `json:"bundle_proof,omitempty"`
+	Error       string       `json:"error,omitempty"`
 }
 
 // Hash return proofMsg content hash.
@@ -204,11 +225,13 @@ type ChunkProof struct {
 
 // BatchProof includes the proof info that are required for batch verification and rollup.
 type BatchProof struct {
+	Protocol  []byte `json:"protocol"`
 	Proof     []byte `json:"proof"`
 	Instances []byte `json:"instances"`
 	Vk        []byte `json:"vk"`
 	// cross-reference between cooridinator computation and prover compution
-	GitVersion string `json:"git_version,omitempty"`
+	BatchHash  common.Hash `json:"batch_hash"`
+	GitVersion string      `json:"git_version,omitempty"`
 }
 
 // SanityCheck checks whether an BatchProof is in a legal format
@@ -240,3 +263,12 @@ func (ap *BatchProof) SanityCheck() error {
 
 	return nil
 }
+
+// BundleProof includes the proof info that are required for verification of a bundle of batch proofs.
+type BundleProof struct {
+	Proof     []byte `json:"proof"`
+	Instances []byte `json:"instances"`
+	Vk        []byte `json:"vk"`
+	// cross-reference between cooridinator computation and prover compution
+	GitVersion string `json:"git_version,omitempty"`
+}

common/types/message/message_test.go

@@ -108,7 +108,7 @@ func TestProofDetailHash(t *testing.T) {
 	}
 	hash, err := proofDetail.Hash()
 	assert.NoError(t, err)
-	expectedHash := "01128ea9006601146ba80dbda959c96ebaefca463e78570e473a57d821db5ec1"
+	expectedHash := "4c291e7582ee773add1c145270a6e704e00ba193b6118ee2c5fd646112bc867c"
 	assert.Equal(t, expectedHash, hex.EncodeToString(hash))
 }
 
@@ -119,8 +119,11 @@ func TestProveTypeString(t *testing.T) {
 	proofTypeBatch := ProofType(2)
 	assert.Equal(t, "proof type batch", proofTypeBatch.String())
 
-	illegalProof := ProofType(3)
-	assert.Equal(t, "illegal proof type: 3", illegalProof.String())
+	proofTypeBundle := ProofType(3)
+	assert.Equal(t, "proof type bundle", proofTypeBundle.String())
+
+	illegalProof := ProofType(4)
+	assert.Equal(t, "illegal proof type: 4", illegalProof.String())
 }
 
 func TestProofMsgPublicKey(t *testing.T) {

coordinator/Makefile

@@ -54,4 +54,4 @@ docker:
 
 docker_push:
 	docker push scrolltech/coordinator-api:${IMAGE_VERSION}
-	docker push scrolltech/coordinator-cron:${IMAGE_VERSION}
+	docker push scrolltech/coordinator-cron:${IMAGE_VERSION}

database/migrate/migrate_test.go

@@ -59,20 +59,20 @@ func testResetDB(t *testing.T) {
 	cur, err := Current(pgDB)
 	assert.NoError(t, err)
 	// total number of tables.
-	assert.Equal(t, int64(20), cur)
+	assert.Equal(t, int64(22), cur)
 }
 
 func testMigrate(t *testing.T) {
 	assert.NoError(t, Migrate(pgDB))
 	cur, err := Current(pgDB)
 	assert.NoError(t, err)
-	assert.Equal(t, int64(20), cur)
+	assert.Equal(t, int64(22), cur)
 }
 
 func testRollback(t *testing.T) {
 	version, err := Current(pgDB)
 	assert.NoError(t, err)
-	assert.Equal(t, int64(20), version)
+	assert.Equal(t, int64(22), version)
 
 	assert.NoError(t, Rollback(pgDB, nil))