Tudor/aggressive header compression

go/common/headers.go

@@ -83,6 +83,24 @@ type RollupHeader struct {
 	LastBatchSeqNo uint64
 }
 
+// CalldataRollupHeader contains all information necessary to reconstruct the batches included in the rollup.
+// This data structure is serialised, compressed, and encrypted, before being serialised again in the rollup.
+type CalldataRollupHeader struct {
+	FirstBatchSequence *big.Int
+	FirstBatchHeight   *big.Int
+	FirstParentHash    L2BatchHash
+
+	StartTime       uint64
+	BatchTimeDeltas []*big.Int // todo - minimize assuming a default of 1 sec and then store only exceptions
+
+	ReOrgs         []*BatchHeader // sparse list of reorged headers - non null only for reorgs.
+	L1HeightDeltas []*big.Int     // Contains the L1 height on the indexes where it changes. Todo - can be optimised with deltas
+
+	// todo - these fields are for debugging the compression. Should be removed.
+	BatchHashes []L2BatchHash
+	// BatchHeaders []*BatchHeader
+}
+
 // MarshalJSON custom marshals the RollupHeader into a json
 func (r *RollupHeader) MarshalJSON() ([]byte, error) {
 	type Alias RollupHeader

go/common/rollups.go

@@ -6,10 +6,10 @@ import (
 
 // ExtRollup is an encrypted form of rollup used when passing the rollup around outside an enclave.
 type ExtRollup struct {
-	Header        *RollupHeader // the fields required by the management contract
-	BatchHeaders  []byte        // compressed batch headers
-	BatchPayloads []byte        // The batches included in the rollup, in external/encrypted form.
-	hash          atomic.Value
+	Header               *RollupHeader // the fields required by the management contract
+	CalldataRollupHeader []byte        // encrypted header useful for recreating the batches
+	BatchPayloads        []byte        // The transactions included in the rollup, in external/encrypted form.
+	hash                 atomic.Value
 }
 
 // Hash returns the keccak256 hash of the rollup's header.

go/common/rpc/converters.go

@@ -208,7 +208,7 @@ func ToExtRollupMsg(rollup *common.ExtRollup) generated.ExtRollupMsg {
 		return generated.ExtRollupMsg{}
 	}
 
-	return generated.ExtRollupMsg{Header: ToRollupHeaderMsg(rollup.Header), BatchPayloads: rollup.BatchPayloads, BatchHeaders: rollup.BatchHeaders}
+	return generated.ExtRollupMsg{Header: ToRollupHeaderMsg(rollup.Header), BatchPayloads: rollup.BatchPayloads, BatchHeaders: rollup.CalldataRollupHeader}
 }
 
 func ToRollupHeaderMsg(header *common.RollupHeader) *generated.RollupHeaderMsg {
@@ -236,9 +236,9 @@ func FromExtRollupMsg(msg *generated.ExtRollupMsg) *common.ExtRollup {
 	}
 
 	return &common.ExtRollup{
-		Header:        FromRollupHeaderMsg(msg.Header),
-		BatchPayloads: msg.BatchPayloads,
-		BatchHeaders:  msg.BatchHeaders,
+		Header:               FromRollupHeaderMsg(msg.Header),
+		BatchPayloads:        msg.BatchPayloads,
+		CalldataRollupHeader: msg.BatchHeaders,
 	}
 }
 

go/enclave/components/batch_executor.go

@@ -49,11 +49,22 @@ func NewBatchExecutor(storage storage.Storage, cc *crosschain.Processors, genesi
 	}
 }
 
+func (executor *batchExecutor) ComputeBatchLight(BlockPtr common.L1BlockHash, ParentPtr common.L2BatchHash, Transactions common.L2Transactions, AtTime uint64, SequencerNo *big.Int) (*ComputedBatch, error) {
+	return executor.ComputeBatch(&BatchExecutionContext{
+		BlockPtr:     BlockPtr,
+		ParentPtr:    ParentPtr,
+		Transactions: Transactions,
+		AtTime:       AtTime,
+		// Creator:      executor,
+		ChainConfig: executor.chainConfig,
+		SequencerNo: SequencerNo,
+	})
+}
+
 func (executor *batchExecutor) ComputeBatch(context *BatchExecutionContext) (*ComputedBatch, error) {
 	defer executor.logger.Info("Batch context processed", log.DurationKey, measure.NewStopwatch())
 
-	// Block is loaded first since if its missing this batch might be based on l1 fork we dont know about
-	// and we want to filter out all fork batches based on not knowing the l1 block
+	// sanity check that the l1 block exists. We don't have to execute batches of forks.
 	block, err := executor.storage.FetchBlock(context.BlockPtr)
 	if errors.Is(err, errutil.ErrNotFound) {
 		return nil, errutil.ErrBlockForBatchNotFound
@@ -64,6 +75,7 @@ func (executor *batchExecutor) ComputeBatch(context *BatchExecutionContext) (*Co
 	// These variables will be used to create the new batch
 	parent, err := executor.storage.FetchBatch(context.ParentPtr)
 	if errors.Is(err, errutil.ErrNotFound) {
+		executor.logger.Error(fmt.Sprintf("can't find parent batch %s. Seq %d", context.ParentPtr, context.SequencerNo))
 		return nil, errutil.ErrAncestorBatchNotFound
 	}
 	if err != nil {

go/enclave/components/batch_registry.go

@@ -107,6 +107,13 @@ func (br *batchRegistry) BatchesAfter(batchSeqNo uint64, rollupLimiter limiters.
 		br.logger.Info("Added batch to rollup", log.BatchHashKey, batch.Hash(), log.BatchSeqNoKey, batch.SeqNo())
 	}
 
+	current := batches[0].SeqNo().Uint64()
+	for i, b := range batches {
+		if current+uint64(i) != b.SeqNo().Uint64() {
+			br.logger.Crit("invalid rollup")
+		}
+	}
+
 	return batches, nil
 }
 

go/enclave/components/interfaces.go

@@ -68,6 +68,8 @@ type BatchExecutor interface {
 	// Should be safe to call in parallel
 	ComputeBatch(*BatchExecutionContext) (*ComputedBatch, error)
 
+	ComputeBatchLight(BlockPtr common.L1BlockHash, ParentPtr common.L2BatchHash, Transactions common.L2Transactions, AtTime uint64, SequencerNo *big.Int) (*ComputedBatch, error)
+
 	// ExecuteBatch - executes the transactions and xchain messages, returns the receipts, and updates the stateDB
 	ExecuteBatch(*core.Batch) (types.Receipts, error)