Merge pull request #9 from availproject/data-availability-verificatio…

…n-v2.3.1

Integration with Avail DA layer in Arbitrum nitro stack

Dockerfile

@@ -40,9 +40,9 @@ WORKDIR /workspace
 RUN apt-get update && apt-get install -y curl build-essential=12.9
 
 FROM wasm-base as wasm-libs-builder
-	# clang / lld used by soft-float wasm
+# clang / lld used by soft-float wasm
 RUN apt-get install -y clang=1:14.0-55.7~deb12u1 lld=1:14.0-55.7~deb12u1
-    # pinned rust 1.70.0
+# pinned rust 1.70.0
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.70.0 --target x86_64-unknown-linux-gnu wasm32-unknown-unknown wasm32-wasi
 COPY ./Makefile ./
 COPY arbitrator/arbutil arbitrator/arbutil
@@ -54,7 +54,7 @@ FROM scratch as wasm-libs-export
 COPY --from=wasm-libs-builder /workspace/ /
 
 FROM wasm-base as wasm-bin-builder
-    # pinned go version
+# pinned go version
 RUN curl -L https://golang.org/dl/go1.20.linux-`dpkg --print-architecture`.tar.gz | tar -C /usr/local -xzf -
 COPY ./Makefile ./go.mod ./go.sum ./
 COPY ./arbcompress ./arbcompress
@@ -66,6 +66,7 @@ COPY ./blsSignatures ./blsSignatures
 COPY ./cmd/chaininfo ./cmd/chaininfo
 COPY ./cmd/replay ./cmd/replay
 COPY ./das/dastree ./das/dastree
+COPY ./das/avail ./das/avail
 COPY ./precompiles ./precompiles
 COPY ./statetransfer ./statetransfer
 COPY ./util ./util
@@ -103,7 +104,7 @@ WORKDIR /workspace
 RUN export DEBIAN_FRONTEND=noninteractive && \
     apt-get update && \
     apt-get install -y make wget gpg software-properties-common zlib1g-dev \
-        libstdc++-11-dev wabt clang llvm-dev libclang-common-14-dev libpolly-14-dev
+    libstdc++-11-dev wabt clang llvm-dev libclang-common-14-dev libpolly-14-dev
 COPY arbitrator/Cargo.* arbitrator/
 COPY arbitrator/arbutil arbitrator/arbutil
 COPY arbitrator/prover/Cargo.toml arbitrator/prover/

arbnode/batch_poster.go

@@ -41,6 +41,7 @@ import (
 	"github.com/offchainlabs/nitro/cmd/chaininfo"
 	"github.com/offchainlabs/nitro/cmd/genericconf"
 	"github.com/offchainlabs/nitro/das"
+	"github.com/offchainlabs/nitro/das/avail"
 	"github.com/offchainlabs/nitro/execution"
 	"github.com/offchainlabs/nitro/solgen/go/bridgegen"
 	"github.com/offchainlabs/nitro/util"
@@ -88,6 +89,7 @@ type BatchPoster struct {
 	gasRefunderAddr    common.Address
 	building           *buildingBatch
 	daWriter           das.DataAvailabilityServiceWriter
+	availDAWriter      avail.DataAvailabilityWriter
 	dataPoster         *dataposter.DataPoster
 	redisLock          *redislock.Simple
 	messagesPerBatch   *arbmath.MovingAverage[uint64]
@@ -264,6 +266,7 @@ type BatchPosterOpts struct {
 	DeployInfo    *chaininfo.RollupAddresses
 	TransactOpts  *bind.TransactOpts
 	DAWriter      das.DataAvailabilityServiceWriter
+	AvailDAWriter avail.DataAvailabilityWriter
 	ParentChainID *big.Int
 }
 
@@ -310,6 +313,7 @@ func NewBatchPoster(ctx context.Context, opts *BatchPosterOpts) (*BatchPoster, e
 		gasRefunderAddr:    opts.Config().gasRefunder,
 		bridgeAddr:         opts.DeployInfo.Bridge,
 		daWriter:           opts.DAWriter,
+		availDAWriter:      opts.AvailDAWriter,
 		redisLock:          redisLock,
 	}
 	b.messagesPerBatch, err = arbmath.NewMovingAverage[uint64](20)
@@ -1168,6 +1172,16 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error)
 		}
 	}
 
+	// ideally we make this part of the above statment by having everything under a single unified interface (soon TM)
+	if b.daWriter == nil && b.availDAWriter != nil {
+		// Store the data on Avail and return a marhsalled BlobPointer, which gets used as the sequencerMsg
+		// which is later used to retrieve the data from Avail
+		sequencerMsg, err = b.availDAWriter.Store(ctx, sequencerMsg)
+		if err != nil {
+			return false, err
+		}
+	}
+
 	data, kzgBlobs, err := b.encodeAddBatch(new(big.Int).SetUint64(batchPosition.NextSeqNum), batchPosition.MessageCount, b.building.msgCount, sequencerMsg, b.building.segments.delayedMsg, b.building.use4844)
 	if err != nil {
 		return false, err

arbnode/delayed_seq_reorg_test.go

@@ -19,7 +19,7 @@ func TestSequencerReorgFromDelayed(t *testing.T) {
 	defer cancel()
 
 	exec, streamer, db, _ := NewTransactionStreamerForTest(t, common.Address{})
-	tracker, err := NewInboxTracker(db, streamer, nil, nil)
+	tracker, err := NewInboxTracker(db, streamer, nil, nil, nil)
 	Require(t, err)
 
 	err = streamer.Start(ctx)

arbnode/inbox_tracker.go

@@ -23,6 +23,7 @@ import (
 	"github.com/offchainlabs/nitro/arbutil"
 	"github.com/offchainlabs/nitro/broadcaster"
 	m "github.com/offchainlabs/nitro/broadcaster/message"
+	"github.com/offchainlabs/nitro/das/avail"
 	"github.com/offchainlabs/nitro/staker"
 	"github.com/offchainlabs/nitro/util/containers"
 )
@@ -33,28 +34,30 @@ var (
 )
 
 type InboxTracker struct {
-	db         ethdb.Database
-	txStreamer *TransactionStreamer
-	mutex      sync.Mutex
-	validator  *staker.BlockValidator
-	das        arbstate.DataAvailabilityReader
-	blobReader arbstate.BlobReader
+	db            ethdb.Database
+	txStreamer    *TransactionStreamer
+	mutex         sync.Mutex
+	validator     *staker.BlockValidator
+	das           arbstate.DataAvailabilityReader
+	availDAReader avail.DataAvailabilityReader
+	blobReader    arbstate.BlobReader
 
 	batchMetaMutex sync.Mutex
 	batchMeta      *containers.LruCache[uint64, BatchMetadata]
 }
 
-func NewInboxTracker(db ethdb.Database, txStreamer *TransactionStreamer, das arbstate.DataAvailabilityReader, blobReader arbstate.BlobReader) (*InboxTracker, error) {
+func NewInboxTracker(db ethdb.Database, txStreamer *TransactionStreamer, das arbstate.DataAvailabilityReader, availDAReader avail.DataAvailabilityReader, blobReader arbstate.BlobReader) (*InboxTracker, error) {
 	// We support a nil txStreamer for the pruning code
 	if txStreamer != nil && txStreamer.chainConfig.ArbitrumChainParams.DataAvailabilityCommittee && das == nil {
 		return nil, errors.New("data availability service required but unconfigured")
 	}
 	tracker := &InboxTracker{
-		db:         db,
-		txStreamer: txStreamer,
-		das:        das,
-		blobReader: blobReader,
-		batchMeta:  containers.NewLruCache[uint64, BatchMetadata](1000),
+		db:            db,
+		txStreamer:    txStreamer,
+		das:           das,
+		availDAReader: availDAReader,
+		blobReader:    blobReader,
+		batchMeta:     containers.NewLruCache[uint64, BatchMetadata](1000),
 	}
 	return tracker, nil
 }
@@ -610,6 +613,9 @@ func (t *InboxTracker) AddSequencerBatches(ctx context.Context, client arbutil.L
 	if t.das != nil {
 		daProviders = append(daProviders, arbstate.NewDAProviderDAS(t.das))
 	}
+	if t.availDAReader != nil {
+		daProviders = append(daProviders, arbstate.NewDAProviderAvail(t.availDAReader))
+	}
 	if t.blobReader != nil {
 		daProviders = append(daProviders, arbstate.NewDAProviderBlobReader(t.blobReader))
 	}

arbnode/node.go

@@ -33,6 +33,7 @@ import (
 	"github.com/offchainlabs/nitro/broadcaster"
 	"github.com/offchainlabs/nitro/cmd/chaininfo"
 	"github.com/offchainlabs/nitro/das"
+	"github.com/offchainlabs/nitro/das/avail"
 	"github.com/offchainlabs/nitro/execution"
 	"github.com/offchainlabs/nitro/execution/gethexec"
 	"github.com/offchainlabs/nitro/solgen/go/bridgegen"
@@ -87,6 +88,7 @@ type Config struct {
 	Staker              staker.L1ValidatorConfig    `koanf:"staker" reload:"hot"`
 	SeqCoordinator      SeqCoordinatorConfig        `koanf:"seq-coordinator"`
 	DataAvailability    das.DataAvailabilityConfig  `koanf:"data-availability"`
+	Avail               avail.DAConfig              `koanf:"avail"`
 	SyncMonitor         SyncMonitorConfig           `koanf:"sync-monitor"`
 	Dangerous           DangerousConfig             `koanf:"dangerous"`
 	TransactionStreamer TransactionStreamerConfig   `koanf:"transaction-streamer" reload:"hot"`
@@ -510,6 +512,8 @@ func createNodeImpl(
 	var daWriter das.DataAvailabilityServiceWriter
 	var daReader das.DataAvailabilityServiceReader
 	var dasLifecycleManager *das.LifecycleManager
+	var availDAWriter avail.DataAvailabilityWriter
+	var availDAReader avail.DataAvailabilityReader
 	if config.DataAvailability.Enable {
 		if config.BatchPoster.Enable {
 			daWriter, daReader, dasLifecycleManager, err = das.CreateBatchPosterDAS(ctx, &config.DataAvailability, dataSigner, l1client, deployInfo.SequencerInbox)
@@ -533,9 +537,16 @@ func createNodeImpl(
 		}
 	} else if l2Config.ArbitrumChainParams.DataAvailabilityCommittee {
 		return nil, errors.New("a data availability service is required for this chain, but it was not configured")
+	} else if config.Avail.Enable {
+		availService, err := avail.NewAvailDA(config.Avail, l1client)
+		if err != nil {
+			return nil, err
+		}
+		availDAWriter = availService
+		availDAReader = availService
 	}
 
-	inboxTracker, err := NewInboxTracker(arbDb, txStreamer, daReader, blobReader)
+	inboxTracker, err := NewInboxTracker(arbDb, txStreamer, daReader, availDAReader, blobReader)
 	if err != nil {
 		return nil, err
 	}
@@ -554,6 +565,7 @@ func createNodeImpl(
 			exec,
 			rawdb.NewTable(arbDb, storage.BlockValidatorPrefix),
 			daReader,
+			availDAReader,
 			blobReader,
 			func() *staker.BlockValidatorConfig { return &configFetcher.Get().BlockValidator },
 			stack,
@@ -672,6 +684,7 @@ func createNodeImpl(
 			DeployInfo:    deployInfo,
 			TransactOpts:  txOptsBatchPoster,
 			DAWriter:      daWriter,
+			AvailDAWriter: availDAWriter,
 			ParentChainID: parentChainID,
 		})
 		if err != nil {

arbstate/das_reader.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/offchainlabs/nitro/arbos/util"
 	"github.com/offchainlabs/nitro/blsSignatures"
+	"github.com/offchainlabs/nitro/das/avail"
 	"github.com/offchainlabs/nitro/das/dastree"
 )
 
@@ -24,6 +25,10 @@ type DataAvailabilityReader interface {
 	ExpirationPolicy(ctx context.Context) (ExpirationPolicy, error)
 }
 
+type AvailDataAvailibilityReader interface {
+	avail.DataAvailabilityReader
+}
+
 var ErrHashMismatch = errors.New("result does not match expected hash")
 
 // DASMessageHeaderFlag indicates that this data is a certificate for the data availability service,

arbstate/inbox.go

@@ -23,6 +23,7 @@ import (
 	"github.com/offchainlabs/nitro/arbos/arbostypes"
 	"github.com/offchainlabs/nitro/arbos/l1pricing"
 	"github.com/offchainlabs/nitro/arbutil"
+	"github.com/offchainlabs/nitro/das/avail"
 	"github.com/offchainlabs/nitro/das/dastree"
 	"github.com/offchainlabs/nitro/util/blobs"
 	"github.com/offchainlabs/nitro/zeroheavy"
@@ -165,6 +166,55 @@ func parseSequencerMessage(ctx context.Context, batchNum uint64, batchBlockHash
 	return parsedMsg, nil
 }
 
+func RecoverPayloadFromAvailBatch(ctx context.Context, batchNum uint64, sequencerMsg []byte, availDAReader avail.DataAvailabilityReader, preimages map[arbutil.PreimageType]map[common.Hash][]byte) ([]byte, error) {
+	var keccakPreimages map[common.Hash][]byte
+	if preimages != nil {
+		if preimages[arbutil.Keccak256PreimageType] == nil {
+			preimages[arbutil.Keccak256PreimageType] = make(map[common.Hash][]byte)
+		}
+		keccakPreimages = preimages[arbutil.Keccak256PreimageType]
+	}
+
+	buf := bytes.NewBuffer(sequencerMsg[40:])
+
+	header, err := buf.ReadByte()
+	if err != nil {
+		log.Error("Couldn't deserialize Avail header byte", "err", err)
+		return nil, err
+	}
+	if !avail.IsAvailMessageHeaderByte(header) {
+		return nil, errors.New("tried to deserialize a message that doesn't have the Avail header")
+	}
+
+	recordPreimage := func(key common.Hash, value []byte) {
+		keccakPreimages[key] = value
+	}
+
+	blobPointer := avail.BlobPointer{}
+	err = blobPointer.UnmarshalFromBinary(buf.Bytes())
+	if err != nil {
+		log.Error("Couldn't unmarshal Avail blob pointer", "err", err)
+		return nil, err
+	}
+
+	log.Info("Attempting to fetch data for", "batchNum", batchNum, "availBlockHash", blobPointer.BlockHash)
+	payload, err := availDAReader.Read(ctx, blobPointer)
+	if err != nil {
+		log.Error("Failed to resolve blob pointer from avail", "err", err)
+		return nil, err
+	}
+
+	log.Info("Succesfully fetched payload from Avail", "batchNum", batchNum, "availBlockHash", blobPointer.BlockHash)
+
+	log.Info("Recording Sha256 preimage for Avail data")
+
+	if keccakPreimages != nil {
+		log.Info("Data is being recorded into the orcale", "length", len(payload))
+		dastree.RecordHash(recordPreimage, payload)
+	}
+	return payload, nil
+}
+
 func RecoverPayloadFromDasBatch(
 	ctx context.Context,
 	batchNum uint64,
@@ -288,6 +338,31 @@ type DataAvailabilityProvider interface {
 	) ([]byte, error)
 }
 
+func NewDAProviderAvail(availDA AvailDataAvailibilityReader) *dAProviderForAvail {
+	return &dAProviderForAvail{
+		availDA: availDA,
+	}
+}
+
+type dAProviderForAvail struct {
+	availDA AvailDataAvailibilityReader
+}
+
+func (a *dAProviderForAvail) IsValidHeaderByte(headerByte byte) bool {
+	return avail.IsAvailMessageHeaderByte(headerByte)
+}
+
+func (a *dAProviderForAvail) RecoverPayloadFromBatch(
+	ctx context.Context,
+	batchNum uint64,
+	batchBlockHash common.Hash,
+	sequencerMsg []byte,
+	preimages map[arbutil.PreimageType]map[common.Hash][]byte,
+	keysetValidationMode KeysetValidationMode,
+) ([]byte, error) {
+	return RecoverPayloadFromAvailBatch(ctx, batchNum, sequencerMsg, a.availDA, preimages)
+}
+
 // NewDAProviderDAS is generally meant to be only used by nitro.
 // DA Providers should implement methods in the DataAvailabilityProvider interface independently
 func NewDAProviderDAS(das DataAvailabilityReader) *dAProviderForDAS {

cmd/pruning/pruning.go

@@ -189,7 +189,7 @@ func findImportantRoots(ctx context.Context, chainDb ethdb.Database, stack *node
 			return nil, fmt.Errorf("failed to get finalized block: %w", err)
 		}
 		l1BlockNum := l1Block.NumberU64()
-		tracker, err := arbnode.NewInboxTracker(arbDb, nil, nil, nil)
+		tracker, err := arbnode.NewInboxTracker(arbDb, nil, nil, nil, nil)
 		if err != nil {
 			return nil, err
 		}