Create a validator job

.gitignore

@@ -4,3 +4,4 @@
 .env
 api/bin
 archiver/bin
+validator/bin

Dockerfile

@@ -15,4 +15,5 @@ RUN make build
 FROM alpine:3.19
 
 COPY --from=builder /app/archiver/bin/blob-archiver /usr/local/bin/blob-archiver
-COPY --from=builder /app/api/bin/blob-api /usr/local/bin/blob-api
+COPY --from=builder /app/api/bin/blob-api /usr/local/bin/blob-api
+COPY --from=builder /app/validator/bin/blob-validator /usr/local/bin/blob-validator

Makefile

@@ -1,6 +1,7 @@
 build:
 	make -C ./archiver blob-archiver
 	make -C ./api blob-api
+	make -C ./validator blob-validator
 .PHONY: build
 
 build-docker:
@@ -10,11 +11,13 @@ build-docker:
 clean:
 	make -C ./archiver clean
 	make -C ./api clean
+	make -C ./validator clean
 .PHONY: clean
 
 test:
 	make -C ./archiver test
 	make -C ./api test
+	make -C ./validator test
 .PHONY: test
 
 integration:

common/beacon/beacontest/stub.go

@@ -3,6 +3,7 @@ package beacontest
 import (
 	"context"
 	"fmt"
+	"strconv"
 	"testing"
 
 	"github.com/attestantio/go-eth2-client/api"
@@ -58,8 +59,14 @@ func NewDefaultStubBeaconClient(t *testing.T) *StubBeaconClient {
 		}
 	}
 
-	headBlobs := blobtest.NewBlobSidecars(t, 6)
-	finalizedBlobs := blobtest.NewBlobSidecars(t, 4)
+	startSlot := blobtest.StartSlot
+
+	originBlobs := blobtest.NewBlobSidecars(t, 1)
+	oneBlobs := blobtest.NewBlobSidecars(t, 2)
+	twoBlobs := blobtest.NewBlobSidecars(t, 0)
+	threeBlobs := blobtest.NewBlobSidecars(t, 4)
+	fourBlobs := blobtest.NewBlobSidecars(t, 5)
+	fiveBlobs := blobtest.NewBlobSidecars(t, 6)
 
 	return &StubBeaconClient{
 		Headers: map[string]*v1.BeaconBlockHeader{
@@ -73,14 +80,25 @@ func NewDefaultStubBeaconClient(t *testing.T) *StubBeaconClient {
 			"finalized":                   makeHeader(13, blobtest.Three, blobtest.Two),
 		},
 		Blobs: map[string][]*deneb.BlobSidecar{
-			blobtest.OriginBlock.String(): blobtest.NewBlobSidecars(t, 1),
-			blobtest.One.String():         blobtest.NewBlobSidecars(t, 2),
-			blobtest.Two.String():         blobtest.NewBlobSidecars(t, 0),
-			blobtest.Three.String():       finalizedBlobs,
-			blobtest.Four.String():        blobtest.NewBlobSidecars(t, 5),
-			blobtest.Five.String():        headBlobs,
-			"head":                        headBlobs,
-			"finalized":                   finalizedBlobs,
+			// Lookup by hash
+			blobtest.OriginBlock.String(): originBlobs,
+			blobtest.One.String():         oneBlobs,
+			blobtest.Two.String():         twoBlobs,
+			blobtest.Three.String():       threeBlobs,
+			blobtest.Four.String():        fourBlobs,
+			blobtest.Five.String():        fiveBlobs,
+
+			// Lookup by identifier
+			"head":      fiveBlobs,
+			"finalized": threeBlobs,
+
+			// Lookup by slot
+			strconv.FormatUint(startSlot, 10):   originBlobs,
+			strconv.FormatUint(startSlot+1, 10): oneBlobs,
+			strconv.FormatUint(startSlot+2, 10): twoBlobs,
+			strconv.FormatUint(startSlot+3, 10): threeBlobs,
+			strconv.FormatUint(startSlot+4, 10): fourBlobs,
+			strconv.FormatUint(startSlot+5, 10): fiveBlobs,
 		},
 	}
 }

common/blobtest/helpers.go

@@ -17,6 +17,9 @@ var (
 	Three       = common.Hash{3}
 	Four        = common.Hash{4}
 	Five        = common.Hash{5}
+
+	StartSlot = uint64(10)
+	EndSlot   = uint64(15)
 )
 
 func RandBytes(t *testing.T, size uint) []byte {

validator/Makefile

@@ -0,0 +1,13 @@
+blob-validator:
+	env GO111MODULE=on GOOS=$(TARGETOS) GOARCH=$(TARGETARCH) go build -v $(LDFLAGS) -o ./bin/blob-validator ./cmd/main.go
+
+clean:
+	rm -f bin/blob-validator
+
+test:
+	go test -v -race ./...
+
+.PHONY: \
+	blob-validator \
+	clean \
+	test

validator/cmd/main.go

@@ -0,0 +1,64 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/base-org/blob-archiver/common/beacon"
+	"github.com/base-org/blob-archiver/validator/flags"
+	"github.com/base-org/blob-archiver/validator/service"
+	opservice "github.com/ethereum-optimism/optimism/op-service"
+	"github.com/ethereum-optimism/optimism/op-service/cliapp"
+	oplog "github.com/ethereum-optimism/optimism/op-service/log"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/urfave/cli/v2"
+)
+
+var (
+	Version   = "v0.0.1"
+	GitCommit = ""
+	GitDate   = ""
+)
+
+func main() {
+	oplog.SetupDefaults()
+
+	app := cli.NewApp()
+	app.Flags = cliapp.ProtectFlags(flags.Flags)
+	app.Version = opservice.FormatVersion(Version, GitCommit, GitDate, "")
+	app.Name = "blob-validator"
+	app.Usage = "Job that checks the validity of blobs"
+	app.Description = "The blob-validator is a job that checks the validity of blobs"
+	app.Action = cliapp.LifecycleCmd(Main())
+
+	err := app.Run(os.Args)
+	if err != nil {
+		log.Crit("Application failed", "message", err)
+	}
+}
+
+// Main is the entrypoint into the API.
+// This method returns a cliapp.LifecycleAction, to create an op-service CLI-lifecycle-managed API Server.
+func Main() cliapp.LifecycleAction {
+	return func(cliCtx *cli.Context, closeApp context.CancelCauseFunc) (cliapp.Lifecycle, error) {
+		cfg := flags.ReadConfig(cliCtx)
+		if err := cfg.Check(); err != nil {
+			return nil, fmt.Errorf("config check failed: %w", err)
+		}
+
+		l := oplog.NewLogger(oplog.AppOut(cliCtx), cfg.LogConfig)
+		oplog.SetGlobalLogHandler(l.GetHandler())
+		opservice.ValidateEnvVars(flags.EnvVarPrefix, flags.Flags, l)
+
+		headerClient, err := beacon.NewBeaconClient(cliCtx.Context, cfg.BeaconConfig)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create beacon client: %w", err)
+		}
+
+		beaconClient := service.NewBlobSidecarClient(cfg.BeaconConfig.BeaconURL)
+		blobClient := service.NewBlobSidecarClient(cfg.BeaconConfig.BeaconURL)
+
+		return service.NewValidator(l, headerClient, beaconClient, blobClient, closeApp), nil
+	}
+}

validator/flags/config.go

@@ -0,0 +1,44 @@
+package flags
+
+import (
+	"fmt"
+	"time"
+
+	common "github.com/base-org/blob-archiver/common/flags"
+	oplog "github.com/ethereum-optimism/optimism/op-service/log"
+	"github.com/urfave/cli/v2"
+)
+
+type ValidatorConfig struct {
+	LogConfig    oplog.CLIConfig
+	BeaconConfig common.BeaconConfig
+	BlobConfig   common.BeaconConfig
+}
+
+func (c ValidatorConfig) Check() error {
+	if err := c.BeaconConfig.Check(); err != nil {
+		return fmt.Errorf("beacon config check failed: %w", err)
+	}
+
+	if err := c.BlobConfig.Check(); err != nil {
+		return fmt.Errorf("blob config check failed: %w", err)
+	}
+
+	return nil
+}
+
+func ReadConfig(cliCtx *cli.Context) ValidatorConfig {
+	timeout, _ := time.ParseDuration(cliCtx.String(BeaconClientTimeoutFlag.Name))
+
+	return ValidatorConfig{
+		LogConfig: oplog.ReadCLIConfig(cliCtx),
+		BeaconConfig: common.BeaconConfig{
+			BeaconURL:           cliCtx.String(L1BeaconClientUrlFlag.Name),
+			BeaconClientTimeout: timeout,
+		},
+		BlobConfig: common.BeaconConfig{
+			BeaconURL:           cliCtx.String(BlobApiClientUrlFlag.Name),
+			BeaconClientTimeout: timeout,
+		},
+	}
+}

validator/flags/flags.go

@@ -0,0 +1,38 @@
+package flags
+
+import (
+	opservice "github.com/ethereum-optimism/optimism/op-service"
+	oplog "github.com/ethereum-optimism/optimism/op-service/log"
+	"github.com/urfave/cli/v2"
+)
+
+const EnvVarPrefix = "BLOB_VALIDATOR"
+
+var (
+	BeaconClientTimeoutFlag = &cli.StringFlag{
+		Name:    "beacon-client-timeout",
+		Usage:   "The timeout duration for the beacon client",
+		Value:   "10s",
+		EnvVars: opservice.PrefixEnvVar(EnvVarPrefix, "CLIENT_TIMEOUT"),
+	}
+	L1BeaconClientUrlFlag = &cli.StringFlag{
+		Name:     "l1-beacon-http",
+		Usage:    "URL for a L1 Beacon-node API",
+		Required: true,
+		EnvVars:  opservice.PrefixEnvVar(EnvVarPrefix, "L1_BEACON_HTTP"),
+	}
+	BlobApiClientUrlFlag = &cli.StringFlag{
+		Name:     "blob-api-http",
+		Usage:    "URL for a Blob API",
+		Required: true,
+		EnvVars:  opservice.PrefixEnvVar(EnvVarPrefix, "BLOB_API_HTTP"),
+	}
+)
+
+func init() {
+	Flags = append(Flags, oplog.CLIFlags(EnvVarPrefix)...)
+	Flags = append(Flags, BeaconClientTimeoutFlag, L1BeaconClientUrlFlag, BlobApiClientUrlFlag)
+}
+
+// Flags contains the list of configuration options available to the binary.
+var Flags []cli.Flag

validator/service/client.go

@@ -0,0 +1,85 @@
+package service
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+
+	"github.com/attestantio/go-eth2-client/api"
+	"github.com/base-org/blob-archiver/common/storage"
+)
+
+type Format string
+
+const (
+	// FormatJson instructs the client to request the response in JSON format
+	FormatJson Format = "application/json"
+	// FormatSSZ instructs the client to request the response in SSZ format
+	FormatSSZ Format = "application/octet-stream"
+)
+
+// BlobSidecarClient is a minimal client for fetching sidecars from the blob service. This client is used instead of an
+// existing client for two reasons.
+// 1) Does not require any endpoints except /eth/v1/blob_sidecar, which is the only endpoint that the Blob API supports
+// 2) Exposes implementation details, e.g. status code, as well as allowing us to specify the format
+type BlobSidecarClient interface {
+	// FetchSidecars fetches the sidecars for a given slot from the blob sidecar API. It returns the HTTP status code and
+	// the sidecars.
+	FetchSidecars(id string, format Format) (int, storage.BlobSidecars, error)
+}
+
+type httpBlobSidecarClient struct {
+	url    string
+	client *http.Client
+}
+
+// NewBlobSidecarClient creates a new BlobSidecarClient that fetches sidecars from the given URL.
+func NewBlobSidecarClient(url string) BlobSidecarClient {
+	return &httpBlobSidecarClient{
+		url:    url,
+		client: &http.Client{},
+	}
+}
+
+func (c *httpBlobSidecarClient) FetchSidecars(id string, format Format) (int, storage.BlobSidecars, error) {
+	url := fmt.Sprintf("%s/eth/v1/beacon/blob_sidecars/%s", c.url, id)
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return http.StatusInternalServerError, storage.BlobSidecars{}, fmt.Errorf("failed to create request: %w", err)
+	}
+
+	req.Header.Set("Accept", string(format))
+
+	response, err := c.client.Do(req)
+	if err != nil {
+		return http.StatusInternalServerError, storage.BlobSidecars{}, fmt.Errorf("failed to fetch sidecars: %w", err)
+	}
+
+	if response.StatusCode != http.StatusOK {
+		return response.StatusCode, storage.BlobSidecars{}, nil
+	}
+
+	defer response.Body.Close()
+
+	var sidecars storage.BlobSidecars
+	if format == FormatJson {
+		if err := json.NewDecoder(response.Body).Decode(&sidecars); err != nil {
+			return response.StatusCode, storage.BlobSidecars{}, fmt.Errorf("failed to decode json response: %w", err)
+		}
+	} else {
+		body, err := io.ReadAll(response.Body)
+		if err != nil {
+			return response.StatusCode, storage.BlobSidecars{}, fmt.Errorf("failed to read response: %w", err)
+		}
+
+		s := api.BlobSidecars{}
+		if err := s.UnmarshalSSZ(body); err != nil {
+			return response.StatusCode, storage.BlobSidecars{}, fmt.Errorf("failed to decode ssz response: %w", err)
+		}
+
+		sidecars.Data = s.Sidecars
+	}
+
+	return response.StatusCode, sidecars, nil
+}