feat: add support for Blobstream API

blob/blob.go

@@ -17,20 +17,6 @@ import (
 
 var errEmptyShares = errors.New("empty shares")
 
-// Commitment is a Merkle Root of the subtree built from shares of the Blob.
-// It is computed by splitting the blob into shares and building the Merkle subtree to be included
-// after Submit.
-type Commitment []byte
-
-func (com Commitment) String() string {
-	return string(com)
-}
-
-// Equal ensures that commitments are the same
-func (com Commitment) Equal(c Commitment) bool {
-	return bytes.Equal(com, c)
-}
-
 // The Proof is a set of nmt proofs that can be verified only through
 // the included method (due to limitation of the nmt https://github.com/celestiaorg/nmt/issues/218).
 // Proof proves the WHOLE namespaced data to the row roots.

blob/commitment_proof.go

@@ -0,0 +1,134 @@
+package blob
+
+import (
+	"bytes"
+	"fmt"
+
+	coretypes "github.com/tendermint/tendermint/types"
+
+	"github.com/celestiaorg/celestia-app/pkg/appconsts"
+	"github.com/celestiaorg/celestia-app/pkg/shares"
+	"github.com/celestiaorg/nmt"
+	"github.com/celestiaorg/nmt/namespace"
+
+	"github.com/celestiaorg/celestia-node/share"
+)
+
+// Commitment is a Merkle Root of the subtree built from shares of the Blob.
+// It is computed by splitting the blob into shares and building the Merkle subtree to be included
+// after Submit.
+type Commitment []byte
+
+// CommitmentProof is an inclusion proof of a commitment to the data root.
+type CommitmentProof struct {
+	// SubtreeRoots are the subtree roots of the blob's data that are
+	// used to create the commitment.
+	SubtreeRoots [][]byte `json:"subtree_roots"`
+	// SubtreeRootProofs are the NMT proofs for the subtree roots
+	// to the row roots.
+	SubtreeRootProofs []*nmt.Proof `json:"subtree_root_proofs"`
+	// NamespaceID is the namespace id of the commitment being proven. This
+	// namespace id is used when verifying the proof. If the namespace id doesn't
+	// match the namespace of the shares, the proof will fail verification.
+	NamespaceID namespace.ID `json:"namespace_id"`
+	// RowProof is the proof of the rows containing the blob's data to the
+	// data root.
+	RowProof         coretypes.RowProof `json:"row_proof"`
+	NamespaceVersion uint8              `json:"namespace_version"`
+}
+
+func (com Commitment) String() string {
+	return string(com)
+}
+
+// Equal ensures that commitments are the same
+func (com Commitment) Equal(c Commitment) bool {
+	return bytes.Equal(com, c)
+}
+
+// Validate performs basic validation to the commitment proof.
+// Note: it doesn't verify if the proof is valid or not.
+// Check Verify() for that.
+func (commitmentProof *CommitmentProof) Validate() error {
+	if len(commitmentProof.SubtreeRoots) < len(commitmentProof.SubtreeRootProofs) {
+		return fmt.Errorf(
+			"the number of subtree roots %d should be bigger than the number of subtree root proofs %d",
+			len(commitmentProof.SubtreeRoots),
+			len(commitmentProof.SubtreeRootProofs),
+		)
+	}
+	if len(commitmentProof.SubtreeRootProofs) != len(commitmentProof.RowProof.Proofs) {
+		return fmt.Errorf(
+			"the number of subtree root proofs %d should be equal to the number of row root proofs %d",
+			len(commitmentProof.SubtreeRootProofs),
+			len(commitmentProof.RowProof.Proofs),
+		)
+	}
+	if int(commitmentProof.RowProof.EndRow-commitmentProof.RowProof.StartRow+1) != len(commitmentProof.RowProof.RowRoots) {
+		return fmt.Errorf(
+			"the number of rows %d must equal the number of row roots %d",
+			int(commitmentProof.RowProof.EndRow-commitmentProof.RowProof.StartRow+1),
+			len(commitmentProof.RowProof.RowRoots),
+		)
+	}
+	if len(commitmentProof.RowProof.Proofs) != len(commitmentProof.RowProof.RowRoots) {
+		return fmt.Errorf(
+			"the number of proofs %d must equal the number of row roots %d",
+			len(commitmentProof.RowProof.Proofs),
+			len(commitmentProof.RowProof.RowRoots),
+		)
+	}
+	return nil
+}
+
+// Verify verifies that a commitment proof is valid, i.e., the subtree roots commit
+// to some data that was posted to a square.
+// Expects the commitment proof to be properly formulated and validated
+// using the Validate() function.
+func (commitmentProof *CommitmentProof) Verify(root []byte, subtreeRootThreshold int) (bool, error) {
+	nmtHasher := nmt.NewNmtHasher(appconsts.NewBaseHashFunc(), share.NamespaceSize, true)
+
+	// computes the total number of shares proven.
+	numberOfShares := 0
+	for _, proof := range commitmentProof.SubtreeRootProofs {
+		numberOfShares += proof.End() - proof.Start()
+	}
+
+	// use the computed total number of shares to calculate the subtree roots
+	// width.
+	// the subtree roots width is defined in ADR-013:
+	//
+	//https://github.com/celestiaorg/celestia-app/blob/main/docs/architecture/adr-013-non-interactive-default-rules-for-zero-padding.md
+	subtreeRootsWidth := shares.SubTreeWidth(numberOfShares, subtreeRootThreshold)
+
+	// verify the proof of the subtree roots
+	subtreeRootsCursor := 0
+	for i, subtreeRootProof := range commitmentProof.SubtreeRootProofs {
+		// calculate the share range that each subtree root commits to.
+		ranges, err := nmt.ToLeafRanges(subtreeRootProof.Start(), subtreeRootProof.End(), subtreeRootsWidth)
+		if err != nil {
+			return false, err
+		}
+		valid, err := subtreeRootProof.VerifySubtreeRootInclusion(
+			nmtHasher,
+			commitmentProof.SubtreeRoots[subtreeRootsCursor:subtreeRootsCursor+len(ranges)],
+			subtreeRootsWidth,
+			commitmentProof.RowProof.RowRoots[i],
+		)
+		if err != nil {
+			return false, err
+		}
+		if !valid {
+			return false,
+				fmt.Errorf(
+					"subtree root proof for range [%d, %d) is invalid",
+					subtreeRootProof.Start(),
+					subtreeRootProof.End(),
+				)
+		}
+		subtreeRootsCursor += len(ranges)
+	}
+
+	// verify row roots to data root proof
+	return commitmentProof.RowProof.VerifyProof(root), nil
+}

blob/service.go

@@ -1,7 +1,9 @@
 package blob
 
 import (
+	bytes2 "bytes"
 	"context"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"slices"
@@ -14,7 +16,12 @@ import (
 	"go.opentelemetry.io/otel/codes"
 	"go.opentelemetry.io/otel/trace"
 
+	"github.com/celestiaorg/celestia-app/pkg/appconsts"
+	appns "github.com/celestiaorg/celestia-app/pkg/namespace"
+	pkgproof "github.com/celestiaorg/celestia-app/pkg/proof"
 	"github.com/celestiaorg/celestia-app/pkg/shares"
+	"github.com/celestiaorg/nmt"
+	"github.com/celestiaorg/rsmt2d"
 
 	"github.com/celestiaorg/celestia-node/header"
 	"github.com/celestiaorg/celestia-node/libs/utils"
@@ -387,3 +394,194 @@ func (s *Service) getBlobs(
 	_, _, err = s.retrieve(ctx, header.Height(), namespace, sharesParser)
 	return blobs, err
 }
+
+func (s *Service) GetCommitmentProof(
+	ctx context.Context,
+	height uint64,
+	namespace share.Namespace,
+	shareCommitment []byte,
+) (*CommitmentProof, error) {
+	log.Debugw("proving share commitment", "height", height, "commitment", shareCommitment, "namespace", namespace)
+	if height == 0 {
+		return nil, fmt.Errorf("height cannot be equal to 0")
+	}
+
+	// get the blob to compute the subtree roots
+	log.Debugw(
+		"getting the blob",
+		"height",
+		height,
+		"commitment",
+		shareCommitment,
+		"namespace",
+		namespace,
+	)
+	blb, err := s.Get(ctx, height, namespace, shareCommitment)
+	if err != nil {
+		return nil, err
+	}
+
+	log.Debugw(
+		"converting the blob to shares",
+		"height",
+		height,
+		"commitment",
+		shareCommitment,
+		"namespace",
+		namespace,
+	)
+	blobShares, err := BlobsToShares(blb)
+	if err != nil {
+		return nil, err
+	}
+	if len(blobShares) == 0 {
+		return nil, fmt.Errorf("the blob shares for commitment %s are empty", hex.EncodeToString(shareCommitment))
+	}
+
+	// get the extended header
+	log.Debugw(
+		"getting the extended header",
+		"height",
+		height,
+	)
+	extendedHeader, err := s.headerGetter(ctx, height)
+	if err != nil {
+		return nil, err
+	}
+
+	log.Debugw("getting eds", "height", height)
+	eds, err := s.shareGetter.GetEDS(ctx, extendedHeader)
+	if err != nil {
+		return nil, err
+	}
+
+	return ProveCommitment(eds, namespace, blobShares)
+}
+
+func ProveCommitment(
+	eds *rsmt2d.ExtendedDataSquare,
+	namespace share.Namespace,
+	blobShares []share.Share,
+) (*CommitmentProof, error) {
+	// find the blob shares in the EDS
+	blobSharesStartIndex := -1
+	for index, share := range eds.FlattenedODS() {
+		if bytes2.Equal(share, blobShares[0]) {
+			blobSharesStartIndex = index
+		}
+	}
+	if blobSharesStartIndex < 0 {
+		return nil, fmt.Errorf("couldn't find the blob shares in the ODS")
+	}
+
+	nID, err := appns.From(namespace)
+	if err != nil {
+		return nil, err
+	}
+
+	log.Debugw(
+		"generating the blob share proof for commitment",
+		"start_share",
+		blobSharesStartIndex,
+		"end_share",
+		blobSharesStartIndex+len(blobShares),
+	)
+	sharesProof, err := pkgproof.NewShareInclusionProofFromEDS(
+		eds,
+		nID,
+		shares.NewRange(blobSharesStartIndex, blobSharesStartIndex+len(blobShares)),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	// convert the shares to row root proofs to nmt proofs
+	nmtProofs := make([]*nmt.Proof, 0)
+	for _, proof := range sharesProof.ShareProofs {
+		nmtProof := nmt.NewInclusionProof(int(proof.Start),
+			int(proof.End),
+			proof.Nodes,
+			true)
+		nmtProofs = append(
+			nmtProofs,
+			&nmtProof,
+		)
+	}
+
+	// compute the subtree roots of the blob shares
+	log.Debugw("computing the subtree roots")
+	subtreeRoots := make([][]byte, 0)
+	dataCursor := 0
+	for _, proof := range nmtProofs {
+		// TODO: do we want directly use the default subtree root threshold
+		// or want to allow specifying which version to use?
+		ranges, err := nmt.ToLeafRanges(
+			proof.Start(),
+			proof.End(),
+			shares.SubTreeWidth(len(blobShares), appconsts.DefaultSubtreeRootThreshold),
+		)
+		if err != nil {
+			return nil, err
+		}
+		roots, err := computeSubtreeRoots(
+			blobShares[dataCursor:dataCursor+proof.End()-proof.Start()],
+			ranges,
+			proof.Start(),
+		)
+		if err != nil {
+			return nil, err
+		}
+		subtreeRoots = append(subtreeRoots, roots...)
+		dataCursor += proof.End() - proof.Start()
+	}
+
+	log.Debugw("successfully proved the share commitment")
+	commitmentProof := CommitmentProof{
+		SubtreeRoots:      subtreeRoots,
+		SubtreeRootProofs: nmtProofs,
+		NamespaceID:       namespace.ID(),
+		RowProof:          sharesProof.RowProof,
+		NamespaceVersion:  namespace.Version(),
+	}
+	return &commitmentProof, nil
+}
+
+// computeSubtreeRoots takes a set of shares and ranges and returns the corresponding subtree roots.
+// the offset is the number of shares that are before the subtree roots we're calculating.
+func computeSubtreeRoots(shares []share.Share, ranges []nmt.LeafRange, offset int) ([][]byte, error) {
+	if len(shares) == 0 {
+		return nil, fmt.Errorf("cannot compute subtree roots for an empty shares list")
+	}
+	if len(ranges) == 0 {
+		return nil, fmt.Errorf("cannot compute subtree roots for an empty ranges list")
+	}
+	if offset < 0 {
+		return nil, fmt.Errorf("the offset %d cannot be stricly negative", offset)
+	}
+
+	// create a tree containing the shares to generate their subtree roots
+	tree := nmt.New(
+		appconsts.NewBaseHashFunc(),
+		nmt.IgnoreMaxNamespace(true),
+		nmt.NamespaceIDSize(share.NamespaceSize),
+	)
+	for _, sh := range shares {
+		leafData := make([]byte, 0)
+		leafData = append(append(leafData, share.GetNamespace(sh)...), sh...)
+		err := tree.Push(leafData)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// generate the subtree roots
+	subtreeRoots := make([][]byte, 0)
+	for _, rg := range ranges {
+		root, err := tree.ComputeSubtreeRoot(rg.Start-offset, rg.End-offset)
+		if err != nil {
+			return nil, err
+		}
+		subtreeRoots = append(subtreeRoots, root)
+	}
+	return subtreeRoots, nil
+}