-
Notifications
You must be signed in to change notification settings - Fork 1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
retrieve and save blobs during backfill
- Loading branch information
Showing
17 changed files
with
503 additions
and
80 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| package backfill | ||
|
|
||
| import ( | ||
| "testing" | ||
|
|
||
| "github.com/prysmaticlabs/prysm/v4/consensus-types/primitives" | ||
| "github.com/prysmaticlabs/prysm/v4/testing/require" | ||
| ) | ||
|
|
||
| func TestSortBatchDesc(t *testing.T) { | ||
| orderIn := []primitives.Slot{100, 10000, 1} | ||
| orderOut := []primitives.Slot{10000, 100, 1} | ||
| batches := make([]batch, len(orderIn)) | ||
| for i := range orderIn { | ||
| batches[i] = batch{end: orderIn[i]} | ||
| } | ||
| sortBatchDesc(batches) | ||
| for i := range orderOut { | ||
| require.Equal(t, orderOut[i], batches[i].end) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,134 @@ | ||
| package backfill | ||
|
|
||
| import ( | ||
| "context" | ||
|
|
||
| "github.com/pkg/errors" | ||
| "github.com/prysmaticlabs/prysm/v4/beacon-chain/das" | ||
| "github.com/prysmaticlabs/prysm/v4/beacon-chain/db/filesystem" | ||
| "github.com/prysmaticlabs/prysm/v4/beacon-chain/verification" | ||
| fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams" | ||
| "github.com/prysmaticlabs/prysm/v4/consensus-types/blocks" | ||
| "github.com/prysmaticlabs/prysm/v4/consensus-types/primitives" | ||
| "github.com/prysmaticlabs/prysm/v4/encoding/bytesutil" | ||
| ) | ||
|
|
||
| var ( | ||
| errUnexpectedResponseSize = errors.New("received more blobs than expected for the requested range") | ||
| errUnexpectedCommitment = errors.New("BlobSidecar commitment does not match block") | ||
| errUnexpectedResponseContent = errors.New("BlobSidecar response does not include expected values in expected order") | ||
| errBatchVerifierMismatch = errors.New("the list of blocks passed to the availability check does not match what was verified") | ||
| ) | ||
|
|
||
| type blobSummary struct { | ||
| blockRoot [32]byte | ||
| index uint64 | ||
| commitment [48]byte | ||
| signature [fieldparams.BLSSignatureLength]byte | ||
| } | ||
|
|
||
| func newBlobSync(cs, retentionStart primitives.Slot, vbs verifiedROBlocks, nbv verification.NewBlobVerifier, st *filesystem.BlobStorage) (*blobSync, error) { | ||
| todo, err := vbs.blobIdents(retentionStart) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| bbv := newBlobBatchVerifier(nbv) | ||
| as := das.NewLazilyPersistentStore(st, bbv) | ||
| return &blobSync{cs: cs, expected: todo, bbv: bbv, store: as}, nil | ||
| } | ||
|
|
||
| type blobVerifierMap map[[32]byte][fieldparams.MaxBlobsPerBlock]verification.BlobVerifier | ||
|
|
||
| type blobSync struct { | ||
| store das.AvailabilityStore | ||
| expected []blobSummary | ||
| next int | ||
| bbv *blobBatchVerifier | ||
| cs primitives.Slot | ||
| } | ||
|
|
||
| func (bs *blobSync) blobsNeeded() int { | ||
| return len(bs.expected) - bs.next | ||
| } | ||
|
|
||
| func (bs *blobSync) validateNext(rb blocks.ROBlob) error { | ||
| if bs.next >= len(bs.expected) { | ||
| return errUnexpectedResponseSize | ||
| } | ||
| next := bs.expected[bs.next] | ||
| bs.next += 1 | ||
| // Get the super cheap verifications out of the way before we init a verifier. | ||
| if next.blockRoot != rb.BlockRoot() { | ||
| return errors.Wrapf(errUnexpectedResponseContent, "next expected root=%#x, saw=%#x", next.blockRoot, rb.BlockRoot()) | ||
| } | ||
| if next.index != rb.Index { | ||
| return errors.Wrapf(errUnexpectedResponseContent, "next expected root=%#x, saw=%#x for root=%#x", next.index, rb.Index, next.blockRoot) | ||
| } | ||
| if next.commitment != bytesutil.ToBytes48(rb.KzgCommitment) { | ||
| return errors.Wrapf(errUnexpectedResponseContent, "next expected commitment=%#x, saw=%#x for root=%#x", next.commitment, rb.KzgCommitment, rb.BlockRoot()) | ||
| } | ||
|
|
||
| if bytesutil.ToBytes96(rb.SignedBlockHeader.Signature) != next.signature { | ||
| return verification.ErrInvalidProposerSignature | ||
| } | ||
| v := bs.bbv.newVerifier(rb) | ||
| if err := v.BlobIndexInBounds(); err != nil { | ||
| return err | ||
| } | ||
| v.SatisfyRequirement(verification.RequireValidProposerSignature) | ||
| if err := v.SidecarInclusionProven(); err != nil { | ||
| return err | ||
| } | ||
| if err := v.SidecarKzgProofVerified(); err != nil { | ||
| return err | ||
| } | ||
| if err := bs.store.Persist(bs.cs, rb); err != nil { | ||
| return err | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func newBlobBatchVerifier(nbv verification.NewBlobVerifier) *blobBatchVerifier { | ||
| return &blobBatchVerifier{newBlobVerifier: nbv, verifiers: make(blobVerifierMap)} | ||
| } | ||
|
|
||
| type blobBatchVerifier struct { | ||
| newBlobVerifier verification.NewBlobVerifier | ||
| verifiers blobVerifierMap | ||
| } | ||
|
|
||
| func (bbv *blobBatchVerifier) newVerifier(rb blocks.ROBlob) verification.BlobVerifier { | ||
| m := bbv.verifiers[rb.BlockRoot()] | ||
| m[rb.Index] = bbv.newBlobVerifier(rb, verification.BackfillSidecarRequirements) | ||
| bbv.verifiers[rb.BlockRoot()] = m | ||
| return m[rb.Index] | ||
| } | ||
|
|
||
| func (bbv blobBatchVerifier) VerifiedROBlobs(_ context.Context, blk blocks.ROBlock, _ []blocks.ROBlob) ([]blocks.VerifiedROBlob, error) { | ||
| m, ok := bbv.verifiers[blk.Root()] | ||
| if !ok { | ||
| return nil, errors.Wrapf(verification.ErrMissingVerification, "no record of verifiers for root %#x", blk.Root()) | ||
| } | ||
| c, err := blk.Block().Body().BlobKzgCommitments() | ||
| if err != nil { | ||
| return nil, errors.Wrapf(errUnexpectedCommitment, "error reading commitments from block root %#x", blk.Root()) | ||
| } | ||
| vbs := make([]blocks.VerifiedROBlob, len(c)) | ||
| for i := range c { | ||
| if m[i] == nil { | ||
| return nil, errors.Wrapf(errBatchVerifierMismatch, "do not have verifier for block root %#x idx %d", blk.Root(), i) | ||
| } | ||
| vb, err := m[i].VerifiedROBlob() | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| if bytesutil.ToBytes48(vb.KzgCommitment) != bytesutil.ToBytes48(c[i]) { | ||
| return nil, errors.Wrapf(errBatchVerifierMismatch, "commitments do not match, verified=%#x da check=%#x for root %#x", vb.KzgCommitment, c[i], vb.BlockRoot()) | ||
| } | ||
| vbs[i] = vb | ||
| } | ||
| return vbs, nil | ||
| } | ||
|
|
||
| var _ das.BlobBatchVerifier = &blobBatchVerifier{} |
Oops, something went wrong.