-
Notifications
You must be signed in to change notification settings - Fork 1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* retrieve and save blobs during backfill * Update beacon-chain/sync/backfill/batch.go Co-authored-by: Preston Van Loon <[email protected]> * Update beacon-chain/sync/backfill/blobs.go Co-authored-by: Preston Van Loon <[email protected]> * Update beacon-chain/sync/backfill/metrics.go Co-authored-by: Preston Van Loon <[email protected]> * make blobSync initialization path a little safer * use bytes.Equal to avoid extra allocation * stop using log.Fatal and other little cleanups * wrap up post blob sync actions in batch mutator * unit test coverage on verifier --------- Co-authored-by: Kasey Kirkham <[email protected]> Co-authored-by: Preston Van Loon <[email protected]>
- Loading branch information
3 people
authored
Feb 14, 2024
1 parent
5de22d2
commit bb66201
Showing
18 changed files
with
690 additions
and
146 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| package backfill | ||
|
|
||
| import ( | ||
| "testing" | ||
|
|
||
| "github.com/prysmaticlabs/prysm/v4/consensus-types/primitives" | ||
| "github.com/prysmaticlabs/prysm/v4/testing/require" | ||
| ) | ||
|
|
||
| func TestSortBatchDesc(t *testing.T) { | ||
| orderIn := []primitives.Slot{100, 10000, 1} | ||
| orderOut := []primitives.Slot{10000, 100, 1} | ||
| batches := make([]batch, len(orderIn)) | ||
| for i := range orderIn { | ||
| batches[i] = batch{end: orderIn[i]} | ||
| } | ||
| sortBatchDesc(batches) | ||
| for i := range orderOut { | ||
| require.Equal(t, orderOut[i], batches[i].end) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,141 @@ | ||
| package backfill | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "context" | ||
|
|
||
| "github.com/pkg/errors" | ||
| "github.com/prysmaticlabs/prysm/v4/beacon-chain/das" | ||
| "github.com/prysmaticlabs/prysm/v4/beacon-chain/db/filesystem" | ||
| "github.com/prysmaticlabs/prysm/v4/beacon-chain/verification" | ||
| fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams" | ||
| "github.com/prysmaticlabs/prysm/v4/consensus-types/blocks" | ||
| "github.com/prysmaticlabs/prysm/v4/consensus-types/primitives" | ||
| "github.com/prysmaticlabs/prysm/v4/encoding/bytesutil" | ||
| ) | ||
|
|
||
| var ( | ||
| errUnexpectedResponseSize = errors.New("received more blobs than expected for the requested range") | ||
| errUnexpectedCommitment = errors.New("BlobSidecar commitment does not match block") | ||
| errUnexpectedResponseContent = errors.New("BlobSidecar response does not include expected values in expected order") | ||
| errBatchVerifierMismatch = errors.New("the list of blocks passed to the availability check does not match what was verified") | ||
| ) | ||
|
|
||
| type blobSummary struct { | ||
| blockRoot [32]byte | ||
| index uint64 | ||
| commitment [48]byte | ||
| signature [fieldparams.BLSSignatureLength]byte | ||
| } | ||
|
|
||
| type blobSyncConfig struct { | ||
| retentionStart primitives.Slot | ||
| nbv verification.NewBlobVerifier | ||
| store *filesystem.BlobStorage | ||
| } | ||
|
|
||
| func newBlobSync(current primitives.Slot, vbs verifiedROBlocks, cfg *blobSyncConfig) (*blobSync, error) { | ||
| expected, err := vbs.blobIdents(cfg.retentionStart) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| bbv := newBlobBatchVerifier(cfg.nbv) | ||
| as := das.NewLazilyPersistentStore(cfg.store, bbv) | ||
| return &blobSync{current: current, expected: expected, bbv: bbv, store: as}, nil | ||
| } | ||
|
|
||
| type blobVerifierMap map[[32]byte][fieldparams.MaxBlobsPerBlock]verification.BlobVerifier | ||
|
|
||
| type blobSync struct { | ||
| store das.AvailabilityStore | ||
| expected []blobSummary | ||
| next int | ||
| bbv *blobBatchVerifier | ||
| current primitives.Slot | ||
| } | ||
|
|
||
| func (bs *blobSync) blobsNeeded() int { | ||
| return len(bs.expected) - bs.next | ||
| } | ||
|
|
||
| func (bs *blobSync) validateNext(rb blocks.ROBlob) error { | ||
| if bs.next >= len(bs.expected) { | ||
| return errUnexpectedResponseSize | ||
| } | ||
| next := bs.expected[bs.next] | ||
| bs.next += 1 | ||
| // Get the super cheap verifications out of the way before we init a verifier. | ||
| if next.blockRoot != rb.BlockRoot() { | ||
| return errors.Wrapf(errUnexpectedResponseContent, "next expected root=%#x, saw=%#x", next.blockRoot, rb.BlockRoot()) | ||
| } | ||
| if next.index != rb.Index { | ||
| return errors.Wrapf(errUnexpectedResponseContent, "next expected root=%#x, saw=%#x for root=%#x", next.index, rb.Index, next.blockRoot) | ||
| } | ||
| if next.commitment != bytesutil.ToBytes48(rb.KzgCommitment) { | ||
| return errors.Wrapf(errUnexpectedResponseContent, "next expected commitment=%#x, saw=%#x for root=%#x", next.commitment, rb.KzgCommitment, rb.BlockRoot()) | ||
| } | ||
|
|
||
| if bytesutil.ToBytes96(rb.SignedBlockHeader.Signature) != next.signature { | ||
| return verification.ErrInvalidProposerSignature | ||
| } | ||
| v := bs.bbv.newVerifier(rb) | ||
| if err := v.BlobIndexInBounds(); err != nil { | ||
| return err | ||
| } | ||
| v.SatisfyRequirement(verification.RequireValidProposerSignature) | ||
| if err := v.SidecarInclusionProven(); err != nil { | ||
| return err | ||
| } | ||
| if err := v.SidecarKzgProofVerified(); err != nil { | ||
| return err | ||
| } | ||
| if err := bs.store.Persist(bs.current, rb); err != nil { | ||
| return err | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func newBlobBatchVerifier(nbv verification.NewBlobVerifier) *blobBatchVerifier { | ||
| return &blobBatchVerifier{newBlobVerifier: nbv, verifiers: make(blobVerifierMap)} | ||
| } | ||
|
|
||
| type blobBatchVerifier struct { | ||
| newBlobVerifier verification.NewBlobVerifier | ||
| verifiers blobVerifierMap | ||
| } | ||
|
|
||
| func (bbv *blobBatchVerifier) newVerifier(rb blocks.ROBlob) verification.BlobVerifier { | ||
| m := bbv.verifiers[rb.BlockRoot()] | ||
| m[rb.Index] = bbv.newBlobVerifier(rb, verification.BackfillSidecarRequirements) | ||
| bbv.verifiers[rb.BlockRoot()] = m | ||
| return m[rb.Index] | ||
| } | ||
|
|
||
| func (bbv blobBatchVerifier) VerifiedROBlobs(_ context.Context, blk blocks.ROBlock, _ []blocks.ROBlob) ([]blocks.VerifiedROBlob, error) { | ||
| m, ok := bbv.verifiers[blk.Root()] | ||
| if !ok { | ||
| return nil, errors.Wrapf(verification.ErrMissingVerification, "no record of verifiers for root %#x", blk.Root()) | ||
| } | ||
| c, err := blk.Block().Body().BlobKzgCommitments() | ||
| if err != nil { | ||
| return nil, errors.Wrapf(errUnexpectedCommitment, "error reading commitments from block root %#x", blk.Root()) | ||
| } | ||
| vbs := make([]blocks.VerifiedROBlob, len(c)) | ||
| for i := range c { | ||
| if m[i] == nil { | ||
| return nil, errors.Wrapf(errBatchVerifierMismatch, "do not have verifier for block root %#x idx %d", blk.Root(), i) | ||
| } | ||
| vb, err := m[i].VerifiedROBlob() | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| if !bytes.Equal(vb.KzgCommitment, c[i]) { | ||
| return nil, errors.Wrapf(errBatchVerifierMismatch, "commitments do not match, verified=%#x da check=%#x for root %#x", vb.KzgCommitment, c[i], vb.BlockRoot()) | ||
| } | ||
| vbs[i] = vb | ||
| } | ||
| return vbs, nil | ||
| } | ||
|
|
||
| var _ das.BlobBatchVerifier = &blobBatchVerifier{} |
Oops, something went wrong.