From 8a8123f630856214ef9342a6b7c1737ecdc9a986 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 30 Nov 2023 07:53:22 +0000 Subject: [PATCH] add batch verification parallelization, fix bug --- core/data.go | 8 +++--- core/encoding/encoder.go | 5 +--- core/validator.go | 37 ++++++++++++++++++------- pkg/encoding/kzgEncoder/multiframe.go | 40 +-------------------------- 4 files changed, 33 insertions(+), 57 deletions(-) diff --git a/core/data.go b/core/data.go index cd518e735..75fbce5e5 100644 --- a/core/data.go +++ b/core/data.go @@ -163,10 +163,10 @@ func (cb Bundles) Serialize() ([][][]byte, error) { // Sample is a chunk with associated metadata used by the Universal Batch Verifier type Sample struct { - Commitment *Commitment - Chunk *Chunk - EvalIndex ChunkNumber - BlobIndex int + Commitment *Commitment + Chunk *Chunk + AssignmentIndex ChunkNumber + BlobIndex int } // SubBatch is a part of the whole Batch with identical Encoding Parameters, i.e. (ChunkLen, NumChunk) diff --git a/core/encoding/encoder.go b/core/encoding/encoder.go index 241dd709e..ca76b84ec 100644 --- a/core/encoding/encoder.go +++ b/core/encoding/encoder.go @@ -2,7 +2,6 @@ package encoding import ( "crypto/sha256" - "fmt" "github.com/Layr-Labs/eigenda/core" "github.com/Layr-Labs/eigenda/pkg/encoding/encoder" @@ -61,7 +60,6 @@ func (e *Encoder) Encode(data []byte, params core.EncodingParams) (core.BlobComm } } encParams := toEncParams(params) - fmt.Println("encParams", encParams) enc, err := e.EncoderGroup.GetKzgEncoder(encParams) if err != nil { @@ -136,7 +134,6 @@ func (e *Encoder) VerifyChunks(chunks []*core.Chunk, indices []core.ChunkNumber, // convert struct understandable by the crypto library func (e *Encoder) UniversalVerifyChunks(params core.EncodingParams, samplesCore []core.Sample, numBlobs int) error { encParams := toEncParams(params) - samples := make([]kzgEncoder.Sample, len(samplesCore)) for i, sc := range samplesCore { @@ -145,7 +142,7 @@ func (e *Encoder) UniversalVerifyChunks(params core.EncodingParams, samplesCore Proof: sc.Chunk.Proof, Row: sc.BlobIndex, Coeffs: sc.Chunk.Coeffs, - X: sc.EvalIndex, + X: sc.AssignmentIndex, } samples[i] = sample } diff --git a/core/validator.go b/core/validator.go index 0497b45c8..a57550a0e 100644 --- a/core/validator.go +++ b/core/validator.go @@ -128,7 +128,6 @@ func (v *chunkValidator) UpdateOperatorID(operatorID OperatorID) { } func (v *chunkValidator) ValidateBatch(blobs []*BlobMessage, operatorState *OperatorState) error { - subBatchMap := make(map[EncodingParams]SubBatch) for i, blob := range blobs { @@ -155,18 +154,19 @@ func (v *chunkValidator) ValidateBatch(blobs []*BlobMessage, operatorState *Oper samples := make([]Sample, len(chunks)) for ind := range chunks { samples[ind] = Sample{ - Commitment: blob.BlobHeader.BlobCommitments.Commitment, - Chunk: chunks[ind], - EvalIndex: uint(indices[ind]), - BlobIndex: i, + Commitment: blob.BlobHeader.BlobCommitments.Commitment, + Chunk: chunks[ind], + AssignmentIndex: uint(indices[ind]), + BlobIndex: i, } } - // Add into subBatch subBatch, ok := subBatchMap[*params] if !ok { - subBatch.Samples = samples - subBatch.NumBlobs = 1 + subBatchMap[*params] = SubBatch{ + Samples: samples, + NumBlobs: 1, + } } else { subBatch.Samples = append(subBatch.Samples, samples...) subBatch.NumBlobs += 1 @@ -175,14 +175,31 @@ func (v *chunkValidator) ValidateBatch(blobs []*BlobMessage, operatorState *Oper } } - // ToDo add parallelization for verification for each subBatch + numSubBatch := len(subBatchMap) + out := make(chan error, numSubBatch) for params, subBatch := range subBatchMap { - err := v.encoder.UniversalVerifyChunks(params, subBatch.Samples, subBatch.NumBlobs) + params := params + subBatch := subBatch + go v.UniversalVerifyWorker(params, &subBatch, out) + } + + for i := 0; i < numSubBatch; i++ { + err := <-out if err != nil { return err } } return nil +} + +func (v *chunkValidator) UniversalVerifyWorker(params EncodingParams, subBatch *SubBatch, out chan error) { + + err := v.encoder.UniversalVerifyChunks(params, subBatch.Samples, subBatch.NumBlobs) + if err != nil { + out <- err + return + } + out <- nil } diff --git a/pkg/encoding/kzgEncoder/multiframe.go b/pkg/encoding/kzgEncoder/multiframe.go index f1e80e0e6..462b8fd66 100644 --- a/pkg/encoding/kzgEncoder/multiframe.go +++ b/pkg/encoding/kzgEncoder/multiframe.go @@ -55,10 +55,7 @@ func (group *KzgEncoderGroup) UniversalVerify(params rs.EncodingParams, samples D := params.ChunkLen n := len(samples) - - //rInt := uint64(22894) - //var r bls.Fr - //bls.AsFr(&r, rInt) + fmt.Printf("Batch verify %v frames of %v symbols out of %v blobs \n", n, params.ChunkLen, m) r, err := GenRandomness(params, samples, m) if err != nil { @@ -66,11 +63,8 @@ func (group *KzgEncoderGroup) UniversalVerify(params rs.EncodingParams, samples } randomsFr := make([]bls.Fr, n) - //bls.AsFr(&randomsFr[0], rInt) bls.CopyFr(&randomsFr[0], &r) - fmt.Println("random", r.String()) - // lhs var tmp bls.Fr @@ -85,7 +79,6 @@ func (group *KzgEncoderGroup) UniversalVerify(params rs.EncodingParams, samples bls.CopyG1(&proofs[i], &samples[i].Proof) } - fmt.Printf("len proof %v len ran %v\n", len(proofs), len(randomsFr)) // lhs g1 lhsG1 := bls.LinCombG1(proofs, randomsFr) @@ -98,9 +91,6 @@ func (group *KzgEncoderGroup) UniversalVerify(params rs.EncodingParams, samples // rhs g1 // get commitments commits := make([]bls.G1Point, m) - //for k := 0 ; k < n ; k++ { - // commits[k] = samples[k].Commitment - //} // get coeffs ftCoeffs := make([]bls.Fr, m) for k := 0; k < n; k++ { @@ -109,7 +99,6 @@ func (group *KzgEncoderGroup) UniversalVerify(params rs.EncodingParams, samples bls.AddModFr(&ftCoeffs[row], &ftCoeffs[row], &randomsFr[k]) bls.CopyG1(&commits[row], &s.Commitment) } - fmt.Printf("len commit %v len coeff %v\n", len(commits), len(ftCoeffs)) ftG1 := bls.LinCombG1(commits, ftCoeffs) @@ -169,30 +158,3 @@ func (group *KzgEncoderGroup) UniversalVerify(params rs.EncodingParams, samples return errors.New("Universal Verify Incorrect paring") } } - -//func SingleVerify(ks *kzg.KZGSettings, commitment *bls.G1Point, x *bls.Fr, coeffs []bls.Fr, proof bls.G1Point) bool { -// var xPow bls.Fr -// bls.CopyFr(&xPow, &bls.ONE) - -// var tmp bls.Fr -// for i := 0; i < len(coeffs); i++ { -// bls.MulModFr(&tmp, &xPow, x) -// bls.CopyFr(&xPow, &tmp) -// } - -// [x^n]_2 -// var xn2 bls.G2Point -// bls.MulG2(&xn2, &bls.GenG2, &xPow) - -// [s^n - x^n]_2 -// var xnMinusYn bls.G2Point -// bls.SubG2(&xnMinusYn, &ks.Srs.G2[len(coeffs)], &xn2) - -// [interpolation_polynomial(s)]_1 -// is1 := bls.LinCombG1(ks.Srs.G1[:len(coeffs)], coeffs) -// [commitment - interpolation_polynomial(s)]_1 = [commit]_1 - [interpolation_polynomial(s)]_1 -// var commitMinusInterpolation bls.G1Point -// bls.SubG1(&commitMinusInterpolation, commitment, is1) - -// return bls.PairingsVerify(&commitMinusInterpolation, &bls.GenG2, &proof, &xnMinusYn) -//}