allow non encoder to not load g2 list

bxue-l2 · Feb 3, 2024 · d2bfac0 · d2bfac0
1 parent 3189917
commit d2bfac0
Show file tree

Hide file tree

Showing 23 changed files with 83 additions and 80 deletions.
diff --git a/clients/tests/retrieval_client_test.go b/clients/tests/retrieval_client_test.go
@@ -34,7 +34,7 @@ func makeTestEncoder() (core.Encoder, error) {
 		NumWorker:       uint64(runtime.GOMAXPROCS(0)),
 	}
 
-	kzgEncoderGroup, err := kzgEncoder.NewKzgEncoderGroup(config)
+	kzgEncoderGroup, err := kzgEncoder.NewKzgEncoderGroup(config, true)
 	if err != nil {
 		return nil, err
 	}

diff --git a/core/encoding/encoder.go b/core/encoding/encoder.go
@@ -26,8 +26,8 @@ type Encoder struct {
 
 var _ core.Encoder = &Encoder{}
 
-func NewEncoder(config EncoderConfig) (*Encoder, error) {
-	kzgEncoderGroup, err := kzgEncoder.NewKzgEncoderGroup(&config.KzgConfig)
+func NewEncoder(config EncoderConfig, isEncoder bool) (*Encoder, error) {
+	kzgEncoderGroup, err := kzgEncoder.NewKzgEncoderGroup(&config.KzgConfig, isEncoder)
 	if err != nil {
 		return nil, err
 	}

diff --git a/core/encoding/encoder_test.go b/core/encoding/encoder_test.go
@@ -37,7 +37,7 @@ func makeTestEncoder() (core.Encoder, error) {
 		NumWorker:       uint64(runtime.GOMAXPROCS(0)),
 	}
 
-	return encoding.NewEncoder(encoding.EncoderConfig{KzgConfig: config})
+	return encoding.NewEncoder(encoding.EncoderConfig{KzgConfig: config}, true)
 }
 
 func TestEncoder(t *testing.T) {

diff --git a/core/test/core_test.go b/core/test/core_test.go
@@ -41,15 +41,15 @@ func setup(m *testing.M) {
 // makeTestEncoder makes an encoder currently using the only supported backend.
 func makeTestEncoder() (core.Encoder, error) {
 	config := kzgEncoder.KzgConfig{
-		G1Path:    "../../inabox/resources/kzg/g1.point",
-		G2Path:    "../../inabox/resources/kzg/g2.point",
-		CacheDir:  "../../inabox/resources/kzg/SRSTables",
-		SRSOrder:  3000,
+		G1Path:          "../../inabox/resources/kzg/g1.point",
+		G2Path:          "../../inabox/resources/kzg/g2.point",
+		CacheDir:        "../../inabox/resources/kzg/SRSTables",
+		SRSOrder:        3000,
 		SRSNumberToLoad: 3000,
-		NumWorker: uint64(runtime.GOMAXPROCS(0)),
+		NumWorker:       uint64(runtime.GOMAXPROCS(0)),
 	}
 
-	return encoding.NewEncoder(encoding.EncoderConfig{KzgConfig: config})
+	return encoding.NewEncoder(encoding.EncoderConfig{KzgConfig: config}, true)
 
 }
 
@@ -125,10 +125,10 @@ func prepareBatch(t *testing.T, cst core.IndexedChainState, blobs []core.Blob, q
 
 		blobHeader := &core.BlobHeader{
 			BlobCommitments: core.BlobCommitments{
-				Commitment:  commitments.Commitment,
+				Commitment:       commitments.Commitment,
 				LengthCommitment: commitments.LengthCommitment,
-				LengthProof: commitments.LengthProof,
-				Length:      commitments.Length,
+				LengthProof:      commitments.LengthProof,
+				Length:           commitments.Length,
 			},
 			QuorumInfos: []*core.BlobQuorumInfo{quorumHeader},
 		}

diff --git a/disperser/batcher/batcher_test.go b/disperser/batcher/batcher_test.go
@@ -43,15 +43,15 @@ type batcherComponents struct {
 // makeTestEncoder makes an encoder currently using the only supported backend.
 func makeTestEncoder() (core.Encoder, error) {
 	config := kzgEncoder.KzgConfig{
-		G1Path:    "../../inabox/resources/kzg/g1.point",
-		G2Path:    "../../inabox/resources/kzg/g2.point",
-		CacheDir:  "../../inabox/resources/kzg/SRSTables",
-		SRSOrder:  3000,
+		G1Path:          "../../inabox/resources/kzg/g1.point",
+		G2Path:          "../../inabox/resources/kzg/g2.point",
+		CacheDir:        "../../inabox/resources/kzg/SRSTables",
+		SRSOrder:        3000,
 		SRSNumberToLoad: 3000,
-		NumWorker: uint64(runtime.GOMAXPROCS(0)),
+		NumWorker:       uint64(runtime.GOMAXPROCS(0)),
 	}
 
-	return encoding.NewEncoder(encoding.EncoderConfig{KzgConfig: config})
+	return encoding.NewEncoder(encoding.EncoderConfig{KzgConfig: config}, true)
 }
 
 func makeTestBlob(securityParams []*core.SecurityParam) core.Blob {

diff --git a/disperser/cmd/encoder/encoder.go b/disperser/cmd/encoder/encoder.go
@@ -15,7 +15,7 @@ type EncoderGRPCServer struct {
 
 func NewEncoderGRPCServer(config Config, logger common.Logger) (*EncoderGRPCServer, error) {
 
-	coreEncoder, err := encoding.NewEncoder(config.EncoderConfig)
+	coreEncoder, err := encoding.NewEncoder(config.EncoderConfig, true)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create encoder: %w", err)
 	}

diff --git a/disperser/encoder/server_test.go b/disperser/encoder/server_test.go
@@ -31,17 +31,17 @@ var logger = &cmock.Logger{}
 
 func makeTestEncoder() (*encoding.Encoder, ServerConfig) {
 	kzgConfig := kzgEncoder.KzgConfig{
-		G1Path:    "../../inabox/resources/kzg/g1.point",
-		G2Path:    "../../inabox/resources/kzg/g2.point",
-		CacheDir:  "../../inabox/resources/kzg/SRSTables",
-		SRSOrder:  3000,
+		G1Path:          "../../inabox/resources/kzg/g1.point",
+		G2Path:          "../../inabox/resources/kzg/g2.point",
+		CacheDir:        "../../inabox/resources/kzg/SRSTables",
+		SRSOrder:        3000,
 		SRSNumberToLoad: 3000,
-		NumWorker: uint64(runtime.GOMAXPROCS(0)),
+		NumWorker:       uint64(runtime.GOMAXPROCS(0)),
 	}
 
 	encodingConfig := encoding.EncoderConfig{KzgConfig: kzgConfig}
 
-	encoder, _ := encoding.NewEncoder(encodingConfig)
+	encoder, _ := encoding.NewEncoder(encodingConfig, true)
 	encoderServerConfig := ServerConfig{
 		GrpcPort:              "3000",
 		MaxConcurrentRequests: 16,

diff --git a/inabox/tests/integration_suite_test.go b/inabox/tests/integration_suite_test.go
@@ -167,7 +167,7 @@ func setupRetrievalClient(testConfig *deploy.Config) error {
 			Verbose:         true,
 			PreloadEncoder:  false,
 		},
-	})
+	}, false)
 	if err != nil {
 		return err
 	}

diff --git a/node/grpc/server_test.go b/node/grpc/server_test.go
@@ -44,15 +44,15 @@ func TestMain(m *testing.M) {
 // makeTestEncoder makes an encoder currently using the only supported backend.
 func makeTestEncoder() (core.Encoder, error) {
 	config := kzgEncoder.KzgConfig{
-		G1Path:    "../../inabox/resources/kzg/g1.point.300000",
-		G2Path:    "../../inabox/resources/kzg/g2.point.300000",
-		CacheDir:  "../../inabox/resources/kzg/SRSTables",
-		SRSOrder:  300000,
+		G1Path:          "../../inabox/resources/kzg/g1.point.300000",
+		G2Path:          "../../inabox/resources/kzg/g2.point.300000",
+		CacheDir:        "../../inabox/resources/kzg/SRSTables",
+		SRSOrder:        300000,
 		SRSNumberToLoad: 300000,
-		NumWorker: uint64(runtime.GOMAXPROCS(0)),
+		NumWorker:       uint64(runtime.GOMAXPROCS(0)),
 	}
 
-	return encoding.NewEncoder(encoding.EncoderConfig{KzgConfig: config})
+	return encoding.NewEncoder(encoding.EncoderConfig{KzgConfig: config}, true)
 }
 
 func newTestServer(t *testing.T, mockValidator bool) *grpc.Server {

diff --git a/node/node.go b/node/node.go
@@ -113,7 +113,7 @@ func NewNode(config *Config, pubIPProvider pubip.Provider, logger common.Logger)
 	nodeApi := nodeapi.NewNodeApi(AppName, SemVer, ":"+config.NodeApiPort, logger)
 
 	// Make validator
-	enc, err := encoding.NewEncoder(config.EncoderConfig)
+	enc, err := encoding.NewEncoder(config.EncoderConfig, false)
 	if err != nil {
 		return nil, err
 	}

diff --git a/pkg/encoding/kzgEncoder/batchCommitEquivalence_test.go b/pkg/encoding/kzgEncoder/batchCommitEquivalence_test.go
@@ -14,7 +14,7 @@ func TestBatchEquivalence(t *testing.T) {
 	teardownSuite := setupSuite(t)
 	defer teardownSuite(t)
 
-	group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig)
+	group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig, true)
 	params := rs.GetEncodingParams(numSys, numPar, uint64(len(GETTYSBURG_ADDRESS_BYTES)))
 	enc, err := group.NewKzgEncoder(params)
 	require.Nil(t, err)

diff --git a/pkg/encoding/kzgEncoder/degree_test.go b/pkg/encoding/kzgEncoder/degree_test.go
@@ -13,7 +13,7 @@ func TestLengthProof(t *testing.T) {
 	teardownSuite := setupSuite(t)
 	defer teardownSuite(t)
 
-	group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig)
+	group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig, true)
 	params := rs.GetEncodingParams(numSys, numPar, uint64(len(GETTYSBURG_ADDRESS_BYTES)))
 	enc, err := group.NewKzgEncoder(params)
 	require.Nil(t, err)

diff --git a/pkg/encoding/kzgEncoder/encoder.go b/pkg/encoding/kzgEncoder/encoder.go
@@ -55,7 +55,7 @@ type KzgEncoder struct {
 	FFTPointsT [][]bls.G1Point // transpose of FFTPoints
 }
 
-func NewKzgEncoderGroup(config *KzgConfig) (*KzgEncoderGroup, error) {
+func NewKzgEncoderGroup(config *KzgConfig, isEncoder bool) (*KzgEncoderGroup, error) {
 
 	if config.SRSNumberToLoad > config.SRSOrder {
 		return nil, errors.New("SRSOrder is less than srsNumberToLoad")
@@ -67,10 +67,27 @@ func NewKzgEncoderGroup(config *KzgConfig) (*KzgEncoderGroup, error) {
 		log.Println("failed to read G1 points", err)
 		return nil, err
 	}
-	s2, err := utils.ReadG2Points(config.G2Path, config.SRSNumberToLoad, config.NumWorker)
-	if err != nil {
-		log.Println("failed to read G2 points", err)
-		return nil, err
+
+	s2 := make([]bls.G2Point, 0)
+	g2Trailing := make([]bls.G2Point, 0)
+
+	// PreloadEncoder is by default not used by operator node, PreloadEncoder
+	if isEncoder {
+		s2, err = utils.ReadG2Points(config.G2Path, config.SRSNumberToLoad, config.NumWorker)
+		if err != nil {
+			log.Println("failed to read G2 points", err)
+			return nil, err
+		}
+
+		g2Trailing, err = utils.ReadG2PointSection(
+			config.G2Path,
+			config.SRSOrder-config.SRSNumberToLoad,
+			config.SRSOrder, // last exclusive
+			config.NumWorker,
+		)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	srs, err := kzg.NewSrs(s1, s2)
@@ -79,16 +96,6 @@ func NewKzgEncoderGroup(config *KzgConfig) (*KzgEncoderGroup, error) {
 		return nil, err
 	}
 
-	g2Trailing, err := utils.ReadG2PointSection(
-		config.G2Path,
-		config.SRSOrder-config.SRSNumberToLoad,
-		config.SRSOrder, // last exclusive
-		config.NumWorker,
-	)
-	if err != nil {
-		return nil, err
-	}
-
 	fmt.Println("numthread", runtime.GOMAXPROCS(0))
 
 	encoderGroup := &KzgEncoderGroup{

diff --git a/pkg/encoding/kzgEncoder/encoder_fuzz_test.go b/pkg/encoding/kzgEncoder/encoder_fuzz_test.go
@@ -13,7 +13,7 @@ func FuzzOnlySystematic(f *testing.F) {
 	f.Add(GETTYSBURG_ADDRESS_BYTES)
 	f.Fuzz(func(t *testing.T, input []byte) {
 
-		group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig)
+		group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig, true)
 
 		params := rs.GetEncodingParams(10, 3, uint64(len(input)))
 		enc, err := group.NewKzgEncoder(params)

diff --git a/pkg/encoding/kzgEncoder/frame_test.go b/pkg/encoding/kzgEncoder/frame_test.go
@@ -16,7 +16,7 @@ func TestEncodeDecodeFrame_AreInverses(t *testing.T) {
 	teardownSuite := setupSuite(t)
 	defer teardownSuite(t)
 
-	group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig)
+	group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig, true)
 
 	params := rs.GetEncodingParams(numSys, numPar, uint64(len(GETTYSBURG_ADDRESS_BYTES)))
 
@@ -44,7 +44,7 @@ func TestVerify(t *testing.T) {
 	teardownSuite := setupSuite(t)
 	defer teardownSuite(t)
 
-	group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig)
+	group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig, true)
 
 	params := rs.GetEncodingParams(numSys, numPar, uint64(len(GETTYSBURG_ADDRESS_BYTES)))
 

diff --git a/pkg/encoding/kzgEncoder/multiframe_test.go b/pkg/encoding/kzgEncoder/multiframe_test.go
@@ -13,7 +13,7 @@ func TestUniversalVerify(t *testing.T) {
 	teardownSuite := setupSuite(t)
 	defer teardownSuite(t)
 
-	group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig)
+	group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig, true)
 	params := rs.GetEncodingParams(numSys, numPar, uint64(len(GETTYSBURG_ADDRESS_BYTES)))
 	enc, err := group.NewKzgEncoder(params)
 	require.Nil(t, err)

diff --git a/pkg/encoding/kzgEncoder/multiprover_test.go b/pkg/encoding/kzgEncoder/multiprover_test.go
@@ -14,7 +14,7 @@ func TestProveAllCosetThreads(t *testing.T) {
 	teardownSuite := setupSuite(t)
 	defer teardownSuite(t)
 
-	group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig)
+	group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig, true)
 
 	params := rs.GetEncodingParams(numSys, numPar, uint64(len(GETTYSBURG_ADDRESS_BYTES)))
 	enc, err := group.NewKzgEncoder(params)

diff --git a/pkg/encoding/kzgEncoder/zero_padding_test.go b/pkg/encoding/kzgEncoder/zero_padding_test.go
@@ -13,7 +13,7 @@ func TestProveZeroPadding(t *testing.T) {
 	teardownSuite := setupSuite(t)
 	defer teardownSuite(t)
 
-	group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig)
+	group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig, true)
 
 	params := rs.GetEncodingParams(numSys, numPar, uint64(len(GETTYSBURG_ADDRESS_BYTES)))
 	enc, err := group.NewKzgEncoder(params)

diff --git a/pkg/encoding/main.go b/pkg/encoding/main.go
@@ -34,7 +34,7 @@ func readpoints() {
 	}
 
 	// create encoding object
-	kzgGroup, _ := kzgRs.NewKzgEncoderGroup(kzgConfig)
+	kzgGroup, _ := kzgRs.NewKzgEncoderGroup(kzgConfig, true)
 	fmt.Println("there are ", len(kzgGroup.Srs.G1), "points")
 	for i := 0; i < len(kzgGroup.Srs.G1); i++ {
 		fmt.Printf("%v %v\n", i, string(kzgGroup.Srs.G1[i].MarshalText()))
@@ -66,7 +66,7 @@ func TestKzgRs() {
 	}
 
 	// create encoding object
-	kzgGroup, _ := kzgRs.NewKzgEncoderGroup(kzgConfig)
+	kzgGroup, _ := kzgRs.NewKzgEncoderGroup(kzgConfig, true)
 
 	params := rs.EncodingParams{NumChunks: 200, ChunkLen: 180}
 	enc, _ := kzgGroup.NewKzgEncoder(params)

diff --git a/pkg/kzg/srs.go b/pkg/kzg/srs.go
@@ -24,11 +24,9 @@
 
 package kzg
 
-import (
-	"errors"
+import bls "github.com/Layr-Labs/eigenda/pkg/kzg/bn254"
 
-	bls "github.com/Layr-Labs/eigenda/pkg/kzg/bn254"
-)
+//bls "github.com/Layr-Labs/eigenda/pkg/kzg/bn254"
 
 type SRS struct {
 
@@ -40,9 +38,6 @@ type SRS struct {
 
 func NewSrs(G1 []bls.G1Point, G2 []bls.G2Point) (*SRS, error) {
 
-	if len(G1) != len(G2) {
-		return nil, errors.New("secret list lengths don't match")
-	}
 	return &SRS{
 		G1: G1,
 		G2: G2,

diff --git a/retriever/cmd/main.go b/retriever/cmd/main.go
@@ -75,7 +75,7 @@ func RetrieverMain(ctx *cli.Context) error {
 	}
 
 	nodeClient := clients.NewNodeClient(config.Timeout)
-	encoder, err := encoding.NewEncoder(config.EncoderConfig)
+	encoder, err := encoding.NewEncoder(config.EncoderConfig, false)
 	if err != nil {
 		log.Fatalln("could not start tcp listener", err)
 	}

diff --git a/retriever/server_test.go b/retriever/server_test.go
@@ -32,15 +32,15 @@ var (
 
 func makeTestEncoder() (core.Encoder, error) {
 	config := &kzgEncoder.KzgConfig{
-		G1Path:    "../inabox/resources/kzg/g1.point",
-		G2Path:    "../inabox/resources/kzg/g2.point",
-		CacheDir:  "../inabox/resources/kzg/SRSTables",
-		SRSOrder:  3000,
-		SRSNumberToLoad:  3000,
-		NumWorker: uint64(runtime.GOMAXPROCS(0)),
+		G1Path:          "../inabox/resources/kzg/g1.point",
+		G2Path:          "../inabox/resources/kzg/g2.point",
+		CacheDir:        "../inabox/resources/kzg/SRSTables",
+		SRSOrder:        3000,
+		SRSNumberToLoad: 3000,
+		NumWorker:       uint64(runtime.GOMAXPROCS(0)),
 	}
 
-	kzgEncoderGroup, err := kzgEncoder.NewKzgEncoderGroup(config)
+	kzgEncoderGroup, err := kzgEncoder.NewKzgEncoderGroup(config, false)
 	if err != nil {
 		return nil, err
 	}

diff --git a/test/integration_test.go b/test/integration_test.go
@@ -76,16 +76,17 @@ func init() {
 // makeTestEncoder makes an encoder currently using the only supported backend.
 func mustMakeTestEncoder() core.Encoder {
 	config := kzgEncoder.KzgConfig{
-		G1Path:    "../inabox/resources/kzg/g1.point",
-		G2Path:    "../inabox/resources/kzg/g2.point",
-		CacheDir:  "../inabox/resources/kzg/SRSTables",
-		SRSOrder:  3000,
-		SRSNumberToLoad:  3000,
-		NumWorker: uint64(runtime.GOMAXPROCS(0)),
+		G1Path:          "../inabox/resources/kzg/g1.point",
+		G2Path:          "../inabox/resources/kzg/g2.point",
+		CacheDir:        "../inabox/resources/kzg/SRSTables",
+		SRSOrder:        3000,
+		SRSNumberToLoad: 3000,
+		NumWorker:       uint64(runtime.GOMAXPROCS(0)),
 	}
 
 	encoder, err := encoding.NewEncoder(
 		encoding.EncoderConfig{KzgConfig: config},
+		true,
 	)
 	if err != nil {
 		log.Fatalln("failed to initialize new encoder")