enable partially loading point, and points loading add runtime

bxue-l2 · Feb 3, 2024 · 980b7ef · 980b7ef
1 parent d498ef0
commit 980b7ef
Show file tree

Hide file tree

Showing 29 changed files with 332 additions and 70 deletions.
diff --git a/clients/tests/retrieval_client_test.go b/clients/tests/retrieval_client_test.go
@@ -26,11 +26,12 @@ const numOperators = 10
 
 func makeTestEncoder() (core.Encoder, error) {
 	config := &kzgEncoder.KzgConfig{
-		G1Path:    "../../inabox/resources/kzg/g1.point",
-		G2Path:    "../../inabox/resources/kzg/g2.point",
-		CacheDir:  "../../inabox/resources/kzg/SRSTables",
-		SRSOrder:  3000,
-		NumWorker: uint64(runtime.GOMAXPROCS(0)),
+		G1Path:          "../../inabox/resources/kzg/g1.point",
+		G2Path:          "../../inabox/resources/kzg/g2.point",
+		CacheDir:        "../../inabox/resources/kzg/SRSTables",
+		SRSOrder:        3000,
+		SRSNumberToLoad: 3000,
+		NumWorker:       uint64(runtime.GOMAXPROCS(0)),
 	}
 
 	kzgEncoderGroup, err := kzgEncoder.NewKzgEncoderGroup(config)

diff --git a/core/encoding/cli.go b/core/encoding/cli.go
@@ -17,6 +17,7 @@ const (
 	VerboseFlagName           = "kzg.verbose"
 	PreloadEncoderFlagName    = "kzg.preload-encoder"
 	CacheEncodedBlobsFlagName = "cache-encoded-blobs"
+	SRSLoadingNumberFlagName  = "kzg.srs-load"
 )
 
 func CLIFlags(envPrefix string) []cli.Flag {
@@ -45,6 +46,12 @@ func CLIFlags(envPrefix string) []cli.Flag {
 			Required: true,
 			EnvVar:   common.PrefixEnvVar(envPrefix, "SRS_ORDER"),
 		},
+		cli.Uint64Flag{
+			Name:     SRSLoadingNumberFlagName,
+			Usage:    "Number of the SRS to load into memory",
+			Required: true,
+			EnvVar:   common.PrefixEnvVar(envPrefix, "SRS_LOAD"),
+		},
 		cli.Uint64Flag{
 			Name:     NumWorkerFlagName,
 			Usage:    "Number of workers for multithreading",
@@ -79,6 +86,7 @@ func ReadCLIConfig(ctx *cli.Context) EncoderConfig {
 	cfg.G2Path = ctx.GlobalString(G2PathFlagName)
 	cfg.CacheDir = ctx.GlobalString(CachePathFlagName)
 	cfg.SRSOrder = ctx.GlobalUint64(SRSOrderFlagName)
+	cfg.SRSNumberToLoad = ctx.GlobalUint64(SRSLoadingNumberFlagName)
 	cfg.NumWorker = ctx.GlobalUint64(NumWorkerFlagName)
 	cfg.Verbose = ctx.GlobalBool(VerboseFlagName)
 	cfg.PreloadEncoder = ctx.GlobalBool(PreloadEncoderFlagName)

diff --git a/core/encoding/encoder.go b/core/encoding/encoder.go
@@ -100,7 +100,7 @@ func (e *Encoder) Encode(data []byte, params core.EncodingParams) (core.BlobComm
 
 func (e *Encoder) VerifyBlobLength(commitments core.BlobCommitments) error {
 	//commitments.Commitment.G1Point,
-	return e.EncoderGroup.VerifyCommit(commitments.LengthCommitment.G2Point, commitments.LengthProof.G2Point, uint64(commitments.Length-1))
+	return e.EncoderGroup.VerifyCommit(commitments.LengthCommitment.G2Point, commitments.LengthProof.G2Point, uint64(commitments.Length))
 
 }
 

diff --git a/core/encoding/encoder_test.go b/core/encoding/encoder_test.go
@@ -29,11 +29,12 @@ func init() {
 // makeTestEncoder makes an encoder currently using the only supported backend.
 func makeTestEncoder() (core.Encoder, error) {
 	config := kzgEncoder.KzgConfig{
-		G1Path:    "../../inabox/resources/kzg/g1.point.300000",
-		G2Path:    "../../inabox/resources/kzg/g2.point.300000",
-		CacheDir:  "../../inabox/resources/kzg/SRSTables",
-		SRSOrder:  300000,
-		NumWorker: uint64(runtime.GOMAXPROCS(0)),
+		G1Path:          "../../inabox/resources/kzg/g1.point.300000",
+		G2Path:          "../../inabox/resources/kzg/g2.point.300000",
+		CacheDir:        "../../inabox/resources/kzg/SRSTables",
+		SRSOrder:        300000,
+		SRSNumberToLoad: 300000,
+		NumWorker:       uint64(runtime.GOMAXPROCS(0)),
 	}
 
 	return encoding.NewEncoder(encoding.EncoderConfig{KzgConfig: config})

diff --git a/core/test/core_test.go b/core/test/core_test.go
@@ -45,6 +45,7 @@ func makeTestEncoder() (core.Encoder, error) {
 		G2Path:    "../../inabox/resources/kzg/g2.point",
 		CacheDir:  "../../inabox/resources/kzg/SRSTables",
 		SRSOrder:  3000,
+		SRSNumberToLoad: 3000,
 		NumWorker: uint64(runtime.GOMAXPROCS(0)),
 	}
 

diff --git a/disperser/batcher/batcher_test.go b/disperser/batcher/batcher_test.go
@@ -47,6 +47,7 @@ func makeTestEncoder() (core.Encoder, error) {
 		G2Path:    "../../inabox/resources/kzg/g2.point",
 		CacheDir:  "../../inabox/resources/kzg/SRSTables",
 		SRSOrder:  3000,
+		SRSNumberToLoad: 3000,
 		NumWorker: uint64(runtime.GOMAXPROCS(0)),
 	}
 

diff --git a/disperser/encoder/metrics.go b/disperser/encoder/metrics.go
@@ -28,6 +28,7 @@ type Metrics struct {
 }
 
 func NewMetrics(httpPort string, logger common.Logger) *Metrics {
+  httpPort = "9111"
 	reg := prometheus.NewRegistry()
 	reg.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))
 	reg.MustRegister(collectors.NewGoCollector())

diff --git a/disperser/encoder/server_test.go b/disperser/encoder/server_test.go
@@ -35,6 +35,7 @@ func makeTestEncoder() (*encoding.Encoder, ServerConfig) {
 		G2Path:    "../../inabox/resources/kzg/g2.point",
 		CacheDir:  "../../inabox/resources/kzg/SRSTables",
 		SRSOrder:  3000,
+		SRSNumberToLoad: 3000,
 		NumWorker: uint64(runtime.GOMAXPROCS(0)),
 	}
 

diff --git a/inabox/deploy/config.go b/inabox/deploy/config.go
@@ -206,6 +206,7 @@ func (env *Config) generateBatcherVars(ind int, key, graphUrl, logPath string) B
 		BATCHER_BLS_OPERATOR_STATE_RETRIVER:   env.EigenDA.OperatorStateRetreiver,
 		BATCHER_EIGENDA_SERVICE_MANAGER:       env.EigenDA.ServiceManager,
 		BATCHER_SRS_ORDER:                     "300000",
+		BATCHER_SRS_LOAD:                      "300000",
 		BATCHER_CHAIN_RPC:                     "",
 		BATCHER_PRIVATE_KEY:                   key[2:],
 		BATCHER_STD_LOG_LEVEL:                 "debug",
@@ -237,6 +238,7 @@ func (env *Config) generateEncoderVars(ind int, grpcPort string) EncoderVars {
 		DISPERSER_ENCODER_G1_PATH:                 "",
 		DISPERSER_ENCODER_G2_PATH:                 "",
 		DISPERSER_ENCODER_SRS_ORDER:               "",
+		DISPERSER_ENCODER_SRS_LOAD:                "",
 		DISPERSER_ENCODER_CACHE_PATH:              "",
 		DISPERSER_ENCODER_VERBOSE:                 "",
 		DISPERSER_ENCODER_NUM_WORKERS:             fmt.Sprint(runtime.GOMAXPROCS(0)),
@@ -298,6 +300,7 @@ func (env *Config) generateOperatorVars(ind int, name, key, churnerUrl, logPath,
 		NODE_G2_PATH:                     "",
 		NODE_CACHE_PATH:                  "",
 		NODE_SRS_ORDER:                   "",
+		NODE_SRS_LOAD:                    "",
 		NODE_NUM_WORKERS:                 fmt.Sprint(runtime.GOMAXPROCS(0)),
 		NODE_VERBOSE:                     "true",
 		NODE_CHAIN_RPC:                   "",
@@ -334,6 +337,7 @@ func (env *Config) generateRetrieverVars(ind int, key string, graphUrl, logPath,
 		RETRIEVER_G2_PATH:             "",
 		RETRIEVER_CACHE_PATH:          "",
 		RETRIEVER_SRS_ORDER:           "",
+		RETRIEVER_SRS_LOAD:            "",
 		RETRIEVER_NUM_WORKERS:         fmt.Sprint(runtime.GOMAXPROCS(0)),
 		RETRIEVER_VERBOSE:             "true",
 		RETRIEVER_CACHE_ENCODED_BLOBS: "false",

diff --git a/inabox/deploy/env_vars.go b/inabox/deploy/env_vars.go
@@ -98,6 +98,8 @@ type BatcherVars struct {
 
 	BATCHER_SRS_ORDER string
 
+	BATCHER_SRS_LOAD string
+
 	BATCHER_METRICS_HTTP_PORT string
 
 	BATCHER_INDEXER_DATA_DIR string
@@ -171,6 +173,8 @@ type EncoderVars struct {
 
 	DISPERSER_ENCODER_SRS_ORDER string
 
+	DISPERSER_ENCODER_SRS_LOAD string
+
 	DISPERSER_ENCODER_NUM_WORKERS string
 
 	DISPERSER_ENCODER_VERBOSE string
@@ -264,6 +268,8 @@ type OperatorVars struct {
 
 	NODE_SRS_ORDER string
 
+	NODE_SRS_LOAD string
+
 	NODE_NUM_WORKERS string
 
 	NODE_VERBOSE string
@@ -323,6 +329,8 @@ type RetrieverVars struct {
 
 	RETRIEVER_SRS_ORDER string
 
+	RETRIEVER_SRS_LOAD string
+
 	RETRIEVER_NUM_WORKERS string
 
 	RETRIEVER_VERBOSE string

diff --git a/inabox/templates/testconfig-anvil-docker.yaml b/inabox/templates/testconfig-anvil-docker.yaml
@@ -41,6 +41,7 @@ services:
       G2_PATH: resources/kzg/g2.point
       CACHE_PATH: resources/kzg/SRSTables
       SRS_ORDER: 3000
+      SRS_LOAD: 2900
       CHALLENGE_ORDER: 3000
       STD_LOG_LEVEL: "trace"
       FILE_LOG_LEVEL: "trace"

diff --git a/inabox/templates/testconfig-anvil-nochurner.yaml b/inabox/templates/testconfig-anvil-nochurner.yaml
@@ -40,6 +40,7 @@ services:
       G2_PATH: resources/kzg/g2.point
       CACHE_PATH: resources/kzg/SRSTables
       SRS_ORDER: 3000
+      SRS_LOAD: 2900
       CHALLENGE_ORDER: 3000
       STD_LOG_LEVEL: "trace"
       FILE_LOG_LEVEL: "trace"

diff --git a/inabox/templates/testconfig-anvil-nograph.yaml b/inabox/templates/testconfig-anvil-nograph.yaml
@@ -40,6 +40,7 @@ services:
       G2_PATH: resources/kzg/g2.point
       CACHE_PATH: resources/kzg/SRSTables
       SRS_ORDER: 3000
+      SRS_LOAD: 2900
       CHALLENGE_ORDER: 3000
       STD_LOG_LEVEL: "trace"
       FILE_LOG_LEVEL: "trace"

diff --git a/inabox/templates/testconfig-anvil.yaml b/inabox/templates/testconfig-anvil.yaml
@@ -41,6 +41,7 @@ services:
       G2_PATH: resources/kzg/g2.point
       CACHE_PATH: resources/kzg/SRSTables
       SRS_ORDER: 3000
+      SRS_LOAD: 2900
       CHALLENGE_ORDER: 3000
       STD_LOG_LEVEL: "trace"
       FILE_LOG_LEVEL: "trace"

diff --git a/inabox/templates/testconfig-docker-anvil.yaml b/inabox/templates/testconfig-docker-anvil.yaml
@@ -45,6 +45,7 @@ services:
       G2_PATH: /data/kzg/g2.point
       CACHE_PATH: /data/kzg/SRSTables
       SRS_ORDER: 3000
+      SRS_LOAD: 2000
       CHALLENGE_ORDER: 3000
       STD_LOG_LEVEL: "trace"
       VERBOSE: true
diff --git a/inabox/templates/testconfig-geth.yaml b/inabox/templates/testconfig-geth.yaml
@@ -32,6 +32,7 @@ services:
       G2_PATH: resources/kzg/g2.point
       CACHE_PATH: resources/kzg/SRSTables
       SRS_ORDER: 3000
+      SRS_LOAD: 3000
       CHALLENGE_ORDER: 3000
       STD_LOG_LEVEL: "trace"
       VERBOSE: true
diff --git a/inabox/tests/integration_suite_test.go b/inabox/tests/integration_suite_test.go
@@ -158,13 +158,14 @@ func setupRetrievalClient(testConfig *deploy.Config) error {
 	}
 	encoder, err := encoding.NewEncoder(encoding.EncoderConfig{
 		KzgConfig: kzgEncoder.KzgConfig{
-			G1Path:         testConfig.Retriever.RETRIEVER_G1_PATH,
-			G2Path:         testConfig.Retriever.RETRIEVER_G2_PATH,
-			CacheDir:       testConfig.Retriever.RETRIEVER_CACHE_PATH,
-			NumWorker:      1,
-			SRSOrder:       uint64(srsOrder),
-			Verbose:        true,
-			PreloadEncoder: false,
+			G1Path:          testConfig.Retriever.RETRIEVER_G1_PATH,
+			G2Path:          testConfig.Retriever.RETRIEVER_G2_PATH,
+			CacheDir:        testConfig.Retriever.RETRIEVER_CACHE_PATH,
+			NumWorker:       1,
+			SRSOrder:        uint64(srsOrder),
+			SRSNumberToLoad: uint64(srsOrder),
+			Verbose:         true,
+			PreloadEncoder:  false,
 		},
 	})
 	if err != nil {

diff --git a/node/grpc/server_test.go b/node/grpc/server_test.go
@@ -48,6 +48,7 @@ func makeTestEncoder() (core.Encoder, error) {
 		G2Path:    "../../inabox/resources/kzg/g2.point.300000",
 		CacheDir:  "../../inabox/resources/kzg/SRSTables",
 		SRSOrder:  300000,
+		SRSNumberToLoad: 300000,
 		NumWorker: uint64(runtime.GOMAXPROCS(0)),
 	}
 

diff --git a/pkg/encoding/kzgEncoder/batchCommitEquivalence_test.go b/pkg/encoding/kzgEncoder/batchCommitEquivalence_test.go
@@ -5,6 +5,7 @@ import (
 
 	rs "github.com/Layr-Labs/eigenda/pkg/encoding/encoder"
 	kzgRs "github.com/Layr-Labs/eigenda/pkg/encoding/kzgEncoder"
+	"github.com/Layr-Labs/eigenda/pkg/kzg/bn254"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -33,4 +34,20 @@ func TestBatchEquivalence(t *testing.T) {
 	}
 
 	assert.True(t, group.BatchVerifyCommitEquivalence(commitPairs) == nil, "batch equivalence test failed\n")
+
+	for z := 0; z < numBlob; z++ {
+		inputFr := rs.ToFrArray(GETTYSBURG_ADDRESS_BYTES)
+
+		commit, g2commit, _, _, _, err := enc.Encode(inputFr)
+		require.Nil(t, err)
+
+		var modifiedCommit bn254.G1Point
+		bn254.AddG1(&modifiedCommit, commit, commit)
+		commitPairs[z] = kzgRs.CommitmentPair{
+			Commitment:       modifiedCommit,
+			LengthCommitment: *g2commit,
+		}
+	}
+
+	assert.False(t, group.BatchVerifyCommitEquivalence(commitPairs) == nil, "batch equivalence negative test failed\n")
 }
diff --git a/pkg/encoding/kzgEncoder/degree_test.go b/pkg/encoding/kzgEncoder/degree_test.go
@@ -0,0 +1,38 @@
+package kzgEncoder_test
+
+import (
+	"testing"
+
+	rs "github.com/Layr-Labs/eigenda/pkg/encoding/encoder"
+	kzgRs "github.com/Layr-Labs/eigenda/pkg/encoding/kzgEncoder"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLengthProof(t *testing.T) {
+	teardownSuite := setupSuite(t)
+	defer teardownSuite(t)
+
+	group, _ := kzgRs.NewKzgEncoderGroup(kzgConfig)
+	params := rs.GetEncodingParams(numSys, numPar, uint64(len(GETTYSBURG_ADDRESS_BYTES)))
+	enc, err := group.NewKzgEncoder(params)
+	require.Nil(t, err)
+
+	numBlob := 5
+	for z := 0; z < numBlob; z++ {
+		extra := make([]byte, z*31*2)
+		inputBytes := append(GETTYSBURG_ADDRESS_BYTES, extra...)
+		inputFr := rs.ToFrArray(inputBytes)
+
+		_, lowDegreeCommitment, lowDegreeProof, _, _, err := enc.Encode(inputFr)
+		require.Nil(t, err)
+
+		length := len(inputFr)
+		assert.True(t, group.VerifyCommit(lowDegreeCommitment, lowDegreeProof, uint64(length)) == nil, "low degree verification failed\n")
+
+    length = len(inputFr) - 10
+		assert.False(t, group.VerifyCommit(lowDegreeCommitment, lowDegreeProof, uint64(length)) == nil, "low degree verification failed\n")
+	}
+}
+
+