From bc29ea5e7057d4b3ef7a03c5808cddcd52b8c9f0 Mon Sep 17 00:00:00 2001 From: Nikita Kryuchkov Date: Thu, 5 Dec 2024 11:09:44 -0300 Subject: [PATCH 01/21] p2p: increase configured max peers if operator has many committees --- cli/operator/node.go | 20 ++++++++++++++++---- network/p2p/config.go | 2 +- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/cli/operator/node.go b/cli/operator/node.go index 39513ce0fe..245d10aa4b 100644 --- a/cli/operator/node.go +++ b/cli/operator/node.go @@ -162,6 +162,20 @@ var StartNodeCmd = &cobra.Command{ nodeStorage, operatorData := setupOperatorStorage(logger, db, operatorPrivKey, operatorPrivKeyText) operatorDataStore := operatordatastore.New(operatorData) + operatorCommittees := nodeStorage.ValidatorStore().OperatorCommittees(operatorData.ID) + // If operator has more than 4 committees, it needs more peers, so we need to override the value from config if it's too low. + // MaxPeers is used only in p2p, so the lines above must be executed before calling setupP2P function. + const committeeThresholdForPeerIncrease = 4 + const minRequiredPeers = 150 + if len(operatorCommittees) > committeeThresholdForPeerIncrease && cfg.P2pNetworkConfig.MaxPeers < minRequiredPeers { + logger.Warn("configured peer count is too low for this operator's committee count, increasing peer count", + zap.Int("configured_max_peers", cfg.P2pNetworkConfig.MaxPeers), + zap.Int("updated_max_peers", minRequiredPeers), + zap.Int("committee_threshold_for_peer_increase", minRequiredPeers), + ) + cfg.P2pNetworkConfig.MaxPeers = minRequiredPeers + } + usingLocalEvents := len(cfg.LocalEventsPath) != 0 if err := validateConfig(nodeStorage, networkConfig.NetworkName(), usingLocalEvents); err != nil { @@ -221,11 +235,9 @@ var StartNodeCmd = &cobra.Command{ signatureVerifier := signatureverifier.NewSignatureVerifier(nodeStorage) - validatorStore := nodeStorage.ValidatorStore() - messageValidator := validation.New( networkConfig, - validatorStore, + nodeStorage.ValidatorStore(), dutyStore, signatureVerifier, validation.WithLogger(logger), @@ -293,7 +305,7 @@ var StartNodeCmd = &cobra.Command{ validatorCtrl := validator.NewController(logger, cfg.SSVOptions.ValidatorOptions) cfg.SSVOptions.ValidatorController = validatorCtrl - cfg.SSVOptions.ValidatorStore = validatorStore + cfg.SSVOptions.ValidatorStore = nodeStorage.ValidatorStore() operatorNode = operator.New(logger, cfg.SSVOptions, slotTickerProvider, storageMap) diff --git a/network/p2p/config.go b/network/p2p/config.go index c659b610ea..fd949b74d5 100644 --- a/network/p2p/config.go +++ b/network/p2p/config.go @@ -46,7 +46,7 @@ type Config struct { RequestTimeout time.Duration `yaml:"RequestTimeout" env:"P2P_REQUEST_TIMEOUT" env-default:"10s"` MaxBatchResponse uint64 `yaml:"MaxBatchResponse" env:"P2P_MAX_BATCH_RESPONSE" env-default:"25" env-description:"Maximum number of returned objects in a batch"` - MaxPeers int `yaml:"MaxPeers" env:"P2P_MAX_PEERS" env-default:"60" env-description:"Connected peers limit for connections"` + MaxPeers int `yaml:"MaxPeers" env:"P2P_MAX_PEERS" env-default:"60" env-description:"Connected peers limit for connections. This value is applied if operator has 1-4 committees. For 5+ committees this value is overridden with 150 to make sure there are enough peers"` TopicMaxPeers int `yaml:"TopicMaxPeers" env:"P2P_TOPIC_MAX_PEERS" env-default:"10" env-description:"Connected peers limit per pubsub topic"` // Subnets is a static bit list of subnets that this node will register upon start. From cfb9e1604290338d0a560b8be9a425cf7a17efa4 Mon Sep 17 00:00:00 2001 From: Nikita Kryuchkov Date: Thu, 5 Dec 2024 11:50:45 -0300 Subject: [PATCH 02/21] fix a typo --- cli/operator/node.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/operator/node.go b/cli/operator/node.go index 245d10aa4b..5b21d4d833 100644 --- a/cli/operator/node.go +++ b/cli/operator/node.go @@ -164,7 +164,7 @@ var StartNodeCmd = &cobra.Command{ operatorCommittees := nodeStorage.ValidatorStore().OperatorCommittees(operatorData.ID) // If operator has more than 4 committees, it needs more peers, so we need to override the value from config if it's too low. - // MaxPeers is used only in p2p, so the lines above must be executed before calling setupP2P function. + // MaxPeers is used only in p2p, so the lines below must be executed before calling setupP2P function. const committeeThresholdForPeerIncrease = 4 const minRequiredPeers = 150 if len(operatorCommittees) > committeeThresholdForPeerIncrease && cfg.P2pNetworkConfig.MaxPeers < minRequiredPeers { From 7e04d75e85e55235c66afac1cc92d1561e1977f3 Mon Sep 17 00:00:00 2001 From: Nikita Kryuchkov Date: Thu, 5 Dec 2024 15:04:09 -0300 Subject: [PATCH 03/21] check unique committees length instead of all committees length --- cli/operator/node.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cli/operator/node.go b/cli/operator/node.go index 5b21d4d833..c4ae4d6ec1 100644 --- a/cli/operator/node.go +++ b/cli/operator/node.go @@ -163,11 +163,16 @@ var StartNodeCmd = &cobra.Command{ operatorDataStore := operatordatastore.New(operatorData) operatorCommittees := nodeStorage.ValidatorStore().OperatorCommittees(operatorData.ID) + // Currently, OperatorCommittees may return several committees with the same committee ID, so we need to filter the unique ones. + uniqueCommittees := make(map[[32]byte]struct{}) + for _, oc := range operatorCommittees { + uniqueCommittees[oc.ID] = struct{}{} + } // If operator has more than 4 committees, it needs more peers, so we need to override the value from config if it's too low. // MaxPeers is used only in p2p, so the lines below must be executed before calling setupP2P function. const committeeThresholdForPeerIncrease = 4 const minRequiredPeers = 150 - if len(operatorCommittees) > committeeThresholdForPeerIncrease && cfg.P2pNetworkConfig.MaxPeers < minRequiredPeers { + if len(uniqueCommittees) > committeeThresholdForPeerIncrease && cfg.P2pNetworkConfig.MaxPeers < minRequiredPeers { logger.Warn("configured peer count is too low for this operator's committee count, increasing peer count", zap.Int("configured_max_peers", cfg.P2pNetworkConfig.MaxPeers), zap.Int("updated_max_peers", minRequiredPeers), From 674f8ed56c61258389a351a7e04a7b210460a193 Mon Sep 17 00:00:00 2001 From: Nikita Kryuchkov Date: Thu, 5 Dec 2024 15:04:57 -0300 Subject: [PATCH 04/21] extend the comment --- cli/operator/node.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cli/operator/node.go b/cli/operator/node.go index c4ae4d6ec1..411f685452 100644 --- a/cli/operator/node.go +++ b/cli/operator/node.go @@ -164,6 +164,7 @@ var StartNodeCmd = &cobra.Command{ operatorCommittees := nodeStorage.ValidatorStore().OperatorCommittees(operatorData.ID) // Currently, OperatorCommittees may return several committees with the same committee ID, so we need to filter the unique ones. + // This might be a bug in validator store: https://github.com/ssvlabs/ssv/issues/1926 uniqueCommittees := make(map[[32]byte]struct{}) for _, oc := range operatorCommittees { uniqueCommittees[oc.ID] = struct{}{} From 3428458488de8e826cd864842e1befef5d25b646 Mon Sep 17 00:00:00 2001 From: Nikita Kryuchkov Date: Thu, 5 Dec 2024 16:35:37 -0300 Subject: [PATCH 05/21] fix committee_threshold_for_peer_increase log --- cli/operator/node.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/operator/node.go b/cli/operator/node.go index 411f685452..085335b1f3 100644 --- a/cli/operator/node.go +++ b/cli/operator/node.go @@ -177,7 +177,7 @@ var StartNodeCmd = &cobra.Command{ logger.Warn("configured peer count is too low for this operator's committee count, increasing peer count", zap.Int("configured_max_peers", cfg.P2pNetworkConfig.MaxPeers), zap.Int("updated_max_peers", minRequiredPeers), - zap.Int("committee_threshold_for_peer_increase", minRequiredPeers), + zap.Int("committee_threshold_for_peer_increase", committeeThresholdForPeerIncrease), ) cfg.P2pNetworkConfig.MaxPeers = minRequiredPeers } From a043cc9b6b44a5dc610e457b1ec1be70d35d8dd0 Mon Sep 17 00:00:00 2001 From: Nikita Kryuchkov Date: Thu, 5 Dec 2024 16:38:57 -0300 Subject: [PATCH 06/21] extend log --- cli/operator/node.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cli/operator/node.go b/cli/operator/node.go index 085335b1f3..7c73ec2f5d 100644 --- a/cli/operator/node.go +++ b/cli/operator/node.go @@ -178,6 +178,7 @@ var StartNodeCmd = &cobra.Command{ zap.Int("configured_max_peers", cfg.P2pNetworkConfig.MaxPeers), zap.Int("updated_max_peers", minRequiredPeers), zap.Int("committee_threshold_for_peer_increase", committeeThresholdForPeerIncrease), + zap.Int("unique_committees", len(uniqueCommittees)), ) cfg.P2pNetworkConfig.MaxPeers = minRequiredPeers } From df47ebfc9d8ad7cf7d0e8036c917302c0e68f029 Mon Sep 17 00:00:00 2001 From: Nikita Kryuchkov Date: Fri, 6 Dec 2024 12:43:36 -0300 Subject: [PATCH 07/21] pass CommitteeThresholdForPeerIncrease in config --- cli/operator/node.go | 5 ++--- network/p2p/config.go | 9 +++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cli/operator/node.go b/cli/operator/node.go index 7c73ec2f5d..63522df701 100644 --- a/cli/operator/node.go +++ b/cli/operator/node.go @@ -171,13 +171,12 @@ var StartNodeCmd = &cobra.Command{ } // If operator has more than 4 committees, it needs more peers, so we need to override the value from config if it's too low. // MaxPeers is used only in p2p, so the lines below must be executed before calling setupP2P function. - const committeeThresholdForPeerIncrease = 4 const minRequiredPeers = 150 - if len(uniqueCommittees) > committeeThresholdForPeerIncrease && cfg.P2pNetworkConfig.MaxPeers < minRequiredPeers { + if len(uniqueCommittees) > cfg.P2pNetworkConfig.CommitteeThresholdForPeerIncrease && cfg.P2pNetworkConfig.MaxPeers < minRequiredPeers { logger.Warn("configured peer count is too low for this operator's committee count, increasing peer count", zap.Int("configured_max_peers", cfg.P2pNetworkConfig.MaxPeers), zap.Int("updated_max_peers", minRequiredPeers), - zap.Int("committee_threshold_for_peer_increase", committeeThresholdForPeerIncrease), + zap.Int("committee_threshold_for_peer_increase", cfg.P2pNetworkConfig.CommitteeThresholdForPeerIncrease), zap.Int("unique_committees", len(uniqueCommittees)), ) cfg.P2pNetworkConfig.MaxPeers = minRequiredPeers diff --git a/network/p2p/config.go b/network/p2p/config.go index fd949b74d5..d6a8797a31 100644 --- a/network/p2p/config.go +++ b/network/p2p/config.go @@ -44,10 +44,11 @@ type Config struct { HostAddress string `yaml:"HostAddress" env:"HOST_ADDRESS" env-description:"External ip node is exposed for discovery"` HostDNS string `yaml:"HostDNS" env:"HOST_DNS" env-description:"External DNS node is exposed for discovery"` - RequestTimeout time.Duration `yaml:"RequestTimeout" env:"P2P_REQUEST_TIMEOUT" env-default:"10s"` - MaxBatchResponse uint64 `yaml:"MaxBatchResponse" env:"P2P_MAX_BATCH_RESPONSE" env-default:"25" env-description:"Maximum number of returned objects in a batch"` - MaxPeers int `yaml:"MaxPeers" env:"P2P_MAX_PEERS" env-default:"60" env-description:"Connected peers limit for connections. This value is applied if operator has 1-4 committees. For 5+ committees this value is overridden with 150 to make sure there are enough peers"` - TopicMaxPeers int `yaml:"TopicMaxPeers" env:"P2P_TOPIC_MAX_PEERS" env-default:"10" env-description:"Connected peers limit per pubsub topic"` + RequestTimeout time.Duration `yaml:"RequestTimeout" env:"P2P_REQUEST_TIMEOUT" env-default:"10s"` + MaxBatchResponse uint64 `yaml:"MaxBatchResponse" env:"P2P_MAX_BATCH_RESPONSE" env-default:"25" env-description:"Maximum number of returned objects in a batch"` + CommitteeThresholdForPeerIncrease int `yaml:"CommitteeThresholdForPeerIncrease" env:"P2P_COMMITTEE_THRESHOLD_FOR_PEER_INCREASE" env-default:"4" env-description:"Defines a threshold for number of committees an operator must belong to before forcing to increase the maximum number of P2P network peers. This ensures that operators participating in multiple committees have adequate peer connections for optimal network performance. Backwards compatibility is not guaranteed for this flag because it's an intermediate solution. Do not set this unless it's needed"` + MaxPeers int `yaml:"MaxPeers" env:"P2P_MAX_PEERS" env-default:"60" env-description:"Connected peers limit for connections. This value is applied if operator has between 1 and CommitteeThresholdForPeerIncrease (default 4) committees. For CommitteeThresholdForPeerIncrease+1 and more committees this value is overridden with 150 to make sure there are enough peers"` + TopicMaxPeers int `yaml:"TopicMaxPeers" env:"P2P_TOPIC_MAX_PEERS" env-default:"10" env-description:"Connected peers limit per pubsub topic"` // Subnets is a static bit list of subnets that this node will register upon start. Subnets string `yaml:"Subnets" env:"SUBNETS" env-description:"Hex string that represents the subnets that this node will join upon start"` From 6a4024fc2d5e8e1b920eb3fa9b913c0bfd4a7105 Mon Sep 17 00:00:00 2001 From: Nikita Kryuchkov Date: Fri, 6 Dec 2024 12:54:40 -0300 Subject: [PATCH 08/21] pass CommitteeThresholdForPeerIncrease in network config --- cli/operator/node.go | 5 +++-- network/p2p/config.go | 9 ++++----- networkconfig/config.go | 17 +++++++++-------- networkconfig/holesky-e2e.go | 15 ++++++++------- networkconfig/holesky-stage.go | 1 + networkconfig/holesky.go | 1 + networkconfig/mainnet.go | 1 + 7 files changed, 27 insertions(+), 22 deletions(-) diff --git a/cli/operator/node.go b/cli/operator/node.go index 63522df701..77d4cba8f5 100644 --- a/cli/operator/node.go +++ b/cli/operator/node.go @@ -172,11 +172,12 @@ var StartNodeCmd = &cobra.Command{ // If operator has more than 4 committees, it needs more peers, so we need to override the value from config if it's too low. // MaxPeers is used only in p2p, so the lines below must be executed before calling setupP2P function. const minRequiredPeers = 150 - if len(uniqueCommittees) > cfg.P2pNetworkConfig.CommitteeThresholdForPeerIncrease && cfg.P2pNetworkConfig.MaxPeers < minRequiredPeers { + thresholdForPeerIncrease := networkConfig.CommitteeThresholdForPeerIncrease + if len(uniqueCommittees) > thresholdForPeerIncrease && cfg.P2pNetworkConfig.MaxPeers < minRequiredPeers { logger.Warn("configured peer count is too low for this operator's committee count, increasing peer count", zap.Int("configured_max_peers", cfg.P2pNetworkConfig.MaxPeers), zap.Int("updated_max_peers", minRequiredPeers), - zap.Int("committee_threshold_for_peer_increase", cfg.P2pNetworkConfig.CommitteeThresholdForPeerIncrease), + zap.Int("committee_threshold_for_peer_increase", thresholdForPeerIncrease), zap.Int("unique_committees", len(uniqueCommittees)), ) cfg.P2pNetworkConfig.MaxPeers = minRequiredPeers diff --git a/network/p2p/config.go b/network/p2p/config.go index d6a8797a31..5b0ae40d90 100644 --- a/network/p2p/config.go +++ b/network/p2p/config.go @@ -44,11 +44,10 @@ type Config struct { HostAddress string `yaml:"HostAddress" env:"HOST_ADDRESS" env-description:"External ip node is exposed for discovery"` HostDNS string `yaml:"HostDNS" env:"HOST_DNS" env-description:"External DNS node is exposed for discovery"` - RequestTimeout time.Duration `yaml:"RequestTimeout" env:"P2P_REQUEST_TIMEOUT" env-default:"10s"` - MaxBatchResponse uint64 `yaml:"MaxBatchResponse" env:"P2P_MAX_BATCH_RESPONSE" env-default:"25" env-description:"Maximum number of returned objects in a batch"` - CommitteeThresholdForPeerIncrease int `yaml:"CommitteeThresholdForPeerIncrease" env:"P2P_COMMITTEE_THRESHOLD_FOR_PEER_INCREASE" env-default:"4" env-description:"Defines a threshold for number of committees an operator must belong to before forcing to increase the maximum number of P2P network peers. This ensures that operators participating in multiple committees have adequate peer connections for optimal network performance. Backwards compatibility is not guaranteed for this flag because it's an intermediate solution. Do not set this unless it's needed"` - MaxPeers int `yaml:"MaxPeers" env:"P2P_MAX_PEERS" env-default:"60" env-description:"Connected peers limit for connections. This value is applied if operator has between 1 and CommitteeThresholdForPeerIncrease (default 4) committees. For CommitteeThresholdForPeerIncrease+1 and more committees this value is overridden with 150 to make sure there are enough peers"` - TopicMaxPeers int `yaml:"TopicMaxPeers" env:"P2P_TOPIC_MAX_PEERS" env-default:"10" env-description:"Connected peers limit per pubsub topic"` + RequestTimeout time.Duration `yaml:"RequestTimeout" env:"P2P_REQUEST_TIMEOUT" env-default:"10s"` + MaxBatchResponse uint64 `yaml:"MaxBatchResponse" env:"P2P_MAX_BATCH_RESPONSE" env-default:"25" env-description:"Maximum number of returned objects in a batch"` + MaxPeers int `yaml:"MaxPeers" env:"P2P_MAX_PEERS" env-default:"60" env-description:"Connected peers limit for connections. This value is applied if operator has between 1 and CommitteeThresholdForPeerIncrease (default 4) committees. For CommitteeThresholdForPeerIncrease+1 and more committees this value is overridden with 150 to make sure there are enough peers"` + TopicMaxPeers int `yaml:"TopicMaxPeers" env:"P2P_TOPIC_MAX_PEERS" env-default:"10" env-description:"Connected peers limit per pubsub topic"` // Subnets is a static bit list of subnets that this node will register upon start. Subnets string `yaml:"Subnets" env:"SUBNETS" env-description:"Hex string that represents the subnets that this node will join upon start"` diff --git a/networkconfig/config.go b/networkconfig/config.go index 58a1faba7b..60356488cb 100644 --- a/networkconfig/config.go +++ b/networkconfig/config.go @@ -31,14 +31,15 @@ func GetNetworkConfigByName(name string) (NetworkConfig, error) { } type NetworkConfig struct { - Name string - Beacon beacon.BeaconNetwork - DomainType spectypes.DomainType - GenesisEpoch phase0.Epoch - RegistrySyncOffset *big.Int - RegistryContractAddr string // TODO: ethcommon.Address - Bootnodes []string - DiscoveryProtocolID [6]byte + Name string + Beacon beacon.BeaconNetwork + DomainType spectypes.DomainType + GenesisEpoch phase0.Epoch + RegistrySyncOffset *big.Int + RegistryContractAddr string // TODO: ethcommon.Address + Bootnodes []string + DiscoveryProtocolID [6]byte + CommitteeThresholdForPeerIncrease int } func (n NetworkConfig) String() string { diff --git a/networkconfig/holesky-e2e.go b/networkconfig/holesky-e2e.go index 0f962b83ff..1a048d466b 100644 --- a/networkconfig/holesky-e2e.go +++ b/networkconfig/holesky-e2e.go @@ -9,11 +9,12 @@ import ( ) var HoleskyE2E = NetworkConfig{ - Name: "holesky-e2e", - Beacon: beacon.NewNetwork(spectypes.HoleskyNetwork), - DomainType: spectypes.DomainType{0x0, 0x0, 0xee, 0x1}, - GenesisEpoch: 1, - RegistryContractAddr: "0x58410bef803ecd7e63b23664c586a6db72daf59c", - RegistrySyncOffset: big.NewInt(405579), - Bootnodes: []string{}, + Name: "holesky-e2e", + Beacon: beacon.NewNetwork(spectypes.HoleskyNetwork), + DomainType: spectypes.DomainType{0x0, 0x0, 0xee, 0x1}, + GenesisEpoch: 1, + RegistryContractAddr: "0x58410bef803ecd7e63b23664c586a6db72daf59c", + RegistrySyncOffset: big.NewInt(405579), + Bootnodes: []string{}, + CommitteeThresholdForPeerIncrease: 7, } diff --git a/networkconfig/holesky-stage.go b/networkconfig/holesky-stage.go index a8a8c90d31..0f3ba1d080 100644 --- a/networkconfig/holesky-stage.go +++ b/networkconfig/holesky-stage.go @@ -23,4 +23,5 @@ var HoleskyStage = NetworkConfig{ // Private bootnode: "enr:-Ja4QDRUBjWOvVfGxpxvv3FqaCy3psm7IsKu5ETb1GXiexGYDFppD33t7AHRfmQddoAkBiyb7pt4t7ZN0sNB9CsW4I-GAZGOmChMgmlkgnY0gmlwhAorXxuJc2VjcDI1NmsxoQP_bBE-ZYvaXKBR3dRYMN5K_lZP-q-YsBzDZEtxH_4T_YNzc3YBg3RjcIITioN1ZHCCD6I", }, + CommitteeThresholdForPeerIncrease: 7, } diff --git a/networkconfig/holesky.go b/networkconfig/holesky.go index f91cd78e4d..7e79f2acc1 100644 --- a/networkconfig/holesky.go +++ b/networkconfig/holesky.go @@ -19,4 +19,5 @@ var Holesky = NetworkConfig{ Bootnodes: []string{ "enr:-Li4QFIQzamdvTxGJhvcXG_DFmCeyggSffDnllY5DiU47pd_K_1MRnSaJimWtfKJ-MD46jUX9TwgW5Jqe0t4pH41RYWGAYuFnlyth2F0dG5ldHOIAAAAAAAAAACEZXRoMpD1pf1CAAAAAP__________gmlkgnY0gmlwhCLdu_SJc2VjcDI1NmsxoQN4v-N9zFYwEqzGPBBX37q24QPFvAVUtokIo1fblIsmTIN0Y3CCE4uDdWRwgg-j", }, + CommitteeThresholdForPeerIncrease: 7, } diff --git a/networkconfig/mainnet.go b/networkconfig/mainnet.go index b5886394c7..651416d410 100644 --- a/networkconfig/mainnet.go +++ b/networkconfig/mainnet.go @@ -29,4 +29,5 @@ var Mainnet = NetworkConfig{ // CryptoManufaktur "enr:-Li4QH7FwJcL8gJj0zHAITXqghMkG-A5bfWh2-3Q7vosy9D1BS8HZk-1ITuhK_rfzG3v_UtBDI6uNJZWpdcWfrQFCxKGAYnQ1DRCh2F0dG5ldHOIAAAAAAAAAACEZXRoMpD1pf1CAAAAAP__________gmlkgnY0gmlwhBLb3g2Jc2VjcDI1NmsxoQKeSDcZWSaY9FC723E9yYX1Li18bswhLNlxBZdLfgOKp4N0Y3CCE4mDdWRwgg-h", }, + CommitteeThresholdForPeerIncrease: 4, } From bf94d86cea26e033b2a3d85358a640759e13cc9b Mon Sep 17 00:00:00 2001 From: Nikita Kryuchkov Date: Fri, 6 Dec 2024 13:03:59 -0300 Subject: [PATCH 09/21] ignore 0 threshold --- cli/operator/node.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/operator/node.go b/cli/operator/node.go index 77d4cba8f5..e592417fa0 100644 --- a/cli/operator/node.go +++ b/cli/operator/node.go @@ -172,12 +172,12 @@ var StartNodeCmd = &cobra.Command{ // If operator has more than 4 committees, it needs more peers, so we need to override the value from config if it's too low. // MaxPeers is used only in p2p, so the lines below must be executed before calling setupP2P function. const minRequiredPeers = 150 - thresholdForPeerIncrease := networkConfig.CommitteeThresholdForPeerIncrease - if len(uniqueCommittees) > thresholdForPeerIncrease && cfg.P2pNetworkConfig.MaxPeers < minRequiredPeers { + th := networkConfig.CommitteeThresholdForPeerIncrease + if th > 0 && len(uniqueCommittees) > th && cfg.P2pNetworkConfig.MaxPeers < minRequiredPeers { logger.Warn("configured peer count is too low for this operator's committee count, increasing peer count", zap.Int("configured_max_peers", cfg.P2pNetworkConfig.MaxPeers), zap.Int("updated_max_peers", minRequiredPeers), - zap.Int("committee_threshold_for_peer_increase", thresholdForPeerIncrease), + zap.Int("committee_threshold_for_peer_increase", th), zap.Int("unique_committees", len(uniqueCommittees)), ) cfg.P2pNetworkConfig.MaxPeers = minRequiredPeers From 67180d399d44f9ea74e2da43692fb26abf1379e6 Mon Sep 17 00:00:00 2001 From: Nikita Kryuchkov Date: Fri, 6 Dec 2024 13:09:19 -0300 Subject: [PATCH 10/21] fix a comment --- cli/operator/node.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cli/operator/node.go b/cli/operator/node.go index e592417fa0..390987e0b6 100644 --- a/cli/operator/node.go +++ b/cli/operator/node.go @@ -169,7 +169,8 @@ var StartNodeCmd = &cobra.Command{ for _, oc := range operatorCommittees { uniqueCommittees[oc.ID] = struct{}{} } - // If operator has more than 4 committees, it needs more peers, so we need to override the value from config if it's too low. + // If operator has more than CommitteeThresholdForPeerIncrease committees, + // it needs more peers, so we need to override the value from config if it's too low. // MaxPeers is used only in p2p, so the lines below must be executed before calling setupP2P function. const minRequiredPeers = 150 th := networkConfig.CommitteeThresholdForPeerIncrease From 10d6a359ecb5df57c7e099248e0adf04ff8ae8dc Mon Sep 17 00:00:00 2001 From: moshe-blox <89339422+moshe-blox@users.noreply.github.com> Date: Mon, 9 Dec 2024 15:26:31 +0200 Subject: [PATCH 11/21] feat(p2p): more subnets more peers (#1929) * wip * subnets instead of committees * fix --- cli/operator/node.go | 42 +++++++++++++++++++--------------- network/p2p/metrics.go | 38 +++++++++++++++++++++++++++--- network/p2p/p2p.go | 2 +- networkconfig/config.go | 17 +++++++------- networkconfig/holesky-e2e.go | 15 ++++++------ networkconfig/holesky-stage.go | 1 - networkconfig/holesky.go | 1 - networkconfig/mainnet.go | 1 - 8 files changed, 74 insertions(+), 43 deletions(-) diff --git a/cli/operator/node.go b/cli/operator/node.go index 390987e0b6..275a17f23a 100644 --- a/cli/operator/node.go +++ b/cli/operator/node.go @@ -43,7 +43,9 @@ import ( "github.com/ssvlabs/ssv/monitoring/metrics" "github.com/ssvlabs/ssv/monitoring/metricsreporter" "github.com/ssvlabs/ssv/network" + networkcommons "github.com/ssvlabs/ssv/network/commons" p2pv1 "github.com/ssvlabs/ssv/network/p2p" + "github.com/ssvlabs/ssv/network/records" "github.com/ssvlabs/ssv/networkconfig" "github.com/ssvlabs/ssv/nodeprobe" "github.com/ssvlabs/ssv/operator" @@ -162,26 +164,28 @@ var StartNodeCmd = &cobra.Command{ nodeStorage, operatorData := setupOperatorStorage(logger, db, operatorPrivKey, operatorPrivKeyText) operatorDataStore := operatordatastore.New(operatorData) - operatorCommittees := nodeStorage.ValidatorStore().OperatorCommittees(operatorData.ID) - // Currently, OperatorCommittees may return several committees with the same committee ID, so we need to filter the unique ones. - // This might be a bug in validator store: https://github.com/ssvlabs/ssv/issues/1926 - uniqueCommittees := make(map[[32]byte]struct{}) - for _, oc := range operatorCommittees { - uniqueCommittees[oc.ID] = struct{}{} - } - // If operator has more than CommitteeThresholdForPeerIncrease committees, - // it needs more peers, so we need to override the value from config if it's too low. - // MaxPeers is used only in p2p, so the lines below must be executed before calling setupP2P function. - const minRequiredPeers = 150 - th := networkConfig.CommitteeThresholdForPeerIncrease - if th > 0 && len(uniqueCommittees) > th && cfg.P2pNetworkConfig.MaxPeers < minRequiredPeers { - logger.Warn("configured peer count is too low for this operator's committee count, increasing peer count", - zap.Int("configured_max_peers", cfg.P2pNetworkConfig.MaxPeers), - zap.Int("updated_max_peers", minRequiredPeers), - zap.Int("committee_threshold_for_peer_increase", th), - zap.Int("unique_committees", len(uniqueCommittees)), + // TODO: use OperatorCommittees when it's fixed. + myValidators := nodeStorage.ValidatorStore().OperatorValidators(operatorData.ID) + mySubnets := make(records.Subnets, networkcommons.SubnetsCount) + myActiveSubnets := 0 + for _, v := range myValidators { + subnet := networkcommons.CommitteeSubnet(v.CommitteeID()) + if mySubnets[subnet] == 0 { + mySubnets[subnet] = 1 + myActiveSubnets++ + } + } + const baseMaxPeers = 60 + const maxPeersLimit = 150 + const peersPerSubnet = 4 + idealMaxPeers := baseMaxPeers + min(peersPerSubnet*myActiveSubnets, maxPeersLimit) + if cfg.P2pNetworkConfig.MaxPeers < idealMaxPeers { + logger.Warn("increasing MaxPeers to match operator's subscribed subnets", + zap.Int("old_max_peers", cfg.P2pNetworkConfig.MaxPeers), + zap.Int("new_max_peers", idealMaxPeers), + zap.Int("subscribed_subnets", myActiveSubnets), ) - cfg.P2pNetworkConfig.MaxPeers = minRequiredPeers + cfg.P2pNetworkConfig.MaxPeers = idealMaxPeers } usingLocalEvents := len(cfg.LocalEventsPath) != 0 diff --git a/network/p2p/metrics.go b/network/p2p/metrics.go index d73e36262e..05f7a6df0c 100644 --- a/network/p2p/metrics.go +++ b/network/p2p/metrics.go @@ -1,6 +1,7 @@ package p2pv1 import ( + "sort" "strconv" "github.com/ssvlabs/ssv/logging/fields" @@ -82,20 +83,51 @@ func (n *p2pNetwork) reportTopics(logger *zap.Logger) func() { topics := n.topicsCtrl.Topics() nTopics := len(topics) logger.Debug("connected topics", fields.Count(nTopics)) + distribution := []int{} for _, name := range topics { - n.reportTopicPeers(logger, name) + distribution = append(distribution, n.reportTopicPeers(logger, name)) } + + // Calculate min, median, max + sort.Ints(distribution) + min := distribution[0] + median := distribution[len(distribution)/2] + max := distribution[len(distribution)-1] + + onePeerTopics := 0 + twoPeerTopics := 0 + threePeerTopics := 0 + for _, peers := range distribution { + if peers == 1 { + onePeerTopics++ + } else if peers == 2 { + twoPeerTopics++ + } else if peers == 3 { + threePeerTopics++ + } + } + + logger.Debug("topic peers distribution", + zap.Ints("distribution", distribution), + zap.Int("min", min), + zap.Int("median", median), + zap.Int("max", max), + zap.Int("one_peer_topics", onePeerTopics), + zap.Int("two_peer_topics", twoPeerTopics), + zap.Int("three_peer_topics", threePeerTopics), + ) } } -func (n *p2pNetwork) reportTopicPeers(logger *zap.Logger, name string) { +func (n *p2pNetwork) reportTopicPeers(logger *zap.Logger, name string) int { peers, err := n.topicsCtrl.Peers(name) if err != nil { logger.Warn("could not get topic peers", fields.Topic(name), zap.Error(err)) - return + return 0 } logger.Debug("topic peers status", fields.Topic(name), fields.Count(len(peers)), zap.Any("peers", peers)) MetricsConnectedPeers.WithLabelValues(name).Set(float64(len(peers))) + return len(peers) } func (n *p2pNetwork) reportPeerIdentity(logger *zap.Logger, pid peer.ID) { diff --git a/network/p2p/p2p.go b/network/p2p/p2p.go index a516204d10..f24d4df99b 100644 --- a/network/p2p/p2p.go +++ b/network/p2p/p2p.go @@ -52,7 +52,7 @@ const ( connManagerBalancingTimeout = time.Minute peersReportingInterval = 60 * time.Second peerIdentitiesReportingInterval = 5 * time.Minute - topicsReportingInterval = 180 * time.Second + topicsReportingInterval = 60 * time.Second // TODO: revert maximumIrrelevantPeersToDisconnect = 3 ) diff --git a/networkconfig/config.go b/networkconfig/config.go index 60356488cb..58a1faba7b 100644 --- a/networkconfig/config.go +++ b/networkconfig/config.go @@ -31,15 +31,14 @@ func GetNetworkConfigByName(name string) (NetworkConfig, error) { } type NetworkConfig struct { - Name string - Beacon beacon.BeaconNetwork - DomainType spectypes.DomainType - GenesisEpoch phase0.Epoch - RegistrySyncOffset *big.Int - RegistryContractAddr string // TODO: ethcommon.Address - Bootnodes []string - DiscoveryProtocolID [6]byte - CommitteeThresholdForPeerIncrease int + Name string + Beacon beacon.BeaconNetwork + DomainType spectypes.DomainType + GenesisEpoch phase0.Epoch + RegistrySyncOffset *big.Int + RegistryContractAddr string // TODO: ethcommon.Address + Bootnodes []string + DiscoveryProtocolID [6]byte } func (n NetworkConfig) String() string { diff --git a/networkconfig/holesky-e2e.go b/networkconfig/holesky-e2e.go index 1a048d466b..0f962b83ff 100644 --- a/networkconfig/holesky-e2e.go +++ b/networkconfig/holesky-e2e.go @@ -9,12 +9,11 @@ import ( ) var HoleskyE2E = NetworkConfig{ - Name: "holesky-e2e", - Beacon: beacon.NewNetwork(spectypes.HoleskyNetwork), - DomainType: spectypes.DomainType{0x0, 0x0, 0xee, 0x1}, - GenesisEpoch: 1, - RegistryContractAddr: "0x58410bef803ecd7e63b23664c586a6db72daf59c", - RegistrySyncOffset: big.NewInt(405579), - Bootnodes: []string{}, - CommitteeThresholdForPeerIncrease: 7, + Name: "holesky-e2e", + Beacon: beacon.NewNetwork(spectypes.HoleskyNetwork), + DomainType: spectypes.DomainType{0x0, 0x0, 0xee, 0x1}, + GenesisEpoch: 1, + RegistryContractAddr: "0x58410bef803ecd7e63b23664c586a6db72daf59c", + RegistrySyncOffset: big.NewInt(405579), + Bootnodes: []string{}, } diff --git a/networkconfig/holesky-stage.go b/networkconfig/holesky-stage.go index 0f3ba1d080..a8a8c90d31 100644 --- a/networkconfig/holesky-stage.go +++ b/networkconfig/holesky-stage.go @@ -23,5 +23,4 @@ var HoleskyStage = NetworkConfig{ // Private bootnode: "enr:-Ja4QDRUBjWOvVfGxpxvv3FqaCy3psm7IsKu5ETb1GXiexGYDFppD33t7AHRfmQddoAkBiyb7pt4t7ZN0sNB9CsW4I-GAZGOmChMgmlkgnY0gmlwhAorXxuJc2VjcDI1NmsxoQP_bBE-ZYvaXKBR3dRYMN5K_lZP-q-YsBzDZEtxH_4T_YNzc3YBg3RjcIITioN1ZHCCD6I", }, - CommitteeThresholdForPeerIncrease: 7, } diff --git a/networkconfig/holesky.go b/networkconfig/holesky.go index 7e79f2acc1..f91cd78e4d 100644 --- a/networkconfig/holesky.go +++ b/networkconfig/holesky.go @@ -19,5 +19,4 @@ var Holesky = NetworkConfig{ Bootnodes: []string{ "enr:-Li4QFIQzamdvTxGJhvcXG_DFmCeyggSffDnllY5DiU47pd_K_1MRnSaJimWtfKJ-MD46jUX9TwgW5Jqe0t4pH41RYWGAYuFnlyth2F0dG5ldHOIAAAAAAAAAACEZXRoMpD1pf1CAAAAAP__________gmlkgnY0gmlwhCLdu_SJc2VjcDI1NmsxoQN4v-N9zFYwEqzGPBBX37q24QPFvAVUtokIo1fblIsmTIN0Y3CCE4uDdWRwgg-j", }, - CommitteeThresholdForPeerIncrease: 7, } diff --git a/networkconfig/mainnet.go b/networkconfig/mainnet.go index 651416d410..b5886394c7 100644 --- a/networkconfig/mainnet.go +++ b/networkconfig/mainnet.go @@ -29,5 +29,4 @@ var Mainnet = NetworkConfig{ // CryptoManufaktur "enr:-Li4QH7FwJcL8gJj0zHAITXqghMkG-A5bfWh2-3Q7vosy9D1BS8HZk-1ITuhK_rfzG3v_UtBDI6uNJZWpdcWfrQFCxKGAYnQ1DRCh2F0dG5ldHOIAAAAAAAAAACEZXRoMpD1pf1CAAAAAP__________gmlkgnY0gmlwhBLb3g2Jc2VjcDI1NmsxoQKeSDcZWSaY9FC723E9yYX1Li18bswhLNlxBZdLfgOKp4N0Y3CCE4mDdWRwgg-h", }, - CommitteeThresholdForPeerIncrease: 4, } From b00e4a81c15a0826c02544b48cf86deeaa4e81ce Mon Sep 17 00:00:00 2001 From: moshe-blox Date: Mon, 9 Dec 2024 15:35:46 +0200 Subject: [PATCH 12/21] comment --- network/p2p/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/p2p/config.go b/network/p2p/config.go index 5b0ae40d90..b777b8213e 100644 --- a/network/p2p/config.go +++ b/network/p2p/config.go @@ -46,7 +46,7 @@ type Config struct { RequestTimeout time.Duration `yaml:"RequestTimeout" env:"P2P_REQUEST_TIMEOUT" env-default:"10s"` MaxBatchResponse uint64 `yaml:"MaxBatchResponse" env:"P2P_MAX_BATCH_RESPONSE" env-default:"25" env-description:"Maximum number of returned objects in a batch"` - MaxPeers int `yaml:"MaxPeers" env:"P2P_MAX_PEERS" env-default:"60" env-description:"Connected peers limit for connections. This value is applied if operator has between 1 and CommitteeThresholdForPeerIncrease (default 4) committees. For CommitteeThresholdForPeerIncrease+1 and more committees this value is overridden with 150 to make sure there are enough peers"` + MaxPeers int `yaml:"MaxPeers" env:"P2P_MAX_PEERS" env-default:"60" env-description:"Connected peers limit. At the moment, this is grown (up to 150) when the node is an operator which is subscribed to many subnets."` TopicMaxPeers int `yaml:"TopicMaxPeers" env:"P2P_TOPIC_MAX_PEERS" env-default:"10" env-description:"Connected peers limit per pubsub topic"` // Subnets is a static bit list of subnets that this node will register upon start. From 317f7a71ae47d4fcca2a6e230c9e367d4a0bb948 Mon Sep 17 00:00:00 2001 From: moshe-blox Date: Mon, 9 Dec 2024 15:37:32 +0200 Subject: [PATCH 13/21] 3 peers per subnet --- cli/operator/node.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/operator/node.go b/cli/operator/node.go index 275a17f23a..badbe4078c 100644 --- a/cli/operator/node.go +++ b/cli/operator/node.go @@ -177,7 +177,7 @@ var StartNodeCmd = &cobra.Command{ } const baseMaxPeers = 60 const maxPeersLimit = 150 - const peersPerSubnet = 4 + const peersPerSubnet = 3 idealMaxPeers := baseMaxPeers + min(peersPerSubnet*myActiveSubnets, maxPeersLimit) if cfg.P2pNetworkConfig.MaxPeers < idealMaxPeers { logger.Warn("increasing MaxPeers to match operator's subscribed subnets", From c63bddea829b31863c82e288b27891d744f0fce9 Mon Sep 17 00:00:00 2001 From: moshe-blox Date: Mon, 9 Dec 2024 15:56:50 +0200 Subject: [PATCH 14/21] fix min() usg --- cli/operator/node.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/operator/node.go b/cli/operator/node.go index badbe4078c..450546182f 100644 --- a/cli/operator/node.go +++ b/cli/operator/node.go @@ -178,9 +178,9 @@ var StartNodeCmd = &cobra.Command{ const baseMaxPeers = 60 const maxPeersLimit = 150 const peersPerSubnet = 3 - idealMaxPeers := baseMaxPeers + min(peersPerSubnet*myActiveSubnets, maxPeersLimit) + idealMaxPeers := min(baseMaxPeers+peersPerSubnet*myActiveSubnets, maxPeersLimit) if cfg.P2pNetworkConfig.MaxPeers < idealMaxPeers { - logger.Warn("increasing MaxPeers to match operator's subscribed subnets", + logger.Warn("increasing MaxPeers to match the operator's subscribed subnets", zap.Int("old_max_peers", cfg.P2pNetworkConfig.MaxPeers), zap.Int("new_max_peers", idealMaxPeers), zap.Int("subscribed_subnets", myActiveSubnets), From 77041626f3885081b546bc96d47a35813f3304f6 Mon Sep 17 00:00:00 2001 From: moshe-blox Date: Mon, 9 Dec 2024 16:02:16 +0200 Subject: [PATCH 15/21] log time --- cli/operator/node.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cli/operator/node.go b/cli/operator/node.go index 450546182f..271147a96c 100644 --- a/cli/operator/node.go +++ b/cli/operator/node.go @@ -164,7 +164,9 @@ var StartNodeCmd = &cobra.Command{ nodeStorage, operatorData := setupOperatorStorage(logger, db, operatorPrivKey, operatorPrivKeyText) operatorDataStore := operatordatastore.New(operatorData) + // Increase MaxPeers if the operator is subscribed to many subnets. // TODO: use OperatorCommittees when it's fixed. + start := time.Now() myValidators := nodeStorage.ValidatorStore().OperatorValidators(operatorData.ID) mySubnets := make(records.Subnets, networkcommons.SubnetsCount) myActiveSubnets := 0 @@ -184,6 +186,7 @@ var StartNodeCmd = &cobra.Command{ zap.Int("old_max_peers", cfg.P2pNetworkConfig.MaxPeers), zap.Int("new_max_peers", idealMaxPeers), zap.Int("subscribed_subnets", myActiveSubnets), + zap.Duration("took", time.Since(start)), ) cfg.P2pNetworkConfig.MaxPeers = idealMaxPeers } From 6f00fee26d3fa99e7d2d6fdf3e1ef9cabf0e6fee Mon Sep 17 00:00:00 2001 From: moshe-blox Date: Mon, 9 Dec 2024 17:43:46 +0200 Subject: [PATCH 16/21] logs --- network/p2p/metrics.go | 46 ++++++++++++++++++------------------------ 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/network/p2p/metrics.go b/network/p2p/metrics.go index 05f7a6df0c..f8cfea2cbb 100644 --- a/network/p2p/metrics.go +++ b/network/p2p/metrics.go @@ -81,45 +81,39 @@ func (n *p2pNetwork) reportPeerIdentities(logger *zap.Logger) func() { func (n *p2pNetwork) reportTopics(logger *zap.Logger) func() { return func() { topics := n.topicsCtrl.Topics() - nTopics := len(topics) - logger.Debug("connected topics", fields.Count(nTopics)) - distribution := []int{} + logger.Debug("connected topics", fields.Count(len(topics))) + + subnetPeerCounts := []int{} + deadSubnets := 0 + unhealthySubnets := 0 for _, name := range topics { - distribution = append(distribution, n.reportTopicPeers(logger, name)) - } + count := n.reportTopicPeers(logger, name) + subnetPeerCounts = append(subnetPeerCounts, count) - // Calculate min, median, max - sort.Ints(distribution) - min := distribution[0] - median := distribution[len(distribution)/2] - max := distribution[len(distribution)-1] - - onePeerTopics := 0 - twoPeerTopics := 0 - threePeerTopics := 0 - for _, peers := range distribution { - if peers == 1 { - onePeerTopics++ - } else if peers == 2 { - twoPeerTopics++ - } else if peers == 3 { - threePeerTopics++ + if count == 0 { + deadSubnets++ + } else if count <= 2 { + unhealthySubnets++ } } + // Calculate min, median, max + sort.Ints(subnetPeerCounts) + min := subnetPeerCounts[0] + median := subnetPeerCounts[len(subnetPeerCounts)/2] + max := subnetPeerCounts[len(subnetPeerCounts)-1] + logger.Debug("topic peers distribution", - zap.Ints("distribution", distribution), zap.Int("min", min), zap.Int("median", median), zap.Int("max", max), - zap.Int("one_peer_topics", onePeerTopics), - zap.Int("two_peer_topics", twoPeerTopics), - zap.Int("three_peer_topics", threePeerTopics), + zap.Int("dead_subnets", deadSubnets), + zap.Int("unhealthy_subnets", unhealthySubnets), ) } } -func (n *p2pNetwork) reportTopicPeers(logger *zap.Logger, name string) int { +func (n *p2pNetwork) reportTopicPeers(logger *zap.Logger, name string) (peerCount int) { peers, err := n.topicsCtrl.Peers(name) if err != nil { logger.Warn("could not get topic peers", fields.Topic(name), zap.Error(err)) From 04b398b0b66063baf64e2425953bb82458e6e35e Mon Sep 17 00:00:00 2001 From: moshe-blox Date: Mon, 9 Dec 2024 18:25:07 +0200 Subject: [PATCH 17/21] comments --- cli/operator/node.go | 8 +++++--- network/p2p/config.go | 2 +- network/p2p/p2p.go | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/cli/operator/node.go b/cli/operator/node.go index 271147a96c..a15ef3421a 100644 --- a/cli/operator/node.go +++ b/cli/operator/node.go @@ -166,6 +166,11 @@ var StartNodeCmd = &cobra.Command{ // Increase MaxPeers if the operator is subscribed to many subnets. // TODO: use OperatorCommittees when it's fixed. + const ( + baseMaxPeers = 60 + maxPeersLimit = 150 + peersPerSubnet = 3 + ) start := time.Now() myValidators := nodeStorage.ValidatorStore().OperatorValidators(operatorData.ID) mySubnets := make(records.Subnets, networkcommons.SubnetsCount) @@ -177,9 +182,6 @@ var StartNodeCmd = &cobra.Command{ myActiveSubnets++ } } - const baseMaxPeers = 60 - const maxPeersLimit = 150 - const peersPerSubnet = 3 idealMaxPeers := min(baseMaxPeers+peersPerSubnet*myActiveSubnets, maxPeersLimit) if cfg.P2pNetworkConfig.MaxPeers < idealMaxPeers { logger.Warn("increasing MaxPeers to match the operator's subscribed subnets", diff --git a/network/p2p/config.go b/network/p2p/config.go index b777b8213e..56b3665a89 100644 --- a/network/p2p/config.go +++ b/network/p2p/config.go @@ -46,7 +46,7 @@ type Config struct { RequestTimeout time.Duration `yaml:"RequestTimeout" env:"P2P_REQUEST_TIMEOUT" env-default:"10s"` MaxBatchResponse uint64 `yaml:"MaxBatchResponse" env:"P2P_MAX_BATCH_RESPONSE" env-default:"25" env-description:"Maximum number of returned objects in a batch"` - MaxPeers int `yaml:"MaxPeers" env:"P2P_MAX_PEERS" env-default:"60" env-description:"Connected peers limit. At the moment, this is grown (up to 150) when the node is an operator which is subscribed to many subnets."` + MaxPeers int `yaml:"MaxPeers" env:"P2P_MAX_PEERS" env-default:"60" env-description:"Connected peers limit. If this is lower than the ideal MaxPeers for the operator's subnets, it will be increased (up to 150)."` TopicMaxPeers int `yaml:"TopicMaxPeers" env:"P2P_TOPIC_MAX_PEERS" env-default:"10" env-description:"Connected peers limit per pubsub topic"` // Subnets is a static bit list of subnets that this node will register upon start. diff --git a/network/p2p/p2p.go b/network/p2p/p2p.go index f24d4df99b..ebd87cc11c 100644 --- a/network/p2p/p2p.go +++ b/network/p2p/p2p.go @@ -52,7 +52,7 @@ const ( connManagerBalancingTimeout = time.Minute peersReportingInterval = 60 * time.Second peerIdentitiesReportingInterval = 5 * time.Minute - topicsReportingInterval = 60 * time.Second // TODO: revert + topicsReportingInterval = 180 * time.Second // TODO: revert maximumIrrelevantPeersToDisconnect = 3 ) From 1fb160d8b0030b60b3960004b7f79b8367496457 Mon Sep 17 00:00:00 2001 From: moshe-blox Date: Mon, 9 Dec 2024 18:30:48 +0200 Subject: [PATCH 18/21] comment --- network/p2p/p2p.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/p2p/p2p.go b/network/p2p/p2p.go index ebd87cc11c..a516204d10 100644 --- a/network/p2p/p2p.go +++ b/network/p2p/p2p.go @@ -52,7 +52,7 @@ const ( connManagerBalancingTimeout = time.Minute peersReportingInterval = 60 * time.Second peerIdentitiesReportingInterval = 5 * time.Minute - topicsReportingInterval = 180 * time.Second // TODO: revert + topicsReportingInterval = 180 * time.Second maximumIrrelevantPeersToDisconnect = 3 ) From c1ed6e5578a3eb0533581d55aa133c60cdfc776d Mon Sep 17 00:00:00 2001 From: Nikita Kryuchkov Date: Mon, 9 Dec 2024 15:18:25 -0300 Subject: [PATCH 19/21] change peer count after setupEventHandling --- cli/operator/node.go | 58 ++++++++++++++++++++++---------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/cli/operator/node.go b/cli/operator/node.go index a15ef3421a..ef88b4718f 100644 --- a/cli/operator/node.go +++ b/cli/operator/node.go @@ -164,35 +164,6 @@ var StartNodeCmd = &cobra.Command{ nodeStorage, operatorData := setupOperatorStorage(logger, db, operatorPrivKey, operatorPrivKeyText) operatorDataStore := operatordatastore.New(operatorData) - // Increase MaxPeers if the operator is subscribed to many subnets. - // TODO: use OperatorCommittees when it's fixed. - const ( - baseMaxPeers = 60 - maxPeersLimit = 150 - peersPerSubnet = 3 - ) - start := time.Now() - myValidators := nodeStorage.ValidatorStore().OperatorValidators(operatorData.ID) - mySubnets := make(records.Subnets, networkcommons.SubnetsCount) - myActiveSubnets := 0 - for _, v := range myValidators { - subnet := networkcommons.CommitteeSubnet(v.CommitteeID()) - if mySubnets[subnet] == 0 { - mySubnets[subnet] = 1 - myActiveSubnets++ - } - } - idealMaxPeers := min(baseMaxPeers+peersPerSubnet*myActiveSubnets, maxPeersLimit) - if cfg.P2pNetworkConfig.MaxPeers < idealMaxPeers { - logger.Warn("increasing MaxPeers to match the operator's subscribed subnets", - zap.Int("old_max_peers", cfg.P2pNetworkConfig.MaxPeers), - zap.Int("new_max_peers", idealMaxPeers), - zap.Int("subscribed_subnets", myActiveSubnets), - zap.Duration("took", time.Since(start)), - ) - cfg.P2pNetworkConfig.MaxPeers = idealMaxPeers - } - usingLocalEvents := len(cfg.LocalEventsPath) != 0 if err := validateConfig(nodeStorage, networkConfig.NetworkName(), usingLocalEvents); err != nil { @@ -367,6 +338,35 @@ var StartNodeCmd = &cobra.Command{ nodeProber.AddNode("event syncer", eventSyncer) } + // Increase MaxPeers if the operator is subscribed to many subnets. + // TODO: use OperatorCommittees when it's fixed. + const ( + baseMaxPeers = 60 + maxPeersLimit = 150 + peersPerSubnet = 3 + ) + start := time.Now() + myValidators := nodeStorage.ValidatorStore().OperatorValidators(operatorData.ID) + mySubnets := make(records.Subnets, networkcommons.SubnetsCount) + myActiveSubnets := 0 + for _, v := range myValidators { + subnet := networkcommons.CommitteeSubnet(v.CommitteeID()) + if mySubnets[subnet] == 0 { + mySubnets[subnet] = 1 + myActiveSubnets++ + } + } + idealMaxPeers := min(baseMaxPeers+peersPerSubnet*myActiveSubnets, maxPeersLimit) + if cfg.P2pNetworkConfig.MaxPeers < idealMaxPeers { + logger.Warn("increasing MaxPeers to match the operator's subscribed subnets", + zap.Int("old_max_peers", cfg.P2pNetworkConfig.MaxPeers), + zap.Int("new_max_peers", idealMaxPeers), + zap.Int("subscribed_subnets", myActiveSubnets), + zap.Duration("took", time.Since(start)), + ) + cfg.P2pNetworkConfig.MaxPeers = idealMaxPeers + } + cfg.P2pNetworkConfig.GetValidatorStats = func() (uint64, uint64, uint64, error) { return validatorCtrl.GetValidatorStats() } From 793b3be391b517dbdbc1d78cded35c666779212c Mon Sep 17 00:00:00 2001 From: moshe-blox Date: Tue, 10 Dec 2024 16:40:07 +0200 Subject: [PATCH 20/21] rename --- cli/operator/node.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cli/operator/node.go b/cli/operator/node.go index ef88b4718f..d2ec45b5df 100644 --- a/cli/operator/node.go +++ b/cli/operator/node.go @@ -341,9 +341,9 @@ var StartNodeCmd = &cobra.Command{ // Increase MaxPeers if the operator is subscribed to many subnets. // TODO: use OperatorCommittees when it's fixed. const ( - baseMaxPeers = 60 - maxPeersLimit = 150 - peersPerSubnet = 3 + baseMaxPeers = 60 + maxPeersLimit = 150 + idealPeersPerSubnet = 3 ) start := time.Now() myValidators := nodeStorage.ValidatorStore().OperatorValidators(operatorData.ID) @@ -356,7 +356,7 @@ var StartNodeCmd = &cobra.Command{ myActiveSubnets++ } } - idealMaxPeers := min(baseMaxPeers+peersPerSubnet*myActiveSubnets, maxPeersLimit) + idealMaxPeers := min(baseMaxPeers+idealPeersPerSubnet*myActiveSubnets, maxPeersLimit) if cfg.P2pNetworkConfig.MaxPeers < idealMaxPeers { logger.Warn("increasing MaxPeers to match the operator's subscribed subnets", zap.Int("old_max_peers", cfg.P2pNetworkConfig.MaxPeers), From 5f599fec79673a32caf7fc2808b0542b02b5a676 Mon Sep 17 00:00:00 2001 From: moshe-blox Date: Tue, 10 Dec 2024 17:02:53 +0200 Subject: [PATCH 21/21] new envs --- cli/operator/node.go | 68 ++++++++++++++++++++++--------------------- network/p2p/config.go | 7 +++-- 2 files changed, 40 insertions(+), 35 deletions(-) diff --git a/cli/operator/node.go b/cli/operator/node.go index d2ec45b5df..ed37573a5f 100644 --- a/cli/operator/node.go +++ b/cli/operator/node.go @@ -340,41 +340,43 @@ var StartNodeCmd = &cobra.Command{ // Increase MaxPeers if the operator is subscribed to many subnets. // TODO: use OperatorCommittees when it's fixed. - const ( - baseMaxPeers = 60 - maxPeersLimit = 150 - idealPeersPerSubnet = 3 - ) - start := time.Now() - myValidators := nodeStorage.ValidatorStore().OperatorValidators(operatorData.ID) - mySubnets := make(records.Subnets, networkcommons.SubnetsCount) - myActiveSubnets := 0 - for _, v := range myValidators { - subnet := networkcommons.CommitteeSubnet(v.CommitteeID()) - if mySubnets[subnet] == 0 { - mySubnets[subnet] = 1 - myActiveSubnets++ - } - } - idealMaxPeers := min(baseMaxPeers+idealPeersPerSubnet*myActiveSubnets, maxPeersLimit) - if cfg.P2pNetworkConfig.MaxPeers < idealMaxPeers { - logger.Warn("increasing MaxPeers to match the operator's subscribed subnets", - zap.Int("old_max_peers", cfg.P2pNetworkConfig.MaxPeers), - zap.Int("new_max_peers", idealMaxPeers), - zap.Int("subscribed_subnets", myActiveSubnets), - zap.Duration("took", time.Since(start)), + if cfg.P2pNetworkConfig.DynamicMaxPeers { + var ( + baseMaxPeers = 60 + maxPeersLimit = cfg.P2pNetworkConfig.DynamicMaxPeersLimit + idealPeersPerSubnet = 3 ) - cfg.P2pNetworkConfig.MaxPeers = idealMaxPeers - } + start := time.Now() + myValidators := nodeStorage.ValidatorStore().OperatorValidators(operatorData.ID) + mySubnets := make(records.Subnets, networkcommons.SubnetsCount) + myActiveSubnets := 0 + for _, v := range myValidators { + subnet := networkcommons.CommitteeSubnet(v.CommitteeID()) + if mySubnets[subnet] == 0 { + mySubnets[subnet] = 1 + myActiveSubnets++ + } + } + idealMaxPeers := min(baseMaxPeers+idealPeersPerSubnet*myActiveSubnets, maxPeersLimit) + if cfg.P2pNetworkConfig.MaxPeers < idealMaxPeers { + logger.Warn("increasing MaxPeers to match the operator's subscribed subnets", + zap.Int("old_max_peers", cfg.P2pNetworkConfig.MaxPeers), + zap.Int("new_max_peers", idealMaxPeers), + zap.Int("subscribed_subnets", myActiveSubnets), + zap.Duration("took", time.Since(start)), + ) + cfg.P2pNetworkConfig.MaxPeers = idealMaxPeers + } - cfg.P2pNetworkConfig.GetValidatorStats = func() (uint64, uint64, uint64, error) { - return validatorCtrl.GetValidatorStats() - } - if err := p2pNetwork.Setup(logger); err != nil { - logger.Fatal("failed to setup network", zap.Error(err)) - } - if err := p2pNetwork.Start(logger); err != nil { - logger.Fatal("failed to start network", zap.Error(err)) + cfg.P2pNetworkConfig.GetValidatorStats = func() (uint64, uint64, uint64, error) { + return validatorCtrl.GetValidatorStats() + } + if err := p2pNetwork.Setup(logger); err != nil { + logger.Fatal("failed to setup network", zap.Error(err)) + } + if err := p2pNetwork.Start(logger); err != nil { + logger.Fatal("failed to start network", zap.Error(err)) + } } if cfg.SSVAPIPort > 0 { diff --git a/network/p2p/config.go b/network/p2p/config.go index 56b3665a89..37a2710c17 100644 --- a/network/p2p/config.go +++ b/network/p2p/config.go @@ -46,8 +46,11 @@ type Config struct { RequestTimeout time.Duration `yaml:"RequestTimeout" env:"P2P_REQUEST_TIMEOUT" env-default:"10s"` MaxBatchResponse uint64 `yaml:"MaxBatchResponse" env:"P2P_MAX_BATCH_RESPONSE" env-default:"25" env-description:"Maximum number of returned objects in a batch"` - MaxPeers int `yaml:"MaxPeers" env:"P2P_MAX_PEERS" env-default:"60" env-description:"Connected peers limit. If this is lower than the ideal MaxPeers for the operator's subnets, it will be increased (up to 150)."` - TopicMaxPeers int `yaml:"TopicMaxPeers" env:"P2P_TOPIC_MAX_PEERS" env-default:"10" env-description:"Connected peers limit per pubsub topic"` + + MaxPeers int `yaml:"MaxPeers" env:"P2P_MAX_PEERS" env-default:"60" env-description:"Connected peers limit. At the time being, this may be increased by DynamicMaxPeers until that is phased out."` + DynamicMaxPeers bool `yaml:"DynamicMaxPeers" env:"P2P_DYNAMIC_MAX_PEERS" env-default:"true" env-description:"If true, MaxPeers will grow with the operator's number of committees."` + DynamicMaxPeersLimit int `yaml:"DynamicMaxPeersLimit" env:"P2P_DYNAMIC_MAX_PEERS_LIMIT" env-default:"150" env-description:"Limit for MaxPeers when DynamicMaxPeers is enabled."` + TopicMaxPeers int `yaml:"TopicMaxPeers" env:"P2P_TOPIC_MAX_PEERS" env-default:"10" env-description:"Connected peers limit per pubsub topic"` // Subnets is a static bit list of subnets that this node will register upon start. Subnets string `yaml:"Subnets" env:"SUBNETS" env-description:"Hex string that represents the subnets that this node will join upon start"`