From 3334ca9536e468afdbff0ca82e03fb166fbe78db Mon Sep 17 00:00:00 2001 From: moshe-blox Date: Sun, 8 Dec 2024 16:50:22 +0200 Subject: [PATCH] wip --- cli/operator/node.go | 41 ++++++++++++++++----------------- network/p2p/metrics.go | 38 +++++++++++++++++++++++++++--- network/p2p/p2p.go | 2 +- network/records/subnets_test.go | 11 +++++++++ networkconfig/config.go | 18 +++++++-------- networkconfig/holesky-e2e.go | 16 ++++++------- networkconfig/holesky-stage.go | 2 +- networkconfig/holesky.go | 2 +- networkconfig/mainnet.go | 2 +- 9 files changed, 87 insertions(+), 45 deletions(-) diff --git a/cli/operator/node.go b/cli/operator/node.go index 390987e0b6..b4c0dd3c94 100644 --- a/cli/operator/node.go +++ b/cli/operator/node.go @@ -162,27 +162,26 @@ var StartNodeCmd = &cobra.Command{ nodeStorage, operatorData := setupOperatorStorage(logger, db, operatorPrivKey, operatorPrivKeyText) operatorDataStore := operatordatastore.New(operatorData) - operatorCommittees := nodeStorage.ValidatorStore().OperatorCommittees(operatorData.ID) - // Currently, OperatorCommittees may return several committees with the same committee ID, so we need to filter the unique ones. - // This might be a bug in validator store: https://github.com/ssvlabs/ssv/issues/1926 - uniqueCommittees := make(map[[32]byte]struct{}) - for _, oc := range operatorCommittees { - uniqueCommittees[oc.ID] = struct{}{} - } - // If operator has more than CommitteeThresholdForPeerIncrease committees, - // it needs more peers, so we need to override the value from config if it's too low. - // MaxPeers is used only in p2p, so the lines below must be executed before calling setupP2P function. - const minRequiredPeers = 150 - th := networkConfig.CommitteeThresholdForPeerIncrease - if th > 0 && len(uniqueCommittees) > th && cfg.P2pNetworkConfig.MaxPeers < minRequiredPeers { - logger.Warn("configured peer count is too low for this operator's committee count, increasing peer count", - zap.Int("configured_max_peers", cfg.P2pNetworkConfig.MaxPeers), - zap.Int("updated_max_peers", minRequiredPeers), - zap.Int("committee_threshold_for_peer_increase", th), - zap.Int("unique_committees", len(uniqueCommittees)), - ) - cfg.P2pNetworkConfig.MaxPeers = minRequiredPeers - } + // operatorCommittees := nodeStorage.ValidatorStore().OperatorCommittees(operatorData.ID) + // // Currently, OperatorCommittees may return several committees with the same committee ID, so we need to filter the unique ones. + // // This might be a bug in validator store: https://github.com/ssvlabs/ssv/issues/1926 + // uniqueCommittees := make(map[[32]byte]struct{}) + // for _, oc := range operatorCommittees { + // uniqueCommittees[oc.ID] = struct{}{} + // } + // // If operator has more than CommitteeThresholdForPeerIncrease committees, + // // it needs more peers, so we need to override the value from config if it's too low. + // // MaxPeers is used only in p2p, so the lines below must be executed before calling setupP2P function. + // const baseMaxPeers = 60 + // idealMaxPeers := baseMaxPeers + (networkConfig.PeersPerSubnet * len(uniqueCommittees)) + // if cfg.P2pNetworkConfig.MaxPeers < idealMaxPeers { + // logger.Warn("increasing MaxPeers to match operator's committee count", + // zap.Int("old_max_peers", cfg.P2pNetworkConfig.MaxPeers), + // zap.Int("new_max_peers", idealMaxPeers), + // zap.Int("unique_committees", len(uniqueCommittees)), + // ) + // cfg.P2pNetworkConfig.MaxPeers = idealMaxPeers + // } usingLocalEvents := len(cfg.LocalEventsPath) != 0 diff --git a/network/p2p/metrics.go b/network/p2p/metrics.go index d73e36262e..05f7a6df0c 100644 --- a/network/p2p/metrics.go +++ b/network/p2p/metrics.go @@ -1,6 +1,7 @@ package p2pv1 import ( + "sort" "strconv" "github.com/ssvlabs/ssv/logging/fields" @@ -82,20 +83,51 @@ func (n *p2pNetwork) reportTopics(logger *zap.Logger) func() { topics := n.topicsCtrl.Topics() nTopics := len(topics) logger.Debug("connected topics", fields.Count(nTopics)) + distribution := []int{} for _, name := range topics { - n.reportTopicPeers(logger, name) + distribution = append(distribution, n.reportTopicPeers(logger, name)) } + + // Calculate min, median, max + sort.Ints(distribution) + min := distribution[0] + median := distribution[len(distribution)/2] + max := distribution[len(distribution)-1] + + onePeerTopics := 0 + twoPeerTopics := 0 + threePeerTopics := 0 + for _, peers := range distribution { + if peers == 1 { + onePeerTopics++ + } else if peers == 2 { + twoPeerTopics++ + } else if peers == 3 { + threePeerTopics++ + } + } + + logger.Debug("topic peers distribution", + zap.Ints("distribution", distribution), + zap.Int("min", min), + zap.Int("median", median), + zap.Int("max", max), + zap.Int("one_peer_topics", onePeerTopics), + zap.Int("two_peer_topics", twoPeerTopics), + zap.Int("three_peer_topics", threePeerTopics), + ) } } -func (n *p2pNetwork) reportTopicPeers(logger *zap.Logger, name string) { +func (n *p2pNetwork) reportTopicPeers(logger *zap.Logger, name string) int { peers, err := n.topicsCtrl.Peers(name) if err != nil { logger.Warn("could not get topic peers", fields.Topic(name), zap.Error(err)) - return + return 0 } logger.Debug("topic peers status", fields.Topic(name), fields.Count(len(peers)), zap.Any("peers", peers)) MetricsConnectedPeers.WithLabelValues(name).Set(float64(len(peers))) + return len(peers) } func (n *p2pNetwork) reportPeerIdentity(logger *zap.Logger, pid peer.ID) { diff --git a/network/p2p/p2p.go b/network/p2p/p2p.go index a516204d10..1445c3e3db 100644 --- a/network/p2p/p2p.go +++ b/network/p2p/p2p.go @@ -52,7 +52,7 @@ const ( connManagerBalancingTimeout = time.Minute peersReportingInterval = 60 * time.Second peerIdentitiesReportingInterval = 5 * time.Minute - topicsReportingInterval = 180 * time.Second + topicsReportingInterval = 30 * time.Second maximumIrrelevantPeersToDisconnect = 3 ) diff --git a/network/records/subnets_test.go b/network/records/subnets_test.go index 25a8de143d..6acdf98bbc 100644 --- a/network/records/subnets_test.go +++ b/network/records/subnets_test.go @@ -2,6 +2,7 @@ package records import ( crand "crypto/rand" + "math/rand/v2" "strings" "testing" @@ -11,6 +12,16 @@ import ( "github.com/ssvlabs/ssv/network/commons" ) +func TestDelete(t *testing.T) { + // TODO: delete + subnets := make([]byte, 128) + for _, i := range rand.Perm(128)[:16] { + subnets[i] = 1 + } + t.Logf("subnets: %v", Subnets(subnets).String()) + return +} + func Test_SubnetsEntry(t *testing.T) { SubnetsCount := 128 priv, _, err := crypto.GenerateSecp256k1Key(crand.Reader) diff --git a/networkconfig/config.go b/networkconfig/config.go index 60356488cb..74ca91390a 100644 --- a/networkconfig/config.go +++ b/networkconfig/config.go @@ -31,15 +31,15 @@ func GetNetworkConfigByName(name string) (NetworkConfig, error) { } type NetworkConfig struct { - Name string - Beacon beacon.BeaconNetwork - DomainType spectypes.DomainType - GenesisEpoch phase0.Epoch - RegistrySyncOffset *big.Int - RegistryContractAddr string // TODO: ethcommon.Address - Bootnodes []string - DiscoveryProtocolID [6]byte - CommitteeThresholdForPeerIncrease int + Name string + Beacon beacon.BeaconNetwork + DomainType spectypes.DomainType + GenesisEpoch phase0.Epoch + RegistrySyncOffset *big.Int + RegistryContractAddr string // TODO: ethcommon.Address + Bootnodes []string + DiscoveryProtocolID [6]byte + PeersPerSubnet int } func (n NetworkConfig) String() string { diff --git a/networkconfig/holesky-e2e.go b/networkconfig/holesky-e2e.go index 1a048d466b..25e34b6d6a 100644 --- a/networkconfig/holesky-e2e.go +++ b/networkconfig/holesky-e2e.go @@ -9,12 +9,12 @@ import ( ) var HoleskyE2E = NetworkConfig{ - Name: "holesky-e2e", - Beacon: beacon.NewNetwork(spectypes.HoleskyNetwork), - DomainType: spectypes.DomainType{0x0, 0x0, 0xee, 0x1}, - GenesisEpoch: 1, - RegistryContractAddr: "0x58410bef803ecd7e63b23664c586a6db72daf59c", - RegistrySyncOffset: big.NewInt(405579), - Bootnodes: []string{}, - CommitteeThresholdForPeerIncrease: 7, + Name: "holesky-e2e", + Beacon: beacon.NewNetwork(spectypes.HoleskyNetwork), + DomainType: spectypes.DomainType{0x0, 0x0, 0xee, 0x1}, + GenesisEpoch: 1, + RegistryContractAddr: "0x58410bef803ecd7e63b23664c586a6db72daf59c", + RegistrySyncOffset: big.NewInt(405579), + Bootnodes: []string{}, + PeersPerSubnet: 7, } diff --git a/networkconfig/holesky-stage.go b/networkconfig/holesky-stage.go index 0f3ba1d080..f94a092cae 100644 --- a/networkconfig/holesky-stage.go +++ b/networkconfig/holesky-stage.go @@ -23,5 +23,5 @@ var HoleskyStage = NetworkConfig{ // Private bootnode: "enr:-Ja4QDRUBjWOvVfGxpxvv3FqaCy3psm7IsKu5ETb1GXiexGYDFppD33t7AHRfmQddoAkBiyb7pt4t7ZN0sNB9CsW4I-GAZGOmChMgmlkgnY0gmlwhAorXxuJc2VjcDI1NmsxoQP_bBE-ZYvaXKBR3dRYMN5K_lZP-q-YsBzDZEtxH_4T_YNzc3YBg3RjcIITioN1ZHCCD6I", }, - CommitteeThresholdForPeerIncrease: 7, + PeersPerSubnet: 7, } diff --git a/networkconfig/holesky.go b/networkconfig/holesky.go index 7e79f2acc1..60bb9b953a 100644 --- a/networkconfig/holesky.go +++ b/networkconfig/holesky.go @@ -19,5 +19,5 @@ var Holesky = NetworkConfig{ Bootnodes: []string{ "enr:-Li4QFIQzamdvTxGJhvcXG_DFmCeyggSffDnllY5DiU47pd_K_1MRnSaJimWtfKJ-MD46jUX9TwgW5Jqe0t4pH41RYWGAYuFnlyth2F0dG5ldHOIAAAAAAAAAACEZXRoMpD1pf1CAAAAAP__________gmlkgnY0gmlwhCLdu_SJc2VjcDI1NmsxoQN4v-N9zFYwEqzGPBBX37q24QPFvAVUtokIo1fblIsmTIN0Y3CCE4uDdWRwgg-j", }, - CommitteeThresholdForPeerIncrease: 7, + PeersPerSubnet: 7, } diff --git a/networkconfig/mainnet.go b/networkconfig/mainnet.go index 651416d410..ec1f824c40 100644 --- a/networkconfig/mainnet.go +++ b/networkconfig/mainnet.go @@ -29,5 +29,5 @@ var Mainnet = NetworkConfig{ // CryptoManufaktur "enr:-Li4QH7FwJcL8gJj0zHAITXqghMkG-A5bfWh2-3Q7vosy9D1BS8HZk-1ITuhK_rfzG3v_UtBDI6uNJZWpdcWfrQFCxKGAYnQ1DRCh2F0dG5ldHOIAAAAAAAAAACEZXRoMpD1pf1CAAAAAP__________gmlkgnY0gmlwhBLb3g2Jc2VjcDI1NmsxoQKeSDcZWSaY9FC723E9yYX1Li18bswhLNlxBZdLfgOKp4N0Y3CCE4mDdWRwgg-h", }, - CommitteeThresholdForPeerIncrease: 4, + PeersPerSubnet: 4, }