From c42ba6a15b428a6f1c09807230b3441537a8e83a Mon Sep 17 00:00:00 2001 From: Ibrahim Kettaneh Date: Thu, 7 Nov 2024 11:38:29 -0500 Subject: [PATCH] raft: add replicas.leaders.not_fortified metric This commit adds the metric replicas.leaders.not_fortified which reports the number of leaders that think they are not fortified. References: #132754 Release note: None --- docs/generated/metrics/metrics.html | 1 + pkg/kv/kvserver/metrics.go | 8 + pkg/kv/kvserver/replica_metrics.go | 11 + pkg/kv/kvserver/replica_test.go | 237 ++++++++++-------- pkg/kv/kvserver/store.go | 5 + .../opentelemetry/cockroachdb_metrics.go | 1 + 6 files changed, 165 insertions(+), 98 deletions(-) diff --git a/docs/generated/metrics/metrics.html b/docs/generated/metrics/metrics.html index cac3e614a9c0..2bb6c4d83025 100644 --- a/docs/generated/metrics/metrics.html +++ b/docs/generated/metrics/metrics.html @@ -606,6 +606,7 @@ STORAGEreplicasNumber of replicasReplicasGAUGECOUNTAVGNONE STORAGEreplicas.leadersNumber of raft leadersRaft LeadersGAUGECOUNTAVGNONE STORAGEreplicas.leaders_invalid_leaseNumber of replicas that are Raft leaders whose lease is invalidReplicasGAUGECOUNTAVGNONE +STORAGEreplicas.leaders_not_fortifiedNumber of replicas that are not fortified Raft leadersReplicasGAUGECOUNTAVGNONE STORAGEreplicas.leaders_not_leaseholdersNumber of replicas that are Raft leaders whose range lease is held by another storeReplicasGAUGECOUNTAVGNONE STORAGEreplicas.leaseholdersNumber of lease holdersReplicasGAUGECOUNTAVGNONE STORAGEreplicas.quiescentNumber of quiesced replicasReplicasGAUGECOUNTAVGNONE diff --git a/pkg/kv/kvserver/metrics.go b/pkg/kv/kvserver/metrics.go index 7a206d9eaf4b..8b95085eb345 100644 --- a/pkg/kv/kvserver/metrics.go +++ b/pkg/kv/kvserver/metrics.go @@ -65,6 +65,12 @@ var ( Measurement: "Replicas", Unit: metric.Unit_COUNT, } + metaRaftLeaderNotFortifiedCount = metric.Metadata{ + Name: "replicas.leaders_not_fortified", + Help: "Number of replicas that are not fortified Raft leaders", + Measurement: "Replicas", + Unit: metric.Unit_COUNT, + } metaRaftLeaderInvalidLeaseCount = metric.Metadata{ Name: "replicas.leaders_invalid_lease", Help: "Number of replicas that are Raft leaders whose lease is invalid", @@ -2585,6 +2591,7 @@ type StoreMetrics struct { ReservedReplicaCount *metric.Gauge RaftLeaderCount *metric.Gauge RaftLeaderNotLeaseHolderCount *metric.Gauge + RaftLeaderNotFortifiedCount *metric.Gauge RaftLeaderInvalidLeaseCount *metric.Gauge LeaseHolderCount *metric.Gauge QuiescentCount *metric.Gauge @@ -3286,6 +3293,7 @@ func newStoreMetrics(histogramWindow time.Duration) *StoreMetrics { ReservedReplicaCount: metric.NewGauge(metaReservedReplicaCount), RaftLeaderCount: metric.NewGauge(metaRaftLeaderCount), RaftLeaderNotLeaseHolderCount: metric.NewGauge(metaRaftLeaderNotLeaseHolderCount), + RaftLeaderNotFortifiedCount: metric.NewGauge(metaRaftLeaderNotFortifiedCount), RaftLeaderInvalidLeaseCount: metric.NewGauge(metaRaftLeaderInvalidLeaseCount), LeaseHolderCount: metric.NewGauge(metaLeaseHolderCount), QuiescentCount: metric.NewGauge(metaQuiescentCount), diff --git a/pkg/kv/kvserver/replica_metrics.go b/pkg/kv/kvserver/replica_metrics.go index 4a167625199a..abbea8981432 100644 --- a/pkg/kv/kvserver/replica_metrics.go +++ b/pkg/kv/kvserver/replica_metrics.go @@ -33,6 +33,10 @@ type ReplicaMetrics struct { ViolatingLeasePreferences bool LessPreferredLease bool + // LeaderNotFortified indicates whether the leader believes itself to be + // fortified or not. + LeaderNotFortified bool + // Quiescent indicates whether the replica believes itself to be quiesced. Quiescent bool // Ticking indicates whether the store is ticking the replica. It should be @@ -95,6 +99,8 @@ func (r *Replica) Metrics( clusterNodes: clusterNodes, desc: r.shMu.state.Desc, raftStatus: r.raftSparseStatusRLocked(), + leadSupportStatus: r.raftLeadSupportStatusRLocked(), + now: now, leaseStatus: r.leaseStatusAtRLocked(ctx, now), storeID: r.store.StoreID(), storeAttrs: storeAttrs, @@ -126,6 +132,8 @@ type calcReplicaMetricsInput struct { clusterNodes int desc *roachpb.RangeDescriptor raftStatus *raft.SparseStatus + leadSupportStatus raft.LeadSupportStatus + now hlc.ClockTimestamp leaseStatus kvserverpb.LeaseStatus storeID roachpb.StoreID storeAttrs, nodeAttrs roachpb.Attributes @@ -176,9 +184,11 @@ func calcReplicaMetrics(d calcReplicaMetricsInput) ReplicaMetrics { // behind. leader := d.raftStatus != nil && d.raftStatus.RaftState == raftpb.StateLeader var leaderBehindCount, leaderPausedFollowerCount int64 + var leaderNotFortified bool if leader { leaderBehindCount = calcBehindCount(d.raftStatus, d.desc, d.vitalityMap) leaderPausedFollowerCount = int64(len(d.paused)) + leaderNotFortified = d.leadSupportStatus.LeadSupportUntil.Less(d.now.ToTimestamp()) } return ReplicaMetrics{ @@ -190,6 +200,7 @@ func calcReplicaMetrics(d calcReplicaMetricsInput) ReplicaMetrics { LivenessLease: livenessLease, ViolatingLeasePreferences: violatingLeasePreferences, LessPreferredLease: lessPreferredLease, + LeaderNotFortified: leaderNotFortified, Quiescent: d.quiescent, Ticking: d.ticking, RangeCounter: rangeCounter, diff --git a/pkg/kv/kvserver/replica_test.go b/pkg/kv/kvserver/replica_test.go index 09a7a07e5364..b8928de9cd70 100644 --- a/pkg/kv/kvserver/replica_test.go +++ b/pkg/kv/kvserver/replica_test.go @@ -9002,6 +9002,10 @@ func TestReplicaMetrics(t *testing.T) { status.HardState.Lead = lead return status } + leadSupportedStatus := func(leadSupportedUntil hlc.Timestamp) raft.LeadSupportStatus { + status := raft.LeadSupportStatus{LeadSupportUntil: leadSupportedUntil} + return status + } desc := func(ids ...int) roachpb.RangeDescriptor { var d roachpb.RangeDescriptor for i, id := range ids { @@ -9025,52 +9029,57 @@ func TestReplicaMetrics(t *testing.T) { tc.StartWithStoreConfig(ctx, t, stopper, cfg) testCases := []struct { - replicas int32 - storeID roachpb.StoreID - desc roachpb.RangeDescriptor - raftStatus *raft.SparseStatus - liveness livenesspb.NodeVitalityInterface - raftLogSize int64 - expected ReplicaMetrics + replicas int32 + storeID roachpb.StoreID + desc roachpb.RangeDescriptor + raftStatus *raft.SparseStatus + liveness livenesspb.NodeVitalityInterface + raftLogSize int64 + leaderSupportStatus raft.LeadSupportStatus + expected ReplicaMetrics }{ // The leader of a 1-replica range is up. - {1, 1, desc(1), status(1, progress(2)), live(1), 0, + {1, 1, desc(1), status(1, progress(2)), live(1), 0, leadSupportedStatus(hlc.Timestamp{}), ReplicaMetrics{ - Leader: true, - RangeCounter: true, - Unavailable: false, - Underreplicated: false, - BehindCount: 10, + Leader: true, + RangeCounter: true, + Unavailable: false, + Underreplicated: false, + BehindCount: 10, + LeaderNotFortified: true, }}, // The leader of a 2-replica range is up (only 1 replica present). - {2, 1, desc(1), status(1, progress(2)), live(1), 0, + {2, 1, desc(1), status(1, progress(2)), live(1), 0, leadSupportedStatus(hlc.Timestamp{}), ReplicaMetrics{ - Leader: true, - RangeCounter: true, - Unavailable: false, - Underreplicated: true, - BehindCount: 10, + Leader: true, + RangeCounter: true, + Unavailable: false, + Underreplicated: true, + BehindCount: 10, + LeaderNotFortified: true, }}, // The leader of a 2-replica range is up. - {2, 1, desc(1, 2), status(1, progress(2)), live(1), 0, + {2, 1, desc(1, 2), status(1, progress(2)), live(1), 0, leadSupportedStatus(hlc.Timestamp{}), ReplicaMetrics{ - Leader: true, - RangeCounter: true, - Unavailable: true, - Underreplicated: true, - BehindCount: 10, + Leader: true, + RangeCounter: true, + Unavailable: true, + Underreplicated: true, + BehindCount: 10, + LeaderNotFortified: true, }}, // Both replicas of a 2-replica range are up to date. - {2, 1, desc(1, 2), status(1, progress(2, 2)), live(1, 2), 0, + {2, 1, desc(1, 2), status(1, progress(2, 2)), live(1, 2), 0, leadSupportedStatus(hlc.Timestamp{}), ReplicaMetrics{ - Leader: true, - RangeCounter: true, - Unavailable: false, - Underreplicated: false, - BehindCount: 20, + Leader: true, + RangeCounter: true, + Unavailable: false, + Underreplicated: false, + BehindCount: 20, + LeaderNotFortified: true, }}, // Both replicas of a 2-replica range are up to date (local replica is not leader) - {2, 2, desc(1, 2), status(2, progress(2, 2)), live(1, 2), 0, + {2, 2, desc(1, 2), status(2, progress(2, 2)), live(1, 2), 0, leadSupportedStatus(hlc.Timestamp{}), ReplicaMetrics{ Leader: false, RangeCounter: false, @@ -9078,90 +9087,99 @@ func TestReplicaMetrics(t *testing.T) { Underreplicated: false, }}, // Both replicas of a 2-replica range are live, but follower is behind. - {2, 1, desc(1, 2), status(1, progress(2, 1)), live(1, 2), 0, + {2, 1, desc(1, 2), status(1, progress(2, 1)), live(1, 2), 0, leadSupportedStatus(hlc.Timestamp{}), ReplicaMetrics{ - Leader: true, - RangeCounter: true, - Unavailable: false, - Underreplicated: false, - BehindCount: 21, + Leader: true, + RangeCounter: true, + Unavailable: false, + Underreplicated: false, + BehindCount: 21, + LeaderNotFortified: true, }}, // Both replicas of a 2-replica range are up to date, but follower is dead. - {2, 1, desc(1, 2), status(1, progress(2, 2)), live(1), 0, + {2, 1, desc(1, 2), status(1, progress(2, 2)), live(1), 0, leadSupportedStatus(hlc.Timestamp{}), ReplicaMetrics{ - Leader: true, - RangeCounter: true, - Unavailable: true, - Underreplicated: true, - BehindCount: 20, + Leader: true, + RangeCounter: true, + Unavailable: true, + Underreplicated: true, + BehindCount: 20, + LeaderNotFortified: true, }}, // The leader of a 3-replica range is up. - {3, 1, desc(1, 2, 3), status(1, progress(1)), live(1), 0, + {3, 1, desc(1, 2, 3), status(1, progress(1)), live(1), 0, leadSupportedStatus(hlc.Timestamp{}), ReplicaMetrics{ - Leader: true, - RangeCounter: true, - Unavailable: true, - Underreplicated: true, - BehindCount: 11, + Leader: true, + RangeCounter: true, + Unavailable: true, + Underreplicated: true, + BehindCount: 11, + LeaderNotFortified: true, }}, // All replicas of a 3-replica range are up to date. - {3, 1, desc(1, 2, 3), status(1, progress(2, 2, 2)), live(1, 2, 3), 0, + {3, 1, desc(1, 2, 3), status(1, progress(2, 2, 2)), live(1, 2, 3), 0, leadSupportedStatus(hlc.Timestamp{}), ReplicaMetrics{ - Leader: true, - RangeCounter: true, - Unavailable: false, - Underreplicated: false, - BehindCount: 30, + Leader: true, + RangeCounter: true, + Unavailable: false, + Underreplicated: false, + BehindCount: 30, + LeaderNotFortified: true, }}, // All replicas of a 3-replica range are up to date (match = 0 is // considered up to date). - {3, 1, desc(1, 2, 3), status(1, progress(2, 2, 0)), live(1, 2, 3), 0, + {3, 1, desc(1, 2, 3), status(1, progress(2, 2, 0)), live(1, 2, 3), 0, leadSupportedStatus(hlc.Timestamp{}), ReplicaMetrics{ - Leader: true, - RangeCounter: true, - Unavailable: false, - Underreplicated: false, - BehindCount: 20, + Leader: true, + RangeCounter: true, + Unavailable: false, + Underreplicated: false, + BehindCount: 20, + LeaderNotFortified: true, }}, // All replicas of a 3-replica range are live but one replica is behind. - {3, 1, desc(1, 2, 3), status(1, progress(2, 2, 1)), live(1, 2, 3), 0, + {3, 1, desc(1, 2, 3), status(1, progress(2, 2, 1)), live(1, 2, 3), 0, leadSupportedStatus(hlc.Timestamp{}), ReplicaMetrics{ - Leader: true, - RangeCounter: true, - Unavailable: false, - Underreplicated: false, - BehindCount: 31, + Leader: true, + RangeCounter: true, + Unavailable: false, + Underreplicated: false, + BehindCount: 31, + LeaderNotFortified: true, }}, // All replicas of a 3-replica range are live but two replicas are behind. - {3, 1, desc(1, 2, 3), status(1, progress(2, 1, 1)), live(1, 2, 3), 0, + {3, 1, desc(1, 2, 3), status(1, progress(2, 1, 1)), live(1, 2, 3), 0, leadSupportedStatus(hlc.Timestamp{}), ReplicaMetrics{ - Leader: true, - RangeCounter: true, - Unavailable: false, - Underreplicated: false, - BehindCount: 32, + Leader: true, + RangeCounter: true, + Unavailable: false, + Underreplicated: false, + BehindCount: 32, + LeaderNotFortified: true, }}, // All replicas of a 3-replica range are up to date, but one replica is dead. - {3, 1, desc(1, 2, 3), status(1, progress(2, 2, 2)), live(1, 2), 0, + {3, 1, desc(1, 2, 3), status(1, progress(2, 2, 2)), live(1, 2), 0, leadSupportedStatus(hlc.Timestamp{}), ReplicaMetrics{ - Leader: true, - RangeCounter: true, - Unavailable: false, - Underreplicated: true, - BehindCount: 30, + Leader: true, + RangeCounter: true, + Unavailable: false, + Underreplicated: true, + BehindCount: 30, + LeaderNotFortified: true, }}, // All replicas of a 3-replica range are up to date, but two replicas are dead. - {3, 1, desc(1, 2, 3), status(1, progress(2, 2, 2)), live(1), 0, + {3, 1, desc(1, 2, 3), status(1, progress(2, 2, 2)), live(1), 0, leadSupportedStatus(hlc.Timestamp{}), ReplicaMetrics{ - Leader: true, - RangeCounter: true, - Unavailable: true, - Underreplicated: true, - BehindCount: 30, + Leader: true, + RangeCounter: true, + Unavailable: true, + Underreplicated: true, + BehindCount: 30, + LeaderNotFortified: true, }}, // All replicas of a 3-replica range are up to date, but two replicas are // dead, including the leader. - {3, 2, desc(1, 2, 3), status(0, progress(2, 2, 2)), live(2), 0, + {3, 2, desc(1, 2, 3), status(0, progress(2, 2, 2)), live(2), 0, leadSupportedStatus(hlc.Timestamp{}), ReplicaMetrics{ Leader: false, RangeCounter: true, @@ -9170,7 +9188,7 @@ func TestReplicaMetrics(t *testing.T) { BehindCount: 0, }}, // Range has no leader, local replica is the range counter. - {3, 1, desc(1, 2, 3), status(0, progress(2, 2, 2)), live(1, 2, 3), 0, + {3, 1, desc(1, 2, 3), status(0, progress(2, 2, 2)), live(1, 2, 3), 0, leadSupportedStatus(hlc.Timestamp{}), ReplicaMetrics{ Leader: false, RangeCounter: true, @@ -9178,7 +9196,7 @@ func TestReplicaMetrics(t *testing.T) { Underreplicated: false, }}, // Range has no leader, local replica is the range counter. - {3, 3, desc(3, 2, 1), status(0, progress(2, 2, 2)), live(1, 2, 3), 0, + {3, 3, desc(3, 2, 1), status(0, progress(2, 2, 2)), live(1, 2, 3), 0, leadSupportedStatus(hlc.Timestamp{}), ReplicaMetrics{ Leader: false, RangeCounter: true, @@ -9186,7 +9204,7 @@ func TestReplicaMetrics(t *testing.T) { Underreplicated: false, }}, // Range has no leader, local replica is not the range counter. - {3, 2, desc(1, 2, 3), status(0, progress(2, 2, 2)), live(1, 2, 3), 0, + {3, 2, desc(1, 2, 3), status(0, progress(2, 2, 2)), live(1, 2, 3), 0, leadSupportedStatus(hlc.Timestamp{}), ReplicaMetrics{ Leader: false, RangeCounter: false, @@ -9194,7 +9212,7 @@ func TestReplicaMetrics(t *testing.T) { Underreplicated: false, }}, // Range has no leader, local replica is not the range counter. - {3, 3, desc(1, 2, 3), status(0, progress(2, 2, 2)), live(1, 2, 3), 0, + {3, 3, desc(1, 2, 3), status(0, progress(2, 2, 2)), live(1, 2, 3), 0, leadSupportedStatus(hlc.Timestamp{}), ReplicaMetrics{ Leader: false, RangeCounter: false, @@ -9202,14 +9220,35 @@ func TestReplicaMetrics(t *testing.T) { Underreplicated: false, }}, // The leader of a 1-replica range is up and raft log is too large. - {1, 1, desc(1), status(1, progress(2)), live(1), 5 * cfg.RaftLogTruncationThreshold, + {1, 1, desc(1), status(1, progress(2)), live(1), 5 * cfg.RaftLogTruncationThreshold, leadSupportedStatus(hlc.Timestamp{}), ReplicaMetrics{ - Leader: true, - RangeCounter: true, - Unavailable: false, - Underreplicated: false, - BehindCount: 10, - RaftLogTooLarge: true, + Leader: true, + RangeCounter: true, + Unavailable: false, + Underreplicated: false, + BehindCount: 10, + RaftLogTooLarge: true, + LeaderNotFortified: true, + }}, + // The leader of a 1-replica range is up, and the leader support expired. + {1, 1, desc(1), status(1, progress(2)), live(1), 0, leadSupportedStatus(hlc.MinTimestamp), + ReplicaMetrics{ + Leader: true, + RangeCounter: true, + Unavailable: false, + Underreplicated: false, + BehindCount: 10, + LeaderNotFortified: true, // the support expired + }}, + // The leader of a 1-replica range is up, and the support hasn't expired. + {1, 1, desc(1), status(1, progress(2)), live(1), 0, leadSupportedStatus(hlc.MaxTimestamp), + ReplicaMetrics{ + Leader: true, + RangeCounter: true, + Unavailable: false, + Underreplicated: false, + BehindCount: 10, + LeaderNotFortified: false, // The support hasn't expired yet }}, } @@ -9233,6 +9272,8 @@ func TestReplicaMetrics(t *testing.T) { ticking: c.expected.Ticking, raftLogSize: c.raftLogSize, raftLogSizeTrusted: true, + now: tc.Clock().NowAsClockTimestamp(), + leadSupportStatus: c.leaderSupportStatus, }) require.Equal(t, c.expected, metrics) }) diff --git a/pkg/kv/kvserver/store.go b/pkg/kv/kvserver/store.go index fbeca39b7c2d..25de6f0bd476 100644 --- a/pkg/kv/kvserver/store.go +++ b/pkg/kv/kvserver/store.go @@ -3288,6 +3288,7 @@ func (s *Store) updateReplicationGauges(ctx context.Context) error { leaseViolatingPreferencesCount int64 leaseLessPreferredCount int64 raftLeaderNotLeaseHolderCount int64 + raftLeaderNotFortifiedCount int64 raftLeaderInvalidLeaseCount int64 quiescentCount int64 uninitializedCount int64 @@ -3353,6 +3354,9 @@ func (s *Store) updateReplicationGauges(ctx context.Context) error { if !metrics.LeaseValid { raftLeaderInvalidLeaseCount++ } + if metrics.LeaderNotFortified { + raftLeaderNotFortifiedCount++ + } kvflowSendStats.Clear() rep.SendStreamStats(&kvflowSendStats) sizeCount, sizeBytes := kvflowSendStats.SumSendQueues() @@ -3443,6 +3447,7 @@ func (s *Store) updateReplicationGauges(ctx context.Context) error { s.metrics.RaftLeaderCount.Update(raftLeaderCount) s.metrics.RaftLeaderNotLeaseHolderCount.Update(raftLeaderNotLeaseHolderCount) s.metrics.RaftLeaderInvalidLeaseCount.Update(raftLeaderInvalidLeaseCount) + s.metrics.RaftLeaderNotFortifiedCount.Update(raftLeaderNotFortifiedCount) s.metrics.LeaseHolderCount.Update(leaseHolderCount) s.metrics.LeaseExpirationCount.Update(leaseExpirationCount) s.metrics.LeaseEpochCount.Update(leaseEpochCount) diff --git a/pkg/roachprod/opentelemetry/cockroachdb_metrics.go b/pkg/roachprod/opentelemetry/cockroachdb_metrics.go index 7b954b986492..074dbbf3d41c 100644 --- a/pkg/roachprod/opentelemetry/cockroachdb_metrics.go +++ b/pkg/roachprod/opentelemetry/cockroachdb_metrics.go @@ -1374,6 +1374,7 @@ var cockroachdbMetrics = map[string]string{ "replicas_leaders": "replicas.leaders", "replicas_leaders_invalid_lease": "replicas.leaders_invalid_lease", "replicas_leaders_not_leaseholders": "replicas.leaders.not_leaseholders", + "replicas_leaders_not_fortified": "replicas.leaders.not_fortified", "replicas_leaseholders": "replicas.leaseholders", "replicas_quiescent": "replicas.quiescent", "replicas_reserved": "replicas.reserved",