Merge #134545

134545: raft: add replicas.leaders.not_fortified metric r=iskettaneh a=iskettaneh This commit adds the metric replicas.leaders.not_fortified which reports the number of leaders that think they are not fortified. References: #132754 Release note: None Co-authored-by: Ibrahim Kettaneh <[email protected]>
cockroachdb · Nov 13, 2024 · 29b34ea · 29b34ea
2 parents 409ba28 + c42ba6a
commit 29b34ea
Show file tree

Hide file tree

Showing 6 changed files with 165 additions and 98 deletions.
diff --git a/docs/generated/metrics/metrics.html b/docs/generated/metrics/metrics.html
@@ -606,6 +606,7 @@
 <tr><td>STORAGE</td><td>replicas</td><td>Number of replicas</td><td>Replicas</td><td>GAUGE</td><td>COUNT</td><td>AVG</td><td>NONE</td></tr>
 <tr><td>STORAGE</td><td>replicas.leaders</td><td>Number of raft leaders</td><td>Raft Leaders</td><td>GAUGE</td><td>COUNT</td><td>AVG</td><td>NONE</td></tr>
 <tr><td>STORAGE</td><td>replicas.leaders_invalid_lease</td><td>Number of replicas that are Raft leaders whose lease is invalid</td><td>Replicas</td><td>GAUGE</td><td>COUNT</td><td>AVG</td><td>NONE</td></tr>
+<tr><td>STORAGE</td><td>replicas.leaders_not_fortified</td><td>Number of replicas that are not fortified Raft leaders</td><td>Replicas</td><td>GAUGE</td><td>COUNT</td><td>AVG</td><td>NONE</td></tr>
 <tr><td>STORAGE</td><td>replicas.leaders_not_leaseholders</td><td>Number of replicas that are Raft leaders whose range lease is held by another store</td><td>Replicas</td><td>GAUGE</td><td>COUNT</td><td>AVG</td><td>NONE</td></tr>
 <tr><td>STORAGE</td><td>replicas.leaseholders</td><td>Number of lease holders</td><td>Replicas</td><td>GAUGE</td><td>COUNT</td><td>AVG</td><td>NONE</td></tr>
 <tr><td>STORAGE</td><td>replicas.quiescent</td><td>Number of quiesced replicas</td><td>Replicas</td><td>GAUGE</td><td>COUNT</td><td>AVG</td><td>NONE</td></tr>

diff --git a/pkg/kv/kvserver/metrics.go b/pkg/kv/kvserver/metrics.go
@@ -65,6 +65,12 @@ var (
 		Measurement: "Replicas",
 		Unit:        metric.Unit_COUNT,
 	}
+	metaRaftLeaderNotFortifiedCount = metric.Metadata{
+		Name:        "replicas.leaders_not_fortified",
+		Help:        "Number of replicas that are not fortified Raft leaders",
+		Measurement: "Replicas",
+		Unit:        metric.Unit_COUNT,
+	}
 	metaRaftLeaderInvalidLeaseCount = metric.Metadata{
 		Name:        "replicas.leaders_invalid_lease",
 		Help:        "Number of replicas that are Raft leaders whose lease is invalid",
@@ -2585,6 +2591,7 @@ type StoreMetrics struct {
 	ReservedReplicaCount          *metric.Gauge
 	RaftLeaderCount               *metric.Gauge
 	RaftLeaderNotLeaseHolderCount *metric.Gauge
+	RaftLeaderNotFortifiedCount   *metric.Gauge
 	RaftLeaderInvalidLeaseCount   *metric.Gauge
 	LeaseHolderCount              *metric.Gauge
 	QuiescentCount                *metric.Gauge
@@ -3286,6 +3293,7 @@ func newStoreMetrics(histogramWindow time.Duration) *StoreMetrics {
 		ReservedReplicaCount:          metric.NewGauge(metaReservedReplicaCount),
 		RaftLeaderCount:               metric.NewGauge(metaRaftLeaderCount),
 		RaftLeaderNotLeaseHolderCount: metric.NewGauge(metaRaftLeaderNotLeaseHolderCount),
+		RaftLeaderNotFortifiedCount:   metric.NewGauge(metaRaftLeaderNotFortifiedCount),
 		RaftLeaderInvalidLeaseCount:   metric.NewGauge(metaRaftLeaderInvalidLeaseCount),
 		LeaseHolderCount:              metric.NewGauge(metaLeaseHolderCount),
 		QuiescentCount:                metric.NewGauge(metaQuiescentCount),

diff --git a/pkg/kv/kvserver/replica_metrics.go b/pkg/kv/kvserver/replica_metrics.go
@@ -33,6 +33,10 @@ type ReplicaMetrics struct {
 	ViolatingLeasePreferences bool
 	LessPreferredLease        bool
 
+	// LeaderNotFortified indicates whether the leader believes itself to be
+	// fortified or not.
+	LeaderNotFortified bool
+
 	// Quiescent indicates whether the replica believes itself to be quiesced.
 	Quiescent bool
 	// Ticking indicates whether the store is ticking the replica. It should be
@@ -95,6 +99,8 @@ func (r *Replica) Metrics(
 		clusterNodes:             clusterNodes,
 		desc:                     r.shMu.state.Desc,
 		raftStatus:               r.raftSparseStatusRLocked(),
+		leadSupportStatus:        r.raftLeadSupportStatusRLocked(),
+		now:                      now,
 		leaseStatus:              r.leaseStatusAtRLocked(ctx, now),
 		storeID:                  r.store.StoreID(),
 		storeAttrs:               storeAttrs,
@@ -126,6 +132,8 @@ type calcReplicaMetricsInput struct {
 	clusterNodes             int
 	desc                     *roachpb.RangeDescriptor
 	raftStatus               *raft.SparseStatus
+	leadSupportStatus        raft.LeadSupportStatus
+	now                      hlc.ClockTimestamp
 	leaseStatus              kvserverpb.LeaseStatus
 	storeID                  roachpb.StoreID
 	storeAttrs, nodeAttrs    roachpb.Attributes
@@ -176,9 +184,11 @@ func calcReplicaMetrics(d calcReplicaMetricsInput) ReplicaMetrics {
 	// behind.
 	leader := d.raftStatus != nil && d.raftStatus.RaftState == raftpb.StateLeader
 	var leaderBehindCount, leaderPausedFollowerCount int64
+	var leaderNotFortified bool
 	if leader {
 		leaderBehindCount = calcBehindCount(d.raftStatus, d.desc, d.vitalityMap)
 		leaderPausedFollowerCount = int64(len(d.paused))
+		leaderNotFortified = d.leadSupportStatus.LeadSupportUntil.Less(d.now.ToTimestamp())
 	}
 
 	return ReplicaMetrics{
@@ -190,6 +200,7 @@ func calcReplicaMetrics(d calcReplicaMetricsInput) ReplicaMetrics {
 		LivenessLease:             livenessLease,
 		ViolatingLeasePreferences: violatingLeasePreferences,
 		LessPreferredLease:        lessPreferredLease,
+		LeaderNotFortified:        leaderNotFortified,
 		Quiescent:                 d.quiescent,
 		Ticking:                   d.ticking,
 		RangeCounter:              rangeCounter,