From d157d931adb44e6d7009cab6191412f786ec4645 Mon Sep 17 00:00:00 2001
From: "Chris S. Kim" <ckim@hashicorp.com>
Date: Thu, 5 Oct 2023 16:01:16 -0400
Subject: [PATCH] Retry tests

---
 agent/metrics_test.go | 69 +++++++++++++++++++------------------------
 1 file changed, 30 insertions(+), 39 deletions(-)

diff --git a/agent/metrics_test.go b/agent/metrics_test.go
index fd4f9f8d2e17..97286cdde989 100644
--- a/agent/metrics_test.go
+++ b/agent/metrics_test.go
@@ -20,6 +20,7 @@ import (
 	"github.com/hashicorp/consul/agent/rpc/middleware"
 	"github.com/hashicorp/consul/lib/retry"
 	"github.com/hashicorp/consul/sdk/testutil"
+	testretry "github.com/hashicorp/consul/sdk/testutil/retry"
 	"github.com/hashicorp/consul/testrpc"
 	"github.com/hashicorp/consul/tlsutil"
 )
@@ -30,8 +31,7 @@ func skipIfShortTesting(t *testing.T) {
 	}
 }
 
-func recordPromMetrics(t *testing.T, a *TestAgent, respRec *httptest.ResponseRecorder) {
-	t.Helper()
+func recordPromMetrics(t require.TestingT, a *TestAgent, respRec *httptest.ResponseRecorder) {
 	req, err := http.NewRequest("GET", "/v1/agent/metrics?format=prometheus", nil)
 	require.NoError(t, err, "Failed to generate new http request.")
 
@@ -483,28 +483,24 @@ func TestHTTPHandlers_AgentMetrics_WAL_Prometheus(t *testing.T) {
 		defer a.Shutdown()
 		testrpc.WaitForLeader(t, a.RPC, "dc1")
 
-		respRec := httptest.NewRecorder()
-		recordPromMetrics(t, a, respRec)
+		testretry.Run(t, func(r *testretry.R) {
+			respRec := httptest.NewRecorder()
+			recordPromMetrics(r, a, respRec)
+
+			out := respRec.Body.String()
+			require.Contains(r, out, "agent_5_raft_wal_head_truncations")
+			require.Contains(r, out, "agent_5_raft_wal_last_segment_age_seconds")
+			require.Contains(r, out, "agent_5_raft_wal_log_appends")
+			require.Contains(r, out, "agent_5_raft_wal_log_entries_read")
+			require.Contains(r, out, "agent_5_raft_wal_log_entries_written")
+			require.Contains(r, out, "agent_5_raft_wal_log_entry_bytes_read")
+			require.Contains(r, out, "agent_5_raft_wal_log_entry_bytes_written")
+			require.Contains(r, out, "agent_5_raft_wal_segment_rotations")
+			require.Contains(r, out, "agent_5_raft_wal_stable_gets")
+			require.Contains(r, out, "agent_5_raft_wal_stable_sets")
+			require.Contains(r, out, "agent_5_raft_wal_tail_truncations")
+		})
 
-		out := respRec.Body.String()
-		defer func() {
-			if t.Failed() {
-				t.Log("--- Failed output START ---")
-				t.Log(out)
-				t.Log("--- Failed output END ---")
-			}
-		}()
-		require.Contains(t, out, "agent_5_raft_wal_head_truncations")
-		require.Contains(t, out, "agent_5_raft_wal_last_segment_age_seconds")
-		require.Contains(t, out, "agent_5_raft_wal_log_appends")
-		require.Contains(t, out, "agent_5_raft_wal_log_entries_read")
-		require.Contains(t, out, "agent_5_raft_wal_log_entries_written")
-		require.Contains(t, out, "agent_5_raft_wal_log_entry_bytes_read")
-		require.Contains(t, out, "agent_5_raft_wal_log_entry_bytes_written")
-		require.Contains(t, out, "agent_5_raft_wal_segment_rotations")
-		require.Contains(t, out, "agent_5_raft_wal_stable_gets")
-		require.Contains(t, out, "agent_5_raft_wal_stable_sets")
-		require.Contains(t, out, "agent_5_raft_wal_tail_truncations")
 	})
 
 	t.Run("server without WAL enabled emits no WAL metrics", func(t *testing.T) {
@@ -590,22 +586,17 @@ func TestHTTPHandlers_AgentMetrics_LogVerifier_Prometheus(t *testing.T) {
 		defer a.Shutdown()
 		testrpc.WaitForLeader(t, a.RPC, "dc1")
 
-		respRec := httptest.NewRecorder()
-		recordPromMetrics(t, a, respRec)
-
-		out := respRec.Body.String()
-		defer func() {
-			if t.Failed() {
-				t.Log("--- Failed output START ---")
-				t.Log(out)
-				t.Log("--- Failed output END ---")
-			}
-		}()
-		require.Contains(t, out, "agent_5_raft_logstore_verifier_checkpoints_written")
-		require.Contains(t, out, "agent_5_raft_logstore_verifier_dropped_reports")
-		require.Contains(t, out, "agent_5_raft_logstore_verifier_ranges_verified")
-		require.Contains(t, out, "agent_5_raft_logstore_verifier_read_checksum_failures")
-		require.Contains(t, out, "agent_5_raft_logstore_verifier_write_checksum_failures")
+		testretry.Run(t, func(r *testretry.R) {
+			respRec := httptest.NewRecorder()
+			recordPromMetrics(r, a, respRec)
+
+			out := respRec.Body.String()
+			require.Contains(r, out, "agent_5_raft_logstore_verifier_checkpoints_written")
+			require.Contains(r, out, "agent_5_raft_logstore_verifier_dropped_reports")
+			require.Contains(r, out, "agent_5_raft_logstore_verifier_ranges_verified")
+			require.Contains(r, out, "agent_5_raft_logstore_verifier_read_checksum_failures")
+			require.Contains(r, out, "agent_5_raft_logstore_verifier_write_checksum_failures")
+		})
 	})
 
 	t.Run("server with verifier disabled emits no extra metrics", func(t *testing.T) {