forget about smart reconnect, just do it in a loop

integration-tests/go.mod

@@ -6,7 +6,6 @@ go 1.21.7
 replace github.com/smartcontractkit/chainlink/v2 => ../
 
 require (
-	github.com/avast/retry-go/v4 v4.5.1
 	github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df
 	github.com/cli/go-gh/v2 v2.0.0
 	github.com/ethereum/go-ethereum v1.13.8
@@ -89,6 +88,7 @@ require (
 	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/avast/retry-go v3.0.0+incompatible // indirect
+	github.com/avast/retry-go/v4 v4.5.1 // indirect
 	github.com/aws/aws-sdk-go v1.45.25 // indirect
 	github.com/aws/constructs-go/constructs/v10 v10.1.255 // indirect
 	github.com/aws/jsii-runtime-go v1.75.0 // indirect

integration-tests/testsetups/keeper_benchmark.go

@@ -12,7 +12,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/avast/retry-go/v4"
 	geth "github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/accounts/abi"
 	"github.com/ethereum/go-ethereum/common"
@@ -280,8 +279,6 @@ func (k *KeeperBenchmarkTest) Run() {
 	// signals all goroutines to stop when subscription error occurs
 	stopAllGoroutinesCh := make(chan struct{})
 
-	retryLimit := 5
-
 	// this goroutine fans out headers to goroutines in the background
 	// and exists when all goroutines are done or when an error occurs
 	go func() {
@@ -308,24 +305,18 @@ func (k *KeeperBenchmarkTest) Run() {
 			case err := <-sub.Err():
 				// no need to unsubscribe, subscripion errored
 				k.log.Error().Err(err).Msg("header subscription failed. Trying to reconnect...")
-				startErr := retry.Do(func() error {
+				connectionLostAt := time.Now()
+				// we use infinite loop here on purposes, these nodes can be down for extended periods of time ¯\_(ツ)_/¯
+			RECONNECT:
+				for {
 					sub, err = k.chainClient.Client.Client().EthSubscribe(context.Background(), headerCh, "newHeads")
-					return err
-				},
-					retry.Attempts(uint(retryLimit)),
-					retry.Delay(1*time.Second),
-					retry.OnRetry(func(n uint, err error) {
-						k.log.Info().
-							Str("Attempt", fmt.Sprintf("%d/%d", n+1, retryLimit)).
-							Msg("Trying to reconnect to header subscription")
-					}),
-				)
-				if startErr != nil {
-					// close channel to signal all goroutines they should exit, but only if we failed to reconnect
-					close(stopAllGoroutinesCh)
-					return
+					if err == nil {
+						break RECONNECT
+					}
+
+					time.Sleep(5 * time.Second)
 				}
-				k.log.Info().Msg("Reconnected to header subscription")
+				k.log.Info().Str("Reconnect Time", time.Since(connectionLostAt).String()).Msg("Reconnected to header subscription")
 			}
 		}
 	}()