Resolves #436 - Handle Connection Errors

cmd/root.go

@@ -124,6 +124,10 @@ func InitializeCmd() {
 	RootCmd.PersistentFlags().Uint16VarP(&rateLimit, "rate-limit", "", 10, "Request limit per second")
 	RootCmd.PersistentFlags().BoolVarP(&httpclient.Retry5xx, "retry-5xx", "", false, "Whether we should retry requests with HTTP 5xx response code")
 	RootCmd.PersistentFlags().BoolVarP(&httpclient.Retry429, "retry-429", "", false, "Whether we should retry requests with HTTP 429 response code")
+	RootCmd.PersistentFlags().BoolVarP(&httpclient.RetryConnectionErrors, "retry-connection-errors", "", false, "Whether we should retry requests with connection errors")
+	RootCmd.PersistentFlags().UintVarP(&httpclient.RetryDelay, "retry-delay", "", 500, "When retrying how long should we delay")
+	RootCmd.PersistentFlags().BoolVarP(&httpclient.RetryAllErrors, "retry-all-errors", "", false, "When enable retries on all errors (i.e., the same as --retry-5xx --retry-429 and --retry-connection-errors")
+
 	RootCmd.PersistentFlags().BoolVarP(&httpclient.DontLog2xxs, "silence-2xx", "", false, "Whether we should silence HTTP 2xx response code logging")
 
 	RootCmd.PersistentFlags().Float32VarP(&requestTimeout, "timeout", "", 60, "Request timeout in seconds (fractional values allowed)")

external/httpclient/httpclient.go

@@ -97,6 +97,11 @@ func Initialize(rateLimit uint16, requestTimeout float32, statisticsFrequency in
 var Retry429 = false
 var Retry5xx = false
 
+var RetryConnectionErrors = false
+
+var RetryAllErrors = false
+var RetryDelay uint = 500
+
 var statsLock = &sync.Mutex{}
 
 var HttpClient = &http.Client{}
@@ -179,9 +184,16 @@ func doRequestInternal(ctx context.Context, method string, contentType string, p
 	reqURL.Path = path
 	reqURL.RawQuery = query
 
-	var bodyBuf bytes.Buffer
+	var bodyBuf []byte
 	if payload != nil {
-		payload = io.TeeReader(payload, &bodyBuf)
+		buf := new(bytes.Buffer)
+		_, err := buf.ReadFrom(payload)
+		if err != nil {
+			log.Warnf("Error reading payload, %s", err)
+		}
+		bodyBuf = buf.Bytes()
+
+		payload = bytes.NewReader(bodyBuf)
 	}
 
 	req, err := http.NewRequest(method, reqURL.String(), payload)
@@ -270,9 +282,25 @@ func doRequestInternal(ctx context.Context, method string, contentType string, p
 	statsLock.Unlock()
 
 	log.Tracef("Stats processing complete")
-
-	if err != nil {
-		return nil, err
+	requestError := err
+	if requestError != nil {
+
+		resp = &http.Response{
+			Status:           "CONNECTION_ERROR",
+			StatusCode:       0,
+			Proto:            "",
+			ProtoMajor:       0,
+			ProtoMinor:       0,
+			Header:           map[string][]string{},
+			Body:             io.NopCloser(strings.NewReader(fmt.Sprintf("%v", err))),
+			ContentLength:    0,
+			TransferEncoding: nil,
+			Close:            true,
+			Uncompressed:     false,
+			Trailer:          nil,
+			Request:          nil,
+			TLS:              nil,
+		}
 	}
 
 	var logf func(string, ...interface{})
@@ -322,7 +350,7 @@ func doRequestInternal(ctx context.Context, method string, contentType string, p
 
 	if displayLongFormRequestAndResponse {
 		if payload != nil {
-			body, _ := io.ReadAll(&bodyBuf)
+			body := bodyBuf
 			if len(body) > 0 {
 				logf("(%0.4d) %s %s%s", requestNumber, method, reqURL.String(), requestHeaders)
 				if contentType == "application/json" {
@@ -352,10 +380,24 @@ func doRequestInternal(ctx context.Context, method string, contentType string, p
 	log.Tracef("Starting log to disk")
 	profiles.LogRequestToDisk(method, path, dumpReq, dumpRes, resp.StatusCode)
 	log.Tracef("Done log to disk")
-	if resp.StatusCode == 429 && Retry429 {
-		return doRequestInternal(ctx, method, contentType, path, query, &bodyBuf)
-	} else if resp.StatusCode >= 500 && Retry5xx {
-		return doRequestInternal(ctx, method, contentType, path, query, &bodyBuf)
+	if resp.StatusCode == 429 && (Retry429 || RetryAllErrors) {
+		if RetryDelay > 0 {
+			log.Debugf("Retrying request in %d ms", RetryDelay)
+			time.Sleep(time.Duration(RetryDelay) * time.Millisecond)
+		}
+		return doRequestInternal(ctx, method, contentType, path, query, bytes.NewReader(bodyBuf))
+	} else if resp.StatusCode >= 500 && (Retry5xx || RetryAllErrors) {
+		if RetryDelay > 0 {
+			log.Debugf("Retrying request in %d ms", RetryDelay)
+			time.Sleep(time.Duration(RetryDelay) * time.Millisecond)
+		}
+		return doRequestInternal(ctx, method, contentType, path, query, bytes.NewReader(bodyBuf))
+	} else if requestError != nil && (RetryConnectionErrors || RetryAllErrors) {
+		if RetryDelay > 0 {
+			log.Debugf("Retrying request in %d ms", RetryDelay)
+			time.Sleep(time.Duration(RetryDelay) * time.Millisecond)
+		}
+		return doRequestInternal(ctx, method, contentType, path, query, bytes.NewReader(bodyBuf))
 	} else {
 		return resp, err
 	}

external/profiles/requestlogs.go

@@ -86,7 +86,12 @@ func LogRequestToDisk(requestMethod string, requestPath string, requestBytes []b
 		responseBytes = regex1.ReplaceAll(responseBytes, []byte("client_secret=*****"))
 	}
 
-	return SaveRequest(fmt.Sprintf("%s %s ==> %d", requestMethod, requestPath, responseCode), requestBytes, responseBytes)
+	statusCode := fmt.Sprintf("%d", responseCode)
+
+	if responseCode == 0 {
+		statusCode = "ERROR"
+	}
+	return SaveRequest(fmt.Sprintf("%s %s ==> %s", requestMethod, requestPath, statusCode), requestBytes, responseBytes)
 }
 
 func SaveRequest(title string, requestBytes []byte, responseBytes []byte) error {

go.mod

@@ -9,12 +9,12 @@ require (
 	github.com/caarlos0/env/v6 v6.10.1
 	github.com/google/uuid v1.5.0
 	github.com/gookit/color v1.5.3
-	github.com/itchyny/gojq v0.12.13
+	github.com/itchyny/gojq v0.12.14
 	github.com/mattn/go-isatty v0.0.20
 	github.com/santhosh-tekuri/jsonschema/v4 v4.0.0
 	github.com/santhosh-tekuri/jsonschema/v5 v5.3.0
 	github.com/sirupsen/logrus v1.9.3
-	github.com/spf13/cobra v1.7.0
+	github.com/spf13/cobra v1.8.0
 	github.com/stretchr/testify v1.8.4
 	github.com/thediveo/enumflag v0.10.1
 	github.com/yosida95/uritemplate/v3 v3.0.2

go.sum

@@ -26,7 +26,7 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
-github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -70,8 +70,8 @@ github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
-github.com/itchyny/gojq v0.12.13 h1:IxyYlHYIlspQHHTE0f3cJF0NKDMfajxViuhBLnHd/QU=
-github.com/itchyny/gojq v0.12.13/go.mod h1:JzwzAqenfhrPUuwbmEz3nu3JQmFLlQTQMUcOdnu/Sf4=
+github.com/itchyny/gojq v0.12.14 h1:6k8vVtsrhQSYgSGg827AD+PVVaB1NLXEdX+dda2oZCc=
+github.com/itchyny/gojq v0.12.14/go.mod h1:y1G7oO7XkcR1LPZO59KyoCRy08T3j9vDYRV0GgYSS+s=
 github.com/itchyny/timefmt-go v0.1.5 h1:G0INE2la8S6ru/ZI5JecgyzbbJNs5lG1RcBqa7Jm6GE=
 github.com/itchyny/timefmt-go v0.1.5/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8=
 github.com/jolestar/go-commons-pool/v2 v2.1.2 h1:E+XGo58F23t7HtZiC/W6jzO2Ux2IccSH/yx4nD+J1CM=
@@ -140,8 +140,8 @@ github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU
 github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
 github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v0.0.7/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
-github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
-github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
+github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
+github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=