Rate limit for HTTP calls (#48)

* test for rate limit

* Add HTTP retry

* Retry test without body

* Adding retry http client to configuration

* lint fix

---------

Co-authored-by: Peter Csajtai <[email protected]>

.gitignore

@@ -13,6 +13,8 @@
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
 
+.idea/
+
 # Dependency directories (remove the comment below to include it)
 vendor/
 

go.mod

@@ -8,15 +8,19 @@ require (
 	github.com/hashicorp/terraform-plugin-go v0.22.1
 	github.com/hashicorp/terraform-plugin-log v0.9.0
 	github.com/hashicorp/terraform-plugin-testing v1.6.0
+	github.com/stretchr/testify v1.8.4
 )
 
 require (
 	github.com/Kunde21/markdownfmt/v3 v3.1.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/hashicorp/cli v1.1.6 // indirect
 	github.com/mattn/go-runewidth v0.0.9 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/yuin/goldmark v1.6.0 // indirect
 	github.com/yuin/goldmark-meta v1.1.0 // indirect
 	gopkg.in/yaml.v2 v2.3.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
 require (

internal/configcat/client/client.go

@@ -3,6 +3,7 @@ package client
 import (
 	"context"
 	"fmt"
+	"net/http"
 
 	configcatpublicapi "github.com/configcat/configcat-publicapi-go-client"
 )
@@ -46,6 +47,9 @@ func NewClient(basePath, basicAuthUsername, basicAuthPassword, version string) (
 	configuration.Servers[0].URL = basePath
 	configuration.UserAgent = "terraform-provider-configcat/" + version
 	configuration.AddDefaultHeader("X-Caller-Id", "terraform-provider-configcat/"+version)
+	configuration.HTTPClient = &http.Client{
+		Transport: Retry(http.DefaultTransport, 5),
+	}
 	apiClient := configcatpublicapi.NewAPIClient(configuration)
 
 	client := &Client{

internal/configcat/client/retry_roundtrip.go

@@ -0,0 +1,80 @@
+package client
+
+import (
+	"bytes"
+	"io"
+	"net/http"
+	"strconv"
+	"time"
+)
+
+type retryRoundTripper struct {
+	http.RoundTripper
+	maxRetries int
+}
+
+func Retry(transport http.RoundTripper, maxRetries int) http.RoundTripper {
+	return &retryRoundTripper{RoundTripper: transport, maxRetries: maxRetries}
+}
+
+func (rt *retryRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) {
+	bodyBuf, err := preReadRequestBodyAndClose(r)
+	if err != nil {
+		return nil, err
+	}
+
+	var attempts int
+	for {
+		if bodyBuf != nil {
+			r.Body = io.NopCloser(bodyBuf)
+		}
+		resp, err := rt.RoundTripper.RoundTrip(r)
+		if err != nil {
+			return resp, err
+		}
+		if resp.StatusCode != http.StatusTooManyRequests {
+			return resp, nil
+		}
+		retryAfterHeader := resp.Header.Get("Retry-After")
+		retryAfter, err := strconv.ParseInt(retryAfterHeader, 10, 64)
+		if err != nil {
+			return resp, err
+		}
+
+		if attempts >= rt.maxRetries {
+			return resp, nil
+		}
+
+		if bodyBuf != nil {
+			if _, err := bodyBuf.Seek(0, io.SeekStart); err != nil {
+				return resp, err
+			}
+		}
+
+		if resp.Body != nil {
+			_, _ = io.Copy(io.Discard, resp.Body)
+			_ = resp.Body.Close()
+		}
+
+		select {
+		case <-time.After(time.Duration(retryAfter) * time.Second):
+			attempts++
+		case <-r.Context().Done():
+			return nil, r.Context().Err()
+		}
+	}
+}
+
+func preReadRequestBodyAndClose(r *http.Request) (*bytes.Reader, error) {
+	var reader *bytes.Reader
+	if r.Body != nil && r.Body != http.NoBody {
+		var buf bytes.Buffer
+		if _, err := io.Copy(&buf, r.Body); err != nil {
+			_ = r.Body.Close()
+			return nil, err
+		}
+		_ = r.Body.Close()
+		reader = bytes.NewReader(buf.Bytes())
+	}
+	return reader, nil
+}

internal/configcat/client/retry_roundtrip_test.go

@@ -0,0 +1,73 @@
+package client
+
+import (
+	"github.com/stretchr/testify/assert"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+func TestRetry_NoBody(t *testing.T) {
+	attempts := 0
+	ts := httptest.NewServer(
+		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			attempts++
+			w.Header().Add("Retry-After", "1")
+			w.WriteHeader(http.StatusTooManyRequests)
+		}))
+	defer ts.Close()
+
+	client := &http.Client{
+		Transport: Retry(http.DefaultTransport, 2),
+	}
+
+	resp, err := client.Get(ts.URL)
+	assert.NoError(t, err)
+	assert.Equal(t, http.StatusTooManyRequests, resp.StatusCode)
+	assert.Equal(t, 3, attempts)
+}
+
+func TestRetry(t *testing.T) {
+	attempts := 0
+	ts := httptest.NewServer(
+		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			attempts++
+			w.Header().Add("Retry-After", "1")
+			w.WriteHeader(http.StatusTooManyRequests)
+		}))
+	defer ts.Close()
+
+	client := &http.Client{
+		Transport: Retry(http.DefaultTransport, 2),
+	}
+
+	resp, err := client.Post(ts.URL, "text/plain", strings.NewReader("body"))
+	assert.NoError(t, err)
+	assert.Equal(t, http.StatusTooManyRequests, resp.StatusCode)
+	assert.Equal(t, 3, attempts)
+}
+
+func TestRetry_Eventually_Ok(t *testing.T) {
+	attempts := 0
+	ts := httptest.NewServer(
+		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			attempts++
+			if attempts <= 2 {
+				w.Header().Add("Retry-After", "1")
+				w.WriteHeader(http.StatusTooManyRequests)
+			} else {
+				w.WriteHeader(http.StatusOK)
+			}
+		}))
+	defer ts.Close()
+
+	client := &http.Client{
+		Transport: Retry(http.DefaultTransport, 10),
+	}
+
+	resp, err := client.Post(ts.URL, "text/plain", strings.NewReader("body"))
+	assert.NoError(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	assert.Equal(t, 3, attempts)
+}

internal/configcat/configs_data_source_test.go

@@ -77,3 +77,22 @@ func TestAccConfigsDataSource(t *testing.T) {
 		},
 	})
 }
+
+func TestAccLotsofConfigsDataSource(t *testing.T) {
+	const productId = "08d86d63-2721-4da6-8c06-584521d516bc"
+
+	resource.Test(t, resource.TestCase{
+		PreCheck:                 func() { testAccPreCheck(t) },
+		ProtoV6ProviderFactories: testAccProtoV6ProviderFactories,
+		Steps: []resource.TestStep{
+			{
+				ConfigFile: config.TestNameFile("main.tf"),
+				ConfigVariables: config.Variables{
+					"product_id":        config.StringVariable(productId),
+					"name_filter_regex": config.StringVariable(""),
+					"data_sources":      config.IntegerVariable(30),
+				},
+			},
+		},
+	})
+}

internal/configcat/testdata/TestAccLotsofConfigsDataSource/main.tf

@@ -0,0 +1,23 @@
+variable "product_id" {
+  type = string
+}
+
+variable "name_filter_regex" {
+  type    = string
+  default = null
+}
+
+variable "data_sources" {
+  type = number
+}
+
+data "configcat_configs" "test" {
+  count = var.data_sources
+
+  product_id        = var.product_id
+  name_filter_regex = var.name_filter_regex
+}
+
+output "config_id" {
+  value = length(data.configcat_configs.test[0].configs) > 0 ? data.configcat_configs.test[0].configs[0].config_id : null
+}