From 34aa782c8bf32e056ad70949b347852d803985d4 Mon Sep 17 00:00:00 2001 From: Lajos Szoke <63732287+laliconfigcat@users.noreply.github.com> Date: Fri, 12 Apr 2024 21:31:39 +0200 Subject: [PATCH] Rate limit for HTTP calls (#48) * test for rate limit * Add HTTP retry * Retry test without body * Adding retry http client to configuration * lint fix --------- Co-authored-by: Peter Csajtai --- .gitignore | 2 + go.mod | 4 + internal/configcat/client/client.go | 4 + internal/configcat/client/retry_roundtrip.go | 80 +++++++++++++++++++ .../configcat/client/retry_roundtrip_test.go | 73 +++++++++++++++++ .../configcat/configs_data_source_test.go | 19 +++++ .../TestAccLotsofConfigsDataSource/main.tf | 23 ++++++ 7 files changed, 205 insertions(+) create mode 100644 internal/configcat/client/retry_roundtrip.go create mode 100644 internal/configcat/client/retry_roundtrip_test.go create mode 100644 internal/configcat/testdata/TestAccLotsofConfigsDataSource/main.tf diff --git a/.gitignore b/.gitignore index fc10fbf6..9bab1b2b 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,8 @@ # Output of the go coverage tool, specifically when used with LiteIDE *.out +.idea/ + # Dependency directories (remove the comment below to include it) vendor/ diff --git a/go.mod b/go.mod index 3f2e5a1e..67cca998 100644 --- a/go.mod +++ b/go.mod @@ -8,15 +8,19 @@ require ( github.com/hashicorp/terraform-plugin-go v0.22.1 github.com/hashicorp/terraform-plugin-log v0.9.0 github.com/hashicorp/terraform-plugin-testing v1.6.0 + github.com/stretchr/testify v1.8.4 ) require ( github.com/Kunde21/markdownfmt/v3 v3.1.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/hashicorp/cli v1.1.6 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/yuin/goldmark v1.6.0 // indirect github.com/yuin/goldmark-meta v1.1.0 // indirect gopkg.in/yaml.v2 v2.3.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( diff --git a/internal/configcat/client/client.go b/internal/configcat/client/client.go index 62c169c3..b1b8dd10 100644 --- a/internal/configcat/client/client.go +++ b/internal/configcat/client/client.go @@ -3,6 +3,7 @@ package client import ( "context" "fmt" + "net/http" configcatpublicapi "github.com/configcat/configcat-publicapi-go-client" ) @@ -46,6 +47,9 @@ func NewClient(basePath, basicAuthUsername, basicAuthPassword, version string) ( configuration.Servers[0].URL = basePath configuration.UserAgent = "terraform-provider-configcat/" + version configuration.AddDefaultHeader("X-Caller-Id", "terraform-provider-configcat/"+version) + configuration.HTTPClient = &http.Client{ + Transport: Retry(http.DefaultTransport, 5), + } apiClient := configcatpublicapi.NewAPIClient(configuration) client := &Client{ diff --git a/internal/configcat/client/retry_roundtrip.go b/internal/configcat/client/retry_roundtrip.go new file mode 100644 index 00000000..7ddb1732 --- /dev/null +++ b/internal/configcat/client/retry_roundtrip.go @@ -0,0 +1,80 @@ +package client + +import ( + "bytes" + "io" + "net/http" + "strconv" + "time" +) + +type retryRoundTripper struct { + http.RoundTripper + maxRetries int +} + +func Retry(transport http.RoundTripper, maxRetries int) http.RoundTripper { + return &retryRoundTripper{RoundTripper: transport, maxRetries: maxRetries} +} + +func (rt *retryRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { + bodyBuf, err := preReadRequestBodyAndClose(r) + if err != nil { + return nil, err + } + + var attempts int + for { + if bodyBuf != nil { + r.Body = io.NopCloser(bodyBuf) + } + resp, err := rt.RoundTripper.RoundTrip(r) + if err != nil { + return resp, err + } + if resp.StatusCode != http.StatusTooManyRequests { + return resp, nil + } + retryAfterHeader := resp.Header.Get("Retry-After") + retryAfter, err := strconv.ParseInt(retryAfterHeader, 10, 64) + if err != nil { + return resp, err + } + + if attempts >= rt.maxRetries { + return resp, nil + } + + if bodyBuf != nil { + if _, err := bodyBuf.Seek(0, io.SeekStart); err != nil { + return resp, err + } + } + + if resp.Body != nil { + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + } + + select { + case <-time.After(time.Duration(retryAfter) * time.Second): + attempts++ + case <-r.Context().Done(): + return nil, r.Context().Err() + } + } +} + +func preReadRequestBodyAndClose(r *http.Request) (*bytes.Reader, error) { + var reader *bytes.Reader + if r.Body != nil && r.Body != http.NoBody { + var buf bytes.Buffer + if _, err := io.Copy(&buf, r.Body); err != nil { + _ = r.Body.Close() + return nil, err + } + _ = r.Body.Close() + reader = bytes.NewReader(buf.Bytes()) + } + return reader, nil +} diff --git a/internal/configcat/client/retry_roundtrip_test.go b/internal/configcat/client/retry_roundtrip_test.go new file mode 100644 index 00000000..67372eef --- /dev/null +++ b/internal/configcat/client/retry_roundtrip_test.go @@ -0,0 +1,73 @@ +package client + +import ( + "github.com/stretchr/testify/assert" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestRetry_NoBody(t *testing.T) { + attempts := 0 + ts := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + attempts++ + w.Header().Add("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer ts.Close() + + client := &http.Client{ + Transport: Retry(http.DefaultTransport, 2), + } + + resp, err := client.Get(ts.URL) + assert.NoError(t, err) + assert.Equal(t, http.StatusTooManyRequests, resp.StatusCode) + assert.Equal(t, 3, attempts) +} + +func TestRetry(t *testing.T) { + attempts := 0 + ts := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + attempts++ + w.Header().Add("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer ts.Close() + + client := &http.Client{ + Transport: Retry(http.DefaultTransport, 2), + } + + resp, err := client.Post(ts.URL, "text/plain", strings.NewReader("body")) + assert.NoError(t, err) + assert.Equal(t, http.StatusTooManyRequests, resp.StatusCode) + assert.Equal(t, 3, attempts) +} + +func TestRetry_Eventually_Ok(t *testing.T) { + attempts := 0 + ts := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + attempts++ + if attempts <= 2 { + w.Header().Add("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + } else { + w.WriteHeader(http.StatusOK) + } + })) + defer ts.Close() + + client := &http.Client{ + Transport: Retry(http.DefaultTransport, 10), + } + + resp, err := client.Post(ts.URL, "text/plain", strings.NewReader("body")) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.Equal(t, 3, attempts) +} diff --git a/internal/configcat/configs_data_source_test.go b/internal/configcat/configs_data_source_test.go index 68d0a7be..ff8e07d0 100644 --- a/internal/configcat/configs_data_source_test.go +++ b/internal/configcat/configs_data_source_test.go @@ -77,3 +77,22 @@ func TestAccConfigsDataSource(t *testing.T) { }, }) } + +func TestAccLotsofConfigsDataSource(t *testing.T) { + const productId = "08d86d63-2721-4da6-8c06-584521d516bc" + + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + ProtoV6ProviderFactories: testAccProtoV6ProviderFactories, + Steps: []resource.TestStep{ + { + ConfigFile: config.TestNameFile("main.tf"), + ConfigVariables: config.Variables{ + "product_id": config.StringVariable(productId), + "name_filter_regex": config.StringVariable(""), + "data_sources": config.IntegerVariable(30), + }, + }, + }, + }) +} diff --git a/internal/configcat/testdata/TestAccLotsofConfigsDataSource/main.tf b/internal/configcat/testdata/TestAccLotsofConfigsDataSource/main.tf new file mode 100644 index 00000000..11fe7d9b --- /dev/null +++ b/internal/configcat/testdata/TestAccLotsofConfigsDataSource/main.tf @@ -0,0 +1,23 @@ +variable "product_id" { + type = string +} + +variable "name_filter_regex" { + type = string + default = null +} + +variable "data_sources" { + type = number +} + +data "configcat_configs" "test" { + count = var.data_sources + + product_id = var.product_id + name_filter_regex = var.name_filter_regex +} + +output "config_id" { + value = length(data.configcat_configs.test[0].configs) > 0 ? data.configcat_configs.test[0].configs[0].config_id : null +}