From 0d0cbecf4d9d194a1e543b019a38bef77d4ee3bb Mon Sep 17 00:00:00 2001 From: Michael Shen Date: Tue, 8 Oct 2024 10:16:13 -0400 Subject: [PATCH] Remove httputil package It was providing a client-side rate limiter that is now built-in functionality in aws-sdk-go-v2 Signed-off-by: Michael Shen --- go.mod | 1 - go.sum | 2 - pkg/httputil/client.go | 37 ---------- pkg/httputil/client_test.go | 134 ------------------------------------ 4 files changed, 174 deletions(-) delete mode 100644 pkg/httputil/client.go delete mode 100644 pkg/httputil/client_test.go diff --git a/go.mod b/go.mod index 5926f788..862565e7 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,6 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.0 go.uber.org/zap v1.19.0 - golang.org/x/time v0.3.0 google.golang.org/grpc v1.65.0 k8s.io/kms v0.31.0 ) diff --git a/go.sum b/go.sum index 8cbef95c..b11d7cf7 100644 --- a/go.sum +++ b/go.sum @@ -402,8 +402,6 @@ golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= diff --git a/pkg/httputil/client.go b/pkg/httputil/client.go deleted file mode 100644 index fc09e43d..00000000 --- a/pkg/httputil/client.go +++ /dev/null @@ -1,37 +0,0 @@ -// Package httputil implements HTTP utilities. -package httputil - -import ( - "fmt" - "net/http" - - "golang.org/x/time/rate" -) - -// NewRateLimitedClient returns a new HTTP client with rate limiter. -func NewRateLimitedClient(qps int, burst int) (*http.Client, error) { - if qps == 0 { - return http.DefaultClient, nil - } - if burst < 1 { - return nil, fmt.Errorf("burst expected >0, got %d", burst) - } - return &http.Client{ - Transport: &rateLimitedRoundTripper{ - rt: http.DefaultTransport, - rl: rate.NewLimiter(rate.Limit(qps), burst), - }, - }, nil -} - -type rateLimitedRoundTripper struct { - rt http.RoundTripper - rl *rate.Limiter -} - -func (rr *rateLimitedRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { - if err := rr.rl.Wait(req.Context()); err != nil { - return nil, err - } - return rr.rt.RoundTrip(req) -} diff --git a/pkg/httputil/client_test.go b/pkg/httputil/client_test.go deleted file mode 100644 index 2e31229d..00000000 --- a/pkg/httputil/client_test.go +++ /dev/null @@ -1,134 +0,0 @@ -package httputil - -import ( - "context" - "fmt" - "net/http" - "net/http/httptest" - "strings" - "testing" - "time" -) - -func TestNewRateLimitedClient(t *testing.T) { - mux := http.NewServeMux() - mux.HandleFunc("/test", testHandler) - - ts := httptest.NewServer(mux) - defer ts.Close() - - u := ts.URL + "/test" - - // requests are to be throttled if qps+burst < reqs - // estimated time: reqs / (qps+burst) seconds - tbs := []struct { - ctxTimeout time.Duration - qps int - burst int - requests int // concurrent requests - err string - }{ - { - qps: 1, - burst: 1, - requests: 10, - }, - { - qps: 15, - burst: 5, - requests: 100, - }, - { - qps: 8, - burst: 2, - requests: 20, - }, - { - // 20 concurrent ec2 API requests should exceed 1 QPS before 10ms - // thus rate limiter returns an error - ctxTimeout: 10 * time.Millisecond, - qps: 1, - burst: 1, - requests: 20, - err: `context deadline`, - // "Wait(n=1) would exceed context deadline" for requests before timeout - // "context deadline exceeded" for requests after timeout - }, - } - for idx, tt := range tbs { - cli, err := NewRateLimitedClient(tt.qps, tt.burst) - if err != nil { - t.Fatalf("#%d: failed to create a new client (%v)", idx, err) - } - - start := time.Now() - - errc := make(chan error, tt.requests) - for i := 0; i < tt.requests; i++ { - go func() { - var ctx context.Context - if tt.ctxTimeout > 0 { - var cancel context.CancelFunc - ctx, cancel = context.WithTimeout(context.TODO(), tt.ctxTimeout) - defer cancel() - } else { - ctx = context.TODO() - } - req, err := http.NewRequest(http.MethodGet, u, nil) - if err != nil { - errc <- err - return - } - _, err = cli.Do(req.WithContext(ctx)) - errc <- err - }() - } - - failed := false - for i := 0; i < tt.requests; i++ { - err = <-errc - switch { - case tt.err == "": // expects no error - if err != nil { - t.Errorf("#%d-%d: unexpected error %v", idx, i, err) - } - case tt.err != "": // expects error - if err == nil { - // this means that the request did not get throttled. - continue - } - if !strings.Contains(err.Error(), tt.err) && - // TODO: why does this happen even when ctx is not canceled - // ref. https://github.com/golang/go/issues/36848 - !strings.Contains(err.Error(), "i/o timeout") { - t.Errorf("#%d-%d: expected %q, got %v", idx, i, tt.err, err) - } - failed = true - } - } - - if tt.err != "" && !failed { - t.Fatalf("#%d: expected failure %q, got no error", idx, tt.err) - } - - if tt.err == "" { - observedDuration := time.Since(start).Round(time.Second) - expectedDuration := time.Duration(0) - if tt.qps+tt.burst < tt.requests { - expectedDuration = (time.Duration(tt.requests/(tt.qps)) * time.Second) - } - if expectedDuration > 0 && observedDuration > expectedDuration { - t.Fatalf("with rate limit, requests expected duration %v, got %v", expectedDuration, observedDuration) - } - } - } -} - -func testHandler(w http.ResponseWriter, req *http.Request) { - switch req.Method { - case "GET": - fmt.Fprint(w, `test`) - default: - http.Error(w, "Method Not Allowed", 405) - } -}