diff --git a/config/tests/samples/create/harness.go b/config/tests/samples/create/harness.go index fe7b86dd63..d7714a3112 100644 --- a/config/tests/samples/create/harness.go +++ b/config/tests/samples/create/harness.go @@ -66,6 +66,7 @@ type Harness struct { *testing.T Ctx context.Context + Events *test.MemoryEventSink Project testgcp.GCPProject client client.Client @@ -275,9 +276,24 @@ func NewHarness(t *testing.T, ctx context.Context) *Harness { h.Project = testgcp.GetDefaultProject(t) } - // Log DCL requests + eventSink := test.NewMemoryEventSink() + ctx = test.AddSinkToContext(ctx, eventSink) + h.Ctx = ctx + + h.Events = eventSink + + eventSinks := test.EventSinksFromContext(ctx) + + // Set up event sink for logging to a file, if ARTIFACTS env var is set if artifacts := os.Getenv("ARTIFACTS"); artifacts != "" { outputDir := filepath.Join(artifacts, "http-logs") + eventSinks = append(eventSinks, test.NewDirectoryEventSink(outputDir)) + } else { + log.Info("env var ARTIFACTS is not set; will not record http log") + } + + // Intercept (and log) DCL requests + if len(eventSinks) != 0 { if kccConfig.HTTPClient == nil { httpClient, err := google.DefaultClient(ctx, gcp.ClientScopes...) if err != nil { @@ -285,37 +301,31 @@ func NewHarness(t *testing.T, ctx context.Context) *Harness { } kccConfig.HTTPClient = httpClient } - t := test.NewHTTPRecorder(kccConfig.HTTPClient.Transport, outputDir) + t := test.NewHTTPRecorder(kccConfig.HTTPClient.Transport, eventSinks...) kccConfig.HTTPClient = &http.Client{Transport: t} } - // Log TF requests + // Intercept (and log) TF requests transport_tpg.DefaultHTTPClientTransformer = func(ctx context.Context, inner *http.Client) *http.Client { ret := inner if t := ctx.Value(httpRoundTripperKey); t != nil { ret = &http.Client{Transport: t.(http.RoundTripper)} } - if artifacts := os.Getenv("ARTIFACTS"); artifacts == "" { - log.Info("env var ARTIFACTS is not set; will not record http log") - } else { - outputDir := filepath.Join(artifacts, "http-logs") - t := test.NewHTTPRecorder(ret.Transport, outputDir) + if len(eventSinks) != 0 { + t := test.NewHTTPRecorder(ret.Transport, eventSinks...) ret = &http.Client{Transport: t} } return ret } - // Log TF oauth requests + // Intercept (and log) TF oauth requests transport_tpg.OAuth2HTTPClientTransformer = func(ctx context.Context, inner *http.Client) *http.Client { ret := inner if t := ctx.Value(httpRoundTripperKey); t != nil { ret = &http.Client{Transport: t.(http.RoundTripper)} } - if artifacts := os.Getenv("ARTIFACTS"); artifacts == "" { - log.Info("env var ARTIFACTS is not set; will not record http log") - } else { - outputDir := filepath.Join(artifacts, "http-logs") - t := test.NewHTTPRecorder(ret.Transport, outputDir) + if len(eventSinks) != 0 { + t := test.NewHTTPRecorder(ret.Transport, eventSinks...) ret = &http.Client{Transport: t} } return ret diff --git a/pkg/cli/gcpclient/client_integration_test.go b/pkg/cli/gcpclient/client_integration_test.go index 80e6a74001..11e02499d1 100644 --- a/pkg/cli/gcpclient/client_integration_test.go +++ b/pkg/cli/gcpclient/client_integration_test.go @@ -84,7 +84,7 @@ func init() { return inner } outputDir := filepath.Join(artifacts, "http-logs") - t := test.NewHTTPRecorder(inner.Transport, outputDir) + t := test.NewHTTPRecorder(inner.Transport, test.NewDirectoryEventSink(outputDir)) return &http.Client{Transport: t} } transport_tpg.OAuth2HTTPClientTransformer = func(ctx context.Context, inner *http.Client) *http.Client { @@ -93,7 +93,7 @@ func init() { return inner } outputDir := filepath.Join(artifacts, "http-logs") - t := test.NewHTTPRecorder(inner.Transport, outputDir) + t := test.NewHTTPRecorder(inner.Transport, test.NewDirectoryEventSink(outputDir)) return &http.Client{Transport: t} } } diff --git a/pkg/controller/dynamic/dynamic_controller_integration_test.go b/pkg/controller/dynamic/dynamic_controller_integration_test.go index 4e36722cf0..a5ccb0ea98 100644 --- a/pkg/controller/dynamic/dynamic_controller_integration_test.go +++ b/pkg/controller/dynamic/dynamic_controller_integration_test.go @@ -88,7 +88,7 @@ func init() { log.Info("env var ARTIFACTS is not set; will not record http log") } else { outputDir := filepath.Join(artifacts, "http-logs") - t := test.NewHTTPRecorder(ret.Transport, outputDir) + t := test.NewHTTPRecorder(ret.Transport, test.NewDirectoryEventSink(outputDir)) ret = &http.Client{Transport: t} } return ret diff --git a/pkg/controller/mocktests/secretmanager_secret_test.go b/pkg/controller/mocktests/secretmanager_secret_test.go index 6d3520e3de..d8a27c0b64 100644 --- a/pkg/controller/mocktests/secretmanager_secret_test.go +++ b/pkg/controller/mocktests/secretmanager_secret_test.go @@ -65,7 +65,7 @@ func TestSecretManagerSecretVersion(t *testing.T) { } else { outputDir := filepath.Join(artifacts, "http-logs") - roundTripper = test.NewHTTPRecorder(mockCloud, outputDir) + roundTripper = test.NewHTTPRecorder(mockCloud, test.NewDirectoryEventSink(outputDir)) } gcpHTTPClient := &http.Client{Transport: roundTripper} diff --git a/pkg/dcl/clientconfig/config.go b/pkg/dcl/clientconfig/config.go index 25f3e29246..2dc00848da 100644 --- a/pkg/dcl/clientconfig/config.go +++ b/pkg/dcl/clientconfig/config.go @@ -96,13 +96,20 @@ func New(ctx context.Context, opt Options) (*dcl.Config, error) { // Deprecated: Prefer using a harness. func NewForIntegrationTest() *dcl.Config { ctx := context.TODO() + eventSinks := test.EventSinksFromContext(ctx) + + if artifacts := os.Getenv("ARTIFACTS"); artifacts != "" { + outputDir := filepath.Join(artifacts, "http-logs") + + eventSinks = append(eventSinks, test.NewDirectoryEventSink(outputDir)) + } + opt := Options{ UserAgent: "kcc/dev", } // Log DCL requests - if artifacts := os.Getenv("ARTIFACTS"); artifacts != "" { - outputDir := filepath.Join(artifacts, "http-logs") + if len(eventSinks) != 0 { if opt.HTTPClient == nil { httpClient, err := google.DefaultClient(ctx, gcp.ClientScopes...) if err != nil { @@ -110,7 +117,7 @@ func NewForIntegrationTest() *dcl.Config { } opt.HTTPClient = httpClient } - t := test.NewHTTPRecorder(opt.HTTPClient.Transport, outputDir) + t := test.NewHTTPRecorder(opt.HTTPClient.Transport, eventSinks...) opt.HTTPClient = &http.Client{Transport: t} } diff --git a/pkg/test/eventsink.go b/pkg/test/eventsink.go new file mode 100644 index 0000000000..cd9c1b76ad --- /dev/null +++ b/pkg/test/eventsink.go @@ -0,0 +1,173 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + + "k8s.io/klog/v2" + "sigs.k8s.io/yaml" +) + +// An EventSink listens for various events we are able to capture during tests, +// currently just http requests/responses. +type EventSink interface { + AddHTTPEvent(ctx context.Context, entry *LogEntry) +} + +type httpEventSinkType int + +var httpEventSinkKey httpEventSinkType + +// EventSinksFromContext gets the EventSink listeners attached to the passed context. +func EventSinksFromContext(ctx context.Context) []EventSink { + v := ctx.Value(httpEventSinkKey) + if v == nil { + return nil + } + return v.([]EventSink) +} + +// AddSinkToContext attaches the sinks to the returned context. +func AddSinkToContext(ctx context.Context, sinks ...EventSink) context.Context { + var eventSinks []EventSink + v := ctx.Value(httpEventSinkKey) + if v != nil { + eventSinks = v.([]EventSink) + } + eventSinks = append(eventSinks, sinks...) + return context.WithValue(ctx, httpEventSinkKey, eventSinks) +} + +func NewMemoryEventSink() *MemoryEventSink { + return &MemoryEventSink{} +} + +// MemoryEventSink is an EventSink that stores events in memory +type MemoryEventSink struct { + mutex sync.Mutex + HTTPEvents []*LogEntry `json:"httpEvents,omitempty"` +} + +func (s *MemoryEventSink) AddHTTPEvent(ctx context.Context, entry *LogEntry) { + s.mutex.Lock() + defer s.mutex.Unlock() + + s.HTTPEvents = append(s.HTTPEvents, entry) +} + +func (s *MemoryEventSink) FormatHTTP() string { + s.mutex.Lock() + defer s.mutex.Unlock() + + var eventStrings []string + for _, entry := range s.HTTPEvents { + s := entry.FormatHTTP() + eventStrings = append(eventStrings, s) + } + return strings.Join(eventStrings, "\n---\n\n") +} + +func (s *MemoryEventSink) PrettifyJSON(mutators ...JSONMutator) { + s.mutex.Lock() + defer s.mutex.Unlock() + + for _, entry := range s.HTTPEvents { + entry.PrettifyJSON(mutators...) + } +} + +func (s *MemoryEventSink) RemoveHTTPResponseHeader(key string) { + s.mutex.Lock() + defer s.mutex.Unlock() + + for _, entry := range s.HTTPEvents { + entry.Response.RemoveHeader(key) + } +} + +func (s *MemoryEventSink) RemoveRequests(pred func(e *LogEntry) bool) { + s.mutex.Lock() + defer s.mutex.Unlock() + + var keep []*LogEntry + for _, entry := range s.HTTPEvents { + if !pred(entry) { + keep = append(keep, entry) + } + } + s.HTTPEvents = keep +} + +type DirectoryEventSink struct { + outputDir string + + // mutex to avoid concurrent writes to the same file + mutex sync.Mutex +} + +func NewDirectoryEventSink(outputDir string) *DirectoryEventSink { + return &DirectoryEventSink{outputDir: outputDir} +} + +func (r *DirectoryEventSink) AddHTTPEvent(ctx context.Context, entry *LogEntry) { + // Write to a log file + t := TestFromContext(ctx) + testName := "unknown" + if t != nil { + testName = t.Name() + } + dirName := sanitizePath(testName) + p := filepath.Join(r.outputDir, dirName, "requests.log") + + if err := r.writeToFile(p, entry); err != nil { + klog.Fatalf("error writing http event: %v", err) + } +} + +func (r *DirectoryEventSink) writeToFile(p string, entry *LogEntry) error { + b, err := yaml.Marshal(entry) + if err != nil { + return fmt.Errorf("failed to marshal data: %w", err) + } + + // Just in case we are writing to the same file concurrently + r.mutex.Lock() + defer r.mutex.Unlock() + + if err := os.MkdirAll(filepath.Dir(p), 0755); err != nil { + return fmt.Errorf("failed to create directory %q: %w", filepath.Dir(p), err) + } + f, err := os.OpenFile(p, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0644) + if err != nil { + return fmt.Errorf("failed to open file %q: %w", p, err) + } + defer f.Close() + + if _, err := f.Write(b); err != nil { + return fmt.Errorf("failed to write to file %q: %w", p, err) + } + delimeter := "\n\n---\n\n" + if _, err := f.Write([]byte(delimeter)); err != nil { + return fmt.Errorf("failed to write to file %q: %w", p, err) + } + + return nil +} diff --git a/pkg/test/http_recorder.go b/pkg/test/http_recorder.go index a41c07e47a..b93bb05ceb 100644 --- a/pkg/test/http_recorder.go +++ b/pkg/test/http_recorder.go @@ -16,18 +16,16 @@ package test import ( "bytes" + "encoding/json" "fmt" "io/ioutil" "net/http" - "os" - "path/filepath" + "sort" "strings" - "sync" "time" "unicode" "k8s.io/klog/v2" - "sigs.k8s.io/yaml" ) type LogEntry struct { @@ -52,15 +50,13 @@ type Response struct { } type HTTPRecorder struct { - outputDir string - inner http.RoundTripper + inner http.RoundTripper - // mutex to avoid concurrent writes to the same file - mutex sync.Mutex + eventSinks []EventSink } -func NewHTTPRecorder(inner http.RoundTripper, outputDir string) *HTTPRecorder { - rt := &HTTPRecorder{outputDir: outputDir, inner: inner} +func NewHTTPRecorder(inner http.RoundTripper, eventSinks ...EventSink) *HTTPRecorder { + rt := &HTTPRecorder{inner: inner, eventSinks: eventSinks} return rt } @@ -127,39 +123,10 @@ func (r *HTTPRecorder) record(entry *LogEntry, req *http.Request, resp *http.Res } } + // If we have event sink(s), write to that sink also ctx := req.Context() - t := TestFromContext(ctx) - testName := "unknown" - if t != nil { - testName = t.Name() - } - dirName := sanitizePath(testName) - p := filepath.Join(r.outputDir, dirName, "requests.log") - - b, err := yaml.Marshal(entry) - if err != nil { - return fmt.Errorf("failed to marshal data: %w", err) - } - - // Just in case we are writing to the same file concurrently - r.mutex.Lock() - defer r.mutex.Unlock() - - if err := os.MkdirAll(filepath.Dir(p), 0755); err != nil { - return fmt.Errorf("failed to create directory %q: %w", filepath.Dir(p), err) - } - f, err := os.OpenFile(p, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0644) - if err != nil { - return fmt.Errorf("failed to open file %q: %w", p, err) - } - defer f.Close() - - if _, err := f.Write(b); err != nil { - return fmt.Errorf("failed to write to file %q: %w", p, err) - } - delimeter := "\n\n---\n\n" - if _, err := f.Write([]byte(delimeter)); err != nil { - return fmt.Errorf("failed to write to file %q: %w", p, err) + for _, eventSink := range r.eventSinks { + eventSink.AddHTTPEvent(ctx, entry) } return nil @@ -176,3 +143,98 @@ func sanitizePath(s string) string { } return out.String() } + +func (e *LogEntry) FormatHTTP() string { + var b strings.Builder + b.WriteString(e.Request.FormatHTTP()) + b.WriteString(e.Response.FormatHTTP()) + return b.String() +} + +func (r *Request) FormatHTTP() string { + var b strings.Builder + b.WriteString(fmt.Sprintf("%s %s\n", r.Method, r.URL)) + var keys []string + for k := range r.Header { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + for _, v := range r.Header[k] { + b.WriteString(fmt.Sprintf("%s: %s\n", k, v)) + } + } + b.WriteString("\n") + if r.Body != "" { + b.WriteString(r.Body) + b.WriteString("\n\n") + } + return b.String() +} + +func (r *Response) FormatHTTP() string { + var b strings.Builder + b.WriteString(fmt.Sprintf("%s\n", r.Status)) + var keys []string + for k := range r.Header { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + for _, v := range r.Header[k] { + b.WriteString(fmt.Sprintf("%s: %s\n", k, v)) + } + } + b.WriteString("\n") + if r.Body != "" { + b.WriteString(r.Body) + b.WriteString("\n") + } + return b.String() +} + +type JSONMutator func(obj map[string]any) + +func (r *LogEntry) PrettifyJSON(mutators ...JSONMutator) { + r.Request.PrettifyJSON(mutators...) + r.Response.PrettifyJSON(mutators...) +} + +func (r *Response) PrettifyJSON(mutators ...JSONMutator) { + r.Body = prettifyJSON(r.Body, mutators...) +} + +func (r *Request) PrettifyJSON(mutators ...JSONMutator) { + r.Body = prettifyJSON(r.Body, mutators...) +} + +func prettifyJSON(s string, mutators ...JSONMutator) string { + if s == "" { + return s + } + + obj := make(map[string]any) + if err := json.Unmarshal([]byte(s), &obj); err != nil { + klog.Fatalf("error from json.Unmarshal(%q): %v", s, err) + return s + } + + for _, mutator := range mutators { + mutator(obj) + } + + b, err := json.MarshalIndent(obj, "", " ") + if err != nil { + klog.Fatalf("error from json.MarshalIndent: %v", err) + return s + } + return string(b) +} + +func (r *Response) RemoveHeader(key string) { + r.Header.Del(key) +} + +func (r *Request) RemoveHeader(key string) { + r.Header.Del(key) +} diff --git a/tests/e2e/unified_test.go b/tests/e2e/unified_test.go index 4ad140ed0d..ef17048454 100644 --- a/tests/e2e/unified_test.go +++ b/tests/e2e/unified_test.go @@ -160,6 +160,57 @@ func TestAllInSeries(t *testing.T) { } create.DeleteResources(h, opt.Create) + + // Verify events against golden file + if os.Getenv("GOLDEN_REQUEST_CHECKS") != "" { + events := h.Events + + // TODO: Fix how we poll / wait for objects being ready. + events.RemoveRequests(func(e *test.LogEntry) bool { + if e.Response.StatusCode == 404 && e.Request.Method == "GET" { + return true + } + return false + }) + + jsonMutators := []test.JSONMutator{} + + jsonMutators = append(jsonMutators, func(obj map[string]any) { + _, found, _ := unstructured.NestedString(obj, "uniqueId") + if found { + unstructured.SetNestedField(obj, "111111111111111111111", "uniqueId") + } + }) + jsonMutators = append(jsonMutators, func(obj map[string]any) { + _, found, _ := unstructured.NestedString(obj, "oauth2ClientId") + if found { + unstructured.SetNestedField(obj, "888888888888888888888", "oauth2ClientId") + } + }) + jsonMutators = append(jsonMutators, func(obj map[string]any) { + _, found, _ := unstructured.NestedString(obj, "etag") + if found { + unstructured.SetNestedField(obj, "abcdef0123A=", "etag") + } + }) + jsonMutators = append(jsonMutators, func(obj map[string]any) { + _, found, _ := unstructured.NestedString(obj, "serviceAccount", "etag") + if found { + unstructured.SetNestedField(obj, "abcdef0123A=", "serviceAccount", "etag") + } + }) + events.PrettifyJSON(jsonMutators...) + + events.RemoveHTTPResponseHeader("Date") + events.RemoveHTTPResponseHeader("Alt-Svc") + got := events.FormatHTTP() + expectedPath := filepath.Join(fixture.SourceDir, "_http.log") + normalizers := []func(string) string{} + normalizers = append(normalizers, h.IgnoreComments) + normalizers = append(normalizers, h.ReplaceString(uniqueID, "${uniqueId}")) + normalizers = append(normalizers, h.ReplaceString(project.ProjectID, "${projectId}")) + h.CompareGoldenFile(expectedPath, got, normalizers...) + } }) } })