Merge pull request GoogleCloudPlatform#957 from justinsb/golden_requests

Record and verify HTTP requests

config/tests/samples/create/harness.go

@@ -66,6 +66,7 @@ type Harness struct {
 	*testing.T
 	Ctx context.Context
 
+	Events  *test.MemoryEventSink
 	Project testgcp.GCPProject
 
 	client     client.Client
@@ -275,47 +276,56 @@ func NewHarness(t *testing.T, ctx context.Context) *Harness {
 		h.Project = testgcp.GetDefaultProject(t)
 	}
 
-	// Log DCL requests
+	eventSink := test.NewMemoryEventSink()
+	ctx = test.AddSinkToContext(ctx, eventSink)
+	h.Ctx = ctx
+
+	h.Events = eventSink
+
+	eventSinks := test.EventSinksFromContext(ctx)
+
+	// Set up event sink for logging to a file, if ARTIFACTS env var is set
 	if artifacts := os.Getenv("ARTIFACTS"); artifacts != "" {
 		outputDir := filepath.Join(artifacts, "http-logs")
+		eventSinks = append(eventSinks, test.NewDirectoryEventSink(outputDir))
+	} else {
+		log.Info("env var ARTIFACTS is not set; will not record http log")
+	}
+
+	// Intercept (and log) DCL requests
+	if len(eventSinks) != 0 {
 		if kccConfig.HTTPClient == nil {
 			httpClient, err := google.DefaultClient(ctx, gcp.ClientScopes...)
 			if err != nil {
 				t.Fatalf("error creating the http client to be used by DCL: %v", err)
 			}
 			kccConfig.HTTPClient = httpClient
 		}
-		t := test.NewHTTPRecorder(kccConfig.HTTPClient.Transport, outputDir)
+		t := test.NewHTTPRecorder(kccConfig.HTTPClient.Transport, eventSinks...)
 		kccConfig.HTTPClient = &http.Client{Transport: t}
 	}
 
-	// Log TF requests
+	// Intercept (and log) TF requests
 	transport_tpg.DefaultHTTPClientTransformer = func(ctx context.Context, inner *http.Client) *http.Client {
 		ret := inner
 		if t := ctx.Value(httpRoundTripperKey); t != nil {
 			ret = &http.Client{Transport: t.(http.RoundTripper)}
 		}
-		if artifacts := os.Getenv("ARTIFACTS"); artifacts == "" {
-			log.Info("env var ARTIFACTS is not set; will not record http log")
-		} else {
-			outputDir := filepath.Join(artifacts, "http-logs")
-			t := test.NewHTTPRecorder(ret.Transport, outputDir)
+		if len(eventSinks) != 0 {
+			t := test.NewHTTPRecorder(ret.Transport, eventSinks...)
 			ret = &http.Client{Transport: t}
 		}
 		return ret
 	}
 
-	// Log TF oauth requests
+	// Intercept (and log) TF oauth requests
 	transport_tpg.OAuth2HTTPClientTransformer = func(ctx context.Context, inner *http.Client) *http.Client {
 		ret := inner
 		if t := ctx.Value(httpRoundTripperKey); t != nil {
 			ret = &http.Client{Transport: t.(http.RoundTripper)}
 		}
-		if artifacts := os.Getenv("ARTIFACTS"); artifacts == "" {
-			log.Info("env var ARTIFACTS is not set; will not record http log")
-		} else {
-			outputDir := filepath.Join(artifacts, "http-logs")
-			t := test.NewHTTPRecorder(ret.Transport, outputDir)
+		if len(eventSinks) != 0 {
+			t := test.NewHTTPRecorder(ret.Transport, eventSinks...)
 			ret = &http.Client{Transport: t}
 		}
 		return ret

mockgcp/common/httpmux/errors.go

@@ -0,0 +1,93 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package httpmux
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+
+	"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"k8s.io/klog/v2"
+)
+
+type wrappedStatus struct {
+	Error *wrappedError `json:"error,omitempty"`
+}
+
+type wrappedError struct {
+	Code    int            `json:"code,omitempty"`
+	Message string         `json:"message,omitempty"`
+	Status  string         `json:"status,omitempty"`
+	Errors  []errorDetails `json:"errors,omitempty"`
+}
+
+type errorDetails struct {
+	Domain  string `json:"domain,omitempty"`
+	Message string `json:"message,omitempty"`
+	Reason  string `json:"reason,omitempty"`
+}
+
+// customErrorHandler wraps errors in an error blockk
+func customErrorHandler(ctx context.Context, mux *runtime.ServeMux, marshaler runtime.Marshaler, w http.ResponseWriter, r *http.Request, err error) {
+	s := status.Convert(err)
+	// pb := s.Proto()
+
+	w.Header().Del("Trailer")
+	w.Header().Del("Transfer-Encoding")
+
+	w.Header().Set("Content-Type", "application/json; charset=UTF-8")
+
+	httpStatusCode := runtime.HTTPStatusFromCode(s.Code())
+	wrapped := &wrappedStatus{
+		Error: &wrappedError{
+			Code:    httpStatusCode,
+			Message: s.Message(),
+		},
+	}
+
+	switch s.Code() {
+	case codes.PermissionDenied:
+		wrapped.Error.Status = "PERMISSION_DENIED"
+	case codes.AlreadyExists:
+		wrapped.Error.Status = "ALREADY_EXISTS"
+	case codes.NotFound:
+		wrapped.Error.Status = "NOT_FOUND"
+		wrapped.Error.Errors = append(wrapped.Error.Errors, errorDetails{
+			Domain:  "global",
+			Message: wrapped.Error.Message,
+			Reason:  "notFound",
+		})
+	}
+
+	buf, merr := json.Marshal(wrapped)
+	if merr != nil {
+		klog.Warningf("Failed to marshal error message %q: %v", s, merr)
+		runtime.DefaultHTTPErrorHandler(ctx, mux, marshaler, w, r, err)
+		return
+	}
+
+	if err := addGCPHeaders(ctx, w, nil); err != nil {
+		klog.Warningf("unexpected error from header filter: %v", err)
+	}
+
+	w.WriteHeader(httpStatusCode)
+	if _, err := w.Write(buf); err != nil {
+		klog.Warningf("Failed to write response: %v", err)
+	}
+
+}

mockgcp/common/httpmux/mux.go

@@ -0,0 +1,79 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package httpmux
+
+import (
+	"context"
+	"net/http"
+
+	"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
+	"google.golang.org/grpc"
+	"google.golang.org/protobuf/encoding/protojson"
+	"google.golang.org/protobuf/proto"
+	"k8s.io/klog/v2"
+)
+
+// NewServeMux constructs an http server with our error handling etc
+func NewServeMux(ctx context.Context, conn *grpc.ClientConn, handlers ...func(ctx context.Context, mux *runtime.ServeMux, conn *grpc.ClientConn) error) (*runtime.ServeMux, error) {
+	marshaler := &runtime.HTTPBodyMarshaler{
+		Marshaler: &runtime.JSONPb{
+			MarshalOptions: protojson.MarshalOptions{
+				EmitUnpopulated: false,
+			},
+			UnmarshalOptions: protojson.UnmarshalOptions{
+				DiscardUnknown: true,
+			},
+		},
+	}
+
+	outgoingHeaderMatcher := func(key string) (string, bool) {
+		switch key {
+		case "content-type":
+			return "", false
+		default:
+			klog.Warningf("unknown grpc metadata header %q", key)
+			return "", false
+		}
+	}
+
+	mux := runtime.NewServeMux(
+		runtime.WithErrorHandler(customErrorHandler),
+		runtime.WithMarshalerOption(runtime.MIMEWildcard, marshaler),
+		runtime.WithOutgoingHeaderMatcher(outgoingHeaderMatcher),
+		runtime.WithForwardResponseOption(addGCPHeaders),
+	)
+
+	for _, handler := range handlers {
+		if err := handler(ctx, mux, conn); err != nil {
+			return nil, err
+		}
+	}
+
+	return mux, nil
+}
+
+func addGCPHeaders(ctx context.Context, w http.ResponseWriter, resp proto.Message) error {
+	if w.Header().Get("Content-Type") == "application/json" {
+		w.Header().Set("Content-Type", "application/json; charset=UTF-8")
+	}
+	w.Header().Set("Cache-Control", "private")
+	w.Header().Set("Server", "ESF")
+	w.Header()["Vary"] = []string{"Origin", "X-Origin", "Referer"}
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+	w.Header().Set("X-Frame-Options", "SAMEORIGIN")
+	w.Header().Set("X-Xss-Protection", "0")
+
+	return nil
+}

mockgcp/mock_http_roundtrip.go

@@ -267,6 +267,10 @@ func (m *mockRoundTripper) RoundTrip(req *http.Request) (*http.Response, error)
 		response := &http.Response{}
 		response.Body = ioutil.NopCloser(&body)
 		response.Header = w.header
+		if w.statusCode == 0 {
+			w.statusCode = 200
+		}
+		response.Status = fmt.Sprintf("%d %s", w.statusCode, http.StatusText(w.statusCode))
 		response.StatusCode = w.statusCode
 		return response, nil
 	}

mockgcp/mockiam/service.go

@@ -22,10 +22,10 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/GoogleCloudPlatform/k8s-config-connector/mockgcp/common"
+	"github.com/GoogleCloudPlatform/k8s-config-connector/mockgcp/common/httpmux"
 	"github.com/GoogleCloudPlatform/k8s-config-connector/mockgcp/common/projects"
 	pb "github.com/GoogleCloudPlatform/k8s-config-connector/mockgcp/generated/mockgcp/iam/admin/v1"
 	"github.com/GoogleCloudPlatform/k8s-config-connector/mockgcp/pkg/storage"
-	"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
 )
 
 // MockService represents a mocked IAM service.
@@ -63,10 +63,5 @@ func (s *MockService) Register(grpcServer *grpc.Server) {
 }
 
 func (s *MockService) NewHTTPMux(ctx context.Context, conn *grpc.ClientConn) (http.Handler, error) {
-	mux := runtime.NewServeMux()
-	if err := pb.RegisterIAMHandler(ctx, mux, conn); err != nil {
-		return nil, err
-	}
-
-	return mux, nil
+	return httpmux.NewServeMux(ctx, conn, pb.RegisterIAMHandler)
 }

mockgcp/mockiam/serviceaccounts.go

@@ -16,6 +16,7 @@ package mockiam
 
 import (
 	"context"
+	"crypto/md5"
 	"regexp"
 	"strconv"
 	"time"
@@ -25,6 +26,7 @@ import (
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/emptypb"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/klog/v2"
 
 	"github.com/GoogleCloudPlatform/k8s-config-connector/mockgcp/common/projects"
 	pb "github.com/GoogleCloudPlatform/k8s-config-connector/mockgcp/generated/mockgcp/iam/admin/v1"
@@ -59,7 +61,7 @@ func (s *ServerV1) GetServiceAccount(ctx context.Context, req *pb.GetServiceAcco
 		}
 
 		if found == nil {
-			return nil, status.Errorf(codes.NotFound, "serviceaccount %q not found", req.Name)
+			return nil, status.Errorf(codes.NotFound, "Service account %q not found", req.Name)
 		}
 
 		return found, nil
@@ -69,7 +71,7 @@ func (s *ServerV1) GetServiceAccount(ctx context.Context, req *pb.GetServiceAcco
 	fqn := name.String()
 	if err := s.storage.Get(ctx, fqn, sa); err != nil {
 		if apierrors.IsNotFound(err) {
-			return nil, status.Errorf(codes.NotFound, "serviceaccount %q not found", req.Name)
+			return nil, status.Errorf(codes.NotFound, "Service account %q not found", req.Name)
 		}
 		return nil, status.Errorf(codes.Internal, "error reading serviceaccount: %v", err)
 	}
@@ -119,6 +121,9 @@ func (s *ServerV1) CreateServiceAccount(ctx context.Context, req *pb.CreateServi
 	sa.UniqueId = strconv.FormatInt(uniqueID, 10)
 	sa.Email = name.Email
 	sa.DisplayName = displayName
+	sa.Oauth2ClientId = sa.UniqueId
+
+	sa.Etag = computeEtag(sa)
 
 	fqn := name.String()
 	if err := s.storage.Create(ctx, fqn, sa); err != nil {
@@ -179,3 +184,13 @@ func (s *ServerV1) PatchServiceAccount(ctx context.Context, req *pb.PatchService
 	}
 	return sa, nil
 }
+
+func computeEtag(obj proto.Message) []byte {
+	// TODO: Do we risk exposing internal fields?  Doesn't matter on a mock, I guess
+	b, err := proto.Marshal(obj)
+	if err != nil {
+		klog.Fatalf("failed to marshal proto object: %v", err)
+	}
+	hash := md5.Sum(b)
+	return hash[:]
+}

pkg/cli/gcpclient/client_integration_test.go

@@ -84,7 +84,7 @@ func init() {
 			return inner
 		}
 		outputDir := filepath.Join(artifacts, "http-logs")
-		t := test.NewHTTPRecorder(inner.Transport, outputDir)
+		t := test.NewHTTPRecorder(inner.Transport, test.NewDirectoryEventSink(outputDir))
 		return &http.Client{Transport: t}
 	}
 	transport_tpg.OAuth2HTTPClientTransformer = func(ctx context.Context, inner *http.Client) *http.Client {
@@ -93,7 +93,7 @@ func init() {
 			return inner
 		}
 		outputDir := filepath.Join(artifacts, "http-logs")
-		t := test.NewHTTPRecorder(inner.Transport, outputDir)
+		t := test.NewHTTPRecorder(inner.Transport, test.NewDirectoryEventSink(outputDir))
 		return &http.Client{Transport: t}
 	}
 }

pkg/controller/dynamic/dynamic_controller_integration_test.go

@@ -88,7 +88,7 @@ func init() {
 			log.Info("env var ARTIFACTS is not set; will not record http log")
 		} else {
 			outputDir := filepath.Join(artifacts, "http-logs")
-			t := test.NewHTTPRecorder(ret.Transport, outputDir)
+			t := test.NewHTTPRecorder(ret.Transport, test.NewDirectoryEventSink(outputDir))
 			ret = &http.Client{Transport: t}
 		}
 		return ret

pkg/controller/mocktests/secretmanager_secret_test.go

@@ -65,7 +65,7 @@ func TestSecretManagerSecretVersion(t *testing.T) {
 	} else {
 		outputDir := filepath.Join(artifacts, "http-logs")
 
-		roundTripper = test.NewHTTPRecorder(mockCloud, outputDir)
+		roundTripper = test.NewHTTPRecorder(mockCloud, test.NewDirectoryEventSink(outputDir))
 	}
 
 	gcpHTTPClient := &http.Client{Transport: roundTripper}