diff --git a/internal/controllers/gateway/gateway_controller.go b/internal/controllers/gateway/gateway_controller.go index 16aae95c7db..990c48d342d 100644 --- a/internal/controllers/gateway/gateway_controller.go +++ b/internal/controllers/gateway/gateway_controller.go @@ -628,7 +628,10 @@ func (r *GatewayReconciler) determineServiceForGateway(ctx context.Context, ref var name k8stypes.NamespacedName switch { case ref == r.PublishServiceRef.String(): - if protocol == gatewayv1beta1.HTTPProtocolType || protocol == gatewayv1beta1.HTTPSProtocolType || protocol == gatewayv1beta1.TCPProtocolType { + if protocol == gatewayv1beta1.HTTPProtocolType || + protocol == gatewayv1beta1.HTTPSProtocolType || + protocol == gatewayv1beta1.TCPProtocolType || + r.PublishServiceTLSRef.IsAbsent() && protocol == TLSProtocolType { name = r.PublishServiceRef } case r.PublishServiceUDPRef.IsPresent() && ref == r.PublishServiceUDPRef.MustGet().String(): diff --git a/internal/dataplane/parser/translate_tlsroute.go b/internal/dataplane/parser/translate_tlsroute.go index 4fa20901e0b..9a5c9d9129d 100644 --- a/internal/dataplane/parser/translate_tlsroute.go +++ b/internal/dataplane/parser/translate_tlsroute.go @@ -133,7 +133,7 @@ func (p *Parser) isTLSRoutePassthrough(tlsroute *gatewayv1alpha2.TLSRoute) (bool return false, err } - // if anyone of the listeners used for the gateway is configured to passthrough + // If any of the gateway's listeners is configured to passthrough // TLS requests, we return true. for _, listener := range gateway.Spec.Listeners { if parentRef.SectionName == nil || listener.Name == *parentRef.SectionName { diff --git a/test/conformance/gateway_conformance_test.go b/test/conformance/gateway_conformance_test.go index 742034e559b..48e79172eec 100644 --- a/test/conformance/gateway_conformance_test.go +++ b/test/conformance/gateway_conformance_test.go @@ -4,6 +4,7 @@ package conformance import ( "fmt" + "strings" "testing" "github.com/google/uuid" @@ -104,9 +105,10 @@ func TestGatewayConformance(t *testing.T) { // This service creation is a temporary solution, intended to be replaced by // https://github.com/Kong/charts/issues/848 t.Log("creating tls service for gateway conformance tests") + svcNameSuffix, _, _ := strings.Cut(uuid.NewString(), "-") tlsService := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: "tls-proxy-" + uuid.NewString(), + Name: "ingress-controller-kong-tls-proxy-" + svcNameSuffix, Namespace: "kong-system", }, Spec: corev1.ServiceSpec{ @@ -179,11 +181,10 @@ func TestGatewayConformance(t *testing.T) { GatewayClassName: gatewayClass.Name, Debug: showDebug, CleanupBaseResources: shouldCleanup, + EnableAllSupportedFeatures: enableAllSupportedFeatures, ExemptFeatures: exemptFeatures, BaseManifests: conformanceTestsBaseManifests, SkipTests: skippedTests, - EnableAllSupportedFeatures: false, - SupportedFeatures: suite.TLSCoreFeatures, }) cSuite.Setup(t) // To work with individual tests only, you can disable the normal Run call and construct a slice containing a diff --git a/test/integration/tlsroute_test.go b/test/integration/tlsroute_test.go index 37806db63c2..97e1a242104 100644 --- a/test/integration/tlsroute_test.go +++ b/test/integration/tlsroute_test.go @@ -42,7 +42,9 @@ const ( tlsEchoPort = 1030 ) -func TestTLSRouteEssentials(t *testing.T) { +// TestTLSRouteReferenceGrant tests cross-namespace certificate references. These are technically implemented within +// Gateway Listeners, but require an attached Route to see the associated certificate behavior on the proxy. +func TestTLSRoutePassthroughReferenceGrant(t *testing.T) { skipTestForExpressionRouter(t) t.Log("locking Gateway TLS ports") tlsMutex.Lock() @@ -54,12 +56,18 @@ func TestTLSRouteEssentials(t *testing.T) { ctx := context.Background() ns, cleaner := helpers.Setup(ctx, t, env) - t.Log("getting gateway client") + otherNs, err := clusters.GenerateNamespace(ctx, env.Cluster(), t.Name()) + require.NoError(t, err) + cleaner.AddNamespace(otherNs) + + t.Log("getting the gateway client") gatewayClient, err := gatewayclient.NewForConfig(env.Cluster().Config()) require.NoError(t, err) t.Log("configuring secrets") tlsRouteExampleTLSCert, tlsRouteExampleTLSKey := certificate.MustGenerateSelfSignedCertPEMFormat(certificate.WithCommonName(tlsRouteHostname)) + extraTLSRouteTLSCert, extraTLSRouteTLSKey := certificate.MustGenerateSelfSignedCertPEMFormat(certificate.WithCommonName(tlsRouteExtraHostname)) + secrets := []*corev1.Secret{ { ObjectMeta: metav1.ObjectMeta{ @@ -72,69 +80,122 @@ func TestTLSRouteEssentials(t *testing.T) { "tls.key": tlsRouteExampleTLSKey, }, }, + { + ObjectMeta: metav1.ObjectMeta{ + UID: k8stypes.UID("7428fb98-180b-4702-a91f-61351a33c6e9"), + Name: "secret2", + }, + Data: map[string][]byte{ + "tls.crt": extraTLSRouteTLSCert, + "tls.key": extraTLSRouteTLSKey, + }, + }, } t.Log("deploying secrets") secret1, err := env.Cluster().Client().CoreV1().Secrets(ns.Name).Create(ctx, secrets[0], metav1.CreateOptions{}) require.NoError(t, err) cleaner.Add(secret1) - - t.Log("deploying a supported gatewayclass to the test cluster") - gatewayClassName := uuid.NewString() - gwc, err := DeployGatewayClass(ctx, gatewayClient, gatewayClassName) + secret2, err := env.Cluster().Client().CoreV1().Secrets(otherNs.Name).Create(ctx, secrets[1], metav1.CreateOptions{}) require.NoError(t, err) - cleaner.Add(gwc) + cleaner.Add(secret2) + // we need to create the secret 2 in the namespace 1 as well because we need to mount in the deployment. The Gateway will be + // using the secret in namespace 1 to test the referenceGrant. + secret3, err := env.Cluster().Client().CoreV1().Secrets(ns.Name).Create(ctx, secrets[1], metav1.CreateOptions{}) + require.NoError(t, err) + cleaner.Add(secret3) - t.Log("deploying a gateway to the test cluster using unmanaged gateway mode and port 8899") - gatewayName := uuid.NewString() modePassthrough := gatewayv1beta1.TLSModePassthrough - gateway, err := DeployGateway(ctx, gatewayClient, ns.Name, gatewayClassName, func(gw *gatewayv1beta1.Gateway) { - gw.Name = gatewayName - - gw.Spec.Listeners = builder.NewListener("tls"). - TLS(). - WithPort(ktfkong.DefaultTLSServicePort). - WithHostname(tlsRouteHostname). - WithTLSConfig(&gatewayv1beta1.GatewayTLSConfig{ - Mode: &modePassthrough, - CertificateRefs: []gatewayv1beta1.SecretObjectReference{ - { - Name: gatewayv1beta1.ObjectName(tlsSecretName), + t.Log("deploying a gateway to the test cluster using unmanaged gateway mode") + gateway, err := DeployGateway(ctx, gatewayClient, ns.Name, unmanagedGatewayClassName, func(gw *gatewayv1beta1.Gateway) { + otherNamespace := gatewayv1beta1.Namespace(otherNs.Name) + gw.Spec.Listeners = []gatewayv1beta1.Listener{ + builder.NewListener("tls"). + TLS(). + WithPort(ktfkong.DefaultTLSServicePort). + WithHostname(tlsRouteHostname). + WithTLSConfig(&gatewayv1beta1.GatewayTLSConfig{ + Mode: &modePassthrough, + CertificateRefs: []gatewayv1beta1.SecretObjectReference{ + { + Name: gatewayv1beta1.ObjectName(secrets[0].Name), + }, }, - }, - }). - IntoSlice() + }).Build(), + builder.NewListener("tlsother"). + TLS(). + WithPort(ktfkong.DefaultTLSServicePort). + WithHostname(tlsRouteExtraHostname). + WithTLSConfig(&gatewayv1beta1.GatewayTLSConfig{ + Mode: &modePassthrough, + CertificateRefs: []gatewayv1beta1.SecretObjectReference{ + { + Name: gatewayv1beta1.ObjectName(secrets[1].Name), + Namespace: &otherNamespace, + }, + }, + }).Build(), + } }) + require.NoError(t, err) cleaner.Add(gateway) - t.Log("creating a tcpecho pod to test TLSRoute traffic routing") + secret2Name := gatewayv1alpha2.ObjectName(secrets[1].Name) + t.Logf("creating a ReferenceGrant that permits gateway access from %s to secrets in %s", ns.Name, otherNs.Name) + grant := &gatewayv1beta1.ReferenceGrant{ + ObjectMeta: metav1.ObjectMeta{ + Name: uuid.NewString(), + Annotations: map[string]string{}, + }, + Spec: gatewayv1beta1.ReferenceGrantSpec{ + From: []gatewayv1beta1.ReferenceGrantFrom{ + { + Group: gatewayv1alpha2.Group("gateway.networking.k8s.io"), + Kind: gatewayv1alpha2.Kind("Gateway"), + Namespace: gatewayv1alpha2.Namespace(gateway.Namespace), + }, + }, + To: []gatewayv1beta1.ReferenceGrantTo{ + { + Group: gatewayv1alpha2.Group(""), + Kind: gatewayv1alpha2.Kind("Secret"), + Name: &secret2Name, + }, + }, + }, + } - container := generators.NewContainer("tcpecho-1", test.EchoImage, test.EchoTCPPort) - // go-echo sends a "Running on Pod ." immediately on connecting + grant, err = gatewayClient.GatewayV1beta1().ReferenceGrants(otherNs.Name).Create(ctx, grant, metav1.CreateOptions{}) + require.NoError(t, err) + cleaner.Add(grant) + + t.Log("creating a tcpecho pod to test TLSRoute traffic routing") testUUID := uuid.NewString() - container.Env = []corev1.EnvVar{ - { - Name: "POD_NAME", - Value: testUUID, + deployment := generators.NewDeploymentForContainer(createTLSEchoContainer(tlsEchoPort, testUUID)) + deployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes, corev1.Volume{ + Name: tlsSecretName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: tlsSecretName, + }, }, - } - deployment := generators.NewDeploymentForContainer(container) + }) deployment, err = env.Cluster().Client().AppsV1().Deployments(ns.Name).Create(ctx, deployment, metav1.CreateOptions{}) require.NoError(t, err) cleaner.Add(deployment) - t.Log("creating an additional tcpecho pod to test TLSRoute multiple backendRef loadbalancing") - container2 := generators.NewContainer("tcpecho-2", test.EchoImage, test.EchoTCPPort) - // go-echo sends a "Running on Pod ." immediately on connecting + t.Log("creating another tcpecho pod to test TLSRoute traffic routing with referenceGrant") testUUID2 := uuid.NewString() - container2.Env = []corev1.EnvVar{ - { - Name: "POD_NAME", - Value: testUUID2, + deployment2 := generators.NewDeploymentForContainer(createTLSEchoContainer(tlsEchoPort, testUUID2)) + deployment2.Spec.Template.Spec.Volumes = append(deployment2.Spec.Template.Spec.Volumes, corev1.Volume{ + Name: tlsSecretName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: string(secret2Name), + }, }, - } - deployment2 := generators.NewDeploymentForContainer(container2) + }) deployment2, err = env.Cluster().Client().AppsV1().Deployments(ns.Name).Create(ctx, deployment2, metav1.CreateOptions{}) require.NoError(t, err) cleaner.Add(deployment2) @@ -151,9 +212,9 @@ func TestTLSRouteEssentials(t *testing.T) { require.NoError(t, err) cleaner.Add(service2) - backendPort := gatewayv1alpha2.PortNumber(test.EchoTCPPort) + backendTLSPort := gatewayv1alpha2.PortNumber(tlsEchoPort) t.Logf("creating a tlsroute to access deployment %s via kong", deployment.Name) - tlsRoute := &gatewayv1alpha2.TLSRoute{ + tlsroute := &gatewayv1alpha2.TLSRoute{ ObjectMeta: metav1.ObjectMeta{ Name: uuid.NewString(), Annotations: map[string]string{}, @@ -161,285 +222,117 @@ func TestTLSRouteEssentials(t *testing.T) { Spec: gatewayv1alpha2.TLSRouteSpec{ CommonRouteSpec: gatewayv1alpha2.CommonRouteSpec{ ParentRefs: []gatewayv1alpha2.ParentReference{{ - Name: gatewayv1alpha2.ObjectName(gatewayName), + Name: gatewayv1alpha2.ObjectName(gateway.Name), }}, }, - Hostnames: []gatewayv1alpha2.Hostname{tlsRouteHostname}, + Hostnames: []gatewayv1alpha2.Hostname{tlsRouteHostname, tlsRouteExtraHostname}, Rules: []gatewayv1alpha2.TLSRouteRule{{ - BackendRefs: []gatewayv1alpha2.BackendRef{{ - BackendObjectReference: gatewayv1alpha2.BackendObjectReference{ - Name: gatewayv1alpha2.ObjectName(service.Name), - Port: &backendPort, + BackendRefs: []gatewayv1alpha2.BackendRef{ + { + BackendObjectReference: gatewayv1alpha2.BackendObjectReference{ + Name: gatewayv1alpha2.ObjectName(service.Name), + Port: &backendTLSPort, + }, }, - }}, + { + BackendObjectReference: gatewayv1alpha2.BackendObjectReference{ + Name: gatewayv1alpha2.ObjectName(service2.Name), + Port: &backendTLSPort, + }, + }, + }, }}, }, } - tlsRoute, err = gatewayClient.GatewayV1alpha2().TLSRoutes(ns.Name).Create(ctx, tlsRoute, metav1.CreateOptions{}) + tlsroute, err = gatewayClient.GatewayV1alpha2().TLSRoutes(ns.Name).Create(ctx, tlsroute, metav1.CreateOptions{}) require.NoError(t, err) - cleaner.Add(tlsRoute) - - t.Log("verifying that the Gateway gets linked to the route via status") - callback := GetGatewayIsLinkedCallback(ctx, t, gatewayClient, gatewayv1beta1.TLSProtocolType, ns.Name, tlsRoute.Name) - require.Eventually(t, callback, ingressWait, waitTick) - t.Log("verifying that the tlsroute contains 'Programmed' condition") - require.Eventually(t, - GetVerifyProgrammedConditionCallback(t, gatewayClient, gatewayv1beta1.TLSProtocolType, ns.Name, tlsRoute.Name, metav1.ConditionTrue), - ingressWait, waitTick, - ) + cleaner.Add(tlsroute) + proxyAddress := fmt.Sprintf("%s:%d", proxyURL.Hostname(), ktfkong.DefaultTLSServicePort) t.Log("verifying that the tcpecho is responding properly over TLS") require.Eventually(t, func() bool { - responded, err := tlsEchoResponds(fmt.Sprintf("%s:%d", proxyURL.Hostname(), ktfkong.DefaultTLSServicePort), - testUUID, tlsRouteHostname, tlsRouteHostname, false) + responded, err := tlsEchoResponds(proxyAddress, testUUID, tlsRouteHostname, tlsRouteHostname, true) + if err != nil { + t.Logf("failed accessing tcpecho at %s, err: %v", proxyAddress, err) + return false + } return err == nil && responded == true }, ingressWait, waitTick) - t.Log("removing the parentrefs from the TLSRoute") - oldParentRefs := tlsRoute.Spec.ParentRefs - require.Eventually(t, func() bool { - tlsRoute, err = gatewayClient.GatewayV1alpha2().TLSRoutes(ns.Name).Get(ctx, tlsRoute.Name, metav1.GetOptions{}) - require.NoError(t, err) - tlsRoute.Spec.ParentRefs = nil - tlsRoute, err = gatewayClient.GatewayV1alpha2().TLSRoutes(ns.Name).Update(ctx, tlsRoute, metav1.UpdateOptions{}) - return err == nil - }, time.Minute, time.Second) - - t.Log("verifying that the Gateway gets unlinked from the route via status") - callback = GetGatewayIsUnlinkedCallback(ctx, t, gatewayClient, gatewayv1beta1.TLSProtocolType, ns.Name, tlsRoute.Name) - require.Eventually(t, callback, ingressWait, waitTick) - - t.Log("verifying that the tcpecho is no longer responding") - require.Eventually(t, func() bool { - responded, err := tlsEchoResponds(fmt.Sprintf("%s:%d", proxyURL.Hostname(), ktfkong.DefaultTLSServicePort), - testUUID, tlsRouteHostname, tlsRouteHostname, false) - return responded == false && errors.Is(err, io.EOF) - }, ingressWait, waitTick) - - t.Log("putting the parentRefs back") - require.Eventually(t, func() bool { - tlsRoute, err = gatewayClient.GatewayV1alpha2().TLSRoutes(ns.Name).Get(ctx, tlsRoute.Name, metav1.GetOptions{}) - require.NoError(t, err) - tlsRoute.Spec.ParentRefs = oldParentRefs - tlsRoute, err = gatewayClient.GatewayV1alpha2().TLSRoutes(ns.Name).Update(ctx, tlsRoute, metav1.UpdateOptions{}) - return err == nil - }, time.Minute, time.Second) - - t.Log("verifying that the Gateway gets linked to the route via status") - callback = GetGatewayIsLinkedCallback(ctx, t, gatewayClient, gatewayv1beta1.TLSProtocolType, ns.Name, tlsRoute.Name) - require.Eventually(t, callback, ingressWait, waitTick) - - t.Log("verifying that putting the parentRefs back results in the routes becoming available again") + t.Log("verifying that the tcpecho route can also serve certificates permitted by a ReferenceGrant with a named To") require.Eventually(t, func() bool { - responded, err := tlsEchoResponds(fmt.Sprintf("%s:%d", proxyURL.Hostname(), ktfkong.DefaultTLSServicePort), - testUUID, tlsRouteHostname, tlsRouteHostname, false) + responded, err := tlsEchoResponds(proxyAddress, testUUID2, tlsRouteExtraHostname, tlsRouteExtraHostname, true) + if err != nil { + t.Logf("failed accessing tcpecho at %s, err: %v", proxyAddress, err) + return false + } return err == nil && responded == true }, ingressWait, waitTick) - t.Log("deleting the GatewayClass") - require.NoError(t, gatewayClient.GatewayV1beta1().GatewayClasses().Delete(ctx, gwc.Name, metav1.DeleteOptions{})) - - t.Log("verifying that the Gateway gets unlinked from the route via status") - callback = GetGatewayIsUnlinkedCallback(ctx, t, gatewayClient, gatewayv1beta1.TLSProtocolType, ns.Name, tlsRoute.Name) - require.Eventually(t, callback, ingressWait, waitTick) + t.Log("verifying that using the wrong name in the ReferenceGrant removes the related certificate") + badName := gatewayv1alpha2.ObjectName("garbage") + grant.Spec.To[0].Name = &badName + grant, err = gatewayClient.GatewayV1beta1().ReferenceGrants(otherNs.Name).Update(ctx, grant, metav1.UpdateOptions{}) + require.NoError(t, err) - t.Log("verifying that the data-plane configuration from the TLSRoute gets dropped with the GatewayClass now removed") require.Eventually(t, func() bool { - responded, err := tlsEchoResponds(fmt.Sprintf("%s:%d", proxyURL.Hostname(), ktfkong.DefaultTLSServicePort), - testUUID, tlsRouteHostname, tlsRouteHostname, false) - return responded == false && errors.Is(err, io.EOF) + responded, err := tlsEchoResponds(proxyAddress, testUUID2, tlsRouteExtraHostname, tlsRouteExtraHostname, true) + return err != nil && responded == false }, ingressWait, waitTick) - t.Log("putting the GatewayClass back") - gwc, err = DeployGatewayClass(ctx, gatewayClient, gatewayClassName) + t.Log("verifying that a Listener has the invalid ref status condition") + gateway, err = gatewayClient.GatewayV1beta1().Gateways(ns.Name).Get(ctx, gateway.Name, metav1.GetOptions{}) require.NoError(t, err) + invalid := false + for _, status := range gateway.Status.Listeners { + if ok := util.CheckCondition( + status.Conditions, + util.ConditionType(gatewayv1beta1.ListenerConditionResolvedRefs), + util.ConditionReason(gatewayv1beta1.ListenerReasonRefNotPermitted), + metav1.ConditionFalse, + gateway.Generation, + ); ok { + invalid = true + } + } + require.True(t, invalid) - t.Log("verifying that the Gateway gets linked to the route via status") - callback = GetGatewayIsLinkedCallback(ctx, t, gatewayClient, gatewayv1beta1.TLSProtocolType, ns.Name, tlsRoute.Name) - require.Eventually(t, callback, ingressWait, waitTick) + t.Log("verifying the certificate returns when using a ReferenceGrant with no name restrictions") + grant.Spec.To[0].Name = nil + _, err = gatewayClient.GatewayV1beta1().ReferenceGrants(otherNs.Name).Update(ctx, grant, metav1.UpdateOptions{}) + require.NoError(t, err) - t.Log("verifying that creating the GatewayClass again triggers reconciliation of TLSRoutes and the route becomes available again") require.Eventually(t, func() bool { - responded, err := tlsEchoResponds(fmt.Sprintf("%s:%d", proxyURL.Hostname(), ktfkong.DefaultTLSServicePort), - testUUID, tlsRouteHostname, tlsRouteHostname, false) + responded, err := tlsEchoResponds(proxyAddress, testUUID2, tlsRouteExtraHostname, tlsRouteExtraHostname, true) + if err != nil { + t.Logf("failed accessing tcpecho at %s, err: %v", proxyAddress, err) + return false + } return err == nil && responded == true }, ingressWait, waitTick) +} - t.Log("deleting the Gateway") - require.NoError(t, gatewayClient.GatewayV1beta1().Gateways(ns.Name).Delete(ctx, gatewayName, metav1.DeleteOptions{})) +func TestTLSRoutePassthrough(t *testing.T) { + skipTestForExpressionRouter(t) + t.Log("locking Gateway TLS ports") + tlsMutex.Lock() + t.Cleanup(func() { + t.Log("unlocking TLS port") + tlsMutex.Unlock() + }) - t.Log("verifying that the Gateway gets unlinked from the route via status") - callback = GetGatewayIsUnlinkedCallback(ctx, t, gatewayClient, gatewayv1beta1.TLSProtocolType, ns.Name, tlsRoute.Name) - require.Eventually(t, callback, ingressWait, waitTick) + ctx := context.Background() + ns, cleaner := helpers.Setup(ctx, t, env) - t.Log("verifying that the data-plane configuration from the TLSRoute gets dropped with the Gateway now removed") - require.Eventually(t, func() bool { - responded, err := tlsEchoResponds(fmt.Sprintf("%s:%d", proxyURL.Hostname(), ktfkong.DefaultTLSServicePort), - testUUID, tlsRouteHostname, tlsRouteHostname, false) - return responded == false && errors.Is(err, io.EOF) - }, ingressWait, waitTick) - - t.Log("putting the Gateway back") - gateway, err = DeployGateway(ctx, gatewayClient, ns.Name, gatewayClassName, func(gw *gatewayv1beta1.Gateway) { - gw.Name = gatewayName - gw.Spec.Listeners = builder.NewListener("tls"). - TLS(). - WithPort(ktfkong.DefaultTLSServicePort). - WithHostname(tlsRouteHostname). - WithTLSConfig(&gatewayv1beta1.GatewayTLSConfig{ - Mode: &modePassthrough, - CertificateRefs: []gatewayv1beta1.SecretObjectReference{ - { - Name: gatewayv1beta1.ObjectName(tlsSecretName), - }, - }, - }). - IntoSlice() - }) - require.NoError(t, err) - - t.Log("verifying that the Gateway gets linked to the route via status") - callback = GetGatewayIsLinkedCallback(ctx, t, gatewayClient, gatewayv1beta1.TLSProtocolType, ns.Name, tlsRoute.Name) - require.Eventually(t, callback, ingressWait, waitTick) - - t.Log("verifying that creating the Gateway again triggers reconciliation of TLSRoutes and the route becomes available again") - require.Eventually(t, func() bool { - responded, err := tlsEchoResponds(fmt.Sprintf("%s:%d", proxyURL.Hostname(), ktfkong.DefaultTLSServicePort), - testUUID, tlsRouteHostname, tlsRouteHostname, false) - return err == nil && responded == true - }, ingressWait, waitTick) - - t.Log("adding an additional backendRef to the TLSRoute") - require.Eventually(t, func() bool { - tlsRoute, err = gatewayClient.GatewayV1alpha2().TLSRoutes(ns.Name).Get(ctx, tlsRoute.Name, metav1.GetOptions{}) - require.NoError(t, err) - - tlsRoute.Spec.Rules[0].BackendRefs = []gatewayv1alpha2.BackendRef{ - { - BackendObjectReference: gatewayv1alpha2.BackendObjectReference{ - Name: gatewayv1alpha2.ObjectName(service.Name), - Port: &backendPort, - }, - }, - { - BackendObjectReference: gatewayv1alpha2.BackendObjectReference{ - Name: gatewayv1alpha2.ObjectName(service2.Name), - Port: &backendPort, - }, - }, - } - - tlsRoute, err = gatewayClient.GatewayV1alpha2().TLSRoutes(ns.Name).Update(ctx, tlsRoute, metav1.UpdateOptions{}) - return err == nil - }, ingressWait, waitTick) - - t.Log("verifying that the TLSRoute is now load-balanced between two services") - require.Eventually(t, func() bool { - responded, err := tlsEchoResponds(fmt.Sprintf("%s:%d", proxyURL.Hostname(), ktfkong.DefaultTLSServicePort), - testUUID, tlsRouteHostname, tlsRouteHostname, false) - return err == nil && responded == true - }, ingressWait, waitTick) - require.Eventually(t, func() bool { - responded, err := tlsEchoResponds(fmt.Sprintf("%s:%d", proxyURL.Hostname(), ktfkong.DefaultTLSServicePort), - testUUID2, tlsRouteHostname, tlsRouteHostname, false) - return err == nil && responded == true - }, ingressWait, waitTick) - - t.Log("deleting both GatewayClass and Gateway rapidly") - require.NoError(t, gatewayClient.GatewayV1beta1().GatewayClasses().Delete(ctx, gwc.Name, metav1.DeleteOptions{})) - require.NoError(t, gatewayClient.GatewayV1beta1().Gateways(ns.Name).Delete(ctx, gateway.Name, metav1.DeleteOptions{})) - - t.Log("verifying that the Gateway gets unlinked from the route via status") - callback = GetGatewayIsUnlinkedCallback(ctx, t, gatewayClient, gatewayv1beta1.TLSProtocolType, ns.Name, tlsRoute.Name) - require.Eventually(t, callback, ingressWait, waitTick) - - t.Log("verifying that the data-plane configuration from the TLSRoute does not get orphaned with the GatewayClass and Gateway gone") - require.Eventually(t, func() bool { - responded, err := tlsEchoResponds(fmt.Sprintf("%s:%d", proxyURL.Hostname(), ktfkong.DefaultTLSServicePort), - testUUID, tlsRouteHostname, tlsRouteHostname, false) - return responded == false && errors.Is(err, io.EOF) - }, ingressWait, waitTick) - - t.Log("testing port matching") - t.Log("putting the Gateway back") - _, err = DeployGateway(ctx, gatewayClient, ns.Name, gatewayClassName, func(gw *gatewayv1beta1.Gateway) { - gw.Name = gatewayName - gw.Spec.Listeners = builder.NewListener("tls"). - TLS(). - WithPort(ktfkong.DefaultTLSServicePort). - WithHostname(tlsRouteHostname). - WithTLSConfig(&gatewayv1beta1.GatewayTLSConfig{ - Mode: &modePassthrough, - CertificateRefs: []gatewayv1beta1.SecretObjectReference{ - { - Name: tlsSecretName, - }, - }, - }). - IntoSlice() - }) - require.NoError(t, err) - - t.Log("putting the GatewayClass back") - _, err = DeployGatewayClass(ctx, gatewayClient, gatewayClassName) - - t.Log("ensuring tls echo responds after recreating gateway and gateway class") - require.Eventually(t, func() bool { - responded, err := tlsEchoResponds(fmt.Sprintf("%s:%d", proxyURL.Hostname(), ktfkong.DefaultTLSServicePort), - testUUID, tlsRouteHostname, tlsRouteHostname, false) - return err == nil && responded == true - }, ingressWait, waitTick) - - t.Log("setting the port in ParentRef which does not have a matching listener in Gateway") - require.Eventually(t, func() bool { - tlsRoute, err = gatewayClient.GatewayV1alpha2().TLSRoutes(ns.Name).Get(ctx, tlsRoute.Name, metav1.GetOptions{}) - if err != nil { - return false - } - notExistingPort := gatewayv1alpha2.PortNumber(81) - tlsRoute.Spec.ParentRefs[0].Port = ¬ExistingPort - tlsRoute, err = gatewayClient.GatewayV1alpha2().TLSRoutes(ns.Name).Update(ctx, tlsRoute, metav1.UpdateOptions{}) - return err == nil - }, time.Minute, time.Second) - - t.Log("ensuring tls echo does not respond after using not existing port") - require.Eventually(t, func() bool { - responded, err := tlsEchoResponds(fmt.Sprintf("%s:%d", proxyURL.Hostname(), ktfkong.DefaultTLSServicePort), - testUUID, tlsRouteHostname, tlsRouteHostname, false) - return responded == false && errors.Is(err, io.EOF) - }, ingressWait, waitTick) -} - -// TestTLSRouteReferenceGrant tests cross-namespace certificate references. These are technically implemented within -// Gateway Listeners, but require an attached Route to see the associated certificate behavior on the proxy. -func TestTLSRouteReferenceGrant(t *testing.T) { - skipTestForExpressionRouter(t) - t.Log("locking Gateway TLS ports") - tlsMutex.Lock() - t.Cleanup(func() { - t.Log("unlocking TLS port") - tlsMutex.Unlock() - }) - - ctx := context.Background() - ns, cleaner := helpers.Setup(ctx, t, env) - - otherNs, err := clusters.GenerateNamespace(ctx, env.Cluster(), t.Name()) - require.NoError(t, err) - cleaner.AddNamespace(otherNs) - - t.Log("getting the gateway client") + t.Log("getting gateway client") gatewayClient, err := gatewayclient.NewForConfig(env.Cluster().Config()) require.NoError(t, err) t.Log("configuring secrets") tlsRouteExampleTLSCert, tlsRouteExampleTLSKey := certificate.MustGenerateSelfSignedCertPEMFormat(certificate.WithCommonName(tlsRouteHostname)) - extraTLSRouteTLSCert, extraTLSRouteTLSKey := certificate.MustGenerateSelfSignedCertPEMFormat(certificate.WithCommonName(tlsRouteExtraHostname)) - secrets := []*corev1.Secret{ { ObjectMeta: metav1.ObjectMeta{ - UID: k8stypes.UID("7428fb98-180b-4702-a91f-61351a33c6e8"), Name: tlsSecretName, Namespace: ns.Name, }, @@ -448,105 +341,74 @@ func TestTLSRouteReferenceGrant(t *testing.T) { "tls.key": tlsRouteExampleTLSKey, }, }, - { - ObjectMeta: metav1.ObjectMeta{ - UID: k8stypes.UID("7428fb98-180b-4702-a91f-61351a33c6e9"), - Name: "secret2", - }, - Data: map[string][]byte{ - "tls.crt": extraTLSRouteTLSCert, - "tls.key": extraTLSRouteTLSKey, - }, - }, } t.Log("deploying secrets") secret1, err := env.Cluster().Client().CoreV1().Secrets(ns.Name).Create(ctx, secrets[0], metav1.CreateOptions{}) require.NoError(t, err) cleaner.Add(secret1) - secret2, err := env.Cluster().Client().CoreV1().Secrets(otherNs.Name).Create(ctx, secrets[1], metav1.CreateOptions{}) + + t.Log("deploying a supported gatewayclass to the test cluster") + gatewayClassName := uuid.NewString() + gwc, err := DeployGatewayClass(ctx, gatewayClient, gatewayClassName) require.NoError(t, err) - cleaner.Add(secret2) + cleaner.Add(gwc) - modePassthrough := gatewayv1beta1.TLSModePassthrough t.Log("deploying a gateway to the test cluster using unmanaged gateway mode") - gateway, err := DeployGateway(ctx, gatewayClient, ns.Name, unmanagedGatewayClassName, func(gw *gatewayv1beta1.Gateway) { - otherNamespace := gatewayv1beta1.Namespace(otherNs.Name) + modePassthrough := gatewayv1beta1.TLSModePassthrough + gatewayName := uuid.NewString() + gateway, err := DeployGateway(ctx, gatewayClient, ns.Name, gatewayClassName, func(gw *gatewayv1beta1.Gateway) { + hostname := gatewayv1beta1.Hostname(tlsRouteHostname) + gw.Name = gatewayName gw.Spec.Listeners = []gatewayv1beta1.Listener{ - builder.NewListener("tls"). - TLS(). - WithPort(ktfkong.DefaultTLSServicePort). - WithHostname(tlsRouteHostname). - WithTLSConfig(&gatewayv1beta1.GatewayTLSConfig{ - Mode: &modePassthrough, + { + Name: "tls-passthrough", + Protocol: gatewayv1beta1.TLSProtocolType, + Port: gatewayv1beta1.PortNumber(ktfkong.DefaultTLSServicePort), + Hostname: &hostname, + TLS: &gatewayv1beta1.GatewayTLSConfig{ CertificateRefs: []gatewayv1beta1.SecretObjectReference{ { - Name: gatewayv1beta1.ObjectName(secrets[0].Name), + Name: gatewayv1beta1.ObjectName(tlsSecretName), }, }, - }).Build(), - builder.NewListener("tlsother"). - TLS(). - WithPort(ktfkong.DefaultTLSServicePort). - WithHostname(tlsRouteExtraHostname). - WithTLSConfig(&gatewayv1beta1.GatewayTLSConfig{ Mode: &modePassthrough, - CertificateRefs: []gatewayv1beta1.SecretObjectReference{ - { - Name: gatewayv1beta1.ObjectName(secrets[1].Name), - Namespace: &otherNamespace, - }, - }, - }).Build(), + }, + }, } }) - require.NoError(t, err) cleaner.Add(gateway) - secret2Name := gatewayv1alpha2.ObjectName(secrets[1].Name) - t.Logf("creating a ReferenceGrant that permits tcproute access from %s to services in %s", ns.Name, otherNs.Name) - grant := &gatewayv1beta1.ReferenceGrant{ - ObjectMeta: metav1.ObjectMeta{ - Name: uuid.NewString(), - Annotations: map[string]string{}, - }, - Spec: gatewayv1beta1.ReferenceGrantSpec{ - From: []gatewayv1beta1.ReferenceGrantFrom{ - { - Group: gatewayv1alpha2.Group("gateway.networking.k8s.io"), - Kind: gatewayv1alpha2.Kind("Gateway"), - Namespace: gatewayv1alpha2.Namespace(gateway.Namespace), - }, - }, - To: []gatewayv1beta1.ReferenceGrantTo{ - { - Group: gatewayv1alpha2.Group(""), - Kind: gatewayv1alpha2.Kind("Secret"), - Name: &secret2Name, - }, + t.Log("creating a tcpecho deployment to test TLSRoute traffic routing") + testUUID := uuid.NewString() // go-echo sends a "Running on Pod ." immediately on connecting + deployment := generators.NewDeploymentForContainer(createTLSEchoContainer(tlsEchoPort, testUUID)) + deployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes, corev1.Volume{ + Name: tlsSecretName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: tlsSecretName, }, }, - } - - grant, err = gatewayClient.GatewayV1beta1().ReferenceGrants(otherNs.Name).Create(ctx, grant, metav1.CreateOptions{}) + }) + deployment, err = env.Cluster().Client().AppsV1().Deployments(ns.Name).Create(ctx, deployment, metav1.CreateOptions{}) require.NoError(t, err) - cleaner.Add(grant) + cleaner.Add(deployment) - t.Log("creating a tcpecho pod to test TLSRoute traffic routing") - container := generators.NewContainer("tcpecho", test.EchoImage, test.EchoTCPPort) - // go-echo sends a "Running on Pod ." immediately on connecting - testUUID := uuid.NewString() - container.Env = []corev1.EnvVar{ - { - Name: "POD_NAME", - Value: testUUID, + t.Log("creating an additional tcpecho pod to test TLSRoute multiple backendRef loadbalancing") + testUUID2 := uuid.NewString() // go-echo sends a "Running on Pod ." immediately on connecting + deployment2 := generators.NewDeploymentForContainer(createTLSEchoContainer(tlsEchoPort, testUUID2)) + deployment2.Spec.Template.Spec.Volumes = append(deployment2.Spec.Template.Spec.Volumes, corev1.Volume{ + Name: tlsSecretName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: tlsSecretName, + }, }, - } - deployment := generators.NewDeploymentForContainer(container) - deployment, err = env.Cluster().Client().AppsV1().Deployments(ns.Name).Create(ctx, deployment, metav1.CreateOptions{}) + }) + deployment2, err = env.Cluster().Client().AppsV1().Deployments(ns.Name).Create(ctx, deployment2, metav1.CreateOptions{}) require.NoError(t, err) - cleaner.Add(deployment) + cleaner.Add(deployment2) t.Logf("exposing deployment %s/%s via service", deployment.Namespace, deployment.Name) service := generators.NewServiceForDeployment(deployment, corev1.ServiceTypeLoadBalancer) @@ -554,9 +416,16 @@ func TestTLSRouteReferenceGrant(t *testing.T) { require.NoError(t, err) cleaner.Add(service) - backendPort := gatewayv1alpha2.PortNumber(test.EchoTCPPort) - t.Logf("creating a tlsroute to access deployment %s via kong", deployment.Name) - tlsroute := &gatewayv1alpha2.TLSRoute{ + t.Logf("exposing deployment %s/%s via service", deployment2.Namespace, deployment2.Name) + service2 := generators.NewServiceForDeployment(deployment2, corev1.ServiceTypeLoadBalancer) + service2, err = env.Cluster().Client().CoreV1().Services(ns.Name).Create(ctx, service2, metav1.CreateOptions{}) + require.NoError(t, err) + cleaner.Add(service2) + + backendTLSPort := gatewayv1alpha2.PortNumber(tlsEchoPort) + t.Logf("create a TLSRoute using passthrough listener") + sectionName := gatewayv1alpha2.SectionName("tls-passthrough") + tlsRoute := &gatewayv1alpha2.TLSRoute{ ObjectMeta: metav1.ObjectMeta{ Name: uuid.NewString(), Annotations: map[string]string{}, @@ -564,28 +433,73 @@ func TestTLSRouteReferenceGrant(t *testing.T) { Spec: gatewayv1alpha2.TLSRouteSpec{ CommonRouteSpec: gatewayv1alpha2.CommonRouteSpec{ ParentRefs: []gatewayv1alpha2.ParentReference{{ - Name: gatewayv1alpha2.ObjectName(gateway.Name), + Name: gatewayv1alpha2.ObjectName(gateway.Name), + SectionName: §ionName, }}, }, - Hostnames: []gatewayv1alpha2.Hostname{tlsRouteHostname, tlsRouteExtraHostname}, + Hostnames: []gatewayv1alpha2.Hostname{tlsRouteHostname}, Rules: []gatewayv1alpha2.TLSRouteRule{{ BackendRefs: []gatewayv1alpha2.BackendRef{{ BackendObjectReference: gatewayv1alpha2.BackendObjectReference{ Name: gatewayv1alpha2.ObjectName(service.Name), - Port: &backendPort, + Port: &backendTLSPort, }, }}, }}, }, } - tlsroute, err = gatewayClient.GatewayV1alpha2().TLSRoutes(ns.Name).Create(ctx, tlsroute, metav1.CreateOptions{}) + tlsRoute, err = gatewayClient.GatewayV1alpha2().TLSRoutes(ns.Name).Create(ctx, tlsRoute, metav1.CreateOptions{}) require.NoError(t, err) - cleaner.Add(tlsroute) + cleaner.Add(tlsRoute) + + proxyAddress := fmt.Sprintf("%s:%d", proxyURL.Hostname(), ktfkong.DefaultTLSServicePort) + t.Log("verifying that the tcpecho is responding properly over TLS") + require.Eventually(t, func() bool { + responded, err := tlsEchoResponds(proxyAddress, testUUID, tlsRouteHostname, tlsRouteHostname, true) + if err != nil { + t.Logf("failed accessing tcpecho at %s, err: %v", proxyAddress, err) + return false + } + return err == nil && responded == true + }, ingressWait, waitTick) + + t.Log("removing the parentrefs from the TLSRoute") + oldParentRefs := tlsRoute.Spec.ParentRefs + require.Eventually(t, func() bool { + tlsRoute, err = gatewayClient.GatewayV1alpha2().TLSRoutes(ns.Name).Get(ctx, tlsRoute.Name, metav1.GetOptions{}) + require.NoError(t, err) + tlsRoute.Spec.ParentRefs = nil + tlsRoute, err = gatewayClient.GatewayV1alpha2().TLSRoutes(ns.Name).Update(ctx, tlsRoute, metav1.UpdateOptions{}) + return err == nil + }, time.Minute, time.Second) + + t.Log("verifying that the Gateway gets unlinked from the route via status") + callback := GetGatewayIsUnlinkedCallback(ctx, t, gatewayClient, gatewayv1beta1.TLSProtocolType, ns.Name, tlsRoute.Name) + require.Eventually(t, callback, ingressWait, waitTick) + + t.Log("verifying that the tcpecho is no longer responding") + require.Eventually(t, func() bool { + responded, err := tlsEchoResponds(fmt.Sprintf("%s:%d", proxyURL.Hostname(), ktfkong.DefaultTLSServicePort), + testUUID, tlsRouteHostname, tlsRouteHostname, false) + return responded == false && errors.Is(err, io.EOF) + }, ingressWait, waitTick) + + t.Log("putting the parentRefs back") + require.Eventually(t, func() bool { + tlsRoute, err = gatewayClient.GatewayV1alpha2().TLSRoutes(ns.Name).Get(ctx, tlsRoute.Name, metav1.GetOptions{}) + require.NoError(t, err) + tlsRoute.Spec.ParentRefs = oldParentRefs + tlsRoute, err = gatewayClient.GatewayV1alpha2().TLSRoutes(ns.Name).Update(ctx, tlsRoute, metav1.UpdateOptions{}) + return err == nil + }, time.Minute, time.Second) - proxyAddress := fmt.Sprintf("%s:%d", proxyURL.Hostname(), ktfkong.DefaultTLSServicePort) - t.Log("verifying that the tcpecho is responding properly over TLS") + t.Log("verifying that the Gateway gets linked to the route via status") + callback = GetGatewayIsLinkedCallback(ctx, t, gatewayClient, gatewayv1beta1.TLSProtocolType, ns.Name, tlsRoute.Name) + require.Eventually(t, callback, ingressWait, waitTick) + + t.Log("verifying that putting the parentRefs back results in the routes becoming available again") require.Eventually(t, func() bool { - responded, err := tlsEchoResponds(proxyAddress, testUUID, tlsRouteHostname, tlsRouteHostname, false) + responded, err := tlsEchoResponds(proxyAddress, testUUID, tlsRouteHostname, tlsRouteHostname, true) if err != nil { t.Logf("failed accessing tcpecho at %s, err: %v", proxyAddress, err) return false @@ -593,9 +507,30 @@ func TestTLSRouteReferenceGrant(t *testing.T) { return err == nil && responded == true }, ingressWait, waitTick) - t.Log("verifying that the tcpecho route can also serve certificates permitted by a ReferenceGrant with a named To") + t.Log("deleting the GatewayClass") + require.NoError(t, gatewayClient.GatewayV1beta1().GatewayClasses().Delete(ctx, gwc.Name, metav1.DeleteOptions{})) + + t.Log("verifying that the Gateway gets unlinked from the route via status") + callback = GetGatewayIsUnlinkedCallback(ctx, t, gatewayClient, gatewayv1beta1.TLSProtocolType, ns.Name, tlsRoute.Name) + require.Eventually(t, callback, ingressWait, waitTick) + + t.Log("verifying that the data-plane configuration from the TLSRoute gets dropped with the GatewayClass now removed") + require.Eventually(t, func() bool { + responded, err := tlsEchoResponds(proxyAddress, testUUID, tlsRouteHostname, tlsRouteHostname, true) + return responded == false && errors.Is(err, io.EOF) + }, ingressWait, waitTick) + + t.Log("putting the GatewayClass back") + gwc, err = DeployGatewayClass(ctx, gatewayClient, gatewayClassName) + require.NoError(t, err) + + t.Log("verifying that the Gateway gets linked to the route via status") + callback = GetGatewayIsLinkedCallback(ctx, t, gatewayClient, gatewayv1beta1.TLSProtocolType, ns.Name, tlsRoute.Name) + require.Eventually(t, callback, ingressWait, waitTick) + + t.Log("verifying that creating the GatewayClass again triggers reconciliation of TLSRoutes and the route becomes available again") require.Eventually(t, func() bool { - responded, err := tlsEchoResponds(proxyAddress, testUUID, tlsRouteExtraHostname, tlsRouteExtraHostname, false) + responded, err := tlsEchoResponds(proxyAddress, testUUID, tlsRouteHostname, tlsRouteHostname, true) if err != nil { t.Logf("failed accessing tcpecho at %s, err: %v", proxyAddress, err) return false @@ -603,91 +538,117 @@ func TestTLSRouteReferenceGrant(t *testing.T) { return err == nil && responded == true }, ingressWait, waitTick) - t.Log("verifying that using the wrong name in the ReferenceGrant removes the related certificate") - badName := gatewayv1alpha2.ObjectName("garbage") - grant.Spec.To[0].Name = &badName - grant, err = gatewayClient.GatewayV1beta1().ReferenceGrants(otherNs.Name).Update(ctx, grant, metav1.UpdateOptions{}) - require.NoError(t, err) + t.Log("deleting the Gateway") + require.NoError(t, gatewayClient.GatewayV1beta1().Gateways(ns.Name).Delete(ctx, gatewayName, metav1.DeleteOptions{})) + + t.Log("verifying that the Gateway gets unlinked from the route via status") + callback = GetGatewayIsUnlinkedCallback(ctx, t, gatewayClient, gatewayv1beta1.TLSProtocolType, ns.Name, tlsRoute.Name) + require.Eventually(t, callback, ingressWait, waitTick) + t.Log("verifying that the data-plane configuration from the TLSRoute gets dropped with the Gateway now removed") require.Eventually(t, func() bool { - responded, err := tlsEchoResponds(proxyAddress, testUUID, tlsRouteExtraHostname, tlsRouteExtraHostname, false) - return err != nil && responded == false + responded, err := tlsEchoResponds(proxyAddress, testUUID, tlsRouteHostname, tlsRouteHostname, true) + return responded == false && errors.Is(err, io.EOF) }, ingressWait, waitTick) - t.Log("verifying that a Listener has the invalid ref status condition") - gateway, err = gatewayClient.GatewayV1beta1().Gateways(ns.Name).Get(ctx, gateway.Name, metav1.GetOptions{}) - require.NoError(t, err) - invalid := false - for _, status := range gateway.Status.Listeners { - if ok := util.CheckCondition( - status.Conditions, - util.ConditionType(gatewayv1beta1.ListenerConditionResolvedRefs), - util.ConditionReason(gatewayv1beta1.ListenerReasonRefNotPermitted), - metav1.ConditionFalse, - gateway.Generation, - ); ok { - invalid = true + t.Log("putting the Gateway back") + gateway, err = DeployGateway(ctx, gatewayClient, ns.Name, gatewayClassName, func(gw *gatewayv1beta1.Gateway) { + hostname := gatewayv1beta1.Hostname(tlsRouteHostname) + gw.Name = gatewayName + gw.Spec.Listeners = []gatewayv1beta1.Listener{ + { + Name: "tls-passthrough", + Protocol: gatewayv1beta1.TLSProtocolType, + Port: gatewayv1beta1.PortNumber(ktfkong.DefaultTLSServicePort), + Hostname: &hostname, + TLS: &gatewayv1beta1.GatewayTLSConfig{ + CertificateRefs: []gatewayv1beta1.SecretObjectReference{ + { + Name: gatewayv1beta1.ObjectName(tlsSecretName), + }, + }, + Mode: &modePassthrough, + }, + }, } - } - require.True(t, invalid) - - t.Log("verifying the certificate returns when using a ReferenceGrant with no name restrictions") - grant.Spec.To[0].Name = nil - _, err = gatewayClient.GatewayV1beta1().ReferenceGrants(otherNs.Name).Update(ctx, grant, metav1.UpdateOptions{}) + }) require.NoError(t, err) + t.Log("verifying that the Gateway gets linked to the route via status") + callback = GetGatewayIsLinkedCallback(ctx, t, gatewayClient, gatewayv1beta1.TLSProtocolType, ns.Name, tlsRoute.Name) + require.Eventually(t, callback, ingressWait, waitTick) + + t.Log("verifying that creating the Gateway again triggers reconciliation of TLSRoutes and the route becomes available again") require.Eventually(t, func() bool { - responded, err := tlsEchoResponds(proxyAddress, testUUID, tlsRouteExtraHostname, tlsRouteExtraHostname, false) + responded, err := tlsEchoResponds(proxyAddress, testUUID, tlsRouteHostname, tlsRouteHostname, true) if err != nil { t.Logf("failed accessing tcpecho at %s, err: %v", proxyAddress, err) return false } return err == nil && responded == true }, ingressWait, waitTick) -} - -func TestTLSRoutePassthrough(t *testing.T) { - skipTestForExpressionRouter(t) - t.Log("locking Gateway TLS ports") - tlsMutex.Lock() - t.Cleanup(func() { - t.Log("unlocking TLS port") - tlsMutex.Unlock() - }) - - ctx := context.Background() - ns, cleaner := helpers.Setup(ctx, t, env) - t.Log("getting gateway client") - gatewayClient, err := gatewayclient.NewForConfig(env.Cluster().Config()) - require.NoError(t, err) + t.Log("adding an additional backendRef to the TLSRoute") + require.Eventually(t, func() bool { + tlsRoute, err = gatewayClient.GatewayV1alpha2().TLSRoutes(ns.Name).Get(ctx, tlsRoute.Name, metav1.GetOptions{}) + require.NoError(t, err) - t.Log("configuring secrets") - tlsRouteExampleTLSCert, tlsRouteExampleTLSKey := certificate.MustGenerateSelfSignedCertPEMFormat(certificate.WithCommonName(tlsRouteHostname)) - secrets := []*corev1.Secret{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: tlsSecretName, - Namespace: ns.Name, + tlsRoute.Spec.Rules[0].BackendRefs = []gatewayv1alpha2.BackendRef{ + { + BackendObjectReference: gatewayv1alpha2.BackendObjectReference{ + Name: gatewayv1alpha2.ObjectName(service.Name), + Port: &backendTLSPort, + }, }, - Data: map[string][]byte{ - "tls.crt": tlsRouteExampleTLSCert, - "tls.key": tlsRouteExampleTLSKey, + { + BackendObjectReference: gatewayv1alpha2.BackendObjectReference{ + Name: gatewayv1alpha2.ObjectName(service2.Name), + Port: &backendTLSPort, + }, }, - }, - } + } - t.Log("deploying secrets") - secret1, err := env.Cluster().Client().CoreV1().Secrets(ns.Name).Create(ctx, secrets[0], metav1.CreateOptions{}) - require.NoError(t, err) - cleaner.Add(secret1) + tlsRoute, err = gatewayClient.GatewayV1alpha2().TLSRoutes(ns.Name).Update(ctx, tlsRoute, metav1.UpdateOptions{}) + return err == nil + }, ingressWait, waitTick) - t.Log("deploying a gateway to the test cluster using unmanaged gateway mode") - gatewayName := uuid.NewString() - gateway, err := DeployGateway(ctx, gatewayClient, ns.Name, unmanagedGatewayClassName, func(gw *gatewayv1beta1.Gateway) { - hostname := gatewayv1beta1.Hostname(tlsRouteHostname) - tlsModePassthrough := gatewayv1beta1.TLSModePassthrough + t.Log("verifying that the TLSRoute is now load-balanced between two services") + require.Eventually(t, func() bool { + responded, err := tlsEchoResponds(proxyAddress, testUUID, tlsRouteHostname, tlsRouteHostname, true) + if err != nil { + t.Logf("failed accessing tcpecho at %s, err: %v", proxyAddress, err) + return false + } + return err == nil && responded == true + }, ingressWait, waitTick) + require.Eventually(t, func() bool { + responded, err := tlsEchoResponds(proxyAddress, testUUID2, tlsRouteHostname, tlsRouteHostname, true) + if err != nil { + t.Logf("failed accessing tcpecho at %s, err: %v", proxyAddress, err) + return false + } + return err == nil && responded == true + }, ingressWait, waitTick) + + t.Log("deleting both GatewayClass and Gateway rapidly") + require.NoError(t, gatewayClient.GatewayV1beta1().GatewayClasses().Delete(ctx, gwc.Name, metav1.DeleteOptions{})) + require.NoError(t, gatewayClient.GatewayV1beta1().Gateways(ns.Name).Delete(ctx, gateway.Name, metav1.DeleteOptions{})) + + t.Log("verifying that the Gateway gets unlinked from the route via status") + callback = GetGatewayIsUnlinkedCallback(ctx, t, gatewayClient, gatewayv1beta1.TLSProtocolType, ns.Name, tlsRoute.Name) + require.Eventually(t, callback, ingressWait, waitTick) + + t.Log("verifying that the data-plane configuration from the TLSRoute does not get orphaned with the GatewayClass and Gateway gone") + require.Eventually(t, func() bool { + responded, err := tlsEchoResponds(fmt.Sprintf("%s:%d", proxyURL.Hostname(), ktfkong.DefaultTLSServicePort), + testUUID, tlsRouteHostname, tlsRouteHostname, true) + return responded == false && errors.Is(err, io.EOF) + }, ingressWait, waitTick) + t.Log("testing port matching") + t.Log("putting the Gateway back") + gateway, err = DeployGateway(ctx, gatewayClient, ns.Name, gatewayClassName, func(gw *gatewayv1beta1.Gateway) { + hostname := gatewayv1beta1.Hostname(tlsRouteHostname) gw.Name = gatewayName gw.Spec.Listeners = []gatewayv1beta1.Listener{ { @@ -701,67 +662,17 @@ func TestTLSRoutePassthrough(t *testing.T) { Name: gatewayv1beta1.ObjectName(tlsSecretName), }, }, - Mode: &tlsModePassthrough, + Mode: &modePassthrough, }, }, } }) require.NoError(t, err) - cleaner.Add(gateway) - - t.Log("creating a tcpecho deployment to test TLSRoute traffic routing") - testUUID := uuid.NewString() // go-echo sends a "Running on Pod ." immediately on connecting - deployment := generators.NewDeploymentForContainer(createTLSEchoContainer(tlsEchoPort, testUUID)) - deployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes, corev1.Volume{ - Name: tlsSecretName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: tlsSecretName, - }, - }, - }) - deployment, err = env.Cluster().Client().AppsV1().Deployments(ns.Name).Create(ctx, deployment, metav1.CreateOptions{}) - require.NoError(t, err) - cleaner.Add(deployment) - - t.Logf("exposing deployment %s/%s via service", deployment.Namespace, deployment.Name) - service := generators.NewServiceForDeployment(deployment, corev1.ServiceTypeLoadBalancer) - service, err = env.Cluster().Client().CoreV1().Services(ns.Name).Create(ctx, service, metav1.CreateOptions{}) - require.NoError(t, err) - cleaner.Add(service) - backendTLSPort := gatewayv1alpha2.PortNumber(tlsEchoPort) - t.Logf("create a TLSRoute using passthrough listener") - sectionName := gatewayv1alpha2.SectionName("tls-passthrough") - tlsroute := &gatewayv1alpha2.TLSRoute{ - ObjectMeta: metav1.ObjectMeta{ - Name: uuid.NewString(), - Annotations: map[string]string{}, - }, - Spec: gatewayv1alpha2.TLSRouteSpec{ - CommonRouteSpec: gatewayv1alpha2.CommonRouteSpec{ - ParentRefs: []gatewayv1alpha2.ParentReference{{ - Name: gatewayv1alpha2.ObjectName(gateway.Name), - SectionName: §ionName, - }}, - }, - Hostnames: []gatewayv1alpha2.Hostname{tlsRouteHostname}, - Rules: []gatewayv1alpha2.TLSRouteRule{{ - BackendRefs: []gatewayv1alpha2.BackendRef{{ - BackendObjectReference: gatewayv1alpha2.BackendObjectReference{ - Name: gatewayv1alpha2.ObjectName(service.Name), - Port: &backendTLSPort, - }, - }}, - }}, - }, - } - tlsroute, err = gatewayClient.GatewayV1alpha2().TLSRoutes(ns.Name).Create(ctx, tlsroute, metav1.CreateOptions{}) - require.NoError(t, err) - cleaner.Add(tlsroute) + t.Log("putting the GatewayClass back") + _, err = DeployGatewayClass(ctx, gatewayClient, gatewayClassName) - proxyAddress := fmt.Sprintf("%s:%d", proxyURL.Hostname(), ktfkong.DefaultTLSServicePort) - t.Log("verifying that the tcpecho is responding properly over TLS") + t.Log("ensuring tls echo responds after recreating gateway and gateway class") require.Eventually(t, func() bool { responded, err := tlsEchoResponds(proxyAddress, testUUID, tlsRouteHostname, tlsRouteHostname, true) if err != nil { @@ -770,6 +681,25 @@ func TestTLSRoutePassthrough(t *testing.T) { } return err == nil && responded == true }, ingressWait, waitTick) + + t.Log("setting the port in ParentRef which does not have a matching listener in Gateway") + require.Eventually(t, func() bool { + tlsRoute, err = gatewayClient.GatewayV1alpha2().TLSRoutes(ns.Name).Get(ctx, tlsRoute.Name, metav1.GetOptions{}) + if err != nil { + return false + } + notExistingPort := gatewayv1alpha2.PortNumber(81) + tlsRoute.Spec.ParentRefs[0].Port = ¬ExistingPort + tlsRoute, err = gatewayClient.GatewayV1alpha2().TLSRoutes(ns.Name).Update(ctx, tlsRoute, metav1.UpdateOptions{}) + return err == nil + }, time.Minute, time.Second) + + t.Log("ensuring tls echo does not respond after using not existing port") + require.Eventually(t, func() bool { + responded, err := tlsEchoResponds(fmt.Sprintf("%s:%d", proxyURL.Hostname(), ktfkong.DefaultTLSServicePort), + testUUID, tlsRouteHostname, tlsRouteHostname, true) + return responded == false && errors.Is(err, io.EOF) + }, ingressWait, waitTick) } // tlsEchoResponds takes a TLS address URL and a Pod name and checks if a @@ -784,8 +714,8 @@ func tlsEchoResponds( "tcp", url, &tls.Config{ - InsecureSkipVerify: true, ServerName: hostname, + InsecureSkipVerify: true, }) if err != nil { return false, err @@ -850,7 +780,7 @@ func tlsEchoResponds( } func createTLSEchoContainer(tlsEchoPort int32, sendMsg string) corev1.Container { - container := generators.NewContainer("tcpecho", test.EchoImage, tlsEchoPort) + container := generators.NewContainer("tcpecho-"+sendMsg, test.EchoImage, tlsEchoPort) const tlsCertDir = "/var/run/certs" container.Env = append(container.Env, corev1.EnvVar{