Gateway API: support backend appProtocol (#5934)

Signed-off-by: gang.liu <[email protected]>
projectcontour · Nov 15, 2023 · 9b4f666 · 9b4f666
1 parent c1b1e0a
commit 9b4f666
Show file tree

Hide file tree

Showing 7 changed files with 238 additions and 15 deletions.
diff --git a/changelogs/unreleased/5934-izturn-minor.md b/changelogs/unreleased/5934-izturn-minor.md
@@ -0,0 +1,3 @@
+## Gateway API Backend Protocol Selection
+
+For Gateway API, Contour now enables end-users to specify backend protocols by setting the backend Service's [ServicePort.AppProtocol](https://kubernetes.io/docs/concepts/services-networking/service/#application-protocol) parameter. The accepted values are `kubernetes.io/h2c` and `kubernetes.io/ws`. Note that websocket upgrades are already enabled by default for Gateway API. If `AppProtocol` is set, any other configurations, such as the annotation: `projectcontour.io/upstream-protocol.{protocol}` will be disregarded.
diff --git a/internal/dag/accessors.go b/internal/dag/accessors.go
@@ -103,13 +103,33 @@ func validateExternalName(svc *v1.Service, enableExternalNameSvc bool) error {
 	return nil
 }
 
+// the ServicePort's AppProtocol must be one of the these.
+const (
+	protoK8sH2C = "kubernetes.io/h2c"
+	protoK8sWS  = "kubernetes.io/ws"
+)
+
+func toContourProtocol(appProtocol string) (string, bool) {
+	proto, ok := map[string]string{
+		// *NOTE: for gateway-api: the websocket is enabled by default
+		protoK8sWS:  "",
+		protoK8sH2C: "h2c",
+	}[appProtocol]
+	return proto, ok
+}
 func upstreamProtocol(svc *v1.Service, port v1.ServicePort) string {
+	// if appProtocol is not nil, check it only
+	if port.AppProtocol != nil {
+		proto, _ := toContourProtocol(*port.AppProtocol)
+		return proto
+	}
+
 	up := annotation.ParseUpstreamProtocols(svc.Annotations)
-	protocol := up[port.Name]
-	if protocol == "" {
-		protocol = up[strconv.Itoa(int(port.Port))]
+	proto := up[port.Name]
+	if proto == "" {
+		proto = up[strconv.Itoa(int(port.Port))]
 	}
-	return protocol
+	return proto
 }
 
 func externalName(svc *v1.Service) string {

diff --git a/internal/dag/accessors_test.go b/internal/dag/accessors_test.go
@@ -18,22 +18,30 @@ import (
 	"testing"
 
 	"github.com/projectcontour/contour/internal/fixture"
+	"github.com/projectcontour/contour/internal/ref"
+
 	"github.com/stretchr/testify/assert"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/intstr"
 )
 
-func makeServicePort(name string, protocol v1.Protocol, port int32, extras ...int) v1.ServicePort {
+func makeServicePort(name string, protocol v1.Protocol, port int32, extras ...any) v1.ServicePort {
 	p := v1.ServicePort{
 		Name:     name,
 		Protocol: protocol,
 		Port:     port,
 	}
-	if len(extras) != 0 {
-		p.TargetPort = intstr.FromInt(extras[0])
+
+	if len(extras) > 0 {
+		p.TargetPort = intstr.FromInt(extras[0].(int))
+	}
+
+	if len(extras) > 1 {
+		p.AppProtocol = ref.To(extras[1].(string))
 	}
+
 	return p
 
 }
@@ -82,11 +90,53 @@ func TestBuilderLookupService(t *testing.T) {
 		},
 	}
 
+	annotatedService := &v1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        "annotated-service",
+			Namespace:   "default",
+			Annotations: map[string]string{"projectcontour.io/upstream-protocol.tls": "8443"},
+		},
+		Spec: v1.ServiceSpec{
+			Ports: []v1.ServicePort{{
+				Name:       "foo",
+				Protocol:   "TCP",
+				Port:       8443,
+				TargetPort: intstr.FromInt(26441),
+			}},
+		},
+	}
+
+	appProtoService := &v1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        "app-protocol-service",
+			Namespace:   "default",
+			Annotations: map[string]string{"projectcontour.io/upstream-protocol.tls": "8443,8444"},
+		},
+		Spec: v1.ServiceSpec{
+			Ports: []v1.ServicePort{
+				{
+					Name:        "k8s-h2c",
+					Protocol:    "TCP",
+					AppProtocol: ref.To("kubernetes.io/h2c"),
+					Port:        8443,
+				},
+				{
+					Name:        "k8s-wss",
+					Protocol:    "TCP",
+					AppProtocol: ref.To("kubernetes.io/wss"),
+					Port:        8444,
+				},
+			},
+		},
+	}
+
 	services := map[types.NamespacedName]*v1.Service{
-		{Name: "service1", Namespace: "default"}:              s1,
-		{Name: "servicehealthcheck", Namespace: "default"}:    s2,
-		{Name: "externalnamevalid", Namespace: "default"}:     externalNameValid,
-		{Name: "externalnamelocalhost", Namespace: "default"}: externalNameLocalhost,
+		{Name: "service1", Namespace: "default"}:                             s1,
+		{Name: "servicehealthcheck", Namespace: "default"}:                   s2,
+		{Name: "externalnamevalid", Namespace: "default"}:                    externalNameValid,
+		{Name: "externalnamelocalhost", Namespace: "default"}:                externalNameLocalhost,
+		{Name: annotatedService.Name, Namespace: annotatedService.Namespace}: annotatedService,
+		{Name: appProtoService.Name, Namespace: appProtoService.Namespace}:   appProtoService,
 	}
 
 	tests := map[string]struct {
@@ -151,6 +201,22 @@ func TestBuilderLookupService(t *testing.T) {
 			wantErr:               errors.New(`default/externalnamelocalhost is an ExternalName service that points to localhost, this is not allowed`),
 			enableExternalNameSvc: true,
 		},
+		"lookup service by port number with annotated number": {
+			NamespacedName: types.NamespacedName{Name: annotatedService.Name, Namespace: annotatedService.Namespace},
+			port:           8443,
+			want:           appProtcolService(annotatedService, "tls"),
+		},
+		"lookup service by port number with k8s app protocol: h2c": {
+			NamespacedName: types.NamespacedName{Name: appProtoService.Name, Namespace: appProtoService.Namespace},
+			port:           8443,
+			want:           appProtcolService(appProtoService, "h2c"),
+		},
+
+		"lookup service by port number with unsupported k8s app protocol: wss": {
+			NamespacedName: types.NamespacedName{Name: appProtoService.Name, Namespace: appProtoService.Namespace},
+			port:           8444,
+			want:           appProtcolService(appProtoService, "", 1),
+		},
 	}
 
 	for name, tc := range tests {

diff --git a/internal/dag/builder_test.go b/internal/dag/builder_test.go
@@ -16108,6 +16108,23 @@ func service(s *v1.Service) *Service {
 	return weightedService(s, 1)
 }
 
+func appProtcolService(s *v1.Service, protocol string, portIndex ...int) *Service {
+	idx := 0
+	if len(portIndex) > 0 {
+		idx = portIndex[0]
+	}
+	return &Service{
+		Weighted: WeightedService{
+			Weight:           1,
+			ServiceName:      s.Name,
+			ServiceNamespace: s.Namespace,
+			ServicePort:      s.Spec.Ports[idx],
+			HealthPort:       s.Spec.Ports[idx],
+		},
+		Protocol: protocol,
+	}
+}
+
 func weightedService(s *v1.Service, weight uint32) *Service {
 	return &Service{
 		Weighted: WeightedService{

diff --git a/internal/dag/gatewayapi_processor.go b/internal/dag/gatewayapi_processor.go
@@ -26,6 +26,7 @@ import (
 	"github.com/projectcontour/contour/internal/status"
 
 	"github.com/sirupsen/logrus"
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/types"
@@ -1711,7 +1712,11 @@ func resolvedRefsFalse(reason gatewayapi_v1beta1.RouteConditionReason, msg strin
 // is valid. Returns a metav1.Condition for the route if any errors are detected.
 // As BackendObjectReference is used in multiple fields, the given field is used
 // to build the message in metav1.Condition.
-func (p *GatewayAPIProcessor) validateBackendObjectRef(backendObjectRef gatewayapi_v1beta1.BackendObjectReference, field string, routeKind, routeNamespace string) (*Service, *metav1.Condition) {
+func (p *GatewayAPIProcessor) validateBackendObjectRef(
+	backendObjectRef gatewayapi_v1beta1.BackendObjectReference,
+	field string,
+	routeKind string,
+	routeNamespace string) (*Service, *metav1.Condition) {
 
 	if !(backendObjectRef.Group == nil || *backendObjectRef.Group == "") {
 		return nil, ref.To(resolvedRefsFalse(gatewayapi_v1beta1.RouteReasonInvalidKind, fmt.Sprintf("%s.Group must be \"\"", field)))
@@ -1761,9 +1766,23 @@ func (p *GatewayAPIProcessor) validateBackendObjectRef(backendObjectRef gatewaya
 		return nil, ref.To(resolvedRefsFalse(gatewayapi_v1beta1.RouteReasonBackendNotFound, fmt.Sprintf("service %q is invalid: %s", meta.Name, err)))
 	}
 
+	if err = validateAppProtocol(&service.Weighted.ServicePort); err != nil {
+		return nil, ref.To(resolvedRefsFalse(gatewayapi_v1.RouteReasonUnsupportedProtocol, err.Error()))
+	}
+
 	return service, nil
 }
 
+func validateAppProtocol(svc *v1.ServicePort) error {
+	if svc.AppProtocol == nil {
+		return nil
+	}
+	if _, ok := toContourProtocol(*svc.AppProtocol); ok {
+		return nil
+	}
+	return fmt.Errorf("AppProtocol: \"%s\" is unsupported", *svc.AppProtocol)
+}
+
 func gatewayPathMatchCondition(match *gatewayapi_v1beta1.HTTPPathMatch, routeAccessor *status.RouteParentStatusUpdate) (MatchCondition, bool) {
 	if match == nil {
 		return &PrefixMatchCondition{Prefix: "/"}, true

diff --git a/internal/dag/status_test.go b/internal/dag/status_test.go
@@ -17,14 +17,15 @@ import (
 	"fmt"
 	"testing"
 
-	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
 	contour_api_v1 "github.com/projectcontour/contour/apis/projectcontour/v1"
 	"github.com/projectcontour/contour/internal/fixture"
 	"github.com/projectcontour/contour/internal/gatewayapi"
 	"github.com/projectcontour/contour/internal/k8s"
 	"github.com/projectcontour/contour/internal/ref"
 	"github.com/projectcontour/contour/internal/status"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/stretchr/testify/assert"
 	v1 "k8s.io/api/core/v1"
 	networking_v1 "k8s.io/api/networking/v1"
@@ -5289,6 +5290,28 @@ func TestGatewayAPIHTTPRouteDAGStatus(t *testing.T) {
 		},
 	}
 
+	kuardService4 := &v1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "kuard4",
+			Namespace: "default",
+		},
+		Spec: v1.ServiceSpec{
+			Ports: []v1.ServicePort{makeServicePort("http", "TCP", 8080, 8080, protoK8sH2C)},
+		},
+	}
+
+	kuardService5 := &v1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "kuard5",
+			Namespace: "default",
+		},
+		Spec: v1.ServiceSpec{
+			Ports: []v1.ServicePort{
+				makeServicePort("wss", "TCP", 8444, 8444, "kubernetes.io/wss"),
+			},
+		},
+	}
+
 	run(t, "simple httproute", testcase{
 		objs: []any{
 			kuardService,
@@ -9163,6 +9186,82 @@ func TestGatewayAPIHTTPRouteDAGStatus(t *testing.T) {
 		}},
 	})
 
+	run(t, "service with supported app protocol: h2c", testcase{
+		objs: []any{
+			kuardService4,
+			&gatewayapi_v1beta1.HTTPRoute{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "basic",
+					Namespace: "default",
+				},
+				Spec: gatewayapi_v1beta1.HTTPRouteSpec{
+					CommonRouteSpec: gatewayapi_v1beta1.CommonRouteSpec{
+						ParentRefs: []gatewayapi_v1beta1.ParentReference{gatewayapi.GatewayParentRef("projectcontour", "contour")},
+					},
+					Hostnames: []gatewayapi_v1beta1.Hostname{
+						"test.projectcontour.io",
+					},
+					Rules: []gatewayapi_v1beta1.HTTPRouteRule{
+						{
+							Matches:     gatewayapi.HTTPRouteMatch(gatewayapi_v1.PathMatchPathPrefix, "/"),
+							BackendRefs: gatewayapi.HTTPBackendRef("kuard4", 8080, 1),
+						},
+					},
+				},
+			}},
+		wantRouteConditions: []*status.RouteStatusUpdate{{
+			FullName: types.NamespacedName{Namespace: "default", Name: "basic"},
+			RouteParentStatuses: []*gatewayapi_v1beta1.RouteParentStatus{
+				{
+					ParentRef: gatewayapi.GatewayParentRef("projectcontour", "contour"),
+					Conditions: []metav1.Condition{
+						routeResolvedRefsCondition(),
+						routeAcceptedHTTPRouteCondition(),
+					},
+				},
+			},
+		}},
+		wantGatewayStatusUpdate: validGatewayStatusUpdate("http", gatewayapi_v1.HTTPProtocolType, 1),
+	})
+
+	run(t, "service with unsupported app protocol: wss", testcase{
+		objs: []any{
+			kuardService5,
+			&gatewayapi_v1beta1.HTTPRoute{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "basic",
+					Namespace: "default",
+				},
+				Spec: gatewayapi_v1beta1.HTTPRouteSpec{
+					CommonRouteSpec: gatewayapi_v1beta1.CommonRouteSpec{
+						ParentRefs: []gatewayapi_v1beta1.ParentReference{gatewayapi.GatewayParentRef("projectcontour", "contour")},
+					},
+					Hostnames: []gatewayapi_v1beta1.Hostname{
+						"test.projectcontour.io",
+					},
+					Rules: []gatewayapi_v1beta1.HTTPRouteRule{
+						{
+							Matches:     gatewayapi.HTTPRouteMatch(gatewayapi_v1.PathMatchPathPrefix, "/"),
+							BackendRefs: gatewayapi.HTTPBackendRef("kuard5", 8444, 1),
+						},
+					},
+				},
+			}},
+		wantRouteConditions: []*status.RouteStatusUpdate{{
+			FullName: types.NamespacedName{Namespace: "default", Name: "basic"},
+			RouteParentStatuses: []*gatewayapi_v1beta1.RouteParentStatus{
+				{
+					ParentRef: gatewayapi.GatewayParentRef("projectcontour", "contour"),
+					Conditions: []metav1.Condition{
+						resolvedRefsFalse(gatewayapi_v1.RouteReasonUnsupportedProtocol, "AppProtocol: \"kubernetes.io/wss\" is unsupported"),
+						routeAcceptedHTTPRouteCondition(),
+					},
+				},
+			},
+		}},
+		wantGatewayStatusUpdate: validGatewayStatusUpdate("http", gatewayapi_v1.HTTPProtocolType, 1),
+	})
+
 }
 
 func TestGatewayAPITLSRouteDAGStatus(t *testing.T) {

diff --git a/test/conformance/gatewayapi/gateway_conformance_test.go b/test/conformance/gatewayapi/gateway_conformance_test.go
@@ -75,7 +75,6 @@ func TestGatewayConformance(t *testing.T) {
 			tests.HTTPRouteRedirectPortAndScheme.ShortName,
 
 			// Tests newly failing with Gateway API 1.0, to be addressed.
-			tests.HTTPRouteBackendProtocolH2C.ShortName,
 			tests.HTTPRouteTimeoutBackendRequest.ShortName,
 			tests.HTTPRouteTimeoutRequest.ShortName,
 			tests.GatewayWithAttachedRoutes.ShortName,