implement Gateway infrastructure labels/annotations (#5968)

Closes #5967.

Signed-off-by: Steve Kriss <[email protected]>

apis/projectcontour/v1alpha1/contourdeployment.go

@@ -68,6 +68,9 @@ type ContourDeploymentSpec struct {
 	RuntimeSettings *ContourConfigurationSpec `json:"runtimeSettings,omitempty"`
 
 	// ResourceLabels is a set of labels to add to the provisioned Contour resources.
+	//
+	// Deprecated: use Gateway.Spec.Infrastructure.Labels instead. This field will be
+	// removed in a future release.
 	// +optional
 	ResourceLabels map[string]string `json:"resourceLabels,omitempty"`
 }

changelogs/unreleased/5968-skriss-deprecation.md

@@ -0,0 +1 @@
+The `ContourDeployment.Spec.ResourceLabels` field is now deprecated. You should use `Gateway.Spec.Infrastructure.Labels` instead. The `ResourceLabels` field will be removed in a future release.

changelogs/unreleased/5968-skriss-small.md

@@ -0,0 +1 @@
+Gateway API: adds support for [Gateway infrastructure labels and annotations](https://gateway-api.sigs.k8s.io/reference/spec/#gateway.networking.k8s.io/v1.GatewayInfrastructure)``.

examples/contour/01-crds.yaml

@@ -3496,8 +3496,9 @@ spec:
               resourceLabels:
                 additionalProperties:
                   type: string
-                description: ResourceLabels is a set of labels to add to the provisioned
-                  Contour resources.
+                description: "ResourceLabels is a set of labels to add to the provisioned
+                  Contour resources. \n Deprecated: use Gateway.Spec.Infrastructure.Labels
+                  instead. This field will be removed in a future release."
                 type: object
               runtimeSettings:
                 description: RuntimeSettings is a ContourConfiguration spec to be

examples/render/contour-deployment.yaml

@@ -3715,8 +3715,9 @@ spec:
               resourceLabels:
                 additionalProperties:
                   type: string
-                description: ResourceLabels is a set of labels to add to the provisioned
-                  Contour resources.
+                description: "ResourceLabels is a set of labels to add to the provisioned
+                  Contour resources. \n Deprecated: use Gateway.Spec.Infrastructure.Labels
+                  instead. This field will be removed in a future release."
                 type: object
               runtimeSettings:
                 description: RuntimeSettings is a ContourConfiguration spec to be

examples/render/contour-gateway-provisioner.yaml

@@ -3507,8 +3507,9 @@ spec:
               resourceLabels:
                 additionalProperties:
                   type: string
-                description: ResourceLabels is a set of labels to add to the provisioned
-                  Contour resources.
+                description: "ResourceLabels is a set of labels to add to the provisioned
+                  Contour resources. \n Deprecated: use Gateway.Spec.Infrastructure.Labels
+                  instead. This field will be removed in a future release."
                 type: object
               runtimeSettings:
                 description: RuntimeSettings is a ContourConfiguration spec to be

examples/render/contour-gateway.yaml

@@ -3718,8 +3718,9 @@ spec:
               resourceLabels:
                 additionalProperties:
                   type: string
-                description: ResourceLabels is a set of labels to add to the provisioned
-                  Contour resources.
+                description: "ResourceLabels is a set of labels to add to the provisioned
+                  Contour resources. \n Deprecated: use Gateway.Spec.Infrastructure.Labels
+                  instead. This field will be removed in a future release."
                 type: object
               runtimeSettings:
                 description: RuntimeSettings is a ContourConfiguration spec to be

examples/render/contour.yaml

@@ -3715,8 +3715,9 @@ spec:
               resourceLabels:
                 additionalProperties:
                   type: string
-                description: ResourceLabels is a set of labels to add to the provisioned
-                  Contour resources.
+                description: "ResourceLabels is a set of labels to add to the provisioned
+                  Contour resources. \n Deprecated: use Gateway.Spec.Infrastructure.Labels
+                  instead. This field will be removed in a future release."
                 type: object
               runtimeSettings:
                 description: RuntimeSettings is a ContourConfiguration spec to be

internal/provisioner/controller/gateway.go

@@ -226,6 +226,7 @@ func (r *gatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 		contourModel.Spec.RuntimeSettings = gatewayClassParams.Spec.RuntimeSettings
 
 		// if there is a same name pair, overwrite it
+		// nolint:staticcheck
 		for k, v := range gatewayClassParams.Spec.ResourceLabels {
 			contourModel.Spec.ResourceLabels[k] = v
 		}
@@ -369,6 +370,16 @@ func (r *gatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 		}
 	}
 
+	if gateway.Spec.Infrastructure != nil {
+		for k, v := range gateway.Spec.Infrastructure.Labels {
+			contourModel.Spec.ResourceLabels[string(k)] = string(v)
+		}
+
+		for k, v := range gateway.Spec.Infrastructure.Annotations {
+			contourModel.Spec.ResourceAnnotations[string(k)] = string(v)
+		}
+	}
+
 	if errs := r.ensureContour(ctx, contourModel, log); len(errs) > 0 {
 		return ctrl.Result{}, fmt.Errorf("failed to ensure resources for gateway: %w", retryable.NewMaybeRetryableAggregate(errs))
 	}

internal/provisioner/controller/gateway_test.go

@@ -26,6 +26,7 @@ import (
 	"github.com/stretchr/testify/require"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
@@ -1266,6 +1267,82 @@ func TestGatewayReconcile(t *testing.T) {
 				assert.True(t, errors.IsNotFound(err))
 			},
 		},
+		"The Gateway's infrastructure labels and annotations are set on all resources": {
+			gatewayClass: reconcilableGatewayClass("gatewayclass-1", controller),
+			gateway: &gatewayv1beta1.Gateway{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "gateway-1",
+					Name:      "gateway-1",
+				},
+				Spec: gatewayv1beta1.GatewaySpec{
+					GatewayClassName: gatewayv1beta1.ObjectName("gatewayclass-1"),
+					Infrastructure: &gatewayapi_v1.GatewayInfrastructure{
+						Labels: map[gatewayapi_v1.AnnotationKey]gatewayapi_v1.AnnotationValue{
+							gatewayapi_v1.AnnotationKey("projectcontour.io/label-1"): gatewayapi_v1.AnnotationValue("label-value-1"),
+							gatewayapi_v1.AnnotationKey("projectcontour.io/label-2"): gatewayapi_v1.AnnotationValue("label-value-2"),
+						},
+						Annotations: map[gatewayapi_v1.AnnotationKey]gatewayapi_v1.AnnotationValue{
+							gatewayapi_v1.AnnotationKey("projectcontour.io/annotation-1"): gatewayapi_v1.AnnotationValue("annotation-value-1"),
+							gatewayapi_v1.AnnotationKey("projectcontour.io/annotation-2"): gatewayapi_v1.AnnotationValue("annotation-value-2"),
+						},
+					},
+				},
+			},
+			assertions: func(t *testing.T, r *gatewayReconciler, gw *gatewayv1beta1.Gateway, reconcileErr error) {
+				require.NoError(t, reconcileErr)
+
+				for _, obj := range []client.Object{
+					&appsv1.Deployment{
+						ObjectMeta: metav1.ObjectMeta{Namespace: "gateway-1", Name: "contour-gateway-1"},
+					},
+					&appsv1.DaemonSet{
+						ObjectMeta: metav1.ObjectMeta{Namespace: "gateway-1", Name: "envoy-gateway-1"},
+					},
+					&corev1.Service{
+						ObjectMeta: metav1.ObjectMeta{Namespace: "gateway-1", Name: "contour-gateway-1"},
+					},
+					&corev1.Service{
+						ObjectMeta: metav1.ObjectMeta{Namespace: "gateway-1", Name: "envoy-gateway-1"},
+					},
+					&contourv1alpha1.ContourConfiguration{
+						ObjectMeta: metav1.ObjectMeta{Namespace: "gateway-1", Name: "contourconfig-gateway-1"},
+					},
+					&corev1.Secret{
+						ObjectMeta: metav1.ObjectMeta{Namespace: "gateway-1", Name: "contourcert-gateway-1"},
+					},
+					&corev1.Secret{
+						ObjectMeta: metav1.ObjectMeta{Namespace: "gateway-1", Name: "envoycert-gateway-1"},
+					},
+					&corev1.ServiceAccount{
+						ObjectMeta: metav1.ObjectMeta{Namespace: "gateway-1", Name: "contour-gateway-1"},
+					},
+					&corev1.ServiceAccount{
+						ObjectMeta: metav1.ObjectMeta{Namespace: "gateway-1", Name: "envoy-gateway-1"},
+					},
+					&rbacv1.ClusterRole{
+						ObjectMeta: metav1.ObjectMeta{Name: "contour-gateway-1-gateway-1"},
+					},
+					&rbacv1.ClusterRoleBinding{
+						ObjectMeta: metav1.ObjectMeta{Name: "contour-gateway-1-gateway-1"},
+					},
+					&rbacv1.Role{
+						ObjectMeta: metav1.ObjectMeta{Namespace: "gateway-1", Name: "contour-gateway-1"},
+					},
+					&rbacv1.RoleBinding{
+						ObjectMeta: metav1.ObjectMeta{Namespace: "gateway-1", Name: "contour-rolebinding-gateway-1"},
+					},
+				} {
+					require.NoError(t, r.client.Get(context.Background(), keyFor(obj), obj))
+
+					for k, v := range gw.Spec.Infrastructure.Labels {
+						assert.Equal(t, obj.GetLabels()[string(k)], string(v))
+					}
+					for k, v := range gw.Spec.Infrastructure.Annotations {
+						assert.Equal(t, obj.GetAnnotations()[string(k)], string(v))
+					}
+				}
+			},
+		},
 	}
 
 	for name, tc := range tests {

internal/provisioner/model/model.go

@@ -71,6 +71,7 @@ func Default(namespace, name string) *Contour {
 				},
 			},
 			ResourceLabels:        map[string]string{},
+			ResourceAnnotations:   map[string]string{},
 			EnvoyPodAnnotations:   map[string]string{},
 			ContourPodAnnotations: map[string]string{},
 		},
@@ -200,9 +201,12 @@ type ContourSpec struct {
 	// when envoy be running as a `DaemonSet`,it's must be nil
 	ContourDeploymentStrategy appsv1.DeploymentStrategy
 
-	// ResourceLabels is a set of labels to add to the provisioned Contour resource(s).
+	// ResourceLabels is a set of labels to add to the provisioned resources.
 	ResourceLabels map[string]string
 
+	// ResourceAnnotations is a set of annotations to add to the provisioned resources.
+	ResourceAnnotations map[string]string
+
 	// EnvoyExtraVolumes holds the extra volumes to add to envoy's pod.
 	EnvoyExtraVolumes []corev1.Volume
 

internal/provisioner/model/names.go

@@ -79,10 +79,11 @@ func (c *Contour) EnvoyRBACNames() RBACNames {
 	}
 }
 
-// AppLabels returns labels for a Contour resources(Deployment/DaemonSet).
-func (c *Contour) AppLabels() map[string]string {
+// WorkloadLabels returns labels to apply to the Contour and Envoy
+// workloads (i.e. deployment(s)/daemonset).
+func (c *Contour) WorkloadLabels() map[string]string {
 	labels := map[string]string{}
-	for k, v := range CommonLabels(c) {
+	for k, v := range c.CommonLabels() {
 		labels[k] = v
 	}
 
@@ -94,8 +95,11 @@ func (c *Contour) AppLabels() map[string]string {
 	return labels
 }
 
-// CommonLabels returns labels for Contour resources.
-func CommonLabels(c *Contour) map[string]string {
+// CommonLabels returns labels to apply to all generated
+// resources. Note that WorkloadLabels should be used in
+// place of CommonLabels for the Contour and Envoy workload
+// resources.
+func (c *Contour) CommonLabels() map[string]string {
 	labels := map[string]string{}
 
 	// Add user-defined labels
@@ -111,6 +115,18 @@ func CommonLabels(c *Contour) map[string]string {
 	return labels
 }
 
+// CommonAnnotations returns annotations to apply to all
+// generated resources.
+func (c *Contour) CommonAnnotations() map[string]string {
+	annotations := map[string]string{}
+
+	for k, v := range c.Spec.ResourceAnnotations {
+		annotations[k] = v
+	}
+
+	return annotations
+}
+
 // RBACNames holds a set of names of related RBAC resources.
 type RBACNames struct {
 	ServiceAccount     string

internal/provisioner/objects/contourconfig/contourconfig.go

@@ -29,9 +29,10 @@ import (
 func EnsureContourConfig(ctx context.Context, cli client.Client, contour *model.Contour) error {
 	desired := &contour_api_v1alpha1.ContourConfiguration{
 		ObjectMeta: metav1.ObjectMeta{
-			Namespace: contour.Namespace,
-			Name:      contour.ContourConfigurationName(),
-			Labels:    model.CommonLabels(contour),
+			Namespace:   contour.Namespace,
+			Name:        contour.ContourConfigurationName(),
+			Labels:      contour.CommonLabels(),
+			Annotations: contour.CommonAnnotations(),
 		},
 	}
 

internal/provisioner/objects/dataplane/dataplane.go

@@ -341,9 +341,10 @@ func DesiredDaemonSet(contour *model.Contour, contourImage, envoyImage string) *
 
 	ds := &appsv1.DaemonSet{
 		ObjectMeta: metav1.ObjectMeta{
-			Namespace: contour.Namespace,
-			Name:      contour.EnvoyDataPlaneName(),
-			Labels:    contour.AppLabels(),
+			Namespace:   contour.Namespace,
+			Name:        contour.EnvoyDataPlaneName(),
+			Labels:      contour.WorkloadLabels(),
+			Annotations: contour.CommonAnnotations(),
 		},
 		Spec: appsv1.DaemonSetSpec{
 			RevisionHistoryLimit: ref.To(int32(10)),
@@ -413,9 +414,10 @@ func desiredDeployment(contour *model.Contour, contourImage, envoyImage string)
 
 	deployment := &appsv1.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
-			Namespace: contour.Namespace,
-			Name:      contour.EnvoyDataPlaneName(),
-			Labels:    contour.AppLabels(),
+			Namespace:   contour.Namespace,
+			Name:        contour.EnvoyDataPlaneName(),
+			Labels:      contour.WorkloadLabels(),
+			Annotations: contour.CommonAnnotations(),
 		},
 		Spec: appsv1.DeploymentSpec{
 			Replicas:             ref.To(contour.Spec.EnvoyReplicas),
@@ -526,7 +528,7 @@ func EnvoyPodSelector(contour *model.Contour) *metav1.LabelSelector {
 // envoyPodLabels returns the labels for envoy's pods
 func envoyPodLabels(contour *model.Contour) map[string]string {
 	labels := EnvoyPodSelector(contour).MatchLabels
-	for k, v := range contour.AppLabels() {
+	for k, v := range contour.WorkloadLabels() {
 		labels[k] = v
 	}
 	return labels
@@ -551,5 +553,11 @@ func envoyPodAnnotations(contour *model.Contour) map[string]string {
 	annotations["prometheus.io/port"] = fmt.Sprint(metricsPort)
 	annotations["prometheus.io/path"] = "/stats/prometheus"
 
+	// Annotations specified on the Gateway take precedence
+	// over annotations specified on the GatewayClass/its parameters.
+	for k, v := range contour.CommonAnnotations() {
+		annotations[k] = v
+	}
+
 	return annotations
 }

internal/provisioner/objects/dataplane/dataplane_test.go

@@ -338,7 +338,7 @@ func TestDesiredDaemonSet(t *testing.T) {
 	checkDaemonSetHasEnvVar(t, ds, EnvoyContainerName, envoyNsEnvVar)
 	checkDaemonSetHasEnvVar(t, ds, EnvoyContainerName, envoyPodEnvVar)
 	checkDaemonSetHasEnvVar(t, ds, envoyInitContainerName, envoyNsEnvVar)
-	checkDaemonSetHasLabels(t, ds, cntr.AppLabels())
+	checkDaemonSetHasLabels(t, ds, cntr.WorkloadLabels())
 	checkContainerHasPort(t, ds, int32(cntr.Spec.RuntimeSettings.Envoy.Metrics.Port))
 
 	checkDaemonSetHasNodeSelector(t, ds, nil)

internal/provisioner/objects/deployment/deployment.go

@@ -195,9 +195,10 @@ func DesiredDeployment(contour *model.Contour, image string) *appsv1.Deployment
 	}
 	deploy := &appsv1.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
-			Namespace: contour.Namespace,
-			Name:      contour.ContourDeploymentName(),
-			Labels:    contour.AppLabels(),
+			Namespace:   contour.Namespace,
+			Name:        contour.ContourDeploymentName(),
+			Labels:      contour.WorkloadLabels(),
+			Annotations: contour.CommonAnnotations(),
 		},
 		Spec: appsv1.DeploymentSpec{
 			ProgressDeadlineSeconds: ref.To(int32(600)),
@@ -293,7 +294,7 @@ func ContourDeploymentPodSelector(contour *model.Contour) *metav1.LabelSelector
 // app labels
 func contourPodLabels(contour *model.Contour) map[string]string {
 	labels := ContourDeploymentPodSelector(contour).MatchLabels
-	for k, v := range contour.AppLabels() {
+	for k, v := range contour.WorkloadLabels() {
 		labels[k] = v
 	}
 	return labels
@@ -316,5 +317,11 @@ func contourPodAnnotations(contour *model.Contour) map[string]string {
 	annotations["prometheus.io/scrape"] = "true"
 	annotations["prometheus.io/port"] = fmt.Sprint(port)
 
+	// Annotations specified on the Gateway take precedence
+	// over annotations specified on the GatewayClass/its parameters.
+	for k, v := range contour.CommonAnnotations() {
+		annotations[k] = v
+	}
+
 	return annotations
 }

internal/provisioner/objects/deployment/deployment_test.go

@@ -183,7 +183,7 @@ func TestDesiredDeployment(t *testing.T) {
 	checkContainerHasImage(t, container, testContourImage)
 	checkDeploymentHasEnvVar(t, deploy, contourNsEnvVar)
 	checkDeploymentHasEnvVar(t, deploy, contourPodEnvVar)
-	checkDeploymentHasLabels(t, deploy, cntr.AppLabels())
+	checkDeploymentHasLabels(t, deploy, cntr.WorkloadLabels())
 	checkPodHasAnnotations(t, &deploy.Spec.Template, contourPodAnnotations(cntr))
 
 	for _, port := range cntr.Spec.NetworkPublishing.Envoy.Ports {

internal/provisioner/objects/rbac/clusterrole/cluster_role.go

@@ -69,8 +69,9 @@ func desiredClusterRole(name string, contour *model.Contour) *rbacv1.ClusterRole
 			Kind: "Role",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:   name,
-			Labels: model.CommonLabels(contour),
+			Name:        name,
+			Labels:      contour.CommonLabels(),
+			Annotations: contour.CommonAnnotations(),
 		},
 		Rules: []rbacv1.PolicyRule{
 			// Core Contour-watched resources.