Add option to ExternalService to control envoy proxy arguments

api/v1/externalservice_types.go

@@ -63,6 +63,19 @@ type ExternalServiceSpec struct {
 	// +optional
 	EnvoyClusterMaxConnections *uint32 `json:"envoyClusterMaxConnections,omitempty"`
 
+	// Input to the --log-level command line option. See the help text for the available log levels and the default.
+	EnvoyLogLevel string `json:"envoyLogLevel,omitempty"`
+
+	// Corresponds to Envoy's dns_refresh_rate config field for this cluster, in seconds
+	// See	https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto
+	// +optional
+	EnvoyDnsRefreshRateS int64 `json:"envoyDnsRefreshRateS,omitempty"`
+
+	// Corresponds to Envoy's respect_dns_ttl config field for this cluster.
+	// See	https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto
+	// +optional
+	EnvoyRespectDnsTTL bool `json:"envoyRespectDnsTTL,omitempty"`
+
 	// Provides a way to override the global default
 	// +optional
 	ServiceTopologyMode string `json:"serviceTopologyMode,omitempty"`

config/crd/bases/egress.monzo.com_externalservices.yaml

@@ -46,6 +46,19 @@ spec:
                   cluster will increment.
                 format: int32
                 type: integer
+              envoyDnsRefreshRateS:
+                description: "Corresponds to Envoy's dns_refresh_rate config field
+                  for this cluster, in seconds See\thttps://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto"
+                format: int64
+                type: integer
+              envoyLogLevel:
+                description: Input to the --log-level command line option. See the
+                  help text for the available log levels and the default.
+                type: string
+              envoyRespectDnsTTL:
+                description: "Corresponds to Envoy's respect_dns_ttl config field
+                  for this cluster. See\thttps://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto"
+                type: boolean
               hijackDns:
                 description: 'If true, add a `egress.monzo.com/hijack-dns: true` label
                   to produced Service objects CoreDNS can watch this label and decide
@@ -131,6 +144,9 @@ spec:
                       More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
                     type: object
                 type: object
+              serviceTopologyMode:
+                description: Provides a way to override the global default
+                type: string
               targetCPUUtilizationPercentage:
                 description: Target average CPU utilization (represented as a percentage
                   of requested CPU) over all the pods. Defaults to 50

controllers/configmap.go

@@ -3,6 +3,7 @@ package controllers
 import (
 	"context"
 	"fmt"
+	"google.golang.org/protobuf/types/known/durationpb"
 	"hash/fnv"
 	"strconv"
 
@@ -109,6 +110,10 @@ func envoyConfig(es *egressv1.ExternalService) (string, error) {
 	}
 
 	for _, port := range es.Spec.Ports {
+		var dnsRefreshRate *duration.Duration
+		if es.Spec.EnvoyDnsRefreshRateS != 0 {
+			dnsRefreshRate = &durationpb.Duration{Seconds: es.Spec.EnvoyDnsRefreshRateS}
+		}
 		var clusters []*envoyv3.Cluster
 		protocol := protocolToEnvoy(port.Protocol)
 		name := fmt.Sprintf("%s_%s_%s", es.Name, envoycorev3.SocketAddress_Protocol_name[int32(protocol)], strconv.Itoa(int(port.Port)))
@@ -130,6 +135,9 @@ func envoyConfig(es *egressv1.ExternalService) (string, error) {
 					KeepaliveInterval: &wrapperspb.UInt32Value{Value: 5},
 				},
 			},
+
+			DnsRefreshRate: dnsRefreshRate,
+			RespectDnsTtl:  es.Spec.EnvoyRespectDnsTTL,
 			LoadAssignment: &envoyendpoint.ClusterLoadAssignment{
 				ClusterName: name,
 				Endpoints: []*envoyendpoint.LocalityLbEndpoints{
@@ -298,6 +306,10 @@ func configmap(es *egressv1.ExternalService) (*corev1.ConfigMap, string, error)
 
 func generateOverrideCluster(name string, spec egressv1.ExternalServiceSpec, port egressv1.ExternalServicePort, protocol envoycorev3.SocketAddress_Protocol) *envoyv3.Cluster {
 	overrideClusterName := fmt.Sprintf("%v-override", name)
+	var dnsRefreshRate *duration.Duration
+	if spec.EnvoyDnsRefreshRateS != 0 {
+		dnsRefreshRate = &durationpb.Duration{Seconds: spec.EnvoyDnsRefreshRateS}
+	}
 	var endpoints []*envoyendpoint.LocalityLbEndpoints
 
 	for _, ip := range spec.IpOverride {
@@ -356,6 +368,9 @@ func generateOverrideCluster(name string, spec egressv1.ExternalServiceSpec, por
 			ClusterName: overrideClusterName,
 			Endpoints:   endpoints,
 		},
+
+		DnsRefreshRate: dnsRefreshRate,
+		RespectDnsTtl:  spec.EnvoyRespectDnsTTL,
 	}
 }
 

controllers/deployment.go

@@ -20,6 +20,17 @@ import (
 
 // +kubebuilder:rbac:namespace=egress-operator-system,groups=apps,resources=deployments,verbs=get;list;watch;create;patch
 
+var validLogLevels = map[string]bool{
+	"trace":    true,
+	"debug":    true,
+	"info":     true,
+	"warning":  true,
+	"warn":     true,
+	"error":    true,
+	"critical": true,
+	"off":      true,
+}
+
 func (r *ExternalServiceReconciler) reconcileDeployment(ctx context.Context, req ctrl.Request, es *egressv1.ExternalService, configHash string) error {
 	desired := deployment(es, configHash)
 	if err := ctrl.SetControllerReference(es, desired, r.Scheme); err != nil {
@@ -157,93 +168,84 @@ func deployment(es *egressv1.ExternalService, configHash string) *appsv1.Deploym
 			},
 		}
 	}
-
-	return &appsv1.Deployment{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:        es.Name,
-			Namespace:   namespace,
-			Labels:      labels(es),
-			Annotations: annotations(es),
+	deploymentSpec := appsv1.DeploymentSpec{
+		ProgressDeadlineSeconds: proto.Int(600),
+		RevisionHistoryLimit:    proto.Int(10),
+		Strategy: appsv1.DeploymentStrategy{
+			Type: appsv1.RollingUpdateDeploymentStrategyType,
+			RollingUpdate: &appsv1.RollingUpdateDeployment{
+				MaxUnavailable: intstr.ValueOrDefault(nil, intstr.FromString("25%")),
+				MaxSurge:       intstr.ValueOrDefault(nil, intstr.FromString("25%")),
+			},
 		},
-		Spec: appsv1.DeploymentSpec{
-			ProgressDeadlineSeconds: proto.Int(600),
-			RevisionHistoryLimit:    proto.Int(10),
-			Strategy: appsv1.DeploymentStrategy{
-				Type: appsv1.RollingUpdateDeploymentStrategyType,
-				RollingUpdate: &appsv1.RollingUpdateDeployment{
-					MaxUnavailable: intstr.ValueOrDefault(nil, intstr.FromString("25%")),
-					MaxSurge:       intstr.ValueOrDefault(nil, intstr.FromString("25%")),
-				},
+		Selector: labelSelector,
+		Template: corev1.PodTemplateSpec{
+			ObjectMeta: metav1.ObjectMeta{
+				Labels:      labels(es),
+				Annotations: a,
 			},
-			Selector: labelSelector,
-			Template: corev1.PodTemplateSpec{
-				ObjectMeta: metav1.ObjectMeta{
-					Labels:      labels(es),
-					Annotations: a,
-				},
-				Spec: corev1.PodSpec{
-					Tolerations:               tolerations,
-					NodeSelector:              nodeSelector,
-					TopologySpreadConstraints: podTopologySpread,
-					Containers: []corev1.Container{
-						{
-							Name:            "gateway",
-							Image:           img,
-							ImagePullPolicy: corev1.PullIfNotPresent,
-							Ports:           deploymentPorts(es),
-							VolumeMounts: []corev1.VolumeMount{
-								{
-									Name:      "envoy-config",
-									MountPath: "/etc/envoy",
-								},
+			Spec: corev1.PodSpec{
+				Tolerations:               tolerations,
+				NodeSelector:              nodeSelector,
+				TopologySpreadConstraints: podTopologySpread,
+				Containers: []corev1.Container{
+					{
+						Name:            "gateway",
+						Image:           img,
+						ImagePullPolicy: corev1.PullIfNotPresent,
+						Ports:           deploymentPorts(es),
+						VolumeMounts: []corev1.VolumeMount{
+							{
+								Name:      "envoy-config",
+								MountPath: "/etc/envoy",
 							},
-							// Copying istio; don't try drain outbound listeners, but after going into terminating state,
-							// wait 25 seconds for connections to naturally close before going ahead with stop.
-							Lifecycle: &corev1.Lifecycle{
-								PreStop: &corev1.LifecycleHandler{
-									Exec: &corev1.ExecAction{
-										Command: []string{"/bin/sleep", "25"},
-									},
+						},
+						// Copying istio; don't try drain outbound listeners, but after going into terminating state,
+						// wait 25 seconds for connections to naturally close before going ahead with stop.
+						Lifecycle: &corev1.Lifecycle{
+							PreStop: &corev1.LifecycleHandler{
+								Exec: &corev1.ExecAction{
+									Command: []string{"/bin/sleep", "25"},
 								},
 							},
-							TerminationMessagePath:   corev1.TerminationMessagePathDefault,
-							TerminationMessagePolicy: corev1.TerminationMessageReadFile,
-							ReadinessProbe: &corev1.Probe{
-								ProbeHandler: corev1.ProbeHandler{
-									HTTPGet: &corev1.HTTPGetAction{
-										Path:   "/ready",
-										Port:   intstr.FromInt(int(adPort)),
-										Scheme: corev1.URISchemeHTTP,
-									},
+						},
+						TerminationMessagePath:   corev1.TerminationMessagePathDefault,
+						TerminationMessagePolicy: corev1.TerminationMessageReadFile,
+						ReadinessProbe: &corev1.Probe{
+							ProbeHandler: corev1.ProbeHandler{
+								HTTPGet: &corev1.HTTPGetAction{
+									Path:   "/ready",
+									Port:   intstr.FromInt(int(adPort)),
+									Scheme: corev1.URISchemeHTTP,
 								},
-								FailureThreshold: 3,
-								PeriodSeconds:    10,
-								SuccessThreshold: 1,
-								TimeoutSeconds:   1,
 							},
-							Resources: resources,
-							Env: []corev1.EnvVar{
-								{
-									Name:  "ENVOY_UID",
-									Value: "0",
-								},
+							FailureThreshold: 3,
+							PeriodSeconds:    10,
+							SuccessThreshold: 1,
+							TimeoutSeconds:   1,
+						},
+						Resources: resources,
+						Env: []corev1.EnvVar{
+							{
+								Name:  "ENVOY_UID",
+								Value: "0",
 							},
 						},
 					},
-					RestartPolicy:                 corev1.RestartPolicyAlways,
-					SchedulerName:                 corev1.DefaultSchedulerName,
-					SecurityContext:               &corev1.PodSecurityContext{},
-					TerminationGracePeriodSeconds: proto.Int64(30),
-					DNSPolicy:                     corev1.DNSDefault,
-					Volumes: []corev1.Volume{
-						{
-							Name: "envoy-config",
-							VolumeSource: corev1.VolumeSource{
-								ConfigMap: &corev1.ConfigMapVolumeSource{
-									DefaultMode: proto.Int(420),
-									LocalObjectReference: corev1.LocalObjectReference{
-										Name: es.Name,
-									},
+				},
+				RestartPolicy:                 corev1.RestartPolicyAlways,
+				SchedulerName:                 corev1.DefaultSchedulerName,
+				SecurityContext:               &corev1.PodSecurityContext{},
+				TerminationGracePeriodSeconds: proto.Int64(30),
+				DNSPolicy:                     corev1.DNSDefault,
+				Volumes: []corev1.Volume{
+					{
+						Name: "envoy-config",
+						VolumeSource: corev1.VolumeSource{
+							ConfigMap: &corev1.ConfigMapVolumeSource{
+								DefaultMode: proto.Int(420),
+								LocalObjectReference: corev1.LocalObjectReference{
+									Name: es.Name,
 								},
 							},
 						},
@@ -252,4 +254,19 @@ func deployment(es *egressv1.ExternalService, configHash string) *appsv1.Deploym
 			},
 		},
 	}
+
+	defaultArgs := []string{"-c", "/etc/envoy/envoy.yaml"}
+	if validLogLevels[es.Spec.EnvoyLogLevel] {
+		deploymentSpec.Template.Spec.Containers[0].Args = append(defaultArgs, "--log-level", es.Spec.EnvoyLogLevel)
+	}
+
+	return &appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        es.Name,
+			Namespace:   namespace,
+			Labels:      labels(es),
+			Annotations: annotations(es),
+		},
+		Spec: deploymentSpec,
+	}
 }