From 3b7ec6b2a8aaef08bf42ef7721248bcd04650a39 Mon Sep 17 00:00:00 2001 From: Cottand Date: Thu, 4 Apr 2024 16:50:25 +0100 Subject: [PATCH 01/11] Add option to ExternalService to control envoy proxy arguments --- api/v1/externalservice_types.go | 3 +++ config/crd/bases/egress.monzo.com_externalservices.yaml | 5 +++++ controllers/deployment.go | 1 + 3 files changed, 9 insertions(+) diff --git a/api/v1/externalservice_types.go b/api/v1/externalservice_types.go index 3b106b4..2aba72c 100644 --- a/api/v1/externalservice_types.go +++ b/api/v1/externalservice_types.go @@ -63,6 +63,9 @@ type ExternalServiceSpec struct { // +optional EnvoyClusterMaxConnections *uint32 `json:"envoyClusterMaxConnections,omitempty"` + // Additional arguments passed to the Envoy proxy image + EnvoyArguments []string `json:"envoyArguments,omitempty"` + // Provides a way to override the global default // +optional ServiceTopologyMode string `json:"serviceTopologyMode,omitempty"` diff --git a/config/crd/bases/egress.monzo.com_externalservices.yaml b/config/crd/bases/egress.monzo.com_externalservices.yaml index bd08478..c6dd4e9 100644 --- a/config/crd/bases/egress.monzo.com_externalservices.yaml +++ b/config/crd/bases/egress.monzo.com_externalservices.yaml @@ -39,6 +39,11 @@ spec: dnsName: description: DnsName is a DNS name target for the external service type: string + envoyArguments: + description: Additional arguments passed to the Envoy proxy image + type: array + items: + type: string envoyClusterMaxConnections: description: The maximum number of connections that Envoy will establish to all hosts in an upstream cluster (defaults to 1024). If this diff --git a/controllers/deployment.go b/controllers/deployment.go index 2a9429c..1fd1531 100644 --- a/controllers/deployment.go +++ b/controllers/deployment.go @@ -189,6 +189,7 @@ func deployment(es *egressv1.ExternalService, configHash string) *appsv1.Deploym { Name: "gateway", Image: img, + Args: es.Spec.EnvoyArguments, ImagePullPolicy: corev1.PullIfNotPresent, Ports: deploymentPorts(es), VolumeMounts: []corev1.VolumeMount{ From c3a39c66133a1d951a99beec50075f660dfb200e Mon Sep 17 00:00:00 2001 From: Cottand Date: Fri, 5 Apr 2024 13:18:21 +0100 Subject: [PATCH 02/11] add cluster options to tweak DNS settings --- api/v1/externalservice_types.go | 15 +++++++++++++-- controllers/configmap.go | 3 +++ controllers/deployment.go | 7 ++++++- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/api/v1/externalservice_types.go b/api/v1/externalservice_types.go index 2aba72c..afb58ed 100644 --- a/api/v1/externalservice_types.go +++ b/api/v1/externalservice_types.go @@ -16,6 +16,7 @@ limitations under the License. package v1 import ( + "github.com/golang/protobuf/ptypes/duration" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -63,8 +64,18 @@ type ExternalServiceSpec struct { // +optional EnvoyClusterMaxConnections *uint32 `json:"envoyClusterMaxConnections,omitempty"` - // Additional arguments passed to the Envoy proxy image - EnvoyArguments []string `json:"envoyArguments,omitempty"` + // Input to the --log-level command line option. See the help text for the available log levels and the default. + EnvoyLogLevel string `json:"envoyArguments,omitempty"` + + // Corresponds to Envoy's dns_refresh_rate config field for this cluster. + // See https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto + // +optional + EnvoyDnsRefreshRate *duration.Duration `json:"envoy_dns_refresh_rate,omitempty"` + + // Corresponds to Envoy's respect_dns_ttl config field for this cluster. + // See https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto + // +optional + EnvoyRespectDnsTTL bool `json:"envoy_respect_dns_ttl,omitempty"` // Provides a way to override the global default // +optional diff --git a/controllers/configmap.go b/controllers/configmap.go index 4a41493..6dda795 100644 --- a/controllers/configmap.go +++ b/controllers/configmap.go @@ -356,6 +356,9 @@ func generateOverrideCluster(name string, spec egressv1.ExternalServiceSpec, por ClusterName: overrideClusterName, Endpoints: endpoints, }, + + DnsRefreshRate: spec.EnvoyDnsRefreshRate, + RespectDnsTtl: spec.EnvoyRespectDnsTTL, } } diff --git a/controllers/deployment.go b/controllers/deployment.go index 1fd1531..b072dbe 100644 --- a/controllers/deployment.go +++ b/controllers/deployment.go @@ -158,6 +158,11 @@ func deployment(es *egressv1.ExternalService, configHash string) *appsv1.Deploym } } + var logLevelArgs []string + if es.Spec.EnvoyLogLevel != "" { + logLevelArgs = []string{"--log-level", es.Spec.EnvoyLogLevel} + } + return &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: es.Name, @@ -189,7 +194,7 @@ func deployment(es *egressv1.ExternalService, configHash string) *appsv1.Deploym { Name: "gateway", Image: img, - Args: es.Spec.EnvoyArguments, + Args: logLevelArgs, ImagePullPolicy: corev1.PullIfNotPresent, Ports: deploymentPorts(es), VolumeMounts: []corev1.VolumeMount{ From bd9cd7e8c02c801ccfe4b7a9d94ebf77019a63da Mon Sep 17 00:00:00 2001 From: Cottand Date: Fri, 5 Apr 2024 13:28:12 +0100 Subject: [PATCH 03/11] update externalservices yaml schema --- .../egress.monzo.com_externalservices.yaml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/config/crd/bases/egress.monzo.com_externalservices.yaml b/config/crd/bases/egress.monzo.com_externalservices.yaml index c6dd4e9..3d614b1 100644 --- a/config/crd/bases/egress.monzo.com_externalservices.yaml +++ b/config/crd/bases/egress.monzo.com_externalservices.yaml @@ -39,11 +39,19 @@ spec: dnsName: description: DnsName is a DNS name target for the external service type: string - envoyArguments: - description: Additional arguments passed to the Envoy proxy image - type: array - items: - type: string + envoyLogLevel: + description: Input to the --log-level command line option. See the help text for the available log levels and the default. + type: string + envoyDnsRefreshRate: + description: | + Corresponds to Envoy's dns_refresh_rate config field for this cluster. + See https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto + type: string + envoyRespectDnsTTL: + description: | + Corresponds to Envoy's respect_dns_ttl config field for this cluster. + See https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto + type: boolean envoyClusterMaxConnections: description: The maximum number of connections that Envoy will establish to all hosts in an upstream cluster (defaults to 1024). If this From 026e93183e709c88238cd686ea0f8618f5403a41 Mon Sep 17 00:00:00 2001 From: Cottand Date: Fri, 5 Apr 2024 14:08:54 +0100 Subject: [PATCH 04/11] use seconds instead of duration in regreshDnsInterval --- api/v1/externalservice_types.go | 5 ++--- api/v1/zz_generated.deepcopy.go | 7 ++++++- config/crd/bases/egress.monzo.com_externalservices.yaml | 6 +++--- controllers/configmap.go | 7 ++++++- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/api/v1/externalservice_types.go b/api/v1/externalservice_types.go index afb58ed..db35bc6 100644 --- a/api/v1/externalservice_types.go +++ b/api/v1/externalservice_types.go @@ -16,7 +16,6 @@ limitations under the License. package v1 import ( - "github.com/golang/protobuf/ptypes/duration" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -67,10 +66,10 @@ type ExternalServiceSpec struct { // Input to the --log-level command line option. See the help text for the available log levels and the default. EnvoyLogLevel string `json:"envoyArguments,omitempty"` - // Corresponds to Envoy's dns_refresh_rate config field for this cluster. + // Corresponds to Envoy's dns_refresh_rate config field for this cluster, in seconds // See https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto // +optional - EnvoyDnsRefreshRate *duration.Duration `json:"envoy_dns_refresh_rate,omitempty"` + EnvoyDnsRefreshRateS int64 `json:"envoy_dns_refresh_rate,omitempty"` // Corresponds to Envoy's respect_dns_ttl config field for this cluster. // See https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 3c597f3..9e690a8 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -1,5 +1,4 @@ //go:build !ignore_autogenerated -// +build !ignore_autogenerated /* @@ -21,6 +20,7 @@ limitations under the License. package v1 import ( + durationpb "google.golang.org/protobuf/types/known/durationpb" corev1 "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -144,6 +144,11 @@ func (in *ExternalServiceSpec) DeepCopyInto(out *ExternalServiceSpec) { *out = new(uint32) **out = **in } + if in.EnvoyDnsRefreshRate != nil { + in, out := &in.EnvoyDnsRefreshRate, &out.EnvoyDnsRefreshRate + *out = new(durationpb.Duration) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExternalServiceSpec. diff --git a/config/crd/bases/egress.monzo.com_externalservices.yaml b/config/crd/bases/egress.monzo.com_externalservices.yaml index 3d614b1..d6300c7 100644 --- a/config/crd/bases/egress.monzo.com_externalservices.yaml +++ b/config/crd/bases/egress.monzo.com_externalservices.yaml @@ -42,11 +42,11 @@ spec: envoyLogLevel: description: Input to the --log-level command line option. See the help text for the available log levels and the default. type: string - envoyDnsRefreshRate: + envoyDnsRefreshRateS: description: | - Corresponds to Envoy's dns_refresh_rate config field for this cluster. + Corresponds to Envoy's dns_refresh_rate config field for this cluster, in seconds See https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto - type: string + type: number envoyRespectDnsTTL: description: | Corresponds to Envoy's respect_dns_ttl config field for this cluster. diff --git a/controllers/configmap.go b/controllers/configmap.go index 6dda795..9725d64 100644 --- a/controllers/configmap.go +++ b/controllers/configmap.go @@ -3,6 +3,7 @@ package controllers import ( "context" "fmt" + "google.golang.org/protobuf/types/known/durationpb" "hash/fnv" "strconv" @@ -298,6 +299,10 @@ func configmap(es *egressv1.ExternalService) (*corev1.ConfigMap, string, error) func generateOverrideCluster(name string, spec egressv1.ExternalServiceSpec, port egressv1.ExternalServicePort, protocol envoycorev3.SocketAddress_Protocol) *envoyv3.Cluster { overrideClusterName := fmt.Sprintf("%v-override", name) + var dnsRefreshRate *duration.Duration + if spec.EnvoyDnsRefreshRateS != 0 { + dnsRefreshRate = &durationpb.Duration{Seconds: spec.EnvoyDnsRefreshRateS} + } var endpoints []*envoyendpoint.LocalityLbEndpoints for _, ip := range spec.IpOverride { @@ -357,7 +362,7 @@ func generateOverrideCluster(name string, spec egressv1.ExternalServiceSpec, por Endpoints: endpoints, }, - DnsRefreshRate: spec.EnvoyDnsRefreshRate, + DnsRefreshRate: dnsRefreshRate, RespectDnsTtl: spec.EnvoyRespectDnsTTL, } } From 9c161a4a5430faf2ea062aea2884fcf7a50964ac Mon Sep 17 00:00:00 2001 From: Cottand Date: Fri, 5 Apr 2024 14:58:22 +0100 Subject: [PATCH 05/11] run codegen --- api/v1/zz_generated.deepcopy.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 9e690a8..3c597f3 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -1,4 +1,5 @@ //go:build !ignore_autogenerated +// +build !ignore_autogenerated /* @@ -20,7 +21,6 @@ limitations under the License. package v1 import ( - durationpb "google.golang.org/protobuf/types/known/durationpb" corev1 "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -144,11 +144,6 @@ func (in *ExternalServiceSpec) DeepCopyInto(out *ExternalServiceSpec) { *out = new(uint32) **out = **in } - if in.EnvoyDnsRefreshRate != nil { - in, out := &in.EnvoyDnsRefreshRate, &out.EnvoyDnsRefreshRate - *out = new(durationpb.Duration) - (*in).DeepCopyInto(*out) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExternalServiceSpec. From 20cd876d3ba3215a2f6918bb8bf5151c2504331a Mon Sep 17 00:00:00 2001 From: Cottand Date: Fri, 5 Apr 2024 15:34:50 +0100 Subject: [PATCH 06/11] make manifests --- api/v1/externalservice_types.go | 6 ++-- .../egress.monzo.com_externalservices.yaml | 29 ++++++++++--------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/api/v1/externalservice_types.go b/api/v1/externalservice_types.go index db35bc6..6b2550d 100644 --- a/api/v1/externalservice_types.go +++ b/api/v1/externalservice_types.go @@ -64,17 +64,17 @@ type ExternalServiceSpec struct { EnvoyClusterMaxConnections *uint32 `json:"envoyClusterMaxConnections,omitempty"` // Input to the --log-level command line option. See the help text for the available log levels and the default. - EnvoyLogLevel string `json:"envoyArguments,omitempty"` + EnvoyLogLevel string `json:"envoyLogLevel,omitempty"` // Corresponds to Envoy's dns_refresh_rate config field for this cluster, in seconds // See https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto // +optional - EnvoyDnsRefreshRateS int64 `json:"envoy_dns_refresh_rate,omitempty"` + EnvoyDnsRefreshRateS int64 `json:"envoyDnsRefreshRateS,omitempty"` // Corresponds to Envoy's respect_dns_ttl config field for this cluster. // See https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto // +optional - EnvoyRespectDnsTTL bool `json:"envoy_respect_dns_ttl,omitempty"` + EnvoyRespectDnsTTL bool `json:"envoyRespectDnsTTL,omitempty"` // Provides a way to override the global default // +optional diff --git a/config/crd/bases/egress.monzo.com_externalservices.yaml b/config/crd/bases/egress.monzo.com_externalservices.yaml index d6300c7..eaf1795 100644 --- a/config/crd/bases/egress.monzo.com_externalservices.yaml +++ b/config/crd/bases/egress.monzo.com_externalservices.yaml @@ -39,19 +39,6 @@ spec: dnsName: description: DnsName is a DNS name target for the external service type: string - envoyLogLevel: - description: Input to the --log-level command line option. See the help text for the available log levels and the default. - type: string - envoyDnsRefreshRateS: - description: | - Corresponds to Envoy's dns_refresh_rate config field for this cluster, in seconds - See https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto - type: number - envoyRespectDnsTTL: - description: | - Corresponds to Envoy's respect_dns_ttl config field for this cluster. - See https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto - type: boolean envoyClusterMaxConnections: description: The maximum number of connections that Envoy will establish to all hosts in an upstream cluster (defaults to 1024). If this @@ -59,6 +46,19 @@ spec: cluster will increment. format: int32 type: integer + envoyDnsRefreshRateS: + description: "Corresponds to Envoy's dns_refresh_rate config field + for this cluster, in seconds See\thttps://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto" + format: int64 + type: integer + envoyLogLevel: + description: Input to the --log-level command line option. See the + help text for the available log levels and the default. + type: string + envoyRespectDnsTTL: + description: "Corresponds to Envoy's respect_dns_ttl config field + for this cluster. See\thttps://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto" + type: boolean hijackDns: description: 'If true, add a `egress.monzo.com/hijack-dns: true` label to produced Service objects CoreDNS can watch this label and decide @@ -144,6 +144,9 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object type: object + serviceTopologyMode: + description: Provides a way to override the global default + type: string targetCPUUtilizationPercentage: description: Target average CPU utilization (represented as a percentage of requested CPU) over all the pods. Defaults to 50 From a9196593f8c24c44fd7df6e9064f5b803eda1498 Mon Sep 17 00:00:00 2001 From: Cottand Date: Fri, 5 Apr 2024 16:12:04 +0100 Subject: [PATCH 07/11] fix params in wrong operator --- controllers/configmap.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/controllers/configmap.go b/controllers/configmap.go index 9725d64..9e975b6 100644 --- a/controllers/configmap.go +++ b/controllers/configmap.go @@ -110,6 +110,10 @@ func envoyConfig(es *egressv1.ExternalService) (string, error) { } for _, port := range es.Spec.Ports { + var dnsRefreshRate *duration.Duration + if es.Spec.EnvoyDnsRefreshRateS != 0 { + dnsRefreshRate = &durationpb.Duration{Seconds: spec.EnvoyDnsRefreshRateS} + } var clusters []*envoyv3.Cluster protocol := protocolToEnvoy(port.Protocol) name := fmt.Sprintf("%s_%s_%s", es.Name, envoycorev3.SocketAddress_Protocol_name[int32(protocol)], strconv.Itoa(int(port.Port))) @@ -131,6 +135,9 @@ func envoyConfig(es *egressv1.ExternalService) (string, error) { KeepaliveInterval: &wrapperspb.UInt32Value{Value: 5}, }, }, + + DnsRefreshRate: dnsRefreshRate, + RespectDnsTtl: es.Spec.EnvoyRespectDnsTTL, LoadAssignment: &envoyendpoint.ClusterLoadAssignment{ ClusterName: name, Endpoints: []*envoyendpoint.LocalityLbEndpoints{ From 6e9a22ce47dbbdb87ca486ede6f1ace77ff29f00 Mon Sep 17 00:00:00 2001 From: Cottand Date: Fri, 5 Apr 2024 16:21:51 +0100 Subject: [PATCH 08/11] fix specify default envoy config path --- Makefile | 2 +- controllers/configmap.go | 2 +- controllers/deployment.go | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 09b8139..386b902 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,7 @@ generate: controller-gen $(CONTROLLER_GEN) object:headerFile=./hack/boilerplate.go.txt paths="./..." # Build the docker image -docker-build: test +docker-build: #test docker build . -t ${IMG} # Push the docker image diff --git a/controllers/configmap.go b/controllers/configmap.go index 9e975b6..eaa9a7d 100644 --- a/controllers/configmap.go +++ b/controllers/configmap.go @@ -112,7 +112,7 @@ func envoyConfig(es *egressv1.ExternalService) (string, error) { for _, port := range es.Spec.Ports { var dnsRefreshRate *duration.Duration if es.Spec.EnvoyDnsRefreshRateS != 0 { - dnsRefreshRate = &durationpb.Duration{Seconds: spec.EnvoyDnsRefreshRateS} + dnsRefreshRate = &durationpb.Duration{Seconds: es.Spec.EnvoyDnsRefreshRateS} } var clusters []*envoyv3.Cluster protocol := protocolToEnvoy(port.Protocol) diff --git a/controllers/deployment.go b/controllers/deployment.go index b072dbe..9605ca2 100644 --- a/controllers/deployment.go +++ b/controllers/deployment.go @@ -158,9 +158,9 @@ func deployment(es *egressv1.ExternalService, configHash string) *appsv1.Deploym } } - var logLevelArgs []string + args := []string{"-c", "/etc/envoy/envoy.yaml"} if es.Spec.EnvoyLogLevel != "" { - logLevelArgs = []string{"--log-level", es.Spec.EnvoyLogLevel} + args = append(args, "--log-level", es.Spec.EnvoyLogLevel) } return &appsv1.Deployment{ @@ -194,7 +194,7 @@ func deployment(es *egressv1.ExternalService, configHash string) *appsv1.Deploym { Name: "gateway", Image: img, - Args: logLevelArgs, + Args: args, ImagePullPolicy: corev1.PullIfNotPresent, Ports: deploymentPorts(es), VolumeMounts: []corev1.VolumeMount{ From 991d9a10ae900c7ddad80ef5035e32ab30c0c6cc Mon Sep 17 00:00:00 2001 From: Cottand Date: Fri, 5 Apr 2024 16:33:26 +0100 Subject: [PATCH 09/11] only set envoy args if a log-level is set --- controllers/deployment.go | 167 +++++++++++++++++++------------------- 1 file changed, 84 insertions(+), 83 deletions(-) diff --git a/controllers/deployment.go b/controllers/deployment.go index 9605ca2..85951e1 100644 --- a/controllers/deployment.go +++ b/controllers/deployment.go @@ -157,99 +157,85 @@ func deployment(es *egressv1.ExternalService, configHash string) *appsv1.Deploym }, } } - - args := []string{"-c", "/etc/envoy/envoy.yaml"} - if es.Spec.EnvoyLogLevel != "" { - args = append(args, "--log-level", es.Spec.EnvoyLogLevel) - } - - return &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: es.Name, - Namespace: namespace, - Labels: labels(es), - Annotations: annotations(es), + deploymentSpec := appsv1.DeploymentSpec{ + ProgressDeadlineSeconds: proto.Int(600), + RevisionHistoryLimit: proto.Int(10), + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &appsv1.RollingUpdateDeployment{ + MaxUnavailable: intstr.ValueOrDefault(nil, intstr.FromString("25%")), + MaxSurge: intstr.ValueOrDefault(nil, intstr.FromString("25%")), + }, }, - Spec: appsv1.DeploymentSpec{ - ProgressDeadlineSeconds: proto.Int(600), - RevisionHistoryLimit: proto.Int(10), - Strategy: appsv1.DeploymentStrategy{ - Type: appsv1.RollingUpdateDeploymentStrategyType, - RollingUpdate: &appsv1.RollingUpdateDeployment{ - MaxUnavailable: intstr.ValueOrDefault(nil, intstr.FromString("25%")), - MaxSurge: intstr.ValueOrDefault(nil, intstr.FromString("25%")), - }, + Selector: labelSelector, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels(es), + Annotations: a, }, - Selector: labelSelector, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: labels(es), - Annotations: a, - }, - Spec: corev1.PodSpec{ - Tolerations: tolerations, - NodeSelector: nodeSelector, - TopologySpreadConstraints: podTopologySpread, - Containers: []corev1.Container{ - { - Name: "gateway", - Image: img, - Args: args, - ImagePullPolicy: corev1.PullIfNotPresent, - Ports: deploymentPorts(es), - VolumeMounts: []corev1.VolumeMount{ - { - Name: "envoy-config", - MountPath: "/etc/envoy", - }, + Spec: corev1.PodSpec{ + Tolerations: tolerations, + NodeSelector: nodeSelector, + TopologySpreadConstraints: podTopologySpread, + Containers: []corev1.Container{ + { + Name: "gateway", + Image: img, + Args: args, + ImagePullPolicy: corev1.PullIfNotPresent, + Ports: deploymentPorts(es), + VolumeMounts: []corev1.VolumeMount{ + { + Name: "envoy-config", + MountPath: "/etc/envoy", }, - // Copying istio; don't try drain outbound listeners, but after going into terminating state, - // wait 25 seconds for connections to naturally close before going ahead with stop. - Lifecycle: &corev1.Lifecycle{ - PreStop: &corev1.LifecycleHandler{ - Exec: &corev1.ExecAction{ - Command: []string{"/bin/sleep", "25"}, - }, + }, + // Copying istio; don't try drain outbound listeners, but after going into terminating state, + // wait 25 seconds for connections to naturally close before going ahead with stop. + Lifecycle: &corev1.Lifecycle{ + PreStop: &corev1.LifecycleHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"/bin/sleep", "25"}, }, }, - TerminationMessagePath: corev1.TerminationMessagePathDefault, - TerminationMessagePolicy: corev1.TerminationMessageReadFile, - ReadinessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/ready", - Port: intstr.FromInt(int(adPort)), - Scheme: corev1.URISchemeHTTP, - }, + }, + TerminationMessagePath: corev1.TerminationMessagePathDefault, + TerminationMessagePolicy: corev1.TerminationMessageReadFile, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/ready", + Port: intstr.FromInt(int(adPort)), + Scheme: corev1.URISchemeHTTP, }, - FailureThreshold: 3, - PeriodSeconds: 10, - SuccessThreshold: 1, - TimeoutSeconds: 1, }, - Resources: resources, - Env: []corev1.EnvVar{ - { - Name: "ENVOY_UID", - Value: "0", - }, + FailureThreshold: 3, + PeriodSeconds: 10, + SuccessThreshold: 1, + TimeoutSeconds: 1, + }, + Resources: resources, + Env: []corev1.EnvVar{ + { + Name: "ENVOY_UID", + Value: "0", }, }, }, - RestartPolicy: corev1.RestartPolicyAlways, - SchedulerName: corev1.DefaultSchedulerName, - SecurityContext: &corev1.PodSecurityContext{}, - TerminationGracePeriodSeconds: proto.Int64(30), - DNSPolicy: corev1.DNSDefault, - Volumes: []corev1.Volume{ - { - Name: "envoy-config", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - DefaultMode: proto.Int(420), - LocalObjectReference: corev1.LocalObjectReference{ - Name: es.Name, - }, + }, + RestartPolicy: corev1.RestartPolicyAlways, + SchedulerName: corev1.DefaultSchedulerName, + SecurityContext: &corev1.PodSecurityContext{}, + TerminationGracePeriodSeconds: proto.Int64(30), + DNSPolicy: corev1.DNSDefault, + Volumes: []corev1.Volume{ + { + Name: "envoy-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + DefaultMode: proto.Int(420), + LocalObjectReference: corev1.LocalObjectReference{ + Name: es.Name, }, }, }, @@ -258,4 +244,19 @@ func deployment(es *egressv1.ExternalService, configHash string) *appsv1.Deploym }, }, } + + defaultArgs := []string{"-c", "/etc/envoy/envoy.yaml"} + if es.Spec.EnvoyLogLevel != "" { + deploymentSpec.Template.Spec.Containers[0].Args = append(defaultArgs, "--log-level", es.Spec.EnvoyLogLevel) + } + + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: es.Name, + Namespace: namespace, + Labels: labels(es), + Annotations: annotations(es), + }, + Spec: deploymentSpec, + } } From 767ef6215d025f43765070e69c4086ce1a6bd4d7 Mon Sep 17 00:00:00 2001 From: Cottand Date: Fri, 5 Apr 2024 16:34:26 +0100 Subject: [PATCH 10/11] fix undefined var --- controllers/deployment.go | 1 - 1 file changed, 1 deletion(-) diff --git a/controllers/deployment.go b/controllers/deployment.go index 85951e1..7988db6 100644 --- a/controllers/deployment.go +++ b/controllers/deployment.go @@ -181,7 +181,6 @@ func deployment(es *egressv1.ExternalService, configHash string) *appsv1.Deploym { Name: "gateway", Image: img, - Args: args, ImagePullPolicy: corev1.PullIfNotPresent, Ports: deploymentPorts(es), VolumeMounts: []corev1.VolumeMount{ From 6cf68f7ee90ff7c02ccb6b56133bcd10965103c5 Mon Sep 17 00:00:00 2001 From: Cottand Date: Fri, 5 Apr 2024 17:34:55 +0100 Subject: [PATCH 11/11] add validation to protect against invalid log levels --- Makefile | 2 +- controllers/deployment.go | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 386b902..09b8139 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,7 @@ generate: controller-gen $(CONTROLLER_GEN) object:headerFile=./hack/boilerplate.go.txt paths="./..." # Build the docker image -docker-build: #test +docker-build: test docker build . -t ${IMG} # Push the docker image diff --git a/controllers/deployment.go b/controllers/deployment.go index 7988db6..26b8019 100644 --- a/controllers/deployment.go +++ b/controllers/deployment.go @@ -20,6 +20,17 @@ import ( // +kubebuilder:rbac:namespace=egress-operator-system,groups=apps,resources=deployments,verbs=get;list;watch;create;patch +var validLogLevels = map[string]bool{ + "trace": true, + "debug": true, + "info": true, + "warning": true, + "warn": true, + "error": true, + "critical": true, + "off": true, +} + func (r *ExternalServiceReconciler) reconcileDeployment(ctx context.Context, req ctrl.Request, es *egressv1.ExternalService, configHash string) error { desired := deployment(es, configHash) if err := ctrl.SetControllerReference(es, desired, r.Scheme); err != nil { @@ -245,7 +256,7 @@ func deployment(es *egressv1.ExternalService, configHash string) *appsv1.Deploym } defaultArgs := []string{"-c", "/etc/envoy/envoy.yaml"} - if es.Spec.EnvoyLogLevel != "" { + if validLogLevels[es.Spec.EnvoyLogLevel] { deploymentSpec.Template.Spec.Containers[0].Args = append(defaultArgs, "--log-level", es.Spec.EnvoyLogLevel) }