Skip to content

Commit

Permalink
Merge pull request #43 from monzo/nico.envoy-optional-args
Browse files Browse the repository at this point in the history 
Add option to ExternalService to control envoy proxy arguments
  • Loading branch information
@cottand
cottand authored Apr 5, 2024
2 parents 77b65bc + 6cf68f7 commit ee0a5d2
Show file tree
Hide file tree
Showing 4 changed files with 138 additions and 77 deletions.
13 changes: 13 additions & 0 deletions api/v1/externalservice_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,19 @@ type ExternalServiceSpec struct {
// +optional
EnvoyClusterMaxConnections *uint32 `json:"envoyClusterMaxConnections,omitempty"`

// Input to the --log-level command line option. See the help text for the available log levels and the default.
EnvoyLogLevel string `json:"envoyLogLevel,omitempty"`

// Corresponds to Envoy's dns_refresh_rate config field for this cluster, in seconds
// See https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto
// +optional
EnvoyDnsRefreshRateS int64 `json:"envoyDnsRefreshRateS,omitempty"`

// Corresponds to Envoy's respect_dns_ttl config field for this cluster.
// See https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto
// +optional
EnvoyRespectDnsTTL bool `json:"envoyRespectDnsTTL,omitempty"`

// Provides a way to override the global default
// +optional
ServiceTopologyMode string `json:"serviceTopologyMode,omitempty"`
Expand Down
16 changes: 16 additions & 0 deletions config/crd/bases/egress.monzo.com_externalservices.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,19 @@ spec:
cluster will increment.
format: int32
type: integer
envoyDnsRefreshRateS:
description: "Corresponds to Envoy's dns_refresh_rate config field
for this cluster, in seconds See\thttps://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto"
format: int64
type: integer
envoyLogLevel:
description: Input to the --log-level command line option. See the
help text for the available log levels and the default.
type: string
envoyRespectDnsTTL:
description: "Corresponds to Envoy's respect_dns_ttl config field
for this cluster. See\thttps://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto"
type: boolean
hijackDns:
description: 'If true, add a `egress.monzo.com/hijack-dns: true` label
to produced Service objects CoreDNS can watch this label and decide
Expand Down Expand Up @@ -131,6 +144,9 @@ spec:
More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
type: object
type: object
serviceTopologyMode:
description: Provides a way to override the global default
type: string
targetCPUUtilizationPercentage:
description: Target average CPU utilization (represented as a percentage
of requested CPU) over all the pods. Defaults to 50
Expand Down
15 changes: 15 additions & 0 deletions controllers/configmap.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package controllers
import (
"context"
"fmt"
"google.golang.org/protobuf/types/known/durationpb"
"hash/fnv"
"strconv"

Expand Down Expand Up @@ -109,6 +110,10 @@ func envoyConfig(es *egressv1.ExternalService) (string, error) {
}

for _, port := range es.Spec.Ports {
var dnsRefreshRate *duration.Duration
if es.Spec.EnvoyDnsRefreshRateS != 0 {
dnsRefreshRate = &durationpb.Duration{Seconds: es.Spec.EnvoyDnsRefreshRateS}
}
var clusters []*envoyv3.Cluster
protocol := protocolToEnvoy(port.Protocol)
name := fmt.Sprintf("%s_%s_%s", es.Name, envoycorev3.SocketAddress_Protocol_name[int32(protocol)], strconv.Itoa(int(port.Port)))
Expand All @@ -130,6 +135,9 @@ func envoyConfig(es *egressv1.ExternalService) (string, error) {
KeepaliveInterval: &wrapperspb.UInt32Value{Value: 5},
},
},

DnsRefreshRate: dnsRefreshRate,
RespectDnsTtl: es.Spec.EnvoyRespectDnsTTL,
LoadAssignment: &envoyendpoint.ClusterLoadAssignment{
ClusterName: name,
Endpoints: []*envoyendpoint.LocalityLbEndpoints{
Expand Down Expand Up @@ -298,6 +306,10 @@ func configmap(es *egressv1.ExternalService) (*corev1.ConfigMap, string, error)

func generateOverrideCluster(name string, spec egressv1.ExternalServiceSpec, port egressv1.ExternalServicePort, protocol envoycorev3.SocketAddress_Protocol) *envoyv3.Cluster {
overrideClusterName := fmt.Sprintf("%v-override", name)
var dnsRefreshRate *duration.Duration
if spec.EnvoyDnsRefreshRateS != 0 {
dnsRefreshRate = &durationpb.Duration{Seconds: spec.EnvoyDnsRefreshRateS}
}
var endpoints []*envoyendpoint.LocalityLbEndpoints

for _, ip := range spec.IpOverride {
Expand Down Expand Up @@ -356,6 +368,9 @@ func generateOverrideCluster(name string, spec egressv1.ExternalServiceSpec, por
ClusterName: overrideClusterName,
Endpoints: endpoints,
},

DnsRefreshRate: dnsRefreshRate,
RespectDnsTtl: spec.EnvoyRespectDnsTTL,
}
}

Expand Down
171 changes: 94 additions & 77 deletions controllers/deployment.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,17 @@ import (

// +kubebuilder:rbac:namespace=egress-operator-system,groups=apps,resources=deployments,verbs=get;list;watch;create;patch

var validLogLevels = map[string]bool{
"trace": true,
"debug": true,
"info": true,
"warning": true,
"warn": true,
"error": true,
"critical": true,
"off": true,
}

func (r *ExternalServiceReconciler) reconcileDeployment(ctx context.Context, req ctrl.Request, es *egressv1.ExternalService, configHash string) error {
desired := deployment(es, configHash)
if err := ctrl.SetControllerReference(es, desired, r.Scheme); err != nil {
Expand Down Expand Up @@ -157,93 +168,84 @@ func deployment(es *egressv1.ExternalService, configHash string) *appsv1.Deploym
},
}
}

return &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: es.Name,
Namespace: namespace,
Labels: labels(es),
Annotations: annotations(es),
deploymentSpec := appsv1.DeploymentSpec{
ProgressDeadlineSeconds: proto.Int(600),
RevisionHistoryLimit: proto.Int(10),
Strategy: appsv1.DeploymentStrategy{
Type: appsv1.RollingUpdateDeploymentStrategyType,
RollingUpdate: &appsv1.RollingUpdateDeployment{
MaxUnavailable: intstr.ValueOrDefault(nil, intstr.FromString("25%")),
MaxSurge: intstr.ValueOrDefault(nil, intstr.FromString("25%")),
},
},
Spec: appsv1.DeploymentSpec{
ProgressDeadlineSeconds: proto.Int(600),
RevisionHistoryLimit: proto.Int(10),
Strategy: appsv1.DeploymentStrategy{
Type: appsv1.RollingUpdateDeploymentStrategyType,
RollingUpdate: &appsv1.RollingUpdateDeployment{
MaxUnavailable: intstr.ValueOrDefault(nil, intstr.FromString("25%")),
MaxSurge: intstr.ValueOrDefault(nil, intstr.FromString("25%")),
},
Selector: labelSelector,
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: labels(es),
Annotations: a,
},
Selector: labelSelector,
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: labels(es),
Annotations: a,
},
Spec: corev1.PodSpec{
Tolerations: tolerations,
NodeSelector: nodeSelector,
TopologySpreadConstraints: podTopologySpread,
Containers: []corev1.Container{
{
Name: "gateway",
Image: img,
ImagePullPolicy: corev1.PullIfNotPresent,
Ports: deploymentPorts(es),
VolumeMounts: []corev1.VolumeMount{
{
Name: "envoy-config",
MountPath: "/etc/envoy",
},
Spec: corev1.PodSpec{
Tolerations: tolerations,
NodeSelector: nodeSelector,
TopologySpreadConstraints: podTopologySpread,
Containers: []corev1.Container{
{
Name: "gateway",
Image: img,
ImagePullPolicy: corev1.PullIfNotPresent,
Ports: deploymentPorts(es),
VolumeMounts: []corev1.VolumeMount{
{
Name: "envoy-config",
MountPath: "/etc/envoy",
},
// Copying istio; don't try drain outbound listeners, but after going into terminating state,
// wait 25 seconds for connections to naturally close before going ahead with stop.
Lifecycle: &corev1.Lifecycle{
PreStop: &corev1.LifecycleHandler{
Exec: &corev1.ExecAction{
Command: []string{"/bin/sleep", "25"},
},
},
// Copying istio; don't try drain outbound listeners, but after going into terminating state,
// wait 25 seconds for connections to naturally close before going ahead with stop.
Lifecycle: &corev1.Lifecycle{
PreStop: &corev1.LifecycleHandler{
Exec: &corev1.ExecAction{
Command: []string{"/bin/sleep", "25"},
},
},
TerminationMessagePath: corev1.TerminationMessagePathDefault,
TerminationMessagePolicy: corev1.TerminationMessageReadFile,
ReadinessProbe: &corev1.Probe{
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/ready",
Port: intstr.FromInt(int(adPort)),
Scheme: corev1.URISchemeHTTP,
},
},
TerminationMessagePath: corev1.TerminationMessagePathDefault,
TerminationMessagePolicy: corev1.TerminationMessageReadFile,
ReadinessProbe: &corev1.Probe{
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/ready",
Port: intstr.FromInt(int(adPort)),
Scheme: corev1.URISchemeHTTP,
},
FailureThreshold: 3,
PeriodSeconds: 10,
SuccessThreshold: 1,
TimeoutSeconds: 1,
},
Resources: resources,
Env: []corev1.EnvVar{
{
Name: "ENVOY_UID",
Value: "0",
},
FailureThreshold: 3,
PeriodSeconds: 10,
SuccessThreshold: 1,
TimeoutSeconds: 1,
},
Resources: resources,
Env: []corev1.EnvVar{
{
Name: "ENVOY_UID",
Value: "0",
},
},
},
RestartPolicy: corev1.RestartPolicyAlways,
SchedulerName: corev1.DefaultSchedulerName,
SecurityContext: &corev1.PodSecurityContext{},
TerminationGracePeriodSeconds: proto.Int64(30),
DNSPolicy: corev1.DNSDefault,
Volumes: []corev1.Volume{
{
Name: "envoy-config",
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
DefaultMode: proto.Int(420),
LocalObjectReference: corev1.LocalObjectReference{
Name: es.Name,
},
},
RestartPolicy: corev1.RestartPolicyAlways,
SchedulerName: corev1.DefaultSchedulerName,
SecurityContext: &corev1.PodSecurityContext{},
TerminationGracePeriodSeconds: proto.Int64(30),
DNSPolicy: corev1.DNSDefault,
Volumes: []corev1.Volume{
{
Name: "envoy-config",
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
DefaultMode: proto.Int(420),
LocalObjectReference: corev1.LocalObjectReference{
Name: es.Name,
},
},
},
Expand All @@ -252,4 +254,19 @@ func deployment(es *egressv1.ExternalService, configHash string) *appsv1.Deploym
},
},
}

defaultArgs := []string{"-c", "/etc/envoy/envoy.yaml"}
if validLogLevels[es.Spec.EnvoyLogLevel] {
deploymentSpec.Template.Spec.Containers[0].Args = append(defaultArgs, "--log-level", es.Spec.EnvoyLogLevel)
}

return &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: es.Name,
Namespace: namespace,
Labels: labels(es),
Annotations: annotations(es),
},
Spec: deploymentSpec,
}
}

0 comments on commit ee0a5d2

Please sign in to comment.