From d17621bee697074b1ee33d03cf9343d4518ab5f7 Mon Sep 17 00:00:00 2001 From: Qi Ni Date: Sun, 8 Nov 2020 12:21:47 +0800 Subject: [PATCH] Support customize load balancer health probe protocol --- .../azure/azure_loadbalancer.go | 51 ++++- .../azure/azure_loadbalancer_test.go | 182 +++++++----------- 2 files changed, 122 insertions(+), 111 deletions(-) diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go index cc1ecacba0b05..ac1343af66217 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go @@ -104,6 +104,15 @@ const ( // TODO(feiskyer): disable-tcp-reset annotations has been depracated since v1.18, it would removed on v1.20. ServiceAnnotationLoadBalancerDisableTCPReset = "service.beta.kubernetes.io/azure-load-balancer-disable-tcp-reset" + // ServiceAnnotationLoadBalancerHealthProbeProtocol determines the network protocol that the load balancer health probe use. + // If not set, the local service would use the HTTP and the cluster service would use the TCP by default. + ServiceAnnotationLoadBalancerHealthProbeProtocol = "service.beta.kubernetes.io/azure-load-balancer-health-probe-protocol" + + // ServiceAnnotationLoadBalancerHealthProbeRequestPath determines the request path of the load balancer health probe. + // This is only useful for the HTTP and HTTPS, and would be ignored when using TCP. If not set, + // `/healthz` would be configured by default. + ServiceAnnotationLoadBalancerHealthProbeRequestPath = "service.beta.kubernetes.io/azure-load-balancer-health-probe-request-path" + // serviceTagKey is the service key applied for public IP tags. serviceTagKey = "service" // clusterNameKey is the cluster name key applied for public IP tags. @@ -1561,6 +1570,23 @@ func (az *Cloud) checkLoadBalancerResourcesConflicted( return nil } +func parseHealthProbeProtocolAndPath(service *v1.Service) (string, string) { + var protocol, path string + if v, ok := service.Annotations[ServiceAnnotationLoadBalancerHealthProbeProtocol]; ok { + protocol = v + } else { + return protocol, path + } + // ignore the request path if using TCP + if strings.EqualFold(protocol, string(network.ProbeProtocolHTTP)) || + strings.EqualFold(protocol, string(network.ProbeProtocolHTTPS)) { + if v, ok := service.Annotations[ServiceAnnotationLoadBalancerHealthProbeRequestPath]; ok { + path = v + } + } + return protocol, path +} + func (az *Cloud) reconcileLoadBalancerRule( service *v1.Service, wantLb bool, @@ -1606,14 +1632,21 @@ func (az *Cloud) reconcileLoadBalancerRule( return expectedProbes, expectedRules, err } + probeProtocol, requestPath := parseHealthProbeProtocolAndPath(service) if servicehelpers.NeedsHealthCheck(service) { podPresencePath, podPresencePort := servicehelpers.GetServiceHealthCheckPathPort(service) + if probeProtocol == "" { + probeProtocol = string(network.ProbeProtocolHTTP) + } + if requestPath == "" { + requestPath = podPresencePath + } expectedProbes = append(expectedProbes, network.Probe{ Name: &lbRuleName, ProbePropertiesFormat: &network.ProbePropertiesFormat{ - RequestPath: to.StringPtr(podPresencePath), - Protocol: network.ProbeProtocolHTTP, + RequestPath: to.StringPtr(requestPath), + Protocol: network.ProbeProtocol(probeProtocol), Port: to.Int32Ptr(podPresencePort), IntervalInSeconds: to.Int32Ptr(5), NumberOfProbes: to.Int32Ptr(2), @@ -1621,10 +1654,22 @@ func (az *Cloud) reconcileLoadBalancerRule( }) } else if protocol != v1.ProtocolUDP && protocol != v1.ProtocolSCTP { // we only add the expected probe if we're doing TCP + if probeProtocol == "" { + probeProtocol = string(*probeProto) + } + var actualPath *string + if !strings.EqualFold(probeProtocol, string(network.ProbeProtocolTCP)) { + if requestPath != "" { + actualPath = to.StringPtr(requestPath) + } else { + actualPath = to.StringPtr("/healthz") + } + } expectedProbes = append(expectedProbes, network.Probe{ Name: &lbRuleName, ProbePropertiesFormat: &network.ProbePropertiesFormat{ - Protocol: *probeProto, + Protocol: network.ProbeProtocol(probeProtocol), + RequestPath: actualPath, Port: to.Int32Ptr(port.NodePort), IntervalInSeconds: to.Int32Ptr(5), NumberOfProbes: to.Int32Ptr(2), diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer_test.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer_test.go index 1b83e38e8db3b..7397f4bfd132e 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer_test.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer_test.go @@ -1547,6 +1547,8 @@ func TestReconcileLoadBalancerRule(t *testing.T) { service v1.Service loadBalancerSku string wantLb bool + probeProtocol string + probePath string expectedProbes []network.Probe expectedRules []network.LoadBalancingRule expectedErr error @@ -1561,131 +1563,46 @@ func TestReconcileLoadBalancerRule(t *testing.T) { service: getTestService("test1", v1.ProtocolTCP, map[string]string{"service.beta.kubernetes.io/azure-load-balancer-disable-tcp-reset": "true"}, false, 80), loadBalancerSku: "basic", wantLb: true, - expectedProbes: []network.Probe{ - { - Name: to.StringPtr("atest1-TCP-80"), - ProbePropertiesFormat: &network.ProbePropertiesFormat{ - Protocol: network.ProbeProtocol("Tcp"), - Port: to.Int32Ptr(10080), - IntervalInSeconds: to.Int32Ptr(5), - NumberOfProbes: to.Int32Ptr(2), - }, - }, - }, - expectedRules: []network.LoadBalancingRule{ - { - Name: to.StringPtr("atest1-TCP-80"), - LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{ - Protocol: network.TransportProtocol("Tcp"), - FrontendIPConfiguration: &network.SubResource{ - ID: to.StringPtr("frontendIPConfigID"), - }, - BackendAddressPool: &network.SubResource{ - ID: to.StringPtr("backendPoolID"), - }, - LoadDistribution: "Default", - FrontendPort: to.Int32Ptr(80), - BackendPort: to.Int32Ptr(80), - EnableFloatingIP: to.BoolPtr(true), - DisableOutboundSnat: to.BoolPtr(false), - IdleTimeoutInMinutes: to.Int32Ptr(0), - Probe: &network.SubResource{ - ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/" + - "Microsoft.Network/loadBalancers/lbname/probes/atest1-TCP-80"), - }, - EnableTCPReset: nil, - }, - }, - }, + expectedProbes: getDefaultTestProbes("Tcp", ""), + expectedRules: getDefaultTestRules(false), }, { desc: "reconcileLoadBalancerRule shall return corresponding probe and lbRule (slb without tcp reset)", service: getTestService("test1", v1.ProtocolTCP, map[string]string{"service.beta.kubernetes.io/azure-load-balancer-disable-tcp-reset": "True"}, false, 80), loadBalancerSku: "standard", wantLb: true, - expectedProbes: []network.Probe{ - { - Name: to.StringPtr("atest1-TCP-80"), - ProbePropertiesFormat: &network.ProbePropertiesFormat{ - Protocol: network.ProbeProtocol("Tcp"), - Port: to.Int32Ptr(10080), - IntervalInSeconds: to.Int32Ptr(5), - NumberOfProbes: to.Int32Ptr(2), - }, - }, - }, - expectedRules: []network.LoadBalancingRule{ - { - Name: to.StringPtr("atest1-TCP-80"), - LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{ - Protocol: network.TransportProtocol("Tcp"), - FrontendIPConfiguration: &network.SubResource{ - ID: to.StringPtr("frontendIPConfigID"), - }, - BackendAddressPool: &network.SubResource{ - ID: to.StringPtr("backendPoolID"), - }, - LoadDistribution: "Default", - FrontendPort: to.Int32Ptr(80), - BackendPort: to.Int32Ptr(80), - EnableFloatingIP: to.BoolPtr(true), - DisableOutboundSnat: to.BoolPtr(false), - IdleTimeoutInMinutes: to.Int32Ptr(0), - Probe: &network.SubResource{ - ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/" + - "Microsoft.Network/loadBalancers/lbname/probes/atest1-TCP-80"), - }, - EnableTCPReset: to.BoolPtr(true), - }, - }, - }, + expectedProbes: getDefaultTestProbes("Tcp", ""), + expectedRules: getDefaultTestRules(true), }, { desc: "reconcileLoadBalancerRule shall return corresponding probe and lbRule(slb with tcp reset)", service: getTestService("test1", v1.ProtocolTCP, nil, false, 80), loadBalancerSku: "standard", wantLb: true, - expectedProbes: []network.Probe{ - { - Name: to.StringPtr("atest1-TCP-80"), - ProbePropertiesFormat: &network.ProbePropertiesFormat{ - Protocol: network.ProbeProtocol("Tcp"), - Port: to.Int32Ptr(10080), - IntervalInSeconds: to.Int32Ptr(5), - NumberOfProbes: to.Int32Ptr(2), - }, - }, - }, - expectedRules: []network.LoadBalancingRule{ - { - Name: to.StringPtr("atest1-TCP-80"), - LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{ - Protocol: network.TransportProtocol("Tcp"), - FrontendIPConfiguration: &network.SubResource{ - ID: to.StringPtr("frontendIPConfigID"), - }, - BackendAddressPool: &network.SubResource{ - ID: to.StringPtr("backendPoolID"), - }, - LoadDistribution: "Default", - FrontendPort: to.Int32Ptr(80), - BackendPort: to.Int32Ptr(80), - EnableFloatingIP: to.BoolPtr(true), - DisableOutboundSnat: to.BoolPtr(false), - IdleTimeoutInMinutes: to.Int32Ptr(0), - Probe: &network.SubResource{ - ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/" + - "Microsoft.Network/loadBalancers/lbname/probes/atest1-TCP-80"), - }, - EnableTCPReset: to.BoolPtr(true), - }, - }, - }, + expectedProbes: getDefaultTestProbes("Tcp", ""), + expectedRules: getDefaultTestRules(true), + }, + { + desc: "reconcileLoadBalancerRule shall respect the probe protocol and path configuration in the config file", + service: getTestService("test1", v1.ProtocolTCP, nil, false, 80), + loadBalancerSku: "standard", + wantLb: true, + probeProtocol: "http", + probePath: "/healthy", + expectedProbes: getDefaultTestProbes("http", "/healthy"), + expectedRules: getDefaultTestRules(true), }, } for i, test := range testCases { az := GetTestCloud(ctrl) az.Config.LoadBalancerSku = test.loadBalancerSku + service := test.service + if test.probeProtocol != "" { + service.Annotations[ServiceAnnotationLoadBalancerHealthProbeProtocol] = test.probeProtocol + } + if test.probePath != "" { + service.Annotations[ServiceAnnotationLoadBalancerHealthProbeRequestPath] = test.probePath + } probe, lbrule, err := az.reconcileLoadBalancerRule(&test.service, test.wantLb, "frontendIPConfigID", "backendPoolID", "lbname", to.Int32Ptr(0)) @@ -1699,6 +1616,55 @@ func TestReconcileLoadBalancerRule(t *testing.T) { } } +func getDefaultTestProbes(protocol, path string) []network.Probe { + expectedProbes := []network.Probe{ + { + Name: to.StringPtr("atest1-TCP-80"), + ProbePropertiesFormat: &network.ProbePropertiesFormat{ + Protocol: network.ProbeProtocol(protocol), + Port: to.Int32Ptr(10080), + IntervalInSeconds: to.Int32Ptr(5), + NumberOfProbes: to.Int32Ptr(2), + }, + }, + } + if path != "" { + expectedProbes[0].RequestPath = to.StringPtr(path) + } + return expectedProbes +} + +func getDefaultTestRules(enableTCPReset bool) []network.LoadBalancingRule { + expectedRules := []network.LoadBalancingRule{ + { + Name: to.StringPtr("atest1-TCP-80"), + LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{ + Protocol: network.TransportProtocol("Tcp"), + FrontendIPConfiguration: &network.SubResource{ + ID: to.StringPtr("frontendIPConfigID"), + }, + BackendAddressPool: &network.SubResource{ + ID: to.StringPtr("backendPoolID"), + }, + LoadDistribution: "Default", + FrontendPort: to.Int32Ptr(80), + BackendPort: to.Int32Ptr(80), + EnableFloatingIP: to.BoolPtr(true), + DisableOutboundSnat: to.BoolPtr(false), + IdleTimeoutInMinutes: to.Int32Ptr(0), + Probe: &network.SubResource{ + ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/" + + "Microsoft.Network/loadBalancers/lbname/probes/atest1-TCP-80"), + }, + }, + }, + } + if enableTCPReset { + expectedRules[0].EnableTCPReset = to.BoolPtr(true) + } + return expectedRules +} + func getTestLoadBalancer(name, rgName, clusterName, identifier *string, service v1.Service, lbSku string) network.LoadBalancer { lb := network.LoadBalancer{ Name: name,