Skip to content

Commit

Permalink
Merge pull request kubernetes#96338 from nilo19/feature/support-addit…
Browse files Browse the repository at this point in the history 
…ional-probe-protocol

Support customize load balancer health probe protocol and request path
  • Loading branch information
@k8s-ci-robot
k8s-ci-robot authored Nov 10, 2020
2 parents 4628c60 + d17621b commit 59405cc
Show file tree
Hide file tree
Showing 2 changed files with 122 additions and 111 deletions.
51 changes: 48 additions & 3 deletions staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,15 @@ const (
// TODO(feiskyer): disable-tcp-reset annotations has been depracated since v1.18, it would removed on v1.20.
ServiceAnnotationLoadBalancerDisableTCPReset = "service.beta.kubernetes.io/azure-load-balancer-disable-tcp-reset"

// ServiceAnnotationLoadBalancerHealthProbeProtocol determines the network protocol that the load balancer health probe use.
// If not set, the local service would use the HTTP and the cluster service would use the TCP by default.
ServiceAnnotationLoadBalancerHealthProbeProtocol = "service.beta.kubernetes.io/azure-load-balancer-health-probe-protocol"

// ServiceAnnotationLoadBalancerHealthProbeRequestPath determines the request path of the load balancer health probe.
// This is only useful for the HTTP and HTTPS, and would be ignored when using TCP. If not set,
// `/healthz` would be configured by default.
ServiceAnnotationLoadBalancerHealthProbeRequestPath = "service.beta.kubernetes.io/azure-load-balancer-health-probe-request-path"

// serviceTagKey is the service key applied for public IP tags.
serviceTagKey = "service"
// clusterNameKey is the cluster name key applied for public IP tags.
Expand Down Expand Up @@ -1561,6 +1570,23 @@ func (az *Cloud) checkLoadBalancerResourcesConflicted(
return nil
}

func parseHealthProbeProtocolAndPath(service *v1.Service) (string, string) {
var protocol, path string
if v, ok := service.Annotations[ServiceAnnotationLoadBalancerHealthProbeProtocol]; ok {
protocol = v
} else {
return protocol, path
}
// ignore the request path if using TCP
if strings.EqualFold(protocol, string(network.ProbeProtocolHTTP)) ||
strings.EqualFold(protocol, string(network.ProbeProtocolHTTPS)) {
if v, ok := service.Annotations[ServiceAnnotationLoadBalancerHealthProbeRequestPath]; ok {
path = v
}
}
return protocol, path
}

func (az *Cloud) reconcileLoadBalancerRule(
service *v1.Service,
wantLb bool,
Expand Down Expand Up @@ -1606,25 +1632,44 @@ func (az *Cloud) reconcileLoadBalancerRule(
return expectedProbes, expectedRules, err
}

probeProtocol, requestPath := parseHealthProbeProtocolAndPath(service)
if servicehelpers.NeedsHealthCheck(service) {
podPresencePath, podPresencePort := servicehelpers.GetServiceHealthCheckPathPort(service)
if probeProtocol == "" {
probeProtocol = string(network.ProbeProtocolHTTP)
}
if requestPath == "" {
requestPath = podPresencePath
}

expectedProbes = append(expectedProbes, network.Probe{
Name: &lbRuleName,
ProbePropertiesFormat: &network.ProbePropertiesFormat{
RequestPath: to.StringPtr(podPresencePath),
Protocol: network.ProbeProtocolHTTP,
RequestPath: to.StringPtr(requestPath),
Protocol: network.ProbeProtocol(probeProtocol),
Port: to.Int32Ptr(podPresencePort),
IntervalInSeconds: to.Int32Ptr(5),
NumberOfProbes: to.Int32Ptr(2),
},
})
} else if protocol != v1.ProtocolUDP && protocol != v1.ProtocolSCTP {
// we only add the expected probe if we're doing TCP
if probeProtocol == "" {
probeProtocol = string(*probeProto)
}
var actualPath *string
if !strings.EqualFold(probeProtocol, string(network.ProbeProtocolTCP)) {
if requestPath != "" {
actualPath = to.StringPtr(requestPath)
} else {
actualPath = to.StringPtr("/healthz")
}
}
expectedProbes = append(expectedProbes, network.Probe{
Name: &lbRuleName,
ProbePropertiesFormat: &network.ProbePropertiesFormat{
Protocol: *probeProto,
Protocol: network.ProbeProtocol(probeProtocol),
RequestPath: actualPath,
Port: to.Int32Ptr(port.NodePort),
IntervalInSeconds: to.Int32Ptr(5),
NumberOfProbes: to.Int32Ptr(2),
Expand Down
182 changes: 74 additions & 108 deletions staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1547,6 +1547,8 @@ func TestReconcileLoadBalancerRule(t *testing.T) {
service v1.Service
loadBalancerSku string
wantLb bool
probeProtocol string
probePath string
expectedProbes []network.Probe
expectedRules []network.LoadBalancingRule
expectedErr error
Expand All @@ -1561,131 +1563,46 @@ func TestReconcileLoadBalancerRule(t *testing.T) {
service: getTestService("test1", v1.ProtocolTCP, map[string]string{"service.beta.kubernetes.io/azure-load-balancer-disable-tcp-reset": "true"}, false, 80),
loadBalancerSku: "basic",
wantLb: true,
expectedProbes: []network.Probe{
{
Name: to.StringPtr("atest1-TCP-80"),
ProbePropertiesFormat: &network.ProbePropertiesFormat{
Protocol: network.ProbeProtocol("Tcp"),
Port: to.Int32Ptr(10080),
IntervalInSeconds: to.Int32Ptr(5),
NumberOfProbes: to.Int32Ptr(2),
},
},
},
expectedRules: []network.LoadBalancingRule{
{
Name: to.StringPtr("atest1-TCP-80"),
LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
Protocol: network.TransportProtocol("Tcp"),
FrontendIPConfiguration: &network.SubResource{
ID: to.StringPtr("frontendIPConfigID"),
},
BackendAddressPool: &network.SubResource{
ID: to.StringPtr("backendPoolID"),
},
LoadDistribution: "Default",
FrontendPort: to.Int32Ptr(80),
BackendPort: to.Int32Ptr(80),
EnableFloatingIP: to.BoolPtr(true),
DisableOutboundSnat: to.BoolPtr(false),
IdleTimeoutInMinutes: to.Int32Ptr(0),
Probe: &network.SubResource{
ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/" +
"Microsoft.Network/loadBalancers/lbname/probes/atest1-TCP-80"),
},
EnableTCPReset: nil,
},
},
},
expectedProbes: getDefaultTestProbes("Tcp", ""),
expectedRules: getDefaultTestRules(false),
},
{
desc: "reconcileLoadBalancerRule shall return corresponding probe and lbRule (slb without tcp reset)",
service: getTestService("test1", v1.ProtocolTCP, map[string]string{"service.beta.kubernetes.io/azure-load-balancer-disable-tcp-reset": "True"}, false, 80),
loadBalancerSku: "standard",
wantLb: true,
expectedProbes: []network.Probe{
{
Name: to.StringPtr("atest1-TCP-80"),
ProbePropertiesFormat: &network.ProbePropertiesFormat{
Protocol: network.ProbeProtocol("Tcp"),
Port: to.Int32Ptr(10080),
IntervalInSeconds: to.Int32Ptr(5),
NumberOfProbes: to.Int32Ptr(2),
},
},
},
expectedRules: []network.LoadBalancingRule{
{
Name: to.StringPtr("atest1-TCP-80"),
LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
Protocol: network.TransportProtocol("Tcp"),
FrontendIPConfiguration: &network.SubResource{
ID: to.StringPtr("frontendIPConfigID"),
},
BackendAddressPool: &network.SubResource{
ID: to.StringPtr("backendPoolID"),
},
LoadDistribution: "Default",
FrontendPort: to.Int32Ptr(80),
BackendPort: to.Int32Ptr(80),
EnableFloatingIP: to.BoolPtr(true),
DisableOutboundSnat: to.BoolPtr(false),
IdleTimeoutInMinutes: to.Int32Ptr(0),
Probe: &network.SubResource{
ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/" +
"Microsoft.Network/loadBalancers/lbname/probes/atest1-TCP-80"),
},
EnableTCPReset: to.BoolPtr(true),
},
},
},
expectedProbes: getDefaultTestProbes("Tcp", ""),
expectedRules: getDefaultTestRules(true),
},
{
desc: "reconcileLoadBalancerRule shall return corresponding probe and lbRule(slb with tcp reset)",
service: getTestService("test1", v1.ProtocolTCP, nil, false, 80),
loadBalancerSku: "standard",
wantLb: true,
expectedProbes: []network.Probe{
{
Name: to.StringPtr("atest1-TCP-80"),
ProbePropertiesFormat: &network.ProbePropertiesFormat{
Protocol: network.ProbeProtocol("Tcp"),
Port: to.Int32Ptr(10080),
IntervalInSeconds: to.Int32Ptr(5),
NumberOfProbes: to.Int32Ptr(2),
},
},
},
expectedRules: []network.LoadBalancingRule{
{
Name: to.StringPtr("atest1-TCP-80"),
LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
Protocol: network.TransportProtocol("Tcp"),
FrontendIPConfiguration: &network.SubResource{
ID: to.StringPtr("frontendIPConfigID"),
},
BackendAddressPool: &network.SubResource{
ID: to.StringPtr("backendPoolID"),
},
LoadDistribution: "Default",
FrontendPort: to.Int32Ptr(80),
BackendPort: to.Int32Ptr(80),
EnableFloatingIP: to.BoolPtr(true),
DisableOutboundSnat: to.BoolPtr(false),
IdleTimeoutInMinutes: to.Int32Ptr(0),
Probe: &network.SubResource{
ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/" +
"Microsoft.Network/loadBalancers/lbname/probes/atest1-TCP-80"),
},
EnableTCPReset: to.BoolPtr(true),
},
},
},
expectedProbes: getDefaultTestProbes("Tcp", ""),
expectedRules: getDefaultTestRules(true),
},
{
desc: "reconcileLoadBalancerRule shall respect the probe protocol and path configuration in the config file",
service: getTestService("test1", v1.ProtocolTCP, nil, false, 80),
loadBalancerSku: "standard",
wantLb: true,
probeProtocol: "http",
probePath: "/healthy",
expectedProbes: getDefaultTestProbes("http", "/healthy"),
expectedRules: getDefaultTestRules(true),
},
}
for i, test := range testCases {
az := GetTestCloud(ctrl)
az.Config.LoadBalancerSku = test.loadBalancerSku
service := test.service
if test.probeProtocol != "" {
service.Annotations[ServiceAnnotationLoadBalancerHealthProbeProtocol] = test.probeProtocol
}
if test.probePath != "" {
service.Annotations[ServiceAnnotationLoadBalancerHealthProbeRequestPath] = test.probePath
}
probe, lbrule, err := az.reconcileLoadBalancerRule(&test.service, test.wantLb,
"frontendIPConfigID", "backendPoolID", "lbname", to.Int32Ptr(0))

Expand All @@ -1699,6 +1616,55 @@ func TestReconcileLoadBalancerRule(t *testing.T) {
}
}

func getDefaultTestProbes(protocol, path string) []network.Probe {
expectedProbes := []network.Probe{
{
Name: to.StringPtr("atest1-TCP-80"),
ProbePropertiesFormat: &network.ProbePropertiesFormat{
Protocol: network.ProbeProtocol(protocol),
Port: to.Int32Ptr(10080),
IntervalInSeconds: to.Int32Ptr(5),
NumberOfProbes: to.Int32Ptr(2),
},
},
}
if path != "" {
expectedProbes[0].RequestPath = to.StringPtr(path)
}
return expectedProbes
}

func getDefaultTestRules(enableTCPReset bool) []network.LoadBalancingRule {
expectedRules := []network.LoadBalancingRule{
{
Name: to.StringPtr("atest1-TCP-80"),
LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
Protocol: network.TransportProtocol("Tcp"),
FrontendIPConfiguration: &network.SubResource{
ID: to.StringPtr("frontendIPConfigID"),
},
BackendAddressPool: &network.SubResource{
ID: to.StringPtr("backendPoolID"),
},
LoadDistribution: "Default",
FrontendPort: to.Int32Ptr(80),
BackendPort: to.Int32Ptr(80),
EnableFloatingIP: to.BoolPtr(true),
DisableOutboundSnat: to.BoolPtr(false),
IdleTimeoutInMinutes: to.Int32Ptr(0),
Probe: &network.SubResource{
ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/" +
"Microsoft.Network/loadBalancers/lbname/probes/atest1-TCP-80"),
},
},
},
}
if enableTCPReset {
expectedRules[0].EnableTCPReset = to.BoolPtr(true)
}
return expectedRules
}

func getTestLoadBalancer(name, rgName, clusterName, identifier *string, service v1.Service, lbSku string) network.LoadBalancer {
lb := network.LoadBalancer{
Name: name,
Expand Down

0 comments on commit 59405cc

Please sign in to comment.