Merge pull request #8 from camaeel/feature/wait-for-cloud-controller-…

…initialization

Feature/wait for cloud controller initialization

go.mod

@@ -29,11 +29,11 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	golang.org/x/net v0.1.0 // indirect
+	golang.org/x/net v0.5.0 // indirect
 	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
-	golang.org/x/sys v0.1.0 // indirect
-	golang.org/x/term v0.1.0 // indirect
-	golang.org/x/text v0.4.0 // indirect
+	golang.org/x/sys v0.4.0 // indirect
+	golang.org/x/term v0.4.0 // indirect
+	golang.org/x/text v0.6.0 // indirect
 	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect

go.sum

@@ -336,8 +336,9 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0=
 golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
+golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw=
+golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -411,13 +412,15 @@ golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18=
+golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.4.0 h1:O7UWfv5+A2qiuulQk30kVinPoMtoIPeVaKLEgLpVkvg=
+golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -427,8 +430,9 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k=
+golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

pkg/controller/controller.go

@@ -37,6 +37,7 @@ const (
 	NodeRoleSpotControlPlaneLabel = "node-role.kubernetes.io/spot-control-plane"
 	NodeRoleWorkerLabel           = "node-role.kubernetes.io/worker"
 	NodeRoleSpotWorkerLabel       = "node-role.kubernetes.io/spot-worker"
+	NodeUninitialziedTaint        = "node.cloudprovider.kubernetes.io/uninitialized"
 )
 
 func NewNodeController(client kubernetes.Interface, spotInstanceDiscovery spotdiscovery.SpotDiscoveryInterface, excludeLoadBalancing bool, includeAlphaLabel bool, excludeEviction bool, controlPlaneTaint string, controlPlaneLegacyLabel bool, customRoleLabel string) NodeController {
@@ -77,7 +78,11 @@ func (c NodeController) handler(obj interface{}) {
 		return
 	}
 	log.Debugf("Received handler event for node %s", node.Name)
-	c.markNode(node)
+	if c.isNodeInitialized(node) {
+		c.markNode(node)
+	} else {
+		log.Warnf("Node %s was not yet initialzied by cloud controller.", node.Name)
+	}
 }
 
 func (c NodeController) markNode(node *v1.Node) {
@@ -168,7 +173,7 @@ func addControlPlaneLabels(node *v1.Node, includeAlphaLabel bool, excludeLoadBal
 	}
 }
 
-//Deprecated. Will be removed in future release
+// Deprecated. Will be removed in future release
 func isAlreadyMarkedMaster(node *v1.Node) bool {
 	if node.Labels != nil {
 		if _, ok := node.Labels[NodeRoleMasterLabel]; ok {
@@ -237,3 +242,12 @@ func (c NodeController) isWorkerNode(node *v1.Node) bool {
 func customRoleLabel(role string) string {
 	return fmt.Sprintf("node-role.kubernetes.io/%s", role)
 }
+
+func (c NodeController) isNodeInitialized(node *v1.Node) bool {
+	for i := range node.Spec.Taints {
+		if node.Spec.Taints[i].Key == NodeUninitialziedTaint {
+			return false
+		}
+	}
+	return true
+}

pkg/controller/controller_test.go

@@ -3,6 +3,7 @@ package controller
 import (
 	"context"
 	"fmt"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -99,6 +100,25 @@ var UnManagedNode = &v1.Node{
 	},
 	Spec: v1.NodeSpec{},
 }
+var UninitializedNode = &v1.Node{
+	ObjectMeta: metav1.ObjectMeta{
+		Name: "test-uninitialized-control-plane-node",
+	},
+	Spec: v1.NodeSpec{
+		ProviderID: "aws:///eu-central-1a/i-123uzu123",
+		Taints: []v1.Taint{
+			{
+				Key:    NodeRoleControlPlaneLabel,
+				Effect: "NoSchedule",
+			},
+			{
+				Key:    NodeUninitialziedTaint,
+				Value:  "true",
+				Effect: "NoSchedule",
+			},
+		},
+	},
+}
 
 func TestHandlerShouldSetNodeRoleMasterAndControlPlaneForMaster(t *testing.T) {
 	clientset := fake.NewSimpleClientset(MasterNode)
@@ -374,3 +394,62 @@ func TestHandlerShouldNotSetCustomRoleIfLabelNotPresent(t *testing.T) {
 		t.Errorf("Expected no label %s on node %s, but was assigned", NodeRoleMasterLabel, "test-worker-node")
 	}
 }
+
+func TestIsNodeInitializedTrue(t *testing.T) {
+	var initializedNode = &v1.Node{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test-spot-control-plane-node",
+		},
+		Spec: v1.NodeSpec{
+			ProviderID: "aws:///eu-central-1/i-123uzu123",
+			Taints: []v1.Taint{
+				{
+					Key:    NodeRoleControlPlaneLabel,
+					Effect: "NoSchedule",
+				},
+				{
+					Key:    "dumm-taint",
+					Value:  "true",
+					Effect: "NoSchedule",
+				},
+			},
+		},
+	}
+	clientset := fake.NewSimpleClientset(initializedNode)
+	testingMockDiscovery := TestingMockDiscovery{}
+	c := NewNodeController(clientset, testingMockDiscovery, false, false, false, NodeRoleControlPlaneLabel, false, "")
+
+	result := c.isNodeInitialized(initializedNode)
+	expected := true
+	if result != expected {
+		t.Errorf("Node is not initialized, but should be.")
+	}
+}
+
+func TestIsNodeInitializedFalse(t *testing.T) {
+
+	clientset := fake.NewSimpleClientset(UninitializedNode)
+	testingMockDiscovery := TestingMockDiscovery{}
+	c := NewNodeController(clientset, testingMockDiscovery, false, false, false, NodeRoleControlPlaneLabel, false, "")
+
+	result := c.isNodeInitialized(UninitializedNode)
+	expected := false
+	if result != expected {
+		t.Errorf("Node is initialized, but shouldn't be.")
+	}
+}
+
+func TestHandlerShouldNotSetLabelIfNodeUninitialized(t *testing.T) {
+	clientset := fake.NewSimpleClientset(UninitializedNode)
+	testingMockDiscovery := TestingMockDiscovery{}
+
+	c := NewNodeController(clientset, testingMockDiscovery, false, false, false, NodeRoleControlPlaneLabel, false, "")
+	c.handler(UninitializedNode)
+
+	foundNode, _ := clientset.CoreV1().Nodes().Get(context.TODO(), "test-uninitialized-control-plane-node", metav1.GetOptions{})
+	for k, _ := range foundNode.Labels {
+		if strings.Contains(k, "node-role.kubernetes.io/") {
+			t.Errorf("Unexpected label %s on node %s, but shouldn't be assigned", k, "test-uninitialized-control-plane-node")
+		}
+	}
+}

pkg/spotdiscovery/awsspotdiscovery_test.go

@@ -14,15 +14,15 @@ var WorkerNode = &v1.Node{
 		Name: "test-worker-node",
 	},
 	Spec: v1.NodeSpec{
-		ProviderID: "aws:///eu-central-1/i-123qwe123",
+		ProviderID: "aws:///eu-central-1c/i-123qwe123",
 	},
 }
 var SpotWorkerNode = &v1.Node{
 	ObjectMeta: metav1.ObjectMeta{
 		Name: "test-spot-node",
 	},
 	Spec: v1.NodeSpec{
-		ProviderID: "aws:///eu-central-1/i-123asd132",
+		ProviderID: "aws:///eu-central-1b/i-123asd132",
 	},
 }
 var UnManagedNode = &v1.Node{
@@ -85,3 +85,11 @@ func (c *MockEC2Client) DescribeSpotInstanceRequests(in *ec2.DescribeSpotInstanc
 		SpotInstanceRequests: spotInstances,
 	}, nil
 }
+
+func TestReceiveInstanceID(t *testing.T) {
+	result := receiveInstanceID(WorkerNode)
+	expectedInstanceId := "i-123qwe123"
+	if *result != expectedInstanceId {
+		t.Errorf("Expected instanceId: %s, got: %s", expectedInstanceId, *result)
+	}
+}