diff --git a/inttest/common/bootloosesuite.go b/inttest/common/bootloosesuite.go index cd930b521d12..68dcab415bda 100644 --- a/inttest/common/bootloosesuite.go +++ b/inttest/common/bootloosesuite.go @@ -49,14 +49,12 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" "sigs.k8s.io/yaml" - "github.com/go-openapi/jsonpointer" "github.com/k0sproject/bootloose/pkg/cluster" "github.com/k0sproject/bootloose/pkg/config" "github.com/stretchr/testify/assert" @@ -935,32 +933,6 @@ func (s *BootlooseSuite) WaitForNodeLabel(kc *kubernetes.Clientset, node, labelK }) } -// GetNodeLabels return the labels of given node -func (s *BootlooseSuite) GetNodeAnnotations(node string, kc *kubernetes.Clientset) (map[string]string, error) { - n, err := kc.CoreV1().Nodes().Get(s.Context(), node, metav1.GetOptions{}) - if err != nil { - return nil, err - } - return n.Annotations, nil -} - -// AddNodeLabel adds a label to the provided node. -func (s *BootlooseSuite) AddNodeLabel(node string, kc *kubernetes.Clientset, key string, value string) (*corev1.Node, error) { - return nodeValuePatchAdd(s.Context(), node, kc, "/metadata/labels", key, value) -} - -// AddNodeAnnotation adds an annotation to the provided node. -func (s *BootlooseSuite) AddNodeAnnotation(node string, kc *kubernetes.Clientset, key string, value string) (*corev1.Node, error) { - return nodeValuePatchAdd(s.Context(), node, kc, "/metadata/annotations", key, value) -} - -// nodeValuePatchAdd patch-adds a key/value to a specific path via the Node API -func nodeValuePatchAdd(ctx context.Context, node string, kc *kubernetes.Clientset, path string, key string, value string) (*corev1.Node, error) { - keyPath := fmt.Sprintf("%s/%s", path, jsonpointer.Escape(key)) - patch := fmt.Sprintf(`[{"op":"add", "path":"%s", "value":"%s" }]`, keyPath, value) - return kc.CoreV1().Nodes().Patch(ctx, node, types.JSONPatchType, []byte(patch), metav1.PatchOptions{}) -} - // WaitForKubeAPI waits until we see kube API online on given node. // Timeouts with error return in 5 mins func (s *BootlooseSuite) WaitForKubeAPI(node string, k0sKubeconfigArgs ...string) error { @@ -995,49 +967,40 @@ func (s *BootlooseSuite) WaitForKubeAPI(node string, k0sKubeconfigArgs ...string }) } -// WaitJoinApi waits until we see k0s join api up-and-running on a given node -// Timeouts with error return in 5 mins +// WaitJoinApi waits until we see k0s join api up-and-running on a given node. func (s *BootlooseSuite) WaitJoinAPI(node string) error { - s.T().Logf("waiting for join api to start on node %s", node) + s.T().Logf("Waiting for k0s join API to start on node %s", node) + + m, err := s.MachineForName(node) + if err != nil { + return err + } + joinPort, err := m.HostPort(s.K0sAPIExternalPort) + if err != nil { + return err + } + client := &http.Client{Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }} + checkURL := fmt.Sprintf("https://localhost:%d/v1beta1/ca", joinPort) + return Poll(s.Context(), func(context.Context) (done bool, err error) { - joinAPIStatus, err := s.GetHTTPStatus(node, "/v1beta1/ca") + resp, err := client.Get(checkURL) if err != nil { return false, nil } + defer resp.Body.Close() + // JoinAPI returns always un-authorized when called with no token, but it's a signal that it properly up-and-running still - if joinAPIStatus != http.StatusUnauthorized { + if resp.StatusCode != http.StatusUnauthorized { return false, nil } - s.T().Logf("join api up-and-running") - + s.T().Logf("K0s join API up-and-running") return true, nil }) } -func (s *BootlooseSuite) GetHTTPStatus(node string, path string) (int, error) { - m, err := s.MachineForName(node) - if err != nil { - return 0, err - } - joinPort, err := m.HostPort(s.K0sAPIExternalPort) - if err != nil { - return 0, err - } - - tr := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - } - client := &http.Client{Transport: tr} - checkURL := fmt.Sprintf("https://localhost:%d/%s", joinPort, path) - resp, err := client.Get(checkURL) - if err != nil { - return 0, err - } - defer resp.Body.Close() - return resp.StatusCode, nil -} - func (s *BootlooseSuite) initializeBootlooseCluster() error { dir, err := os.MkdirTemp("", s.T().Name()+"-bootloose.") if err != nil { diff --git a/inttest/k0scloudprovider/k0scloudprovider_test.go b/inttest/k0scloudprovider/k0scloudprovider_test.go index b144465cbaf5..df90bf302c30 100644 --- a/inttest/k0scloudprovider/k0scloudprovider_test.go +++ b/inttest/k0scloudprovider/k0scloudprovider_test.go @@ -18,15 +18,20 @@ package k0scloudprovider import ( "context" + "fmt" "testing" - "time" "github.com/k0sproject/k0s/inttest/common" "github.com/k0sproject/k0s/pkg/k0scloudprovider" - "github.com/stretchr/testify/suite" - v1 "k8s.io/api/core/v1" + "github.com/k0sproject/k0s/pkg/kubernetes/watch" + + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" + + "github.com/go-openapi/jsonpointer" + "github.com/stretchr/testify/suite" ) type K0sCloudProviderSuite struct { @@ -34,7 +39,9 @@ type K0sCloudProviderSuite struct { } func (s *K0sCloudProviderSuite) TestK0sGetsUp() { - s.Require().NoError(s.InitController(0, "--enable-k0s-cloud-provider", "--k0s-cloud-provider-update-frequency=5s")) + ctx := s.Context() + + s.Require().NoError(s.InitController(0, "--enable-k0s-cloud-provider", "--k0s-cloud-provider-update-frequency=1s")) s.Require().NoError(s.RunWorkers("--enable-cloud-provider")) kc, err := s.KubeClient(s.ControllerNode(0)) @@ -44,76 +51,41 @@ func (s *K0sCloudProviderSuite) TestK0sGetsUp() { err = s.WaitForNodeReady(s.WorkerNode(0), kc) s.Require().NoError(err) - // Test the adding of various addresses using addition via annotations - w0Helper := defaultNodeAddValueHelper(s.AddNodeAnnotation) - s.testAddAddress(kc, s.WorkerNode(0), "1.2.3.4", w0Helper) - s.testAddAddress(kc, s.WorkerNode(0), "2041:0000:140F::875B:131B", w0Helper) - s.testAddAddress(kc, s.WorkerNode(0), "GIGO", w0Helper) -} - -// nodeAddValueFunc defines how a key/value can be added to a node -type nodeAddValueFunc func(node string, kc *kubernetes.Clientset, key string, value string) (*v1.Node, error) - -// nodeAddValueHelper provides all of the callback functions needed to test -// the addition of addresses into the provider (pre, add, post) -type nodeAddValueHelper struct { - addressFoundPre func(ctx context.Context, kc *kubernetes.Clientset, node string, addr string, addrType v1.NodeAddressType) (bool, error) - addressAdd nodeAddValueFunc - addressFoundPost func(ctx context.Context, kc *kubernetes.Clientset, node string, addr string, addrType v1.NodeAddressType) (bool, error) -} - -// defaultNodeAddValueHelper creates a nodeAddValueHelper using the provided -// adder function (ie. labels, or annotation add functions) -func defaultNodeAddValueHelper(adder nodeAddValueFunc) nodeAddValueHelper { - return nodeAddValueHelper{ - addressFoundPre: nodeHasAddressWithType, - addressAdd: adder, - addressFoundPost: nodeHasAddressWithType, - } -} - -// testAddAddress adds the provided address to a node via a helper. This ensures that -// the address doesn't already exist, can be added successfully, and exists after addition. -func (s *K0sCloudProviderSuite) testAddAddress(kc *kubernetes.Clientset, node string, addr string, helper nodeAddValueHelper) { - s.T().Logf("Testing add address - node=%s, addr=%s", node, addr) - - addrFound, err := helper.addressFoundPre(s.Context(), kc, node, addr, v1.NodeExternalIP) - s.Require().NoError(err) - s.Require().False(addrFound, "ExternalIP=%s already exists on node=%s", addr, node) - - // Now, add the special 'k0sproject.io/node-ip-external' key with an IP address to the - // worker node, and after a few seconds the IP address should be listed as an 'ExternalIP' - w, err := helper.addressAdd(node, kc, k0scloudprovider.ExternalIPAnnotation, addr) - s.Require().NoError(err) - s.Require().NotNil(w) - - // The k0s-cloud-provider is configured to update every 5s, so wait for 10s and then - // look for the external IP. - s.T().Logf("waiting 10s for the next k0s-cloud-provider update (testAddAddress: node=%s, addr=%s)", node, addr) - time.Sleep(10 * time.Second) - - // Need to ensure that a matching 'ExternalIP' address has been added, indicating that - // k0s-cloud-provider properly processed the annotation. - foundPostUpdate, err := helper.addressFoundPost(s.Context(), kc, node, addr, v1.NodeExternalIP) - s.Require().NoError(err) - s.Require().True(foundPostUpdate, "unable to find ExternalIP=%s on node=%s", addr, node) + s.testAddAddress(ctx, kc, s.WorkerNode(0), "1.2.3.4") + s.testAddAddress(ctx, kc, s.WorkerNode(0), "2041:0000:140F::875B:131B") + s.testAddAddress(ctx, kc, s.WorkerNode(0), "GIGO") } -// nodeHasAddressWithType is a helper for fetching all of the addresses associated to -// the provided node, and asserting that an IP matches by address + type. -func nodeHasAddressWithType(ctx context.Context, kc *kubernetes.Clientset, node string, addr string, addrType v1.NodeAddressType) (bool, error) { - n, err := kc.CoreV1().Nodes().Get(ctx, node, metav1.GetOptions{}) - if err != nil { - return false, err - } - - for _, naddr := range n.Status.Addresses { - if naddr.Type == addrType && naddr.Address == addr { - return true, nil - } - } - - return false, nil +// testAddAddress adds the provided address to a node and ensures that the +// cloud-provider will set it on the node. +func (s *K0sCloudProviderSuite) testAddAddress(ctx context.Context, client kubernetes.Interface, nodeName string, ip string) { + // Adds or sets the special ExternalIPAnnotation with an IP address to the worker + // node, and after a few seconds the IP address should be listed as a NodeExternalIP. + patch := fmt.Sprintf( + `[{"op":"add", "path":"/metadata/annotations/%s", "value":"%s"}]`, + jsonpointer.Escape(k0scloudprovider.ExternalIPAnnotation), jsonpointer.Escape(ip), + ) + _, err := client.CoreV1().Nodes().Patch(ctx, nodeName, types.JSONPatchType, []byte(patch), metav1.PatchOptions{}) + s.Require().NoError(err, "Failed to add or set the annotation for the external IP address") + + // Need to ensure that a matching 'ExternalIP' address has been added, + // indicating that k0s-cloud-provider properly processed the annotation. + s.T().Logf("Waiting for k0s-cloud-provider to update the external IP on node %s to %s", nodeName, ip) + s.Require().NoError(watch.Nodes(client.CoreV1().Nodes()). + WithObjectName(nodeName). + WithErrorCallback(common.RetryWatchErrors(s.T().Logf)). + Until(ctx, func(node *corev1.Node) (bool, error) { + for _, nodeAddr := range node.Status.Addresses { + if nodeAddr.Type == corev1.NodeExternalIP { + if nodeAddr.Address == ip { + return true, nil + } + break + } + } + + return false, nil + }), "While waiting for k0s-cloud-provider to update the external IP on node %s to %s", nodeName, ip) } func TestK0sCloudProviderSuite(t *testing.T) {