Skip to content

Commit

Permalink
Remove unused methods from BootlooseSuite
Browse files Browse the repository at this point in the history 
Inline the single use of AddAnnotation in the k0scloudprovider inttest
and simplify that test a lot. Inline the GetHTTPStatus method into its
only caller and pull out common setup stuff out of the retry loop.

Signed-off-by: Tom Wieczorek <[email protected]>
  • Loading branch information
@twz123
twz123 committed Nov 27, 2023
1 parent 9096897 commit 549c2fe
Show file tree
Hide file tree
Showing 2 changed files with 65 additions and 130 deletions.
79 changes: 21 additions & 58 deletions inttest/common/bootloosesuite.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,14 +49,12 @@ import (

corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"sigs.k8s.io/yaml"

"github.com/go-openapi/jsonpointer"
"github.com/k0sproject/bootloose/pkg/cluster"
"github.com/k0sproject/bootloose/pkg/config"
"github.com/stretchr/testify/assert"
Expand Down Expand Up @@ -935,32 +933,6 @@ func (s *BootlooseSuite) WaitForNodeLabel(kc *kubernetes.Clientset, node, labelK
})
}

// GetNodeLabels return the labels of given node
func (s *BootlooseSuite) GetNodeAnnotations(node string, kc *kubernetes.Clientset) (map[string]string, error) {
n, err := kc.CoreV1().Nodes().Get(s.Context(), node, metav1.GetOptions{})
if err != nil {
return nil, err
}
return n.Annotations, nil
}

// AddNodeLabel adds a label to the provided node.
func (s *BootlooseSuite) AddNodeLabel(node string, kc *kubernetes.Clientset, key string, value string) (*corev1.Node, error) {
return nodeValuePatchAdd(s.Context(), node, kc, "/metadata/labels", key, value)
}

// AddNodeAnnotation adds an annotation to the provided node.
func (s *BootlooseSuite) AddNodeAnnotation(node string, kc *kubernetes.Clientset, key string, value string) (*corev1.Node, error) {
return nodeValuePatchAdd(s.Context(), node, kc, "/metadata/annotations", key, value)
}

// nodeValuePatchAdd patch-adds a key/value to a specific path via the Node API
func nodeValuePatchAdd(ctx context.Context, node string, kc *kubernetes.Clientset, path string, key string, value string) (*corev1.Node, error) {
keyPath := fmt.Sprintf("%s/%s", path, jsonpointer.Escape(key))
patch := fmt.Sprintf(`[{"op":"add", "path":"%s", "value":"%s" }]`, keyPath, value)
return kc.CoreV1().Nodes().Patch(ctx, node, types.JSONPatchType, []byte(patch), metav1.PatchOptions{})
}

// WaitForKubeAPI waits until we see kube API online on given node.
// Timeouts with error return in 5 mins
func (s *BootlooseSuite) WaitForKubeAPI(node string, k0sKubeconfigArgs ...string) error {
Expand Down Expand Up @@ -995,49 +967,40 @@ func (s *BootlooseSuite) WaitForKubeAPI(node string, k0sKubeconfigArgs ...string
})
}

// WaitJoinApi waits until we see k0s join api up-and-running on a given node
// Timeouts with error return in 5 mins
// WaitJoinApi waits until we see k0s join api up-and-running on a given node.
func (s *BootlooseSuite) WaitJoinAPI(node string) error {
s.T().Logf("waiting for join api to start on node %s", node)
s.T().Logf("Waiting for k0s join API to start on node %s", node)

m, err := s.MachineForName(node)
if err != nil {
return err
}
joinPort, err := m.HostPort(s.K0sAPIExternalPort)
if err != nil {
return err
}
client := &http.Client{Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}}
checkURL := fmt.Sprintf("https://localhost:%d/v1beta1/ca", joinPort)

return Poll(s.Context(), func(context.Context) (done bool, err error) {
joinAPIStatus, err := s.GetHTTPStatus(node, "/v1beta1/ca")
resp, err := client.Get(checkURL)
if err != nil {
return false, nil
}
defer resp.Body.Close()

// JoinAPI returns always un-authorized when called with no token, but it's a signal that it properly up-and-running still
if joinAPIStatus != http.StatusUnauthorized {
if resp.StatusCode != http.StatusUnauthorized {
return false, nil
}

s.T().Logf("join api up-and-running")

s.T().Logf("K0s join API up-and-running")
return true, nil
})
}

func (s *BootlooseSuite) GetHTTPStatus(node string, path string) (int, error) {
m, err := s.MachineForName(node)
if err != nil {
return 0, err
}
joinPort, err := m.HostPort(s.K0sAPIExternalPort)
if err != nil {
return 0, err
}

tr := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
client := &http.Client{Transport: tr}
checkURL := fmt.Sprintf("https://localhost:%d/%s", joinPort, path)
resp, err := client.Get(checkURL)
if err != nil {
return 0, err
}
defer resp.Body.Close()
return resp.StatusCode, nil
}

func (s *BootlooseSuite) initializeBootlooseCluster() error {
dir, err := os.MkdirTemp("", s.T().Name()+"-bootloose.")
if err != nil {
Expand Down
116 changes: 44 additions & 72 deletions inttest/k0scloudprovider/k0scloudprovider_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,23 +18,30 @@ package k0scloudprovider

import (
"context"
"fmt"
"testing"
"time"

"github.com/k0sproject/k0s/inttest/common"
"github.com/k0sproject/k0s/pkg/k0scloudprovider"
"github.com/stretchr/testify/suite"
v1 "k8s.io/api/core/v1"
"github.com/k0sproject/k0s/pkg/kubernetes/watch"

corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes"

"github.com/go-openapi/jsonpointer"
"github.com/stretchr/testify/suite"
)

type K0sCloudProviderSuite struct {
common.BootlooseSuite
}

func (s *K0sCloudProviderSuite) TestK0sGetsUp() {
s.Require().NoError(s.InitController(0, "--enable-k0s-cloud-provider", "--k0s-cloud-provider-update-frequency=5s"))
ctx := s.Context()

s.Require().NoError(s.InitController(0, "--enable-k0s-cloud-provider", "--k0s-cloud-provider-update-frequency=1s"))
s.Require().NoError(s.RunWorkers("--enable-cloud-provider"))

kc, err := s.KubeClient(s.ControllerNode(0))
Expand All @@ -44,76 +51,41 @@ func (s *K0sCloudProviderSuite) TestK0sGetsUp() {
err = s.WaitForNodeReady(s.WorkerNode(0), kc)
s.Require().NoError(err)

// Test the adding of various addresses using addition via annotations
w0Helper := defaultNodeAddValueHelper(s.AddNodeAnnotation)
s.testAddAddress(kc, s.WorkerNode(0), "1.2.3.4", w0Helper)
s.testAddAddress(kc, s.WorkerNode(0), "2041:0000:140F::875B:131B", w0Helper)
s.testAddAddress(kc, s.WorkerNode(0), "GIGO", w0Helper)
}

// nodeAddValueFunc defines how a key/value can be added to a node
type nodeAddValueFunc func(node string, kc *kubernetes.Clientset, key string, value string) (*v1.Node, error)

// nodeAddValueHelper provides all of the callback functions needed to test
// the addition of addresses into the provider (pre, add, post)
type nodeAddValueHelper struct {
addressFoundPre func(ctx context.Context, kc *kubernetes.Clientset, node string, addr string, addrType v1.NodeAddressType) (bool, error)
addressAdd nodeAddValueFunc
addressFoundPost func(ctx context.Context, kc *kubernetes.Clientset, node string, addr string, addrType v1.NodeAddressType) (bool, error)
}

// defaultNodeAddValueHelper creates a nodeAddValueHelper using the provided
// adder function (ie. labels, or annotation add functions)
func defaultNodeAddValueHelper(adder nodeAddValueFunc) nodeAddValueHelper {
return nodeAddValueHelper{
addressFoundPre: nodeHasAddressWithType,
addressAdd: adder,
addressFoundPost: nodeHasAddressWithType,
}
}

// testAddAddress adds the provided address to a node via a helper. This ensures that
// the address doesn't already exist, can be added successfully, and exists after addition.
func (s *K0sCloudProviderSuite) testAddAddress(kc *kubernetes.Clientset, node string, addr string, helper nodeAddValueHelper) {
s.T().Logf("Testing add address - node=%s, addr=%s", node, addr)

addrFound, err := helper.addressFoundPre(s.Context(), kc, node, addr, v1.NodeExternalIP)
s.Require().NoError(err)
s.Require().False(addrFound, "ExternalIP=%s already exists on node=%s", addr, node)

// Now, add the special 'k0sproject.io/node-ip-external' key with an IP address to the
// worker node, and after a few seconds the IP address should be listed as an 'ExternalIP'
w, err := helper.addressAdd(node, kc, k0scloudprovider.ExternalIPAnnotation, addr)
s.Require().NoError(err)
s.Require().NotNil(w)

// The k0s-cloud-provider is configured to update every 5s, so wait for 10s and then
// look for the external IP.
s.T().Logf("waiting 10s for the next k0s-cloud-provider update (testAddAddress: node=%s, addr=%s)", node, addr)
time.Sleep(10 * time.Second)

// Need to ensure that a matching 'ExternalIP' address has been added, indicating that
// k0s-cloud-provider properly processed the annotation.
foundPostUpdate, err := helper.addressFoundPost(s.Context(), kc, node, addr, v1.NodeExternalIP)
s.Require().NoError(err)
s.Require().True(foundPostUpdate, "unable to find ExternalIP=%s on node=%s", addr, node)
s.testAddAddress(ctx, kc, s.WorkerNode(0), "1.2.3.4")
s.testAddAddress(ctx, kc, s.WorkerNode(0), "2041:0000:140F::875B:131B")
s.testAddAddress(ctx, kc, s.WorkerNode(0), "GIGO")
}

// nodeHasAddressWithType is a helper for fetching all of the addresses associated to
// the provided node, and asserting that an IP matches by address + type.
func nodeHasAddressWithType(ctx context.Context, kc *kubernetes.Clientset, node string, addr string, addrType v1.NodeAddressType) (bool, error) {
n, err := kc.CoreV1().Nodes().Get(ctx, node, metav1.GetOptions{})
if err != nil {
return false, err
}

for _, naddr := range n.Status.Addresses {
if naddr.Type == addrType && naddr.Address == addr {
return true, nil
}
}

return false, nil
// testAddAddress adds the provided address to a node and ensures that the
// cloud-provider will set it on the node.
func (s *K0sCloudProviderSuite) testAddAddress(ctx context.Context, client kubernetes.Interface, nodeName string, ip string) {
// Adds or sets the special ExternalIPAnnotation with an IP address to the worker
// node, and after a few seconds the IP address should be listed as a NodeExternalIP.
patch := fmt.Sprintf(
`[{"op":"add", "path":"/metadata/annotations/%s", "value":"%s"}]`,
jsonpointer.Escape(k0scloudprovider.ExternalIPAnnotation), jsonpointer.Escape(ip),
)
_, err := client.CoreV1().Nodes().Patch(ctx, nodeName, types.JSONPatchType, []byte(patch), metav1.PatchOptions{})
s.Require().NoError(err, "Failed to add or set the annotation for the external IP address")

// Need to ensure that a matching 'ExternalIP' address has been added,
// indicating that k0s-cloud-provider properly processed the annotation.
s.T().Logf("Waiting for k0s-cloud-provider to update the external IP on node %s to %s", nodeName, ip)
s.Require().NoError(watch.Nodes(client.CoreV1().Nodes()).
WithObjectName(nodeName).
WithErrorCallback(common.RetryWatchErrors(s.T().Logf)).
Until(ctx, func(node *corev1.Node) (bool, error) {
for _, nodeAddr := range node.Status.Addresses {
if nodeAddr.Type == corev1.NodeExternalIP {
if nodeAddr.Address == ip {
return true, nil
}
break
}
}

return false, nil
}), "While waiting for k0s-cloud-provider to update the external IP on node %s to %s", nodeName, ip)
}

func TestK0sCloudProviderSuite(t *testing.T) {
Expand Down

0 comments on commit 549c2fe

Please sign in to comment.