Fix timing issue in EtcdMember inttest

Signed-off-by: Jussi Nummelin <[email protected]>

inttest/common/bootloosesuite.go

@@ -43,6 +43,7 @@ import (
 
 	"github.com/k0sproject/k0s/internal/pkg/file"
 	apclient "github.com/k0sproject/k0s/pkg/client/clientset"
+	etcdmemberclient "github.com/k0sproject/k0s/pkg/client/clientset/typed/etcd/v1beta1"
 	"github.com/k0sproject/k0s/pkg/constant"
 	"github.com/k0sproject/k0s/pkg/k0scontext"
 	"github.com/k0sproject/k0s/pkg/kubernetes/watch"
@@ -908,6 +909,16 @@ func (s *BootlooseSuite) ExtensionsClient(node string, k0sKubeconfigArgs ...stri
 	return extclient.NewForConfig(cfg)
 }
 
+// EtcdMemberClient return a client for accessing etcd member CRDs
+func (s *BootlooseSuite) EtcdMemberClient(node string, k0sKubeconfigArgs ...string) (*etcdmemberclient.EtcdV1beta1Client, error) {
+	cfg, err := s.GetKubeConfig(node, k0sKubeconfigArgs...)
+	if err != nil {
+		return nil, err
+	}
+
+	return etcdmemberclient.NewForConfig(cfg)
+}
+
 // WaitForNodeReady wait that we see the given node in "Ready" state in kubernetes API
 func (s *BootlooseSuite) WaitForNodeReady(name string, kc kubernetes.Interface) error {
 	s.T().Logf("waiting to see %s ready in kube API", name)

inttest/etcdmember/etcdmember_test.go

@@ -20,13 +20,15 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"sync"
 	"testing"
 	"time"
 
 	"github.com/k0sproject/k0s/inttest/common"
 	"github.com/stretchr/testify/suite"
 
 	etcdv1beta1 "github.com/k0sproject/k0s/pkg/apis/etcd/v1beta1"
+	"github.com/k0sproject/k0s/pkg/kubernetes/watch"
 )
 
 type EtcdMemberSuite struct {
@@ -76,18 +78,46 @@ func (s *EtcdMemberSuite) TestDeregistration() {
 	kc, err := s.KubeClient(s.ControllerNode(0))
 	s.Require().NoError(err)
 
+	etcdMemberClient, err := s.EtcdMemberClient(s.ControllerNode(0))
+
 	// Check each node is present in the etcd cluster and reports joined state
 	expectedObjects := []string{"controller0", "controller1", "controller2"}
+	var wg sync.WaitGroup
 	for i, obj := range expectedObjects {
-		em := &etcdv1beta1.EtcdMember{}
-		err = kc.RESTClient().Get().AbsPath(fmt.Sprintf(basePath, obj)).Do(s.Context()).Into(em)
-		s.Require().NoError(err)
-		s.Require().Equal(em.PeerAddress, s.GetControllerIPAddress(i))
-		c := em.Status.GetCondition(etcdv1beta1.ConditionTypeJoined)
-		s.Require().NotEmpty(c)
-		s.Require().Equal(etcdv1beta1.ConditionTrue, c.Status)
-	}
+		wg.Add(1)
+		ctrlName := obj
+		ctrlIndex := i
+		go func() {
+			defer wg.Done()
+			s.T().Logf("verifying initial status of %s", ctrlName)
+			em := &etcdv1beta1.EtcdMember{}
+			ctx, cancel := context.WithTimeout(s.Context(), 20*time.Second)
+			defer cancel()
+
+			err = watch.EtcdMembers(etcdMemberClient.EtcdMembers()).
+				WithObjectName(ctrlName).
+				WithErrorCallback(common.RetryWatchErrors(s.T().Logf)).
+				Until(ctx, func(item *etcdv1beta1.EtcdMember) (done bool, err error) {
+					c := item.Status.GetCondition(etcdv1beta1.ConditionTypeJoined)
+					if c != nil {
+						// We have the condition so we can bail out
+						em = item
+					}
+					return c != nil, nil
+				})
+
+			// We've got the condition, verify status details
+			s.Require().NoError(err)
+			s.Require().Equal(em.PeerAddress, s.GetControllerIPAddress(ctrlIndex))
+			c := em.Status.GetCondition(etcdv1beta1.ConditionTypeJoined)
+			s.Require().NotEmpty(c)
+			s.Require().Equal(etcdv1beta1.ConditionTrue, c.Status)
+		}()
 
+	}
+	s.T().Log("waiting to see correct statuses on EtcdMembers")
+	wg.Wait()
+	s.T().Log("All statuses found")
 	// Make one of the nodes leave
 	s.leaveNode("controller2")