diff --git a/inttest/common/bootloosesuite.go b/inttest/common/bootloosesuite.go index e5fc3b248271..797c6df56d83 100644 --- a/inttest/common/bootloosesuite.go +++ b/inttest/common/bootloosesuite.go @@ -43,6 +43,7 @@ import ( "github.com/k0sproject/k0s/internal/pkg/file" apclient "github.com/k0sproject/k0s/pkg/client/clientset" + etcdmemberclient "github.com/k0sproject/k0s/pkg/client/clientset/typed/etcd/v1beta1" "github.com/k0sproject/k0s/pkg/constant" "github.com/k0sproject/k0s/pkg/k0scontext" "github.com/k0sproject/k0s/pkg/kubernetes/watch" @@ -908,6 +909,16 @@ func (s *BootlooseSuite) ExtensionsClient(node string, k0sKubeconfigArgs ...stri return extclient.NewForConfig(cfg) } +// EtcdMemberClient return a client for accessing etcd member CRDs +func (s *BootlooseSuite) EtcdMemberClient(node string, k0sKubeconfigArgs ...string) (*etcdmemberclient.EtcdV1beta1Client, error) { + cfg, err := s.GetKubeConfig(node, k0sKubeconfigArgs...) + if err != nil { + return nil, err + } + + return etcdmemberclient.NewForConfig(cfg) +} + // WaitForNodeReady wait that we see the given node in "Ready" state in kubernetes API func (s *BootlooseSuite) WaitForNodeReady(name string, kc kubernetes.Interface) error { s.T().Logf("waiting to see %s ready in kube API", name) diff --git a/inttest/etcdmember/etcdmember_test.go b/inttest/etcdmember/etcdmember_test.go index df606223cc57..9eb6a333d4d3 100644 --- a/inttest/etcdmember/etcdmember_test.go +++ b/inttest/etcdmember/etcdmember_test.go @@ -20,6 +20,7 @@ import ( "context" "encoding/json" "fmt" + "sync" "testing" "time" @@ -27,6 +28,7 @@ import ( "github.com/stretchr/testify/suite" etcdv1beta1 "github.com/k0sproject/k0s/pkg/apis/etcd/v1beta1" + "github.com/k0sproject/k0s/pkg/kubernetes/watch" ) type EtcdMemberSuite struct { @@ -76,18 +78,46 @@ func (s *EtcdMemberSuite) TestDeregistration() { kc, err := s.KubeClient(s.ControllerNode(0)) s.Require().NoError(err) + etcdMemberClient, err := s.EtcdMemberClient(s.ControllerNode(0)) + // Check each node is present in the etcd cluster and reports joined state expectedObjects := []string{"controller0", "controller1", "controller2"} + var wg sync.WaitGroup for i, obj := range expectedObjects { - em := &etcdv1beta1.EtcdMember{} - err = kc.RESTClient().Get().AbsPath(fmt.Sprintf(basePath, obj)).Do(s.Context()).Into(em) - s.Require().NoError(err) - s.Require().Equal(em.PeerAddress, s.GetControllerIPAddress(i)) - c := em.Status.GetCondition(etcdv1beta1.ConditionTypeJoined) - s.Require().NotEmpty(c) - s.Require().Equal(etcdv1beta1.ConditionTrue, c.Status) - } + wg.Add(1) + ctrlName := obj + ctrlIndex := i + go func() { + defer wg.Done() + s.T().Logf("verifying initial status of %s", ctrlName) + em := &etcdv1beta1.EtcdMember{} + ctx, cancel := context.WithTimeout(s.Context(), 20*time.Second) + defer cancel() + + err = watch.EtcdMembers(etcdMemberClient.EtcdMembers()). + WithObjectName(ctrlName). + WithErrorCallback(common.RetryWatchErrors(s.T().Logf)). + Until(ctx, func(item *etcdv1beta1.EtcdMember) (done bool, err error) { + c := item.Status.GetCondition(etcdv1beta1.ConditionTypeJoined) + if c != nil { + // We have the condition so we can bail out + em = item + } + return c != nil, nil + }) + + // We've got the condition, verify status details + s.Require().NoError(err) + s.Require().Equal(em.PeerAddress, s.GetControllerIPAddress(ctrlIndex)) + c := em.Status.GetCondition(etcdv1beta1.ConditionTypeJoined) + s.Require().NotEmpty(c) + s.Require().Equal(etcdv1beta1.ConditionTrue, c.Status) + }() + } + s.T().Log("waiting to see correct statuses on EtcdMembers") + wg.Wait() + s.T().Log("All statuses found") // Make one of the nodes leave s.leaveNode("controller2")