From 3392f6d358d52121da88b48413a8422115edf938 Mon Sep 17 00:00:00 2001 From: Jussi Nummelin Date: Thu, 28 Mar 2024 10:38:42 +0200 Subject: [PATCH] This PR adds new functionality to be able to manage Etcd peers using CRDs. One of the challenges for automation (Terraform, cluster API, k0sctl) when removing nodes is that when removing a controller node one has to also remove the Etcd peer entry to it in Etcd cluster. In this PR there's few things going on: - Each controller creates their own `EtcdMember` object with memberID and peer address fields - Each controller watches the changes on `EtcdMember` objects - When the leader controller sees `EtcdMember` object being deleted it will remove the corresponding peer from Etcd cluster. - If the peer removal fails for some reason it will write an event into `kube-system` namespace Very DRAFT still, need to refactor some of the code for better readability and error handling. Signed-off-by: Jussi Nummelin Temp commit for state mgmt with conditions Signed-off-by: Jussi Nummelin Fix timing issue in EtcdMember inttest Signed-off-by: Jussi Nummelin Finetune EtcdMember.joinstatus to be validated as enum Co-authored-by: Tom Wieczorek Signed-off-by: Jussi Nummelin Update pkg/component/controller/etcd_member_reconciler.go Co-authored-by: Tom Wieczorek Signed-off-by: Jussi Nummelin Use net.JoinHostPort Signed-off-by: Jussi Nummelin Fix JoinHostPort Signed-off-by: Jussi Nummelin --- Makefile | 8 +- cmd/controller/controller.go | 13 + internal/testutil/kube_client.go | 9 + inttest/Makefile.variables | 1 + inttest/common/bootloosesuite.go | 23 ++ inttest/common/util.go | 10 + inttest/etcdmember/etcdmember_test.go | 216 +++++++++++ pkg/apis/etcd/doc.go | 18 + pkg/apis/etcd/v1beta1/doc.go | 21 ++ pkg/apis/etcd/v1beta1/types.go | 173 +++++++++ pkg/apis/etcd/v1beta1/types_test.go | 60 +++ .../etcd/v1beta1/zz_generated.deepcopy.go | 121 ++++++ pkg/client/clientset/clientset.go | 13 + .../clientset/fake/clientset_generated.go | 7 + pkg/client/clientset/fake/register.go | 2 + pkg/client/clientset/scheme/register.go | 2 + .../clientset/typed/etcd/v1beta1/doc.go | 20 + .../typed/etcd/v1beta1/etcd_client.go | 107 ++++++ .../typed/etcd/v1beta1/etcdmember.go | 168 +++++++++ .../clientset/typed/etcd/v1beta1/fake/doc.go | 20 + .../etcd/v1beta1/fake/fake_etcd_client.go | 40 ++ .../etcd/v1beta1/fake/fake_etcdmember.go | 124 ++++++ .../typed/etcd/v1beta1/generated_expansion.go | 21 ++ .../controller/etcd_member_reconciler.go | 352 ++++++++++++++++++ pkg/kubernetes/client.go | 34 +- pkg/kubernetes/watch/k0s.go | 5 + .../etcd.k0sproject.io_etcdmembers.yaml | 100 +++++ 27 files changed, 1682 insertions(+), 6 deletions(-) create mode 100644 inttest/etcdmember/etcdmember_test.go create mode 100644 pkg/apis/etcd/doc.go create mode 100644 pkg/apis/etcd/v1beta1/doc.go create mode 100644 pkg/apis/etcd/v1beta1/types.go create mode 100644 pkg/apis/etcd/v1beta1/types_test.go create mode 100644 pkg/apis/etcd/v1beta1/zz_generated.deepcopy.go create mode 100644 pkg/client/clientset/typed/etcd/v1beta1/doc.go create mode 100644 pkg/client/clientset/typed/etcd/v1beta1/etcd_client.go create mode 100644 pkg/client/clientset/typed/etcd/v1beta1/etcdmember.go create mode 100644 pkg/client/clientset/typed/etcd/v1beta1/fake/doc.go create mode 100644 pkg/client/clientset/typed/etcd/v1beta1/fake/fake_etcd_client.go create mode 100644 pkg/client/clientset/typed/etcd/v1beta1/fake/fake_etcdmember.go create mode 100644 pkg/client/clientset/typed/etcd/v1beta1/generated_expansion.go create mode 100644 pkg/component/controller/etcd_member_reconciler.go create mode 100644 static/manifests/etcd/CustomResourceDefinition/etcd.k0sproject.io_etcdmembers.yaml diff --git a/Makefile b/Makefile index f3dd2730fa6e..bd53cb1309cb 100644 --- a/Makefile +++ b/Makefile @@ -124,7 +124,12 @@ controllergen_targets += pkg/apis/autopilot/v1beta2/.controller-gen.stamp pkg/apis/autopilot/v1beta2/.controller-gen.stamp: $(shell find pkg/apis/autopilot/v1beta2/ -maxdepth 1 -type f -name \*.go) pkg/apis/autopilot/v1beta2/.controller-gen.stamp: gen_output_dir = autopilot +controllergen_targets += pkg/apis/etcd/v1beta1/.controller-gen.stamp +pkg/apis/etcd/v1beta1/.controller-gen.stamp: $(shell find pkg/apis/etcd/v1beta1/ -maxdepth 1 -type f -name \*.go) +pkg/apis/etcd/v1beta1/.controller-gen.stamp: gen_output_dir = etcd + codegen_targets += $(controllergen_targets) + pkg/apis/%/.controller-gen.stamp: .k0sbuild.docker-image.k0s hack/tools/boilerplate.go.txt hack/tools/Makefile.variables rm -rf 'static/manifests/$(gen_output_dir)/CustomResourceDefinition' mkdir -p 'static/manifests/$(gen_output_dir)' @@ -137,7 +142,7 @@ pkg/apis/%/.controller-gen.stamp: .k0sbuild.docker-image.k0s hack/tools/boilerpl && mv -f -- "$$gendir"/zz_generated.deepcopy.go '$(dir $@).' touch -- '$@' -clientset_input_dirs := pkg/apis/autopilot/v1beta2 pkg/apis/k0s/v1beta1 pkg/apis/helm/v1beta1 +clientset_input_dirs := pkg/apis/autopilot/v1beta2 pkg/apis/k0s/v1beta1 pkg/apis/helm/v1beta1 pkg/apis/etcd/v1beta1 codegen_targets += pkg/client/clientset/.client-gen.stamp pkg/client/clientset/.client-gen.stamp: $(shell find $(clientset_input_dirs) -type f -name \*.go -not -name \*_test.go -not -name zz_\*) pkg/client/clientset/.client-gen.stamp: .k0sbuild.docker-image.k0s hack/tools/boilerplate.go.txt embedded-bins/Makefile.variables @@ -163,6 +168,7 @@ static/zz_generated_assets.go: .k0sbuild.docker-image.k0s hack/tools/Makefile.va static/manifests/helm/CustomResourceDefinition/... \ static/manifests/v1beta1/CustomResourceDefinition/... \ static/manifests/autopilot/CustomResourceDefinition/... \ + static/manifests/etcd/CustomResourceDefinition/... \ static/manifests/calico/... \ static/manifests/windows/... \ static/misc/... diff --git a/cmd/controller/controller.go b/cmd/controller/controller.go index 51fa8a7b7e7a..13d29d0fce39 100644 --- a/cmd/controller/controller.go +++ b/cmd/controller/controller.go @@ -325,6 +325,19 @@ func (c *command) start(ctx context.Context) error { CertManager: worker.NewCertificateManager(ctx, c.K0sVars.KubeletAuthConfigPath), }) + if nodeConfig.Spec.Storage.Type == v1beta1.EtcdStorageType && nodeConfig.Spec.Storage.Etcd.ExternalCluster == nil { + etcdReconciler, err := controller.NewEtcdMemberReconciler(adminClientFactory, c.K0sVars, nodeConfig.Spec.Storage.Etcd, leaderElector) + if err != nil { + return err + } + etcdCRDSaver, err := controller.NewManifestsSaver("etcd-member", c.K0sVars.DataDir) + if err != nil { + return fmt.Errorf("failed to initialize etcd-member manifests saver: %w", err) + } + clusterComponents.Add(ctx, controller.NewCRD(etcdCRDSaver, []string{"etcd"})) + nodeComponents.Add(ctx, etcdReconciler) + } + perfTimer.Checkpoint("starting-certificates-init") certs := &Certificates{ ClusterSpec: nodeConfig.Spec, diff --git a/internal/testutil/kube_client.go b/internal/testutil/kube_client.go index 5dedba4d38e0..966ae417f2cb 100644 --- a/internal/testutil/kube_client.go +++ b/internal/testutil/kube_client.go @@ -18,6 +18,7 @@ package testutil import ( "fmt" + "k8s.io/client-go/rest" "k8s.io/apimachinery/pkg/runtime" @@ -32,9 +33,13 @@ import ( restfake "k8s.io/client-go/rest/fake" kubetesting "k8s.io/client-go/testing" + etcdMemberClient "github.com/k0sproject/k0s/pkg/client/clientset/typed/etcd/v1beta1" cfgClient "github.com/k0sproject/k0s/pkg/client/clientset/typed/k0s/v1beta1" + kubeutil "github.com/k0sproject/k0s/pkg/kubernetes" ) +var _ kubeutil.ClientFactoryInterface = (*FakeClientFactory)(nil) + // NewFakeClientFactory creates new client factory which uses internally only the kube fake client interface func NewFakeClientFactory(objects ...runtime.Object) FakeClientFactory { rawDiscovery := &discoveryfake.FakeDiscovery{Fake: &kubetesting.Fake{}} @@ -89,3 +94,7 @@ func (f FakeClientFactory) GetRESTClient() (rest.Interface, error) { func (f FakeClientFactory) GetRESTConfig() *rest.Config { return &rest.Config{} } + +func (f FakeClientFactory) GetEtcdMemberClient() (etcdMemberClient.EtcdMemberInterface, error) { + return nil, fmt.Errorf("NOT IMPLEMENTED") +} diff --git a/inttest/Makefile.variables b/inttest/Makefile.variables index d436fb689147..da03d9101ddf 100644 --- a/inttest/Makefile.variables +++ b/inttest/Makefile.variables @@ -58,3 +58,4 @@ smoketests := \ check-singlenode \ check-statussocket \ check-upgrade \ + check-etcdmember \ diff --git a/inttest/common/bootloosesuite.go b/inttest/common/bootloosesuite.go index c57f8e655c0a..c684439d822e 100644 --- a/inttest/common/bootloosesuite.go +++ b/inttest/common/bootloosesuite.go @@ -43,6 +43,7 @@ import ( "github.com/k0sproject/k0s/internal/pkg/file" apclient "github.com/k0sproject/k0s/pkg/client/clientset" + etcdmemberclient "github.com/k0sproject/k0s/pkg/client/clientset/typed/etcd/v1beta1" "github.com/k0sproject/k0s/pkg/constant" "github.com/k0sproject/k0s/pkg/k0scontext" "github.com/k0sproject/k0s/pkg/kubernetes/watch" @@ -774,6 +775,18 @@ func (s *BootlooseSuite) StopController(name string) error { return s.launchDelegate.StopController(s.Context(), ssh) } +func (s *BootlooseSuite) RestartController(name string) error { + ssh, err := s.SSH(s.Context(), name) + s.Require().NoError(err) + defer ssh.Disconnect() + s.T().Log("killing k0s") + err = s.launchDelegate.StopController(s.Context(), ssh) + if err != nil { + return err + } + return s.launchDelegate.StartController(s.Context(), ssh) +} + func (s *BootlooseSuite) StartController(name string) error { ssh, err := s.SSH(s.Context(), name) s.Require().NoError(err) @@ -896,6 +909,16 @@ func (s *BootlooseSuite) ExtensionsClient(node string, k0sKubeconfigArgs ...stri return extclient.NewForConfig(cfg) } +// EtcdMemberClient return a client for accessing etcd member CRDs +func (s *BootlooseSuite) EtcdMemberClient(node string, k0sKubeconfigArgs ...string) (*etcdmemberclient.EtcdV1beta1Client, error) { + cfg, err := s.GetKubeConfig(node, k0sKubeconfigArgs...) + if err != nil { + return nil, err + } + + return etcdmemberclient.NewForConfig(cfg) +} + // WaitForNodeReady wait that we see the given node in "Ready" state in kubernetes API func (s *BootlooseSuite) WaitForNodeReady(name string, kc kubernetes.Interface) error { s.T().Logf("waiting to see %s ready in kube API", name) diff --git a/inttest/common/util.go b/inttest/common/util.go index 2e220cec7f5d..6d3e618b1702 100644 --- a/inttest/common/util.go +++ b/inttest/common/util.go @@ -334,6 +334,16 @@ func VerifyKubeletMetrics(ctx context.Context, kc *kubernetes.Clientset, node st }) } +func ResetNode(name string, suite *BootlooseSuite) error { + ssh, err := suite.SSH(suite.Context(), name) + if err != nil { + return err + } + defer ssh.Disconnect() + _, err = ssh.ExecWithOutput(suite.Context(), fmt.Sprintf("%s reset --debug", suite.K0sFullPath)) + return err +} + // Retrieves the LogfFn stored in context, falling back to use testing.T's Logf // if the context has a *testing.T or logrus's Infof as a last resort. func logfFrom(ctx context.Context) LogfFn { diff --git a/inttest/etcdmember/etcdmember_test.go b/inttest/etcdmember/etcdmember_test.go new file mode 100644 index 000000000000..00cd462bf4e5 --- /dev/null +++ b/inttest/etcdmember/etcdmember_test.go @@ -0,0 +1,216 @@ +/* +Copyright 2024 k0s authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package hacontrolplane + +import ( + "context" + "encoding/json" + "fmt" + "sync" + "testing" + "time" + + "github.com/k0sproject/k0s/inttest/common" + "github.com/stretchr/testify/suite" + + etcdv1beta1 "github.com/k0sproject/k0s/pkg/apis/etcd/v1beta1" + "github.com/k0sproject/k0s/pkg/kubernetes/watch" +) + +type EtcdMemberSuite struct { + common.BootlooseSuite +} + +func (s *EtcdMemberSuite) getMembers(fromControllerIdx int) map[string]string { + // our etcd instances doesn't listen on public IP, so test is performed by calling CLI tools over ssh + // which in general even makes sense, we can test tooling as well + sshCon, err := s.SSH(s.Context(), s.ControllerNode(fromControllerIdx)) + s.Require().NoError(err) + defer sshCon.Disconnect() + output, err := sshCon.ExecWithOutput(s.Context(), "/usr/local/bin/k0s etcd member-list 2>/dev/null") + s.T().Logf("k0s etcd member-list output: %s", output) + s.Require().NoError(err) + + members := struct { + Members map[string]string `json:"members"` + }{} + + s.NoError(json.Unmarshal([]byte(output), &members)) + return members.Members +} + +func (s *EtcdMemberSuite) TestDeregistration() { + + var joinToken string + for idx := 0; idx < s.BootlooseSuite.ControllerCount; idx++ { + s.Require().NoError(s.WaitForSSH(s.ControllerNode(idx), 2*time.Minute, 1*time.Second)) + + // Note that the token is intentionally empty for the first controller + s.Require().NoError(s.InitController(idx, joinToken)) + s.Require().NoError(s.WaitJoinAPI(s.ControllerNode(idx))) + s.Require().NoError(s.WaitForKubeAPI(s.ControllerNode(idx))) + // With the primary controller running, create the join token for subsequent controllers. + if idx == 0 { + token, err := s.GetJoinToken("controller") + s.Require().NoError(err) + joinToken = token + } + } + + // Final sanity -- ensure all nodes see each other according to etcd + for idx := 0; idx < s.BootlooseSuite.ControllerCount; idx++ { + s.Require().Len(s.GetMembers(idx), s.BootlooseSuite.ControllerCount) + } + kc, err := s.KubeClient(s.ControllerNode(0)) + s.Require().NoError(err) + + etcdMemberClient, err := s.EtcdMemberClient(s.ControllerNode(0)) + + // Check each node is present in the etcd cluster and reports joined state + expectedObjects := []string{"controller0", "controller1", "controller2"} + var wg sync.WaitGroup + for i, obj := range expectedObjects { + wg.Add(1) + ctrlName := obj + ctrlIndex := i + go func() { + defer wg.Done() + s.T().Logf("verifying initial status of %s", ctrlName) + em := &etcdv1beta1.EtcdMember{} + ctx, cancel := context.WithTimeout(s.Context(), 30*time.Second) + defer cancel() + + err = watch.EtcdMembers(etcdMemberClient.EtcdMembers()). + WithObjectName(ctrlName). + WithErrorCallback(common.RetryWatchErrors(s.T().Logf)). + Until(ctx, func(item *etcdv1beta1.EtcdMember) (done bool, err error) { + c := item.Status.GetCondition(etcdv1beta1.ConditionTypeJoined) + if c != nil { + // We have the condition so we can bail out + em = item + } + return c != nil, nil + }) + + // We've got the condition, verify status details + s.Require().NoError(err) + s.Require().Equal(em.PeerAddress, s.GetControllerIPAddress(ctrlIndex)) + c := em.Status.GetCondition(etcdv1beta1.ConditionTypeJoined) + s.Require().NotEmpty(c) + s.Require().Equal(etcdv1beta1.ConditionTrue, c.Status) + }() + + } + s.T().Log("waiting to see correct statuses on EtcdMembers") + wg.Wait() + s.T().Log("All statuses found") + // Make one of the nodes leave + s.leaveNode("controller2") + + // Check that the node is gone from the etcd cluster according to etcd itself + members := s.getMembers(0) + s.Require().Len(members, s.BootlooseSuite.ControllerCount-1) + s.Require().NotContains(members, "controller2") + + // Make sure the EtcdMember CR status is successfully updated + em := s.getMember("controller2") + s.Require().Equal(em.Status.ReconcileStatus, "Success") + s.Require().Equal(etcdv1beta1.ConditionFalse, em.Status.GetCondition(etcdv1beta1.ConditionTypeJoined).Status) + + // Stop k0s and reset the node + s.Require().NoError(s.StopController(s.ControllerNode(2))) + s.Require().NoError(common.ResetNode(s.ControllerNode(2), &s.BootlooseSuite)) + + // Make the node rejoin + s.Require().NoError(s.InitController(2, joinToken)) + s.Require().NoError(s.WaitForKubeAPI(s.ControllerNode(2))) + + // Final sanity -- ensure all nodes see each other according to etcd + members = s.getMembers(0) + s.Require().Len(members, s.BootlooseSuite.ControllerCount) + s.Require().Contains(members, "controller2") + + // Check the CR is present again + em = s.getMember("controller2") + s.Require().Equal(em.PeerAddress, s.GetControllerIPAddress(2)) + s.Require().Equal(false, em.Leave) + s.Require().Equal(etcdv1beta1.ConditionTrue, em.Status.GetCondition(etcdv1beta1.ConditionTypeJoined).Status) + + // Check that after restarting the controller, the member is still present + s.Require().NoError(s.RestartController(s.ControllerNode(2))) + em = &etcdv1beta1.EtcdMember{} + err = kc.RESTClient().Get().AbsPath(fmt.Sprintf(basePath, "controller2")).Do(s.Context()).Into(em) + s.Require().NoError(err) + s.Require().Equal(em.PeerAddress, s.GetControllerIPAddress(2)) + +} + +const basePath = "apis/etcd.k0sproject.io/v1beta1/etcdmembers/%s" + +func (s *EtcdMemberSuite) leaveNode(name string) { + kc, err := s.KubeClient(s.ControllerNode(0)) + s.Require().NoError(err) + + // Patch the EtcdMember CR to set the Leave flag + path := fmt.Sprintf(basePath, name) + patch := []byte(`{"leave":true}`) + result := kc.RESTClient().Patch("application/merge-patch+json").AbsPath(path).Body(patch).Do(s.Context()) + + s.Require().NoError(result.Error()) + s.T().Logf("marked %s for leaving, waiting to see the state updated", name) + ctx, cancel := context.WithTimeout(s.Context(), 10*time.Second) + defer cancel() + + err = common.Poll(ctx, func(ctx context.Context) (done bool, err error) { + em := &etcdv1beta1.EtcdMember{} + err = kc.RESTClient().Get().AbsPath(fmt.Sprintf(basePath, name)).Do(s.Context()).Into(em) //DoRaw(s.Context()) + s.Require().NoError(err) + + c := em.Status.GetCondition(etcdv1beta1.ConditionTypeJoined) + if c == nil { + return false, nil + } + s.T().Logf("JoinStatus = %s, waiting for %s", c.Status, etcdv1beta1.ConditionFalse) + return c.Status == etcdv1beta1.ConditionFalse, nil + + }) + s.Require().NoError(err) + +} + +// getMember returns the etcd member CR for the given name +func (s *EtcdMemberSuite) getMember(name string) *etcdv1beta1.EtcdMember { + kc, err := s.KubeClient(s.ControllerNode(0)) + s.Require().NoError(err) + + em := &etcdv1beta1.EtcdMember{} + err = kc.RESTClient().Get().AbsPath(fmt.Sprintf(basePath, name)).Do(s.Context()).Into(em) + s.Require().NoError(err) + return em +} + +func TestEtcdMemberSuite(t *testing.T) { + s := EtcdMemberSuite{ + common.BootlooseSuite{ + ControllerCount: 3, + LaunchMode: common.LaunchModeOpenRC, + }, + } + + suite.Run(t, &s) + +} diff --git a/pkg/apis/etcd/doc.go b/pkg/apis/etcd/doc.go new file mode 100644 index 000000000000..993918f6413e --- /dev/null +++ b/pkg/apis/etcd/doc.go @@ -0,0 +1,18 @@ +// Copyright 2024 k0s authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package k0s contains API Schema definitions for the etcd.k0sproject.io API group. +package etcd + +const GroupName = "etcd.k0sproject.io" diff --git a/pkg/apis/etcd/v1beta1/doc.go b/pkg/apis/etcd/v1beta1/doc.go new file mode 100644 index 000000000000..3c439402de16 --- /dev/null +++ b/pkg/apis/etcd/v1beta1/doc.go @@ -0,0 +1,21 @@ +// Copyright 2024 k0s authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +kubebuilder:object:generate=true +// +groupName=etcd.k0sproject.io + +// Package k0s contains API Schema definitions for the etcd.k0sproject.io API group. +package v1beta1 + +const Version = "v1beta1" diff --git a/pkg/apis/etcd/v1beta1/types.go b/pkg/apis/etcd/v1beta1/types.go new file mode 100644 index 000000000000..c33e75382cfe --- /dev/null +++ b/pkg/apis/etcd/v1beta1/types.go @@ -0,0 +1,173 @@ +/* +Copyright 2024 k0s authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "time" + + "github.com/k0sproject/k0s/pkg/apis/etcd" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var ( + // GroupVersion is group version used to register these objects + SchemeGroupVersion = schema.GroupVersion{Group: etcd.GroupName, Version: Version} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: SchemeGroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) + +func init() { + SchemeBuilder.Register(&EtcdMember{}, &EtcdMemberList{}) +} + +// EtcdMember describes the nodes etcd membership status +// +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Cluster +// +kubebuilder:printcolumn:name="PeerAddress",type=string,JSONPath=`.peerAddress` +// +kubebuilder:printcolumn:name="MemberID",type=string,JSONPath=`.memberID` +// +kubebuilder:printcolumn:name="Joined",type=string,JSONPath=`.status.conditions[?(@.type=="Joined")].status` +// +kubebuilder:printcolumn:name="ReconcileStatus",type=string,JSONPath=`.status.reconcileStatus` +// +genclient +// +genclient:onlyVerbs=create,delete,list,get,watch,update,updateStatus,patch +// +genclient:nonNamespaced +type EtcdMember struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + // PeerAddress is the address of the etcd peer + PeerAddress string `json:"peerAddress"` + // MemberID is the unique identifier of the etcd member. + // The hex form ID is stored as string + // +kubebuilder:validation:Pattern="^[a-fA-F0-9]+$" + MemberID string `json:"memberID"` + + // Leave is a flag to indicate that the member should be removed from the cluster + Leave bool `json:"leave,omitempty"` + + Status Status `json:"status,omitempty"` +} + +// +kubebuilder:validation:Enum=Joined;Left +// +kubebuilder:validation:Enum=Joined;Left +type JoinStatus string + +const JoinStatusJoined JoinStatus = "Joined" +const JoinStatusLeft JoinStatus = "Left" + +type Status struct { + ReconcileStatus string `json:"reconcileStatus,omitempty"` + Message string `json:"message,omitempty"` + // JoinStatus JoinStatus `json:"joinStatus,omitempty"` + + Conditions []JoinCondition `json:"conditions,omitempty"` +} + +type ConditionType string + +const ( + ConditionTypeJoined ConditionType = "Joined" +) + +type ConditionStatus string + +// These are valid condition statuses. "ConditionTrue" means a resource is in the condition. +// "ConditionFalse" means a resource is not in the condition. "ConditionUnknown" means kubernetes +// can't decide if a resource is in the condition or not. +const ( + ConditionTrue ConditionStatus = "True" + ConditionFalse ConditionStatus = "False" + ConditionUnknown ConditionStatus = "Unknown" +) + +type JoinCondition struct { + Type ConditionType `json:"type"` + // +kubebuilder + Status ConditionStatus `json:"status"` + // Last time the condition transitioned from one status to another. + // +optional + LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"` + // Human-readable message indicating details about last transition. + // +optional + Message string `json:"message,omitempty" protobuf:"bytes,6,opt,name=message"` +} + +func (s *Status) GetCondition(conditionType ConditionType) *JoinCondition { + for _, c := range s.Conditions { + if c.Type == conditionType { + return &c + } + } + return nil +} + +func (s *Status) SetCondition(t ConditionType, status ConditionStatus, msg string, time time.Time) { + var joinCondition JoinCondition + for i, j := range s.Conditions { + if j.Type == t { + jc := &s.Conditions[i] + // We found the matchin type, update it + // Also if the status changes, update the timestamp + if jc.Status != status { + jc.LastTransitionTime = metav1.NewTime(time) + } + jc.Status = status + jc.Message = msg + + return + } + } + + joinCondition = JoinCondition{ + Type: t, + Status: status, + Message: msg, + LastTransitionTime: metav1.Now(), + } + + // We did not find the right typed condition + if len(s.Conditions) == 0 { + s.Conditions = []JoinCondition{joinCondition} + } else { + s.Conditions = append(s.Conditions, joinCondition) + } +} + +// EtcdMemberList contains a list of EtcdMembers +// +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Cluster +type EtcdMemberList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []EtcdMember `json:"items"` +} + +// func foo() { +// p := &corev1.Pod{} + +// p.Status.Conditions. + +// } diff --git a/pkg/apis/etcd/v1beta1/types_test.go b/pkg/apis/etcd/v1beta1/types_test.go new file mode 100644 index 000000000000..58b97a95b16b --- /dev/null +++ b/pkg/apis/etcd/v1beta1/types_test.go @@ -0,0 +1,60 @@ +/* +Copyright 2024 k0s authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestStatus_SetConditionWhenEmpty(t *testing.T) { + em := &EtcdMember{} + + em.Status.SetCondition(ConditionTypeJoined, ConditionTrue, "foobar", time.Now()) + + r := require.New(t) + r.Len(em.Status.Conditions, 1) + r.NotEmpty(em.Status.Conditions[0].LastTransitionTime) +} + +func TestStatus_SetConditionChange(t *testing.T) { + start := time.Now() + em := &EtcdMember{ + Status: Status{ + Conditions: []JoinCondition{ + { + Type: ConditionTypeJoined, + Status: ConditionTrue, + LastTransitionTime: metav1.NewTime(start), + }, + }, + }, + } + + em.Status.SetCondition(ConditionTypeJoined, ConditionFalse, "foobar", start.Add(12*time.Second)) + + r := require.New(t) + c := em.Status.Conditions[0] + r.Len(em.Status.Conditions, 1) + t.Logf("original time: %s", metav1.NewTime(start)) + t.Logf("latest time: %s", c.LastTransitionTime.Time) + r.True(start.Before(c.LastTransitionTime.Time)) + r.Equal(ConditionFalse, c.Status) +} diff --git a/pkg/apis/etcd/v1beta1/zz_generated.deepcopy.go b/pkg/apis/etcd/v1beta1/zz_generated.deepcopy.go new file mode 100644 index 000000000000..59183e2b45c4 --- /dev/null +++ b/pkg/apis/etcd/v1beta1/zz_generated.deepcopy.go @@ -0,0 +1,121 @@ +//go:build !ignore_autogenerated + +/* +Copyright k0s authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1beta1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMember) DeepCopyInto(out *EtcdMember) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMember. +func (in *EtcdMember) DeepCopy() *EtcdMember { + if in == nil { + return nil + } + out := new(EtcdMember) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *EtcdMember) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMemberList) DeepCopyInto(out *EtcdMemberList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]EtcdMember, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMemberList. +func (in *EtcdMemberList) DeepCopy() *EtcdMemberList { + if in == nil { + return nil + } + out := new(EtcdMemberList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *EtcdMemberList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *JoinCondition) DeepCopyInto(out *JoinCondition) { + *out = *in + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JoinCondition. +func (in *JoinCondition) DeepCopy() *JoinCondition { + if in == nil { + return nil + } + out := new(JoinCondition) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Status) DeepCopyInto(out *Status) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]JoinCondition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Status. +func (in *Status) DeepCopy() *Status { + if in == nil { + return nil + } + out := new(Status) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/client/clientset/clientset.go b/pkg/client/clientset/clientset.go index beb51fece218..941905801941 100644 --- a/pkg/client/clientset/clientset.go +++ b/pkg/client/clientset/clientset.go @@ -23,6 +23,7 @@ import ( "net/http" autopilotv1beta2 "github.com/k0sproject/k0s/pkg/client/clientset/typed/autopilot/v1beta2" + etcdv1beta1 "github.com/k0sproject/k0s/pkg/client/clientset/typed/etcd/v1beta1" helmv1beta1 "github.com/k0sproject/k0s/pkg/client/clientset/typed/helm/v1beta1" k0sv1beta1 "github.com/k0sproject/k0s/pkg/client/clientset/typed/k0s/v1beta1" discovery "k8s.io/client-go/discovery" @@ -33,6 +34,7 @@ import ( type Interface interface { Discovery() discovery.DiscoveryInterface AutopilotV1beta2() autopilotv1beta2.AutopilotV1beta2Interface + EtcdV1beta1() etcdv1beta1.EtcdV1beta1Interface HelmV1beta1() helmv1beta1.HelmV1beta1Interface K0sV1beta1() k0sv1beta1.K0sV1beta1Interface } @@ -41,6 +43,7 @@ type Interface interface { type Clientset struct { *discovery.DiscoveryClient autopilotV1beta2 *autopilotv1beta2.AutopilotV1beta2Client + etcdV1beta1 *etcdv1beta1.EtcdV1beta1Client helmV1beta1 *helmv1beta1.HelmV1beta1Client k0sV1beta1 *k0sv1beta1.K0sV1beta1Client } @@ -50,6 +53,11 @@ func (c *Clientset) AutopilotV1beta2() autopilotv1beta2.AutopilotV1beta2Interfac return c.autopilotV1beta2 } +// EtcdV1beta1 retrieves the EtcdV1beta1Client +func (c *Clientset) EtcdV1beta1() etcdv1beta1.EtcdV1beta1Interface { + return c.etcdV1beta1 +} + // HelmV1beta1 retrieves the HelmV1beta1Client func (c *Clientset) HelmV1beta1() helmv1beta1.HelmV1beta1Interface { return c.helmV1beta1 @@ -108,6 +116,10 @@ func NewForConfigAndClient(c *rest.Config, httpClient *http.Client) (*Clientset, if err != nil { return nil, err } + cs.etcdV1beta1, err = etcdv1beta1.NewForConfigAndClient(&configShallowCopy, httpClient) + if err != nil { + return nil, err + } cs.helmV1beta1, err = helmv1beta1.NewForConfigAndClient(&configShallowCopy, httpClient) if err != nil { return nil, err @@ -138,6 +150,7 @@ func NewForConfigOrDie(c *rest.Config) *Clientset { func New(c rest.Interface) *Clientset { var cs Clientset cs.autopilotV1beta2 = autopilotv1beta2.New(c) + cs.etcdV1beta1 = etcdv1beta1.New(c) cs.helmV1beta1 = helmv1beta1.New(c) cs.k0sV1beta1 = k0sv1beta1.New(c) diff --git a/pkg/client/clientset/fake/clientset_generated.go b/pkg/client/clientset/fake/clientset_generated.go index d7006bde0172..4a175fa9c6c9 100644 --- a/pkg/client/clientset/fake/clientset_generated.go +++ b/pkg/client/clientset/fake/clientset_generated.go @@ -22,6 +22,8 @@ import ( clientset "github.com/k0sproject/k0s/pkg/client/clientset" autopilotv1beta2 "github.com/k0sproject/k0s/pkg/client/clientset/typed/autopilot/v1beta2" fakeautopilotv1beta2 "github.com/k0sproject/k0s/pkg/client/clientset/typed/autopilot/v1beta2/fake" + etcdv1beta1 "github.com/k0sproject/k0s/pkg/client/clientset/typed/etcd/v1beta1" + fakeetcdv1beta1 "github.com/k0sproject/k0s/pkg/client/clientset/typed/etcd/v1beta1/fake" helmv1beta1 "github.com/k0sproject/k0s/pkg/client/clientset/typed/helm/v1beta1" fakehelmv1beta1 "github.com/k0sproject/k0s/pkg/client/clientset/typed/helm/v1beta1/fake" k0sv1beta1 "github.com/k0sproject/k0s/pkg/client/clientset/typed/k0s/v1beta1" @@ -88,6 +90,11 @@ func (c *Clientset) AutopilotV1beta2() autopilotv1beta2.AutopilotV1beta2Interfac return &fakeautopilotv1beta2.FakeAutopilotV1beta2{Fake: &c.Fake} } +// EtcdV1beta1 retrieves the EtcdV1beta1Client +func (c *Clientset) EtcdV1beta1() etcdv1beta1.EtcdV1beta1Interface { + return &fakeetcdv1beta1.FakeEtcdV1beta1{Fake: &c.Fake} +} + // HelmV1beta1 retrieves the HelmV1beta1Client func (c *Clientset) HelmV1beta1() helmv1beta1.HelmV1beta1Interface { return &fakehelmv1beta1.FakeHelmV1beta1{Fake: &c.Fake} diff --git a/pkg/client/clientset/fake/register.go b/pkg/client/clientset/fake/register.go index 6a20e119b37f..94d1b64e4eec 100644 --- a/pkg/client/clientset/fake/register.go +++ b/pkg/client/clientset/fake/register.go @@ -20,6 +20,7 @@ package fake import ( autopilotv1beta2 "github.com/k0sproject/k0s/pkg/apis/autopilot/v1beta2" + etcdv1beta1 "github.com/k0sproject/k0s/pkg/apis/etcd/v1beta1" helmv1beta1 "github.com/k0sproject/k0s/pkg/apis/helm/v1beta1" k0sv1beta1 "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -34,6 +35,7 @@ var codecs = serializer.NewCodecFactory(scheme) var localSchemeBuilder = runtime.SchemeBuilder{ autopilotv1beta2.AddToScheme, + etcdv1beta1.AddToScheme, helmv1beta1.AddToScheme, k0sv1beta1.AddToScheme, } diff --git a/pkg/client/clientset/scheme/register.go b/pkg/client/clientset/scheme/register.go index 48310aaf7f30..27d3cc64cc11 100644 --- a/pkg/client/clientset/scheme/register.go +++ b/pkg/client/clientset/scheme/register.go @@ -20,6 +20,7 @@ package scheme import ( autopilotv1beta2 "github.com/k0sproject/k0s/pkg/apis/autopilot/v1beta2" + etcdv1beta1 "github.com/k0sproject/k0s/pkg/apis/etcd/v1beta1" helmv1beta1 "github.com/k0sproject/k0s/pkg/apis/helm/v1beta1" k0sv1beta1 "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -34,6 +35,7 @@ var Codecs = serializer.NewCodecFactory(Scheme) var ParameterCodec = runtime.NewParameterCodec(Scheme) var localSchemeBuilder = runtime.SchemeBuilder{ autopilotv1beta2.AddToScheme, + etcdv1beta1.AddToScheme, helmv1beta1.AddToScheme, k0sv1beta1.AddToScheme, } diff --git a/pkg/client/clientset/typed/etcd/v1beta1/doc.go b/pkg/client/clientset/typed/etcd/v1beta1/doc.go new file mode 100644 index 000000000000..f6d3c8a5f3fc --- /dev/null +++ b/pkg/client/clientset/typed/etcd/v1beta1/doc.go @@ -0,0 +1,20 @@ +/* +Copyright k0s authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +// This package has the automatically generated typed clients. +package v1beta1 diff --git a/pkg/client/clientset/typed/etcd/v1beta1/etcd_client.go b/pkg/client/clientset/typed/etcd/v1beta1/etcd_client.go new file mode 100644 index 000000000000..e5a53664c4c9 --- /dev/null +++ b/pkg/client/clientset/typed/etcd/v1beta1/etcd_client.go @@ -0,0 +1,107 @@ +/* +Copyright k0s authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1beta1 + +import ( + "net/http" + + v1beta1 "github.com/k0sproject/k0s/pkg/apis/etcd/v1beta1" + "github.com/k0sproject/k0s/pkg/client/clientset/scheme" + rest "k8s.io/client-go/rest" +) + +type EtcdV1beta1Interface interface { + RESTClient() rest.Interface + EtcdMembersGetter +} + +// EtcdV1beta1Client is used to interact with features provided by the etcd.k0sproject.io group. +type EtcdV1beta1Client struct { + restClient rest.Interface +} + +func (c *EtcdV1beta1Client) EtcdMembers() EtcdMemberInterface { + return newEtcdMembers(c) +} + +// NewForConfig creates a new EtcdV1beta1Client for the given config. +// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), +// where httpClient was generated with rest.HTTPClientFor(c). +func NewForConfig(c *rest.Config) (*EtcdV1beta1Client, error) { + config := *c + if err := setConfigDefaults(&config); err != nil { + return nil, err + } + httpClient, err := rest.HTTPClientFor(&config) + if err != nil { + return nil, err + } + return NewForConfigAndClient(&config, httpClient) +} + +// NewForConfigAndClient creates a new EtcdV1beta1Client for the given config and http client. +// Note the http client provided takes precedence over the configured transport values. +func NewForConfigAndClient(c *rest.Config, h *http.Client) (*EtcdV1beta1Client, error) { + config := *c + if err := setConfigDefaults(&config); err != nil { + return nil, err + } + client, err := rest.RESTClientForConfigAndClient(&config, h) + if err != nil { + return nil, err + } + return &EtcdV1beta1Client{client}, nil +} + +// NewForConfigOrDie creates a new EtcdV1beta1Client for the given config and +// panics if there is an error in the config. +func NewForConfigOrDie(c *rest.Config) *EtcdV1beta1Client { + client, err := NewForConfig(c) + if err != nil { + panic(err) + } + return client +} + +// New creates a new EtcdV1beta1Client for the given RESTClient. +func New(c rest.Interface) *EtcdV1beta1Client { + return &EtcdV1beta1Client{c} +} + +func setConfigDefaults(config *rest.Config) error { + gv := v1beta1.SchemeGroupVersion + config.GroupVersion = &gv + config.APIPath = "/apis" + config.NegotiatedSerializer = scheme.Codecs.WithoutConversion() + + if config.UserAgent == "" { + config.UserAgent = rest.DefaultKubernetesUserAgent() + } + + return nil +} + +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *EtcdV1beta1Client) RESTClient() rest.Interface { + if c == nil { + return nil + } + return c.restClient +} diff --git a/pkg/client/clientset/typed/etcd/v1beta1/etcdmember.go b/pkg/client/clientset/typed/etcd/v1beta1/etcdmember.go new file mode 100644 index 000000000000..bb80a3a2da84 --- /dev/null +++ b/pkg/client/clientset/typed/etcd/v1beta1/etcdmember.go @@ -0,0 +1,168 @@ +/* +Copyright k0s authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1beta1 + +import ( + "context" + "time" + + v1beta1 "github.com/k0sproject/k0s/pkg/apis/etcd/v1beta1" + scheme "github.com/k0sproject/k0s/pkg/client/clientset/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + rest "k8s.io/client-go/rest" +) + +// EtcdMembersGetter has a method to return a EtcdMemberInterface. +// A group's client should implement this interface. +type EtcdMembersGetter interface { + EtcdMembers() EtcdMemberInterface +} + +// EtcdMemberInterface has methods to work with EtcdMember resources. +type EtcdMemberInterface interface { + Create(ctx context.Context, etcdMember *v1beta1.EtcdMember, opts v1.CreateOptions) (*v1beta1.EtcdMember, error) + Update(ctx context.Context, etcdMember *v1beta1.EtcdMember, opts v1.UpdateOptions) (*v1beta1.EtcdMember, error) + UpdateStatus(ctx context.Context, etcdMember *v1beta1.EtcdMember, opts v1.UpdateOptions) (*v1beta1.EtcdMember, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*v1beta1.EtcdMember, error) + List(ctx context.Context, opts v1.ListOptions) (*v1beta1.EtcdMemberList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1beta1.EtcdMember, err error) + EtcdMemberExpansion +} + +// etcdMembers implements EtcdMemberInterface +type etcdMembers struct { + client rest.Interface +} + +// newEtcdMembers returns a EtcdMembers +func newEtcdMembers(c *EtcdV1beta1Client) *etcdMembers { + return &etcdMembers{ + client: c.RESTClient(), + } +} + +// Get takes name of the etcdMember, and returns the corresponding etcdMember object, and an error if there is any. +func (c *etcdMembers) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1beta1.EtcdMember, err error) { + result = &v1beta1.EtcdMember{} + err = c.client.Get(). + Resource("etcdmembers"). + Name(name). + VersionedParams(&options, scheme.ParameterCodec). + Do(ctx). + Into(result) + return +} + +// List takes label and field selectors, and returns the list of EtcdMembers that match those selectors. +func (c *etcdMembers) List(ctx context.Context, opts v1.ListOptions) (result *v1beta1.EtcdMemberList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + result = &v1beta1.EtcdMemberList{} + err = c.client.Get(). + Resource("etcdmembers"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Do(ctx). + Into(result) + return +} + +// Watch returns a watch.Interface that watches the requested etcdMembers. +func (c *etcdMembers) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + opts.Watch = true + return c.client.Get(). + Resource("etcdmembers"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Watch(ctx) +} + +// Create takes the representation of a etcdMember and creates it. Returns the server's representation of the etcdMember, and an error, if there is any. +func (c *etcdMembers) Create(ctx context.Context, etcdMember *v1beta1.EtcdMember, opts v1.CreateOptions) (result *v1beta1.EtcdMember, err error) { + result = &v1beta1.EtcdMember{} + err = c.client.Post(). + Resource("etcdmembers"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(etcdMember). + Do(ctx). + Into(result) + return +} + +// Update takes the representation of a etcdMember and updates it. Returns the server's representation of the etcdMember, and an error, if there is any. +func (c *etcdMembers) Update(ctx context.Context, etcdMember *v1beta1.EtcdMember, opts v1.UpdateOptions) (result *v1beta1.EtcdMember, err error) { + result = &v1beta1.EtcdMember{} + err = c.client.Put(). + Resource("etcdmembers"). + Name(etcdMember.Name). + VersionedParams(&opts, scheme.ParameterCodec). + Body(etcdMember). + Do(ctx). + Into(result) + return +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *etcdMembers) UpdateStatus(ctx context.Context, etcdMember *v1beta1.EtcdMember, opts v1.UpdateOptions) (result *v1beta1.EtcdMember, err error) { + result = &v1beta1.EtcdMember{} + err = c.client.Put(). + Resource("etcdmembers"). + Name(etcdMember.Name). + SubResource("status"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(etcdMember). + Do(ctx). + Into(result) + return +} + +// Delete takes name of the etcdMember and deletes it. Returns an error if one occurs. +func (c *etcdMembers) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + return c.client.Delete(). + Resource("etcdmembers"). + Name(name). + Body(&opts). + Do(ctx). + Error() +} + +// Patch applies the patch and returns the patched etcdMember. +func (c *etcdMembers) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1beta1.EtcdMember, err error) { + result = &v1beta1.EtcdMember{} + err = c.client.Patch(pt). + Resource("etcdmembers"). + Name(name). + SubResource(subresources...). + VersionedParams(&opts, scheme.ParameterCodec). + Body(data). + Do(ctx). + Into(result) + return +} diff --git a/pkg/client/clientset/typed/etcd/v1beta1/fake/doc.go b/pkg/client/clientset/typed/etcd/v1beta1/fake/doc.go new file mode 100644 index 000000000000..f7926ec7323e --- /dev/null +++ b/pkg/client/clientset/typed/etcd/v1beta1/fake/doc.go @@ -0,0 +1,20 @@ +/* +Copyright k0s authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +// Package fake has the automatically generated clients. +package fake diff --git a/pkg/client/clientset/typed/etcd/v1beta1/fake/fake_etcd_client.go b/pkg/client/clientset/typed/etcd/v1beta1/fake/fake_etcd_client.go new file mode 100644 index 000000000000..fd01757511af --- /dev/null +++ b/pkg/client/clientset/typed/etcd/v1beta1/fake/fake_etcd_client.go @@ -0,0 +1,40 @@ +/* +Copyright k0s authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + v1beta1 "github.com/k0sproject/k0s/pkg/client/clientset/typed/etcd/v1beta1" + rest "k8s.io/client-go/rest" + testing "k8s.io/client-go/testing" +) + +type FakeEtcdV1beta1 struct { + *testing.Fake +} + +func (c *FakeEtcdV1beta1) EtcdMembers() v1beta1.EtcdMemberInterface { + return &FakeEtcdMembers{c} +} + +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *FakeEtcdV1beta1) RESTClient() rest.Interface { + var ret *rest.RESTClient + return ret +} diff --git a/pkg/client/clientset/typed/etcd/v1beta1/fake/fake_etcdmember.go b/pkg/client/clientset/typed/etcd/v1beta1/fake/fake_etcdmember.go new file mode 100644 index 000000000000..e097aafe64a6 --- /dev/null +++ b/pkg/client/clientset/typed/etcd/v1beta1/fake/fake_etcdmember.go @@ -0,0 +1,124 @@ +/* +Copyright k0s authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + v1beta1 "github.com/k0sproject/k0s/pkg/apis/etcd/v1beta1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeEtcdMembers implements EtcdMemberInterface +type FakeEtcdMembers struct { + Fake *FakeEtcdV1beta1 +} + +var etcdmembersResource = v1beta1.SchemeGroupVersion.WithResource("etcdmembers") + +var etcdmembersKind = v1beta1.SchemeGroupVersion.WithKind("EtcdMember") + +// Get takes name of the etcdMember, and returns the corresponding etcdMember object, and an error if there is any. +func (c *FakeEtcdMembers) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1beta1.EtcdMember, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootGetAction(etcdmembersResource, name), &v1beta1.EtcdMember{}) + if obj == nil { + return nil, err + } + return obj.(*v1beta1.EtcdMember), err +} + +// List takes label and field selectors, and returns the list of EtcdMembers that match those selectors. +func (c *FakeEtcdMembers) List(ctx context.Context, opts v1.ListOptions) (result *v1beta1.EtcdMemberList, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootListAction(etcdmembersResource, etcdmembersKind, opts), &v1beta1.EtcdMemberList{}) + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1beta1.EtcdMemberList{ListMeta: obj.(*v1beta1.EtcdMemberList).ListMeta} + for _, item := range obj.(*v1beta1.EtcdMemberList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested etcdMembers. +func (c *FakeEtcdMembers) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewRootWatchAction(etcdmembersResource, opts)) +} + +// Create takes the representation of a etcdMember and creates it. Returns the server's representation of the etcdMember, and an error, if there is any. +func (c *FakeEtcdMembers) Create(ctx context.Context, etcdMember *v1beta1.EtcdMember, opts v1.CreateOptions) (result *v1beta1.EtcdMember, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootCreateAction(etcdmembersResource, etcdMember), &v1beta1.EtcdMember{}) + if obj == nil { + return nil, err + } + return obj.(*v1beta1.EtcdMember), err +} + +// Update takes the representation of a etcdMember and updates it. Returns the server's representation of the etcdMember, and an error, if there is any. +func (c *FakeEtcdMembers) Update(ctx context.Context, etcdMember *v1beta1.EtcdMember, opts v1.UpdateOptions) (result *v1beta1.EtcdMember, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootUpdateAction(etcdmembersResource, etcdMember), &v1beta1.EtcdMember{}) + if obj == nil { + return nil, err + } + return obj.(*v1beta1.EtcdMember), err +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeEtcdMembers) UpdateStatus(ctx context.Context, etcdMember *v1beta1.EtcdMember, opts v1.UpdateOptions) (*v1beta1.EtcdMember, error) { + obj, err := c.Fake. + Invokes(testing.NewRootUpdateSubresourceAction(etcdmembersResource, "status", etcdMember), &v1beta1.EtcdMember{}) + if obj == nil { + return nil, err + } + return obj.(*v1beta1.EtcdMember), err +} + +// Delete takes name of the etcdMember and deletes it. Returns an error if one occurs. +func (c *FakeEtcdMembers) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewRootDeleteActionWithOptions(etcdmembersResource, name, opts), &v1beta1.EtcdMember{}) + return err +} + +// Patch applies the patch and returns the patched etcdMember. +func (c *FakeEtcdMembers) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1beta1.EtcdMember, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootPatchSubresourceAction(etcdmembersResource, name, pt, data, subresources...), &v1beta1.EtcdMember{}) + if obj == nil { + return nil, err + } + return obj.(*v1beta1.EtcdMember), err +} diff --git a/pkg/client/clientset/typed/etcd/v1beta1/generated_expansion.go b/pkg/client/clientset/typed/etcd/v1beta1/generated_expansion.go new file mode 100644 index 000000000000..e7281ce72016 --- /dev/null +++ b/pkg/client/clientset/typed/etcd/v1beta1/generated_expansion.go @@ -0,0 +1,21 @@ +/* +Copyright k0s authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1beta1 + +type EtcdMemberExpansion interface{} diff --git a/pkg/component/controller/etcd_member_reconciler.go b/pkg/component/controller/etcd_member_reconciler.go new file mode 100644 index 000000000000..1636dabdbc50 --- /dev/null +++ b/pkg/component/controller/etcd_member_reconciler.go @@ -0,0 +1,352 @@ +/* +Copyright 2024 k0s authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "fmt" + "net" + "os" + "strconv" + "time" + + "github.com/k0sproject/k0s/inttest/common" + + etcdv1beta1 "github.com/k0sproject/k0s/pkg/apis/etcd/v1beta1" + "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1" + etcdmemberclient "github.com/k0sproject/k0s/pkg/client/clientset/typed/etcd/v1beta1" + "github.com/k0sproject/k0s/pkg/component/controller/leaderelector" + "github.com/k0sproject/k0s/pkg/component/manager" + "github.com/k0sproject/k0s/pkg/config" + "github.com/k0sproject/k0s/pkg/etcd" + kubeutil "github.com/k0sproject/k0s/pkg/kubernetes" + "github.com/k0sproject/k0s/pkg/kubernetes/watch" + "github.com/sirupsen/logrus" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + extensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + extclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1" +) + +var _ manager.Component = (*EtcdMemberReconciler)(nil) + +// TODO: DO we need REady probing for this component? +// var _ manager.Ready = (*EtcdMemberReconciler)(nil) + +func NewEtcdMemberReconciler(kubeClientFactory kubeutil.ClientFactoryInterface, k0sVars *config.CfgVars, etcdConfig *v1beta1.EtcdConfig, leaderElector leaderelector.Interface) (*EtcdMemberReconciler, error) { + + return &EtcdMemberReconciler{ + clientFactory: kubeClientFactory, + k0sVars: k0sVars, + etcdConfig: etcdConfig, + leaderElector: leaderElector, + }, nil +} + +type EtcdMemberReconciler struct { + clientFactory kubeutil.ClientFactoryInterface + k0sVars *config.CfgVars + etcdConfig *v1beta1.EtcdConfig + etcdMemberClient etcdmemberclient.EtcdMemberInterface + leaderElector leaderelector.Interface +} + +func (e *EtcdMemberReconciler) Init(_ context.Context) error { + return nil +} + +func (e *EtcdMemberReconciler) Start(ctx context.Context) error { + log := logrus.WithField("component", "etcdMemberReconciler") + + etcdMemberClient, err := e.clientFactory.GetEtcdMemberClient() + if err != nil { + return err + } + e.etcdMemberClient = etcdMemberClient + + // Run the watch in go routine so it keeps running till the context ends + go func() { + err = e.waitForCRD(ctx) + if err != nil { + log.WithError(err).Errorf("didn't see EtcdMember CRD ready in time") + return + } + + // Create the object for this node + err = e.createMemberObject(ctx) + if err != nil { + log.WithError(err).Error("failed to create EtcdMember object") + } + var lastObservedVersion string + err = watch.EtcdMembers(etcdMemberClient). + WithErrorCallback(func(err error) (time.Duration, error) { + retryDelay, e := watch.IsRetryable(err) + if e == nil { + log.WithError(err).Debugf( + "Encountered transient error while watching etcd members"+ + ", last observed resource version was %q"+ + ", retrying in %s", + lastObservedVersion, retryDelay, + ) + return retryDelay, nil + } + log.WithError(e).Error("bailing out watch") + return 0, err + }). + IncludingDeletions(). + Until(ctx, func(member *etcdv1beta1.EtcdMember) (bool, error) { + e.reconcileMember(ctx, member) + // Never stop the watch + return false, nil + }) + if err != nil { + log.WithError(err).Warn("watch terminated") + } + }() + + return nil +} + +func (e *EtcdMemberReconciler) Stop() error { + return nil +} + +type ( + crd = extensionsv1.CustomResourceDefinition + crdList = extensionsv1.CustomResourceDefinitionList +) + +func (e *EtcdMemberReconciler) waitForCRD(ctx context.Context) error { + rc := e.clientFactory.GetRESTConfig() + + ec, err := extclient.NewForConfig(rc) + if err != nil { + return err + } + log := logrus.WithField("component", "etcdMemberReconciler") + log.Info("waiting to see EtcdMember CRD ready") + return watch.FromClient[*crdList, crd](ec.CustomResourceDefinitions()). + WithObjectName(fmt.Sprintf("%s.%s", "etcdmembers", "etcd.k0sproject.io")). + WithErrorCallback(common.RetryWatchErrors(logrus.Infof)). + Until(ctx, func(item *crd) (bool, error) { + for _, cond := range item.Status.Conditions { + if cond.Type == extensionsv1.Established { + log.Infof("EtcdMember CRD status: %s", cond.Status) + return cond.Status == extensionsv1.ConditionTrue, nil + } + } + + return false, nil + }) + +} + +func (e *EtcdMemberReconciler) createMemberObject(ctx context.Context) error { + log := logrus.WithFields(logrus.Fields{"component": "etcdMemberReconciler", "phase": "createMemberObject"}) + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + // find the member ID for this node + etcdClient, err := etcd.NewClient(e.k0sVars.CertRootDir, e.k0sVars.EtcdCertDir, e.etcdConfig) + if err != nil { + return err + } + + peerURL := fmt.Sprintf("https://%s", net.JoinHostPort(e.etcdConfig.PeerAddress, "2380")) + + memberID, err := etcdClient.GetPeerIDByAddress(ctx, peerURL) + if err != nil { + return err + } + + // Convert the memberID to hex string + memberIDStr := fmt.Sprintf("%x", memberID) + + name, err := os.Hostname() + if err != nil { + return err + } + var em *etcdv1beta1.EtcdMember + + log.WithField("name", name).WithField("memberID", memberID).Info("creating EtcdMember object") + + // Check if the object already exists + em, err = e.etcdMemberClient.Get(ctx, name, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + log.Debug("EtcdMember object not found, creating it") + em = &etcdv1beta1.EtcdMember{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + PeerAddress: e.etcdConfig.PeerAddress, + MemberID: memberIDStr, + Leave: false, + Status: etcdv1beta1.Status{ + Conditions: []etcdv1beta1.JoinCondition{ + { + Type: etcdv1beta1.ConditionTypeJoined, + Status: etcdv1beta1.ConditionTrue, + }, + }, + }, + } + em, err = e.etcdMemberClient.Create(ctx, em, v1.CreateOptions{}) + if err != nil { + return err + } + em.Status = etcdv1beta1.Status{ + Conditions: []etcdv1beta1.JoinCondition{ + { + Type: etcdv1beta1.ConditionTypeJoined, + Status: etcdv1beta1.ConditionTrue, + }, + }, + } + _, err = e.etcdMemberClient.UpdateStatus(ctx, em, v1.UpdateOptions{}) + if err != nil { + log.WithError(err).Error("failed to update member status") + } + return nil + } else { + return err + } + } + + em.PeerAddress = e.etcdConfig.PeerAddress + em.MemberID = memberIDStr + em.Leave = false + + log.Debug("EtcdMember object already exists, updating it") + // Update the object if it already exists + em, err = e.etcdMemberClient.Update(ctx, em, v1.UpdateOptions{}) + if err != nil { + return err + } + em.Status = etcdv1beta1.Status{ + Conditions: []etcdv1beta1.JoinCondition{ + { + Type: etcdv1beta1.ConditionTypeJoined, + Status: etcdv1beta1.ConditionTrue, + }, + }, + ReconcileStatus: "", + Message: "", + } + _, err = e.etcdMemberClient.UpdateStatus(ctx, em, v1.UpdateOptions{}) + if err != nil { + return err + } + + return nil +} + +func (e *EtcdMemberReconciler) reconcileMember(ctx context.Context, member *etcdv1beta1.EtcdMember) { + log := logrus.WithFields(logrus.Fields{ + "component": "etcdMemberReconciler", + "phase": "reconcile", + "name": member.Name, + "memberID": member.MemberID, + "peerAddress": member.PeerAddress, + }) + + if !e.leaderElector.IsLeader() { + log.Debug("not the leader, skipping reconcile") + return + } + + log.Debugf("reconciling EtcdMember: %+v", member) + + if !member.Leave { + log.Debug("member not marked for leave, no action needed") + return + } + + etcdClient, err := etcd.NewClient(e.k0sVars.CertRootDir, e.k0sVars.EtcdCertDir, e.etcdConfig) + if err != nil { + log.WithError(err).Warn("failed to create etcd client") + member.Status.ReconcileStatus = "Failed" + member.Status.Message = err.Error() + if _, err = e.etcdMemberClient.UpdateStatus(ctx, member, v1.UpdateOptions{}); err != nil { + log.WithError(err).Error("failed to update member state") + } + + return + } + + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + // Verify that the member is actualy still present in etcd + members, err := etcdClient.ListMembers(ctx) + if err != nil { + member.Status.ReconcileStatus = "Failed" + member.Status.Message = err.Error() + if _, err = e.etcdMemberClient.UpdateStatus(ctx, member, v1.UpdateOptions{}); err != nil { + log.WithError(err).Error("failed to update member state") + } + + return + } + + // Member marked for leave but no member found in etcd, mark for leaved + _, ok := members[member.Name] + if !ok { + log.Debug("member marked for leave but not in actual member list, updating state to reflect that") + member.Status.SetCondition(etcdv1beta1.ConditionTypeJoined, etcdv1beta1.ConditionFalse, member.Status.Message, time.Now()) + member, err = e.etcdMemberClient.UpdateStatus(ctx, member, v1.UpdateOptions{}) + if err != nil { + log.WithError(err).Error("failed to update EtcdMember status") + } + } + + if member.Status.GetCondition(etcdv1beta1.ConditionTypeJoined).Status == etcdv1beta1.ConditionFalse && !ok { + log.Debug("member already left, no action needed") + return + } + + // Convert the memberID to uint64 + memberID, err := strconv.ParseUint(member.MemberID, 16, 64) + if err != nil { + log.WithError(err).Error("failed to parse memberID") + return + } + + if err = etcdClient.DeleteMember(ctx, memberID); err != nil { + logrus. + WithError(err). + Errorf("Failed to delete etcd peer from cluster") + member.Status.ReconcileStatus = "Failed" + member.Status.Message = err.Error() + _, err = e.etcdMemberClient.UpdateStatus(ctx, member, v1.UpdateOptions{}) + if err != nil { + log.WithError(err).Error("failed to update EtcdMember status") + } + return + } + + // Peer removed succesfully, update status + log.Info("reconcile succeeded") + member.Status.ReconcileStatus = "Success" + member.Status.Message = "Member removed from cluster" + member.Status.SetCondition(etcdv1beta1.ConditionTypeJoined, etcdv1beta1.ConditionFalse, member.Status.Message, time.Now()) + _, err = e.etcdMemberClient.UpdateStatus(ctx, member, v1.UpdateOptions{}) + if err != nil { + log.WithError(err).Error("failed to update EtcdMember status") + } +} diff --git a/pkg/kubernetes/client.go b/pkg/kubernetes/client.go index 69f6babd2548..1bd04d442476 100644 --- a/pkg/kubernetes/client.go +++ b/pkg/kubernetes/client.go @@ -20,6 +20,7 @@ import ( "fmt" "sync" + etcdMemberClient "github.com/k0sproject/k0s/pkg/client/clientset/typed/etcd/v1beta1" cfgClient "github.com/k0sproject/k0s/pkg/client/clientset/typed/k0s/v1beta1" "github.com/k0sproject/k0s/pkg/constant" @@ -39,6 +40,7 @@ type ClientFactoryInterface interface { GetConfigClient() (cfgClient.ClusterConfigInterface, error) GetRESTClient() (rest.Interface, error) GetRESTConfig() *rest.Config + GetEtcdMemberClient() (etcdMemberClient.EtcdMemberInterface, error) } // NewAdminClientFactory creates a new factory that loads the admin kubeconfig based client @@ -54,11 +56,12 @@ func NewAdminClientFactory(kubeconfigPath string) ClientFactoryInterface { type ClientFactory struct { configPath string - client kubernetes.Interface - dynamicClient dynamic.Interface - discoveryClient discovery.CachedDiscoveryInterface - restConfig *rest.Config - configClient cfgClient.ClusterConfigInterface + client kubernetes.Interface + dynamicClient dynamic.Interface + discoveryClient discovery.CachedDiscoveryInterface + restConfig *rest.Config + configClient cfgClient.ClusterConfigInterface + etcdMemberClient etcdMemberClient.EtcdMemberInterface mutex sync.Mutex } @@ -171,6 +174,27 @@ func (c *ClientFactory) GetConfigClient() (cfgClient.ClusterConfigInterface, err return c.configClient, nil } +func (c *ClientFactory) GetEtcdMemberClient() (etcdMemberClient.EtcdMemberInterface, error) { + c.mutex.Lock() + defer c.mutex.Unlock() + var err error + if c.restConfig == nil { + c.restConfig, err = clientcmd.BuildConfigFromFlags("", c.configPath) + if err != nil { + return nil, fmt.Errorf("failed to load kubeconfig: %w", err) + } + } + if c.etcdMemberClient != nil { + return c.etcdMemberClient, nil + } + + etcdMemberClient, err := etcdMemberClient.NewForConfig(c.restConfig) + if err != nil { + return nil, err + } + return etcdMemberClient.EtcdMembers(), nil +} + func (c *ClientFactory) GetRESTClient() (rest.Interface, error) { cs, ok := c.client.(*kubernetes.Clientset) if !ok { diff --git a/pkg/kubernetes/watch/k0s.go b/pkg/kubernetes/watch/k0s.go index fb9ac3236503..9b70bc2ae6df 100644 --- a/pkg/kubernetes/watch/k0s.go +++ b/pkg/kubernetes/watch/k0s.go @@ -18,6 +18,7 @@ package watch import ( autopilotv1beta2 "github.com/k0sproject/k0s/pkg/apis/autopilot/v1beta2" + etcdv1beta1 "github.com/k0sproject/k0s/pkg/apis/etcd/v1beta1" helmv1beta1 "github.com/k0sproject/k0s/pkg/apis/helm/v1beta1" k0sv1beta1 "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1" ) @@ -33,3 +34,7 @@ func Plans(client Provider[*autopilotv1beta2.PlanList]) *Watcher[autopilotv1beta func Charts(client Provider[*helmv1beta1.ChartList]) *Watcher[helmv1beta1.Chart] { return FromClient[*helmv1beta1.ChartList, helmv1beta1.Chart](client) } + +func EtcdMembers(client Provider[*etcdv1beta1.EtcdMemberList]) *Watcher[etcdv1beta1.EtcdMember] { + return FromClient[*etcdv1beta1.EtcdMemberList, etcdv1beta1.EtcdMember](client) +} diff --git a/static/manifests/etcd/CustomResourceDefinition/etcd.k0sproject.io_etcdmembers.yaml b/static/manifests/etcd/CustomResourceDefinition/etcd.k0sproject.io_etcdmembers.yaml new file mode 100644 index 000000000000..2a3918f077b8 --- /dev/null +++ b/static/manifests/etcd/CustomResourceDefinition/etcd.k0sproject.io_etcdmembers.yaml @@ -0,0 +1,100 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: etcdmembers.etcd.k0sproject.io +spec: + group: etcd.k0sproject.io + names: + kind: EtcdMember + listKind: EtcdMemberList + plural: etcdmembers + singular: etcdmember + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .peerAddress + name: PeerAddress + type: string + - jsonPath: .memberID + name: MemberID + type: string + - jsonPath: .status.conditions[?(@.type=="Joined")].status + name: Joined + type: string + - jsonPath: .status.reconcileStatus + name: ReconcileStatus + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: EtcdMember describes the nodes etcd membership status + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + leave: + description: Leave is a flag to indicate that the member should be removed + from the cluster + type: boolean + memberID: + description: |- + MemberID is the unique identifier of the etcd member. + The hex form ID is stored as string + pattern: ^[a-fA-F0-9]+$ + type: string + metadata: + type: object + peerAddress: + description: PeerAddress is the address of the etcd peer + type: string + status: + properties: + conditions: + items: + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + format: date-time + type: string + message: + description: Human-readable message indicating details about + last transition. + type: string + status: + type: string + type: + type: string + required: + - status + - type + type: object + type: array + message: + type: string + reconcileStatus: + type: string + type: object + required: + - memberID + - peerAddress + type: object + served: true + storage: true + subresources: + status: {}