diff --git a/mocks/service/k8s/Services.go b/mocks/service/k8s/Services.go index e734f6f47..f507c2437 100644 --- a/mocks/service/k8s/Services.go +++ b/mocks/service/k8s/Services.go @@ -887,6 +887,20 @@ func (_m *Services) UpdatePodLabels(namespace string, podName string, labels map return r0 } +// UpdatePodAnnotations provides a mock function with given fields: namespace, podName, annotations +func (_m *Services) UpdatePodAnnotations(namespace string, podName string, annotations map[string]string) error { + ret := _m.Called(namespace, podName, annotations) + + var r0 error + if rf, ok := ret.Get(0).(func(string, string, map[string]string) error); ok { + r0 = rf(namespace, podName, annotations) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // UpdateRole provides a mock function with given fields: namespace, role func (_m *Services) UpdateRole(namespace string, role *rbacv1.Role) error { ret := _m.Called(namespace, role) diff --git a/operator/redisfailover/service/check.go b/operator/redisfailover/service/check.go index 86401f0f9..dd73d2a93 100644 --- a/operator/redisfailover/service/check.go +++ b/operator/redisfailover/service/check.go @@ -102,6 +102,26 @@ func (r *RedisFailoverChecker) setSlaveLabelIfNecessary(namespace string, pod co return r.k8sService.UpdatePodLabels(namespace, pod.ObjectMeta.Name, generateRedisSlaveRoleLabel()) } +func (r *RedisFailoverChecker) setMasterAnnotationIfNecessary(namespace string, pod corev1.Pod) error { + for annotationKey, annotationValue := range pod.ObjectMeta.Annotations { + if annotationKey == clusterAutoscalerSafeToEvictAnnotationKey && + annotationValue == clusterAutoscalerSafeToEvictAnnotationMaster { + return nil + } + } + return r.k8sService.UpdatePodAnnotations(namespace, pod.ObjectMeta.Name, generateRedisMasterAnnotations()) +} + +func (r *RedisFailoverChecker) setSlaveAnnotationIfNecessary(namespace string, pod corev1.Pod) error { + for annotationKey, annotationValue := range pod.ObjectMeta.Annotations { + if annotationKey == clusterAutoscalerSafeToEvictAnnotationKey && + annotationValue == clusterAutoscalerSafeToEvictAnnotationSlave { + return nil + } + } + return r.k8sService.UpdatePodAnnotations(namespace, pod.ObjectMeta.Name, generateRedisSlaveAnnotations()) +} + // CheckAllSlavesFromMaster controlls that all slaves have the same master (the real one) func (r *RedisFailoverChecker) CheckAllSlavesFromMaster(master string, rf *redisfailoverv1.RedisFailover) error { rps, err := r.k8sService.GetStatefulSetPods(rf.Namespace, GetRedisName(rf)) @@ -121,11 +141,19 @@ func (r *RedisFailoverChecker) CheckAllSlavesFromMaster(master string, rf *redis if err != nil { return err } + err = r.setMasterAnnotationIfNecessary(rf.Namespace, rp) + if err != nil { + return err + } } else { err = r.setSlaveLabelIfNecessary(rf.Namespace, rp) if err != nil { return err } + err = r.setSlaveAnnotationIfNecessary(rf.Namespace, rp) + if err != nil { + return err + } } slave, err := r.redisClient.GetSlaveOf(rp.Status.PodIP, rport, password) diff --git a/operator/redisfailover/service/check_test.go b/operator/redisfailover/service/check_test.go index af64d7e13..c76f88ce8 100644 --- a/operator/redisfailover/service/check_test.go +++ b/operator/redisfailover/service/check_test.go @@ -159,6 +159,7 @@ func TestCheckAllSlavesFromMasterGetStatefulSetError(t *testing.T) { ms := &mK8SService.Services{} ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(nil, errors.New("")) ms.On("UpdatePodLabels", namespace, mock.AnythingOfType("string"), mock.Anything).Once().Return(nil) + ms.On("UpdatePodAnnotations", namespace, mock.AnythingOfType("string"), mock.Anything).Once().Return(nil) mr := &mRedisService.Client{} checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}, metrics.Dummy) @@ -186,6 +187,7 @@ func TestCheckAllSlavesFromMasterGetSlaveOfError(t *testing.T) { ms := &mK8SService.Services{} ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil) ms.On("UpdatePodLabels", namespace, mock.AnythingOfType("string"), mock.Anything).Once().Return(nil) + ms.On("UpdatePodAnnotations", namespace, mock.AnythingOfType("string"), mock.Anything).Once().Return(nil) mr := &mRedisService.Client{} mr.On("GetSlaveOf", "", "0", "").Once().Return("", errors.New("")) @@ -214,6 +216,7 @@ func TestCheckAllSlavesFromMasterDifferentMaster(t *testing.T) { ms := &mK8SService.Services{} ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil) ms.On("UpdatePodLabels", namespace, mock.AnythingOfType("string"), mock.Anything).Once().Return(nil) + ms.On("UpdatePodAnnotations", namespace, mock.AnythingOfType("string"), mock.Anything).Once().Return(nil) mr := &mRedisService.Client{} mr.On("GetSlaveOf", "0.0.0.0", "0", "").Once().Return("1.1.1.1", nil) @@ -242,6 +245,7 @@ func TestCheckAllSlavesFromMaster(t *testing.T) { ms := &mK8SService.Services{} ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil) ms.On("UpdatePodLabels", namespace, mock.AnythingOfType("string"), mock.Anything).Once().Return(nil) + ms.On("UpdatePodAnnotations", namespace, mock.AnythingOfType("string"), mock.Anything).Once().Return(nil) mr := &mRedisService.Client{} mr.On("GetSlaveOf", "0.0.0.0", "0", "").Once().Return("1.1.1.1", nil) diff --git a/operator/redisfailover/service/client.go b/operator/redisfailover/service/client.go index 4cf48e6f9..1c83a49e8 100644 --- a/operator/redisfailover/service/client.go +++ b/operator/redisfailover/service/client.go @@ -67,6 +67,18 @@ func generateRedisSlaveRoleLabel() map[string]string { } } +func generateRedisMasterAnnotations() map[string]string { + return map[string]string{ + clusterAutoscalerSafeToEvictAnnotationKey: clusterAutoscalerSafeToEvictAnnotationMaster, + } +} + +func generateRedisSlaveAnnotations() map[string]string { + return map[string]string{ + clusterAutoscalerSafeToEvictAnnotationKey: clusterAutoscalerSafeToEvictAnnotationSlave, + } +} + // EnsureSentinelService makes sure the sentinel service exists func (r *RedisFailoverKubeClient) EnsureSentinelService(rf *redisfailoverv1.RedisFailover, labels map[string]string, ownerRefs []metav1.OwnerReference) error { svc := generateSentinelService(rf, labels, ownerRefs) diff --git a/operator/redisfailover/service/constants.go b/operator/redisfailover/service/constants.go index feb39ec6d..5baffbf2e 100644 --- a/operator/redisfailover/service/constants.go +++ b/operator/redisfailover/service/constants.go @@ -33,4 +33,8 @@ const ( redisRoleLabelKey = "redisfailovers-role" redisRoleLabelMaster = "master" redisRoleLabelSlave = "slave" + + clusterAutoscalerSafeToEvictAnnotationKey = "cluster-autoscaler.kubernetes.io~1safe-to-evict" + clusterAutoscalerSafeToEvictAnnotationMaster = "false" + clusterAutoscalerSafeToEvictAnnotationSlave = "true" ) diff --git a/operator/redisfailover/service/heal.go b/operator/redisfailover/service/heal.go index 390a6ac48..9a433cba8 100644 --- a/operator/redisfailover/service/heal.go +++ b/operator/redisfailover/service/heal.go @@ -61,6 +61,26 @@ func (r *RedisFailoverHealer) setSlaveLabelIfNecessary(namespace string, pod v1. return r.k8sService.UpdatePodLabels(namespace, pod.ObjectMeta.Name, generateRedisSlaveRoleLabel()) } +func (r *RedisFailoverHealer) setMasterAnnotationIfNecessary(namespace string, pod v1.Pod) error { + for annotationKey, annotationValue := range pod.ObjectMeta.Annotations { + if annotationKey == clusterAutoscalerSafeToEvictAnnotationKey && + annotationValue == clusterAutoscalerSafeToEvictAnnotationMaster { + return nil + } + } + return r.k8sService.UpdatePodAnnotations(namespace, pod.ObjectMeta.Name, generateRedisMasterAnnotations()) +} + +func (r *RedisFailoverHealer) setSlaveAnnotationIfNecessary(namespace string, pod v1.Pod) error { + for annotationKey, annotationValue := range pod.ObjectMeta.Annotations { + if annotationKey == clusterAutoscalerSafeToEvictAnnotationKey && + annotationValue == clusterAutoscalerSafeToEvictAnnotationSlave { + return nil + } + } + return r.k8sService.UpdatePodAnnotations(namespace, pod.ObjectMeta.Name, generateRedisSlaveAnnotations()) +} + func (r *RedisFailoverHealer) MakeMaster(ip string, rf *redisfailoverv1.RedisFailover) error { password, err := k8s.GetRedisPassword(r.k8sService, rf) if err != nil { @@ -79,7 +99,14 @@ func (r *RedisFailoverHealer) MakeMaster(ip string, rf *redisfailoverv1.RedisFai } for _, rp := range rps.Items { if rp.Status.PodIP == ip { - return r.setMasterLabelIfNecessary(rf.Namespace, rp) + err = r.setMasterLabelIfNecessary(rf.Namespace, rp) + if err != nil { + return err + } + err = r.setMasterAnnotationIfNecessary(rf.Namespace, rp) + if err != nil { + return err + } } } return nil @@ -121,6 +148,10 @@ func (r *RedisFailoverHealer) SetOldestAsMaster(rf *redisfailoverv1.RedisFailove if err != nil { return err } + err = r.setMasterAnnotationIfNecessary(rf.Namespace, pod) + if err != nil { + return err + } newMasterIP = pod.Status.PodIP } else { @@ -133,6 +164,11 @@ func (r *RedisFailoverHealer) SetOldestAsMaster(rf *redisfailoverv1.RedisFailove if err != nil { return err } + + err = r.setSlaveAnnotationIfNecessary(rf.Namespace, pod) + if err != nil { + return err + } } } if newMasterIP == "" { @@ -175,6 +211,10 @@ func (r *RedisFailoverHealer) SetMasterOnAll(masterIP string, rf *redisfailoverv if err != nil { return err } + err = r.setSlaveAnnotationIfNecessary(rf.Namespace, pod) + if err != nil { + return err + } } } return nil diff --git a/operator/redisfailover/service/heal_test.go b/operator/redisfailover/service/heal_test.go index 27bd8cc11..2cd573b5a 100644 --- a/operator/redisfailover/service/heal_test.go +++ b/operator/redisfailover/service/heal_test.go @@ -35,6 +35,7 @@ func TestSetOldestAsMasterNewMasterError(t *testing.T) { ms := &mK8SService.Services{} ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil) ms.On("UpdatePodLabels", namespace, mock.AnythingOfType("string"), mock.Anything).Return(nil) + ms.On("UpdatePodAnnotations", namespace, mock.AnythingOfType("string"), mock.Anything).Return(nil) mr := &mRedisService.Client{} mr.On("MakeMaster", "0.0.0.0", "0", "").Once().Return(errors.New("")) @@ -62,6 +63,7 @@ func TestSetOldestAsMaster(t *testing.T) { ms := &mK8SService.Services{} ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil) ms.On("UpdatePodLabels", namespace, mock.AnythingOfType("string"), mock.Anything).Once().Return(nil) + ms.On("UpdatePodAnnotations", namespace, mock.AnythingOfType("string"), mock.Anything).Return(nil) mr := &mRedisService.Client{} mr.On("MakeMaster", "0.0.0.0", "0", "").Once().Return(nil) @@ -94,6 +96,7 @@ func TestSetOldestAsMasterMultiplePodsMakeSlaveOfError(t *testing.T) { ms := &mK8SService.Services{} ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil) ms.On("UpdatePodLabels", namespace, mock.AnythingOfType("string"), mock.Anything).Return(nil) + ms.On("UpdatePodAnnotations", namespace, mock.AnythingOfType("string"), mock.Anything).Return(nil) mr := &mRedisService.Client{} mr.On("MakeMaster", "0.0.0.0", "0", "").Once().Return(nil) mr.On("MakeSlaveOfWithPort", "1.1.1.1", "0.0.0.0", "0", "").Once().Return(errors.New("")) @@ -127,6 +130,7 @@ func TestSetOldestAsMasterMultiplePods(t *testing.T) { ms := &mK8SService.Services{} ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil) ms.On("UpdatePodLabels", namespace, mock.AnythingOfType("string"), mock.Anything).Return(nil) + ms.On("UpdatePodAnnotations", namespace, mock.AnythingOfType("string"), mock.Anything).Return(nil) mr := &mRedisService.Client{} mr.On("MakeMaster", "0.0.0.0", "0", "").Once().Return(nil) mr.On("MakeSlaveOfWithPort", "1.1.1.1", "0.0.0.0", "0", "").Once().Return(nil) @@ -170,6 +174,7 @@ func TestSetOldestAsMasterOrdering(t *testing.T) { ms := &mK8SService.Services{} ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil) ms.On("UpdatePodLabels", namespace, mock.AnythingOfType("string"), mock.Anything).Return(nil) + ms.On("UpdatePodAnnotations", namespace, mock.AnythingOfType("string"), mock.Anything).Return(nil) mr := &mRedisService.Client{} mr.On("MakeMaster", "1.1.1.1", "0", "").Once().Return(nil) mr.On("MakeSlaveOfWithPort", "0.0.0.0", "1.1.1.1", "0", "").Once().Return(nil) @@ -203,6 +208,7 @@ func TestSetMasterOnAllMakeMasterError(t *testing.T) { ms := &mK8SService.Services{} ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil) ms.On("UpdatePodLabels", namespace, mock.AnythingOfType("string"), mock.Anything).Once().Return(nil) + ms.On("UpdatePodAnnotations", namespace, mock.AnythingOfType("string"), mock.Anything).Return(nil) mr := &mRedisService.Client{} mr.On("IsMaster", "0.0.0.0", "0", "").Return(false, errors.New("")) healer := rfservice.NewRedisFailoverHealer(ms, mr, log.DummyLogger{}) @@ -234,6 +240,7 @@ func TestSetMasterOnAllMakeSlaveOfError(t *testing.T) { ms := &mK8SService.Services{} ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil) ms.On("UpdatePodLabels", namespace, mock.AnythingOfType("string"), mock.Anything).Return(nil) + ms.On("UpdatePodAnnotations", namespace, mock.AnythingOfType("string"), mock.Anything).Return(nil) mr := &mRedisService.Client{} mr.On("IsMaster", "0.0.0.0", "0", "").Return(true, nil) mr.On("MakeSlaveOfWithPort", "1.1.1.1", "0.0.0.0", "0", "").Once().Return(errors.New("")) @@ -267,6 +274,7 @@ func TestSetMasterOnAll(t *testing.T) { ms := &mK8SService.Services{} ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil) ms.On("UpdatePodLabels", namespace, mock.AnythingOfType("string"), mock.Anything).Return(nil) + ms.On("UpdatePodAnnotations", namespace, mock.AnythingOfType("string"), mock.Anything).Return(nil) mr := &mRedisService.Client{} mr.On("IsMaster", "0.0.0.0", "0", "").Return(true, nil) mr.On("MakeSlaveOfWithPort", "1.1.1.1", "0.0.0.0", "0", "").Once().Return(nil) diff --git a/service/k8s/pod.go b/service/k8s/pod.go index 0583302ce..c371708ba 100644 --- a/service/k8s/pod.go +++ b/service/k8s/pod.go @@ -24,6 +24,7 @@ type Pod interface { DeletePod(namespace string, name string) error ListPods(namespace string) (*corev1.PodList, error) UpdatePodLabels(namespace, podName string, labels map[string]string) error + UpdatePodAnnotations(namespace, podName string, annotations map[string]string) error } // PodService is the pod service implementation using API calls to kubernetes. @@ -128,3 +129,25 @@ func (p *PodService) UpdatePodLabels(namespace, podName string, labels map[strin } return err } + +func (p *PodService) UpdatePodAnnotations(namespace, podName string, annotations map[string]string) error { + p.logger.Infof("Update pod annotation, namespace: %s, pod name: %s, annotations: %v", namespace, podName, annotations) + + var payloads []interface{} + for annotationKey, annotationValue := range annotations { + payload := PatchStringValue{ + Op: "replace", + Path: "/metadata/annotations/" + annotationKey, + Value: annotationValue, + } + payloads = append(payloads, payload) + } + payloadBytes, _ := json.Marshal(payloads) + + _, err := p.kubeClient.CoreV1().Pods(namespace).Patch(context.TODO(), podName, types.JSONPatchType, payloadBytes, metav1.PatchOptions{}) + recordMetrics(namespace, "Pod", podName, "PATCH", err, p.metricsRecorder) + if err != nil { + p.logger.Errorf("Update pod annotations failed, namespace: %s, pod name: %s, error: %v", namespace, podName, err) + } + return err +}