From 2babf9f6cc1ad3f82c371b0ffadf2e4c1e9384b0 Mon Sep 17 00:00:00 2001 From: pingjiang Date: Fri, 28 Jul 2023 13:59:02 +0800 Subject: [PATCH] SidecarSet add upgrade state in pod annotation Signed-off-by: pingjiang --- pkg/control/sidecarcontrol/util.go | 51 ++++++- pkg/control/sidecarcontrol/util_test.go | 134 +++++++++++++++++- .../sidecarset/sidecarset_processor.go | 7 +- .../sidecarset/sidecarset_strategy_test.go | 2 +- pkg/webhook/pod/mutating/sidecarset.go | 1 + 5 files changed, 180 insertions(+), 15 deletions(-) diff --git a/pkg/control/sidecarcontrol/util.go b/pkg/control/sidecarcontrol/util.go index 4801e26f29..1814d7a858 100644 --- a/pkg/control/sidecarcontrol/util.go +++ b/pkg/control/sidecarcontrol/util.go @@ -81,6 +81,7 @@ type SidecarSetUpgradeSpec struct { SidecarSetName string `json:"sidecarSetName"` SidecarList []string `json:"sidecarList"` // sidecarSet container list SidecarSetControllerRevision string `json:"controllerRevision,omitempty"` // sidecarSet controllerRevision name + State string `json:"state"` // enum: Normal, Updating } // PodMatchSidecarSet determines if pod match Selector of sidecar. @@ -210,7 +211,7 @@ func IsPodSidecarUpdated(sidecarSet *appsv1alpha1.SidecarSet, pod *corev1.Pod) b } // UpdatePodSidecarSetHash when sidecarSet in-place update sidecar container, Update sidecarSet hash in Pod annotations[kruise.io/sidecarset-hash] -func UpdatePodSidecarSetHash(pod *corev1.Pod, sidecarSet *appsv1alpha1.SidecarSet) { +func UpdatePodSidecarSetHash(pod *corev1.Pod, sidecarSet *appsv1alpha1.SidecarSet, onlyState bool) { hashKey := SidecarSetHashAnnotation sidecarSetHash := make(map[string]SidecarSetUpgradeSpec) if err := json.Unmarshal([]byte(pod.Annotations[hashKey]), &sidecarSetHash); err != nil { @@ -247,17 +248,53 @@ func UpdatePodSidecarSetHash(pod *corev1.Pod, sidecarSet *appsv1alpha1.SidecarSe sidecarList.Insert(sidecar.Name) } - sidecarSetHash[sidecarSet.Name] = SidecarSetUpgradeSpec{ - UpdateTimestamp: metav1.Now(), - SidecarSetHash: GetSidecarSetRevision(sidecarSet), - SidecarSetName: sidecarSet.Name, - SidecarList: sidecarList.List(), - SidecarSetControllerRevision: sidecarSet.Status.LatestRevision, + if onlyState { + var state string + state = "Updating" + sidecars := GetSidecarContainersInPod(sidecarSet) + if _, ok := pod.Annotations[SidecarsetInplaceUpdateStateKey]; ok { + if IsSidecarContainerUpdateCompleted(pod, sets.NewString(sidecarSet.Name), sidecars) && IsSiderCarContainersReady(pod, sidecars) { + state = "Normal" + } + } else { + if IsSiderCarContainersReady(pod, sidecars) { + state = "Normal" + } + } + UpdateSidecarSetHashState(sidecarSetHash, sidecarSet.Name, state) + } else { + sidecarSetHash[sidecarSet.Name] = SidecarSetUpgradeSpec{ + UpdateTimestamp: metav1.Now(), + SidecarSetHash: GetSidecarSetRevision(sidecarSet), + SidecarSetName: sidecarSet.Name, + SidecarList: sidecarList.List(), + SidecarSetControllerRevision: sidecarSet.Status.LatestRevision, + State: sidecarSetHash[sidecarSet.Name].State, + } } newHash, _ := json.Marshal(sidecarSetHash) pod.Annotations[hashKey] = string(newHash) } +func UpdateSidecarSetHashState(sidecarSetHash map[string]SidecarSetUpgradeSpec, sidecarSetName string, State string) { + upgradeSpec := sidecarSetHash[sidecarSetName] + upgradeSpec.State = State + sidecarSetHash[sidecarSetName] = upgradeSpec +} + +func IsSiderCarContainersReady(pod *corev1.Pod, containers sets.String) bool { + for _, cs := range pod.Status.ContainerStatuses { + // only check containers set + if !containers.Has(cs.Name) { + continue + } + if !cs.Ready { + return false + } + } + return true +} + func GetSidecarContainersInPod(sidecarSet *appsv1alpha1.SidecarSet) sets.String { names := sets.NewString() for _, sidecarContainer := range sidecarSet.Spec.Containers { diff --git a/pkg/control/sidecarcontrol/util_test.go b/pkg/control/sidecarcontrol/util_test.go index a01bd126d1..5ee22f7bce 100644 --- a/pkg/control/sidecarcontrol/util_test.go +++ b/pkg/control/sidecarcontrol/util_test.go @@ -176,7 +176,7 @@ func TestIsSidecarContainerUpdateCompleted(t *testing.T) { pod := podDemo.DeepCopy() control := New(sidecarSetDemo.DeepCopy()) pod.Spec.Containers[1].Image = "cold-sidecar:v2" - UpdatePodSidecarSetHash(pod, control.GetSidecarset()) + UpdatePodSidecarSetHash(pod, control.GetSidecarset(), false) control.UpdatePodAnnotationsInUpgrade([]string{"cold-sidecar"}, pod) return pod }, @@ -191,7 +191,7 @@ func TestIsSidecarContainerUpdateCompleted(t *testing.T) { pod := podDemo.DeepCopy() control := New(sidecarSetDemo.DeepCopy()) pod.Spec.Containers[1].Image = "cold-sidecar:v2" - UpdatePodSidecarSetHash(pod, control.GetSidecarset()) + UpdatePodSidecarSetHash(pod, control.GetSidecarset(), false) control.UpdatePodAnnotationsInUpgrade([]string{"cold-sidecar"}, pod) pod.Status.ContainerStatuses[1].ImageID = ImageIds["cold-sidecar:v2"] return pod @@ -208,7 +208,7 @@ func TestIsSidecarContainerUpdateCompleted(t *testing.T) { control := New(sidecarSetDemo.DeepCopy()) // upgrade cold sidecar completed pod.Spec.Containers[1].Image = "cold-sidecar:v2" - UpdatePodSidecarSetHash(pod, control.GetSidecarset()) + UpdatePodSidecarSetHash(pod, control.GetSidecarset(), false) control.UpdatePodAnnotationsInUpgrade([]string{"cold-sidecar"}, pod) pod.Status.ContainerStatuses[1].ImageID = ImageIds["cold-sidecar:v2"] // start upgrading hot sidecar @@ -228,7 +228,7 @@ func TestIsSidecarContainerUpdateCompleted(t *testing.T) { control := New(sidecarSetDemo.DeepCopy()) // upgrade cold sidecar completed pod.Spec.Containers[1].Image = "cold-sidecar:v2" - UpdatePodSidecarSetHash(pod, control.GetSidecarset()) + UpdatePodSidecarSetHash(pod, control.GetSidecarset(), false) control.UpdatePodAnnotationsInUpgrade([]string{"cold-sidecar"}, pod) pod.Status.ContainerStatuses[1].ImageID = ImageIds["cold-sidecar:v2"] // start upgrading hot sidecar @@ -251,7 +251,7 @@ func TestIsSidecarContainerUpdateCompleted(t *testing.T) { control := New(sidecarSetDemo.DeepCopy()) // upgrade cold sidecar completed pod.Spec.Containers[1].Image = "cold-sidecar:v2" - UpdatePodSidecarSetHash(pod, control.GetSidecarset()) + UpdatePodSidecarSetHash(pod, control.GetSidecarset(), false) control.UpdatePodAnnotationsInUpgrade([]string{"cold-sidecar"}, pod) pod.Status.ContainerStatuses[1].ImageID = ImageIds["cold-sidecar:v2"] // start upgrading hot sidecar @@ -414,7 +414,7 @@ func TestUpdatePodSidecarSetHash(t *testing.T) { t.Run(cs.name, func(t *testing.T) { podInput := cs.getPod() sidecarSetInput := cs.getSidecarSet() - UpdatePodSidecarSetHash(podInput, sidecarSetInput) + UpdatePodSidecarSetHash(podInput, sidecarSetInput, false) // sidecarSet hash sidecarSetHash := make(map[string]SidecarSetUpgradeSpec) err := json.Unmarshal([]byte(podInput.Annotations[SidecarSetHashAnnotation]), &sidecarSetHash) @@ -446,6 +446,128 @@ func TestUpdatePodSidecarSetHash(t *testing.T) { } } +func TestUpdatePodSidecarSetHashState(t *testing.T) { + cases := []struct { + name string + getPod func() *corev1.Pod + getSidecarSet func() *appsv1alpha1.SidecarSet + exceptRevision map[string]SidecarSetUpgradeSpec + exceptWithoutImageRevision map[string]SidecarSetUpgradeSpec + }{ + { + name: "sidecarConatiner inplaceUpdate, not ready", + getPod: func() *corev1.Pod { + pod := podDemo.DeepCopy() + pod.Annotations[SidecarSetHashAnnotation] = `{"test-sidecarset":{"hash":"aaa"}}` + pod.Annotations[SidecarSetHashWithoutImageAnnotation] = `{"test-sidecarset":{"hash":"without-image-aaa"}}` + pod.Annotations[SidecarsetInplaceUpdateStateKey] = `{"test-sidecarset": {"revision":"new-revision","lastContainerStatuses":{"sidecar-mesh":{"imageID":"envoy@sha256:1ba0da74b20aad52b091877b0e0ece503c563f39e37aa6b0e46777c4d820a2ae"}}}}` + pod.Status.ContainerStatuses[1].Ready = false + return pod + }, + getSidecarSet: func() *appsv1alpha1.SidecarSet { + return sidecarSetDemo.DeepCopy() + }, + exceptRevision: map[string]SidecarSetUpgradeSpec{ + "test-sidecarset": { + SidecarSetHash: "aaa", + }, + }, + exceptWithoutImageRevision: map[string]SidecarSetUpgradeSpec{ + "test-sidecarset": { + State: "Updating", + }, + }, + }, + { + name: "sidecarConatiner inplaceUpdate, not ready", + getPod: func() *corev1.Pod { + pod := podDemo.DeepCopy() + pod.Annotations[SidecarSetHashAnnotation] = `{"test-sidecarset":{"hash":"aaa"}}` + pod.Annotations[SidecarSetHashWithoutImageAnnotation] = `{"test-sidecarset":{"hash":"without-image-aaa"}}` + pod.Annotations[SidecarsetInplaceUpdateStateKey] = `{"test-sidecarset": {"revision":"new-revision","lastContainerStatuses":{"sidecar-mesh":{"imageID":"envoy@sha256:1ba0da74b20aad52b091877b0e0ece503c563f39e37aa6b0e46777c4d820a2ae"}}}}` + return pod + }, + getSidecarSet: func() *appsv1alpha1.SidecarSet { + return sidecarSetDemo.DeepCopy() + }, + exceptRevision: map[string]SidecarSetUpgradeSpec{ + "test-sidecarset": { + SidecarSetHash: "aaa", + }, + }, + exceptWithoutImageRevision: map[string]SidecarSetUpgradeSpec{ + "test-sidecarset": { + State: "Normal", + }, + }, + }, + { + name: "create pod, sidecarConatiner not ready", + getPod: func() *corev1.Pod { + pod := podDemo.DeepCopy() + pod.Annotations[SidecarSetHashAnnotation] = `{"test-sidecarset":{"hash":"aaa"}}` + pod.Annotations[SidecarSetHashWithoutImageAnnotation] = `{"test-sidecarset": "without-image-aaa"}` + pod.Status.ContainerStatuses[1].Ready = false + return pod + }, + getSidecarSet: func() *appsv1alpha1.SidecarSet { + return sidecarSetDemo.DeepCopy() + }, + exceptRevision: map[string]SidecarSetUpgradeSpec{ + "test-sidecarset": { + SidecarSetHash: "aaa", + }, + }, + exceptWithoutImageRevision: map[string]SidecarSetUpgradeSpec{ + "test-sidecarset": { + State: "Updating", + }, + }, + }, + { + name: "create pod, sidecarConatiner ready", + getPod: func() *corev1.Pod { + pod := podDemo.DeepCopy() + pod.Annotations[SidecarSetHashAnnotation] = `{"test-sidecarset":{"hash":"aaa"}}` + pod.Annotations[SidecarSetHashWithoutImageAnnotation] = `{"test-sidecarset":{"hash":"without-image-aaa"}}` + return pod + }, + getSidecarSet: func() *appsv1alpha1.SidecarSet { + return sidecarSetDemo.DeepCopy() + }, + exceptRevision: map[string]SidecarSetUpgradeSpec{ + "test-sidecarset": { + SidecarSetHash: "aaa", + }, + }, + exceptWithoutImageRevision: map[string]SidecarSetUpgradeSpec{ + "test-sidecarset": { + State: "Normal", + }, + }, + }, + } + + for _, cs := range cases { + t.Run(cs.name, func(t *testing.T) { + podInput := cs.getPod() + sidecarSetInput := cs.getSidecarSet() + UpdatePodSidecarSetHash(podInput, sidecarSetInput, true) + sidecarSetHash := make(map[string]SidecarSetUpgradeSpec) + err := json.Unmarshal([]byte(podInput.Annotations[SidecarSetHashAnnotation]), &sidecarSetHash) + if err != nil { + t.Fatalf("parse pod sidecarSet hash failed: %s", err.Error()) + } + for k, o := range sidecarSetHash { + eo := cs.exceptRevision[k] + if o.SidecarSetHash != eo.SidecarSetHash { + t.Fatalf("except sidecar container %s revision %s, but get revision %s", k, eo.SidecarSetHash, o.SidecarSetHash) + } + } + }) + } +} + func TestConvertDownwardAPIFieldLabel(t *testing.T) { testCases := []struct { version string diff --git a/pkg/controller/sidecarset/sidecarset_processor.go b/pkg/controller/sidecarset/sidecarset_processor.go index 54233be433..f937482658 100644 --- a/pkg/controller/sidecarset/sidecarset_processor.go +++ b/pkg/controller/sidecarset/sidecarset_processor.go @@ -102,6 +102,11 @@ func (p *Processor) UpdateSidecarSet(sidecarSet *appsv1alpha1.SidecarSet) (recon klog.V(3).Infof("sidecarset %s matched pods has some update in flight: %v, will sync later", sidecarSet.Name, inflightPods) return reconcile.Result{RequeueAfter: time.Second}, nil } + // update PodSidecarSetHashState + for _, pod := range pods { + sidecarcontrol.UpdatePodSidecarSetHash(pod, sidecarSet, true) + p.Client.Update(context.TODO(), pod) + } // 3. If sidecar container hot upgrade complete, then set the other one(empty sidecar container) image to HotUpgradeEmptyImage if isSidecarSetHasHotUpgradeContainer(sidecarSet) { @@ -601,7 +606,7 @@ func updatePodSidecarContainer(control sidecarcontrol.SidecarControl, pod *corev } } // update sidecarSet hash in pod annotations[kruise.io/sidecarset-hash] - sidecarcontrol.UpdatePodSidecarSetHash(pod, sidecarSet) + sidecarcontrol.UpdatePodSidecarSetHash(pod, sidecarSet, false) // update pod information in upgrade // UpdatePodAnnotationsInUpgrade needs to be called when Update Container, including hot-upgrade reset empty image. // However, reset empty image should not update pod sidecarSet hash annotation, so UpdatePodSidecarSetHash needs to be called additionally diff --git a/pkg/controller/sidecarset/sidecarset_strategy_test.go b/pkg/controller/sidecarset/sidecarset_strategy_test.go index b13d8ebfd6..2c22217bbb 100644 --- a/pkg/controller/sidecarset/sidecarset_strategy_test.go +++ b/pkg/controller/sidecarset/sidecarset_strategy_test.go @@ -93,7 +93,7 @@ func factoryPodsCommon(count, upgraded int, sidecarSet *appsv1alpha1.SidecarSet) } for i := 0; i < upgraded; i++ { pods[i].Spec.Containers[1].Image = "test-image:v2" - sidecarcontrol.UpdatePodSidecarSetHash(pods[i], control.GetSidecarset()) + sidecarcontrol.UpdatePodSidecarSetHash(pods[i], control.GetSidecarset(), false) control.UpdatePodAnnotationsInUpgrade([]string{"test-sidecar"}, pods[i]) } return pods diff --git a/pkg/webhook/pod/mutating/sidecarset.go b/pkg/webhook/pod/mutating/sidecarset.go index df85689b04..0e9926d5b6 100644 --- a/pkg/webhook/pod/mutating/sidecarset.go +++ b/pkg/webhook/pod/mutating/sidecarset.go @@ -450,6 +450,7 @@ func buildSidecars(isUpdated bool, pod *corev1.Pod, oldPod *corev1.Pod, matchedS sidecarSetHash[sidecarSet.Name] = setUpgrade1 sidecarSetHashWithoutImage[sidecarSet.Name] = setUpgrade2 } + sidecarcontrol.UpdateSidecarSetHashState(sidecarSetHash, sidecarSet.Name, "Updating") } // store sidecarset hash in pod annotations