SidecarSet add upgrade state in pod annotation

Signed-off-by: pingjiang <[email protected]>

pkg/control/sidecarcontrol/util.go

@@ -81,6 +81,7 @@ type SidecarSetUpgradeSpec struct {
  SidecarSetName string `json:"sidecarSetName"`
  SidecarList []string `json:"sidecarList"` // sidecarSet container list
  SidecarSetControllerRevision string `json:"controllerRevision,omitempty"` // sidecarSet controllerRevision name
+ State string `json:"state"` // enum: Normal, Updating
 }
 
 // PodMatchSidecarSet determines if pod match Selector of sidecar.
@@ -210,7 +211,7 @@ func IsPodSidecarUpdated(sidecarSet *appsv1alpha1.SidecarSet, pod *corev1.Pod) b
 }
 
 // UpdatePodSidecarSetHash when sidecarSet in-place update sidecar container, Update sidecarSet hash in Pod annotations[kruise.io/sidecarset-hash]
-func UpdatePodSidecarSetHash(pod *corev1.Pod, sidecarSet *appsv1alpha1.SidecarSet) {
+func UpdatePodSidecarSetHash(pod *corev1.Pod, sidecarSet *appsv1alpha1.SidecarSet, onlyState bool) {
  hashKey := SidecarSetHashAnnotation
  sidecarSetHash := make(map[string]SidecarSetUpgradeSpec)
  if err := json.Unmarshal([]byte(pod.Annotations[hashKey]), &sidecarSetHash); err != nil {
@@ -247,17 +248,53 @@ func UpdatePodSidecarSetHash(pod *corev1.Pod, sidecarSet *appsv1alpha1.SidecarSe
  sidecarList.Insert(sidecar.Name)
  }
 
- sidecarSetHash[sidecarSet.Name] = SidecarSetUpgradeSpec{
- UpdateTimestamp: metav1.Now(),
- SidecarSetHash: GetSidecarSetRevision(sidecarSet),
- SidecarSetName: sidecarSet.Name,
- SidecarList: sidecarList.List(),
- SidecarSetControllerRevision: sidecarSet.Status.LatestRevision,
+ if onlyState {
+ var state string
+ state = "Updating"
+ sidecars := GetSidecarContainersInPod(sidecarSet)
+ if _, ok := pod.Annotations[SidecarsetInplaceUpdateStateKey]; ok {
+ if IsSidecarContainerUpdateCompleted(pod, sets.NewString(sidecarSet.Name), sidecars) && IsSiderCarContainersReady(pod, sidecars) {
+ state = "Normal"
+ }
+ } else {
+ if IsSiderCarContainersReady(pod, sidecars) {
+ state = "Normal"
+ }
+ }
+ UpdateSidecarSetHashState(sidecarSetHash, sidecarSet.Name, state)
+ } else {
+ sidecarSetHash[sidecarSet.Name] = SidecarSetUpgradeSpec{
+ UpdateTimestamp: metav1.Now(),
+ SidecarSetHash: GetSidecarSetRevision(sidecarSet),
+ SidecarSetName: sidecarSet.Name,
+ SidecarList: sidecarList.List(),
+ SidecarSetControllerRevision: sidecarSet.Status.LatestRevision,
+ State: sidecarSetHash[sidecarSet.Name].State,
+ }
  }
  newHash, _ := json.Marshal(sidecarSetHash)
  pod.Annotations[hashKey] = string(newHash)
 }
 
+func UpdateSidecarSetHashState(sidecarSetHash map[string]SidecarSetUpgradeSpec, sidecarSetName string, State string) {
+ upgradeSpec := sidecarSetHash[sidecarSetName]
+ upgradeSpec.State = State
+ sidecarSetHash[sidecarSetName] = upgradeSpec
+}
+
+func IsSiderCarContainersReady(pod *corev1.Pod, containers sets.String) bool {
+ for _, cs := range pod.Status.ContainerStatuses {
+ // only check containers set
+ if !containers.Has(cs.Name) {
+ continue
+ }
+ if !cs.Ready {
+ return false
+ }
+ }
+ return true
+}
+
 func GetSidecarContainersInPod(sidecarSet *appsv1alpha1.SidecarSet) sets.String {
  names := sets.NewString()
  for _, sidecarContainer := range sidecarSet.Spec.Containers {

pkg/control/sidecarcontrol/util_test.go

@@ -176,7 +176,7 @@ func TestIsSidecarContainerUpdateCompleted(t *testing.T) {
  pod := podDemo.DeepCopy()
  control := New(sidecarSetDemo.DeepCopy())
  pod.Spec.Containers[1].Image = "cold-sidecar:v2"
- UpdatePodSidecarSetHash(pod, control.GetSidecarset())
+ UpdatePodSidecarSetHash(pod, control.GetSidecarset(), false)
  control.UpdatePodAnnotationsInUpgrade([]string{"cold-sidecar"}, pod)
  return pod
  },
@@ -191,7 +191,7 @@ func TestIsSidecarContainerUpdateCompleted(t *testing.T) {
  pod := podDemo.DeepCopy()
  control := New(sidecarSetDemo.DeepCopy())
  pod.Spec.Containers[1].Image = "cold-sidecar:v2"
- UpdatePodSidecarSetHash(pod, control.GetSidecarset())
+ UpdatePodSidecarSetHash(pod, control.GetSidecarset(), false)
  control.UpdatePodAnnotationsInUpgrade([]string{"cold-sidecar"}, pod)
  pod.Status.ContainerStatuses[1].ImageID = ImageIds["cold-sidecar:v2"]
  return pod
@@ -208,7 +208,7 @@ func TestIsSidecarContainerUpdateCompleted(t *testing.T) {
  control := New(sidecarSetDemo.DeepCopy())
  // upgrade cold sidecar completed
  pod.Spec.Containers[1].Image = "cold-sidecar:v2"
- UpdatePodSidecarSetHash(pod, control.GetSidecarset())
+ UpdatePodSidecarSetHash(pod, control.GetSidecarset(), false)
  control.UpdatePodAnnotationsInUpgrade([]string{"cold-sidecar"}, pod)
  pod.Status.ContainerStatuses[1].ImageID = ImageIds["cold-sidecar:v2"]
  // start upgrading hot sidecar
@@ -228,7 +228,7 @@ func TestIsSidecarContainerUpdateCompleted(t *testing.T) {
  control := New(sidecarSetDemo.DeepCopy())
  // upgrade cold sidecar completed
  pod.Spec.Containers[1].Image = "cold-sidecar:v2"
- UpdatePodSidecarSetHash(pod, control.GetSidecarset())
+ UpdatePodSidecarSetHash(pod, control.GetSidecarset(), false)
  control.UpdatePodAnnotationsInUpgrade([]string{"cold-sidecar"}, pod)
  pod.Status.ContainerStatuses[1].ImageID = ImageIds["cold-sidecar:v2"]
  // start upgrading hot sidecar
@@ -251,7 +251,7 @@ func TestIsSidecarContainerUpdateCompleted(t *testing.T) {
  control := New(sidecarSetDemo.DeepCopy())
  // upgrade cold sidecar completed
  pod.Spec.Containers[1].Image = "cold-sidecar:v2"
- UpdatePodSidecarSetHash(pod, control.GetSidecarset())
+ UpdatePodSidecarSetHash(pod, control.GetSidecarset(), false)
  control.UpdatePodAnnotationsInUpgrade([]string{"cold-sidecar"}, pod)
  pod.Status.ContainerStatuses[1].ImageID = ImageIds["cold-sidecar:v2"]
  // start upgrading hot sidecar
@@ -414,7 +414,7 @@ func TestUpdatePodSidecarSetHash(t *testing.T) {
  t.Run(cs.name, func(t *testing.T) {
  podInput := cs.getPod()
  sidecarSetInput := cs.getSidecarSet()
- UpdatePodSidecarSetHash(podInput, sidecarSetInput)
+ UpdatePodSidecarSetHash(podInput, sidecarSetInput, false)
  // sidecarSet hash
  sidecarSetHash := make(map[string]SidecarSetUpgradeSpec)
  err := json.Unmarshal([]byte(podInput.Annotations[SidecarSetHashAnnotation]), &sidecarSetHash)
@@ -446,6 +446,128 @@ func TestUpdatePodSidecarSetHash(t *testing.T) {
  }
 }
 
+func TestUpdatePodSidecarSetHashState(t *testing.T) {
+ cases := []struct {
+ name string
+ getPod func() *corev1.Pod
+ getSidecarSet func() *appsv1alpha1.SidecarSet
+ exceptRevision map[string]SidecarSetUpgradeSpec
+ exceptWithoutImageRevision map[string]SidecarSetUpgradeSpec
+ }{
+ {
+ name: "sidecarConatiner inplaceUpdate, not ready",
+ getPod: func() *corev1.Pod {
+ pod := podDemo.DeepCopy()
+ pod.Annotations[SidecarSetHashAnnotation] = `{"test-sidecarset":{"hash":"aaa"}}`
+ pod.Annotations[SidecarSetHashWithoutImageAnnotation] = `{"test-sidecarset":{"hash":"without-image-aaa"}}`
+ pod.Annotations[SidecarsetInplaceUpdateStateKey] = `{"test-sidecarset": {"revision":"new-revision","lastContainerStatuses":{"sidecar-mesh":{"imageID":"envoy@sha256:1ba0da74b20aad52b091877b0e0ece503c563f39e37aa6b0e46777c4d820a2ae"}}}}`
+ pod.Status.ContainerStatuses[1].Ready = false
+ return pod
+ },
+ getSidecarSet: func() *appsv1alpha1.SidecarSet {
+ return sidecarSetDemo.DeepCopy()
+ },
+ exceptRevision: map[string]SidecarSetUpgradeSpec{
+ "test-sidecarset": {
+ SidecarSetHash: "aaa",
+ },
+ },
+ exceptWithoutImageRevision: map[string]SidecarSetUpgradeSpec{
+ "test-sidecarset": {
+ State: "Updating",
+ },
+ },
+ },
+ {
+ name: "sidecarConatiner inplaceUpdate, not ready",
+ getPod: func() *corev1.Pod {
+ pod := podDemo.DeepCopy()
+ pod.Annotations[SidecarSetHashAnnotation] = `{"test-sidecarset":{"hash":"aaa"}}`
+ pod.Annotations[SidecarSetHashWithoutImageAnnotation] = `{"test-sidecarset":{"hash":"without-image-aaa"}}`
+ pod.Annotations[SidecarsetInplaceUpdateStateKey] = `{"test-sidecarset": {"revision":"new-revision","lastContainerStatuses":{"sidecar-mesh":{"imageID":"envoy@sha256:1ba0da74b20aad52b091877b0e0ece503c563f39e37aa6b0e46777c4d820a2ae"}}}}`
+ return pod
+ },
+ getSidecarSet: func() *appsv1alpha1.SidecarSet {
+ return sidecarSetDemo.DeepCopy()
+ },
+ exceptRevision: map[string]SidecarSetUpgradeSpec{
+ "test-sidecarset": {
+ SidecarSetHash: "aaa",
+ },
+ },
+ exceptWithoutImageRevision: map[string]SidecarSetUpgradeSpec{
+ "test-sidecarset": {
+ State: "Normal",
+ },
+ },
+ },
+ {
+ name: "create pod, sidecarConatiner not ready",
+ getPod: func() *corev1.Pod {
+ pod := podDemo.DeepCopy()
+ pod.Annotations[SidecarSetHashAnnotation] = `{"test-sidecarset":{"hash":"aaa"}}`
+ pod.Annotations[SidecarSetHashWithoutImageAnnotation] = `{"test-sidecarset": "without-image-aaa"}`
+ pod.Status.ContainerStatuses[1].Ready = false
+ return pod
+ },
+ getSidecarSet: func() *appsv1alpha1.SidecarSet {
+ return sidecarSetDemo.DeepCopy()
+ },
+ exceptRevision: map[string]SidecarSetUpgradeSpec{
+ "test-sidecarset": {
+ SidecarSetHash: "aaa",
+ },
+ },
+ exceptWithoutImageRevision: map[string]SidecarSetUpgradeSpec{
+ "test-sidecarset": {
+ State: "Updating",
+ },
+ },
+ },
+ {
+ name: "create pod, sidecarConatiner ready",
+ getPod: func() *corev1.Pod {
+ pod := podDemo.DeepCopy()
+ pod.Annotations[SidecarSetHashAnnotation] = `{"test-sidecarset":{"hash":"aaa"}}`
+ pod.Annotations[SidecarSetHashWithoutImageAnnotation] = `{"test-sidecarset":{"hash":"without-image-aaa"}}`
+ return pod
+ },
+ getSidecarSet: func() *appsv1alpha1.SidecarSet {
+ return sidecarSetDemo.DeepCopy()
+ },
+ exceptRevision: map[string]SidecarSetUpgradeSpec{
+ "test-sidecarset": {
+ SidecarSetHash: "aaa",
+ },
+ },
+ exceptWithoutImageRevision: map[string]SidecarSetUpgradeSpec{
+ "test-sidecarset": {
+ State: "Normal",
+ },
+ },
+ },
+ }
+
+ for _, cs := range cases {
+ t.Run(cs.name, func(t *testing.T) {
+ podInput := cs.getPod()
+ sidecarSetInput := cs.getSidecarSet()
+ UpdatePodSidecarSetHash(podInput, sidecarSetInput, true)
+ sidecarSetHash := make(map[string]SidecarSetUpgradeSpec)
+ err := json.Unmarshal([]byte(podInput.Annotations[SidecarSetHashAnnotation]), &sidecarSetHash)
+ if err != nil {
+ t.Fatalf("parse pod sidecarSet hash failed: %s", err.Error())
+ }
+ for k, o := range sidecarSetHash {
+ eo := cs.exceptRevision[k]
+ if o.SidecarSetHash != eo.SidecarSetHash {
+ t.Fatalf("except sidecar container %s revision %s, but get revision %s", k, eo.SidecarSetHash, o.SidecarSetHash)
+ }
+ }
+ })
+ }
+}
+
 func TestConvertDownwardAPIFieldLabel(t *testing.T) {
  testCases := []struct {
  version string

pkg/controller/sidecarset/sidecarset_processor.go

@@ -102,6 +102,11 @@ func (p *Processor) UpdateSidecarSet(sidecarSet *appsv1alpha1.SidecarSet) (recon
  klog.V(3).Infof("sidecarset %s matched pods has some update in flight: %v, will sync later", sidecarSet.Name, inflightPods)
  return reconcile.Result{RequeueAfter: time.Second}, nil
  }
+ // update PodSidecarSetHashState
+ for _, pod := range pods {
+ sidecarcontrol.UpdatePodSidecarSetHash(pod, sidecarSet, true)
+ p.Client.Update(context.TODO(), pod)
+ }
 
  // 3. If sidecar container hot upgrade complete, then set the other one(empty sidecar container) image to HotUpgradeEmptyImage
  if isSidecarSetHasHotUpgradeContainer(sidecarSet) {
@@ -601,7 +606,7 @@ func updatePodSidecarContainer(control sidecarcontrol.SidecarControl, pod *corev
  }
  }
  // update sidecarSet hash in pod annotations[kruise.io/sidecarset-hash]
- sidecarcontrol.UpdatePodSidecarSetHash(pod, sidecarSet)
+ sidecarcontrol.UpdatePodSidecarSetHash(pod, sidecarSet, false)
  // update pod information in upgrade
  // UpdatePodAnnotationsInUpgrade needs to be called when Update Container, including hot-upgrade reset empty image.
  // However, reset empty image should not update pod sidecarSet hash annotation, so UpdatePodSidecarSetHash needs to be called additionally

pkg/controller/sidecarset/sidecarset_strategy_test.go

@@ -93,7 +93,7 @@ func factoryPodsCommon(count, upgraded int, sidecarSet *appsv1alpha1.SidecarSet)
  }
  for i := 0; i < upgraded; i++ {
  pods[i].Spec.Containers[1].Image = "test-image:v2"
- sidecarcontrol.UpdatePodSidecarSetHash(pods[i], control.GetSidecarset())
+ sidecarcontrol.UpdatePodSidecarSetHash(pods[i], control.GetSidecarset(), false)
  control.UpdatePodAnnotationsInUpgrade([]string{"test-sidecar"}, pods[i])
  }
  return pods

pkg/webhook/pod/mutating/sidecarset.go

@@ -450,6 +450,7 @@ func buildSidecars(isUpdated bool, pod *corev1.Pod, oldPod *corev1.Pod, matchedS
  sidecarSetHash[sidecarSet.Name] = setUpgrade1
  sidecarSetHashWithoutImage[sidecarSet.Name] = setUpgrade2
  }
+ sidecarcontrol.UpdateSidecarSetHashState(sidecarSetHash, sidecarSet.Name, "Updating")
  }
 
  // store sidecarset hash in pod annotations