SidecarSet add upgrade state in pod annotation (openkruise#1312)

pkg/control/sidecarcontrol/util.go

@@ -75,12 +75,20 @@ var (
  UpdateExpectations = expectations.NewUpdateExpectations(RevisionAdapterImpl)
 )
 
+type SidecarSetUpgradeState string
+
+const (
+ SidecarSetUpgradeStateNormal SidecarSetUpgradeState = "Normal"
+ SidecarSetUpgradeStateUpdating SidecarSetUpgradeState = "Updating"
+)
+
 type SidecarSetUpgradeSpec struct {
- UpdateTimestamp metav1.Time `json:"updateTimestamp"`
- SidecarSetHash string `json:"hash"`
- SidecarSetName string `json:"sidecarSetName"`
- SidecarList []string `json:"sidecarList"` // sidecarSet container list
- SidecarSetControllerRevision string `json:"controllerRevision,omitempty"` // sidecarSet controllerRevision name
+ UpdateTimestamp metav1.Time `json:"updateTimestamp"`
+ SidecarSetHash string `json:"hash"`
+ SidecarSetName string `json:"sidecarSetName"`
+ SidecarList []string `json:"sidecarList"` // sidecarSet container list
+ SidecarSetControllerRevision string `json:"controllerRevision,omitempty"` // sidecarSet controllerRevision name
+ State SidecarSetUpgradeState `json:"state,omitempty"` // sidecarSet upgrade state
 }
 
 // PodMatchSidecarSet determines if pod match Selector of sidecar.
@@ -210,7 +218,7 @@ func IsPodSidecarUpdated(sidecarSet *appsv1alpha1.SidecarSet, pod *corev1.Pod) b
 }
 
 // UpdatePodSidecarSetHash when sidecarSet in-place update sidecar container, Update sidecarSet hash in Pod annotations[kruise.io/sidecarset-hash]
-func UpdatePodSidecarSetHash(pod *corev1.Pod, sidecarSet *appsv1alpha1.SidecarSet) {
+func UpdatePodSidecarSetHash(pod *corev1.Pod, sidecarSet *appsv1alpha1.SidecarSet, state SidecarSetUpgradeState) {
  hashKey := SidecarSetHashAnnotation
  sidecarSetHash := make(map[string]SidecarSetUpgradeSpec)
  if err := json.Unmarshal([]byte(pod.Annotations[hashKey]), &sidecarSetHash); err != nil {
@@ -256,11 +264,34 @@ func UpdatePodSidecarSetHash(pod *corev1.Pod, sidecarSet *appsv1alpha1.SidecarSe
  SidecarSetName: sidecarSet.Name,
  SidecarList: sidecarList.List(),
  SidecarSetControllerRevision: sidecarSet.Status.LatestRevision,
+ State: state,
  }
  newHash, _ := json.Marshal(sidecarSetHash)
  pod.Annotations[hashKey] = string(newHash)
 }
 
+// GetPodSidecarSetHashUpdatedState retrieves the SidecarSet hash annotation after the state is updated.
+func GetPodSidecarSetHashUpdatedState(pod *corev1.Pod, sidecarSet *appsv1alpha1.SidecarSet, state SidecarSetUpgradeState) (string, error) {
+ sidecarSetHash := make(map[string]SidecarSetUpgradeSpec)
+ if err := json.Unmarshal([]byte(pod.Annotations[SidecarSetHashAnnotation]), &sidecarSetHash); err != nil {
+ return "", err
+ }
+ spec, exists := sidecarSetHash[sidecarSet.Name]
+ if !exists {
+ return "", fmt.Errorf("SidecarSet %s not found", sidecarSet.Name)
+ }
+
+ spec.State = state
+ sidecarSetHash[sidecarSet.Name] = spec
+ newHash, _ := json.Marshal(sidecarSetHash)
+
+ updatedAnnotation, err := json.Marshal(metav1.ObjectMeta{Annotations: map[string]string{SidecarSetHashAnnotation: string(newHash)}})
+ if err != nil {
+ return "", err
+ }
+ return string(updatedAnnotation), nil
+}
+
 func GetSidecarContainersInPod(sidecarSet *appsv1alpha1.SidecarSet) sets.String {
  names := sets.NewString()
  for _, sidecarContainer := range sidecarSet.Spec.Containers {
@@ -575,3 +606,12 @@ func IsSidecarContainer(container corev1.Container) bool {
  }
  return false
 }
+
+// IsPodSidecarSetUpgradeStateAwaitingPatchNormal checks whether the SidecarSet upgrade state needs to be patched to Normal
+func IsPodSidecarSetUpgradeStateAwaitingPatchNormal(sidecarSet *appsv1alpha1.SidecarSet, pod *corev1.Pod) bool {
+ if pod.Annotations == nil || pod.Annotations[SidecarSetHashAnnotation] == "" {
+ return false
+ }
+ spec := GetPodSidecarSetUpgradeSpecInAnnotations(sidecarSet.Name, SidecarSetHashAnnotation, pod)
+ return spec.State == SidecarSetUpgradeStateUpdating
+}

pkg/control/sidecarcontrol/util_test.go

@@ -22,6 +22,7 @@ import (
  "reflect"
  "testing"
 
+ "github.com/stretchr/testify/assert"
  utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 
  appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
@@ -178,7 +179,7 @@ func TestIsSidecarContainerUpdateCompleted(t *testing.T) {
  pod := podDemo.DeepCopy()
  control := New(sidecarSetDemo.DeepCopy())
  pod.Spec.Containers[1].Image = "cold-sidecar:v2"
- UpdatePodSidecarSetHash(pod, control.GetSidecarset())
+ UpdatePodSidecarSetHash(pod, control.GetSidecarset(), SidecarSetUpgradeStateUpdating)
  control.UpdatePodAnnotationsInUpgrade([]string{"cold-sidecar"}, pod)
  return pod
  },
@@ -193,7 +194,7 @@ func TestIsSidecarContainerUpdateCompleted(t *testing.T) {
  pod := podDemo.DeepCopy()
  control := New(sidecarSetDemo.DeepCopy())
  pod.Spec.Containers[1].Image = "cold-sidecar:v2"
- UpdatePodSidecarSetHash(pod, control.GetSidecarset())
+ UpdatePodSidecarSetHash(pod, control.GetSidecarset(), SidecarSetUpgradeStateUpdating)
  control.UpdatePodAnnotationsInUpgrade([]string{"cold-sidecar"}, pod)
  pod.Status.ContainerStatuses[1].ImageID = ImageIds["cold-sidecar:v2"]
  return pod
@@ -210,7 +211,7 @@ func TestIsSidecarContainerUpdateCompleted(t *testing.T) {
  control := New(sidecarSetDemo.DeepCopy())
  // upgrade cold sidecar completed
  pod.Spec.Containers[1].Image = "cold-sidecar:v2"
- UpdatePodSidecarSetHash(pod, control.GetSidecarset())
+ UpdatePodSidecarSetHash(pod, control.GetSidecarset(), SidecarSetUpgradeStateUpdating)
  control.UpdatePodAnnotationsInUpgrade([]string{"cold-sidecar"}, pod)
  pod.Status.ContainerStatuses[1].ImageID = ImageIds["cold-sidecar:v2"]
  // start upgrading hot sidecar
@@ -230,7 +231,7 @@ func TestIsSidecarContainerUpdateCompleted(t *testing.T) {
  control := New(sidecarSetDemo.DeepCopy())
  // upgrade cold sidecar completed
  pod.Spec.Containers[1].Image = "cold-sidecar:v2"
- UpdatePodSidecarSetHash(pod, control.GetSidecarset())
+ UpdatePodSidecarSetHash(pod, control.GetSidecarset(), SidecarSetUpgradeStateUpdating)
  control.UpdatePodAnnotationsInUpgrade([]string{"cold-sidecar"}, pod)
  pod.Status.ContainerStatuses[1].ImageID = ImageIds["cold-sidecar:v2"]
  // start upgrading hot sidecar
@@ -253,7 +254,7 @@ func TestIsSidecarContainerUpdateCompleted(t *testing.T) {
  control := New(sidecarSetDemo.DeepCopy())
  // upgrade cold sidecar completed
  pod.Spec.Containers[1].Image = "cold-sidecar:v2"
- UpdatePodSidecarSetHash(pod, control.GetSidecarset())
+ UpdatePodSidecarSetHash(pod, control.GetSidecarset(), SidecarSetUpgradeStateUpdating)
  control.UpdatePodAnnotationsInUpgrade([]string{"cold-sidecar"}, pod)
  pod.Status.ContainerStatuses[1].ImageID = ImageIds["cold-sidecar:v2"]
  // start upgrading hot sidecar
@@ -397,7 +398,6 @@ func TestUpdatePodSidecarSetHash(t *testing.T) {
  getPod: func() *corev1.Pod {
  pod := podDemo.DeepCopy()
  pod.Annotations[SidecarSetHashAnnotation] = "failed-sidecarset-hash"
- pod.Annotations[SidecarSetHashWithoutImageAnnotation] = "failed-sidecarset-hash"
  return pod
  },
  getSidecarSet: func() *appsv1alpha1.SidecarSet {
@@ -416,7 +416,7 @@ func TestUpdatePodSidecarSetHash(t *testing.T) {
  t.Run(cs.name, func(t *testing.T) {
  podInput := cs.getPod()
  sidecarSetInput := cs.getSidecarSet()
- UpdatePodSidecarSetHash(podInput, sidecarSetInput)
+ UpdatePodSidecarSetHash(podInput, sidecarSetInput, SidecarSetUpgradeStateUpdating)
  // sidecarSet hash
  sidecarSetHash := make(map[string]SidecarSetUpgradeSpec)
  err := json.Unmarshal([]byte(podInput.Annotations[SidecarSetHashAnnotation]), &sidecarSetHash)
@@ -448,6 +448,127 @@ func TestUpdatePodSidecarSetHash(t *testing.T) {
  }
 }
 
+func TestGetPodSidecarSetHashUpdatedState(t *testing.T) {
+ cases := []struct {
+ name string
+ getPod func() *corev1.Pod
+ getSidecarSet func() *appsv1alpha1.SidecarSet
+ getState SidecarSetUpgradeState
+ expectState map[string]SidecarSetUpgradeSpec
+ expectWithoutImageRevision map[string]SidecarSetUpgradeSpec
+ expectError bool
+ }{
+ {
+ name: "update normal sidecarSet state",
+ getPod: func() *corev1.Pod {
+ pod := podDemo.DeepCopy()
+ pod.Annotations[SidecarSetHashAnnotation] = `{"test-sidecarset":{"hash":"aaa","state":"Updating"}}`
+ pod.Annotations[SidecarSetHashWithoutImageAnnotation] = `{"test-sidecarset":{"hash":"without-image-aaa"}}`
+ return pod
+ },
+ getSidecarSet: func() *appsv1alpha1.SidecarSet {
+ return sidecarSetDemo.DeepCopy()
+ },
+ getState: SidecarSetUpgradeStateNormal,
+ expectState: map[string]SidecarSetUpgradeSpec{
+ "test-sidecarset": {
+ State: SidecarSetUpgradeStateNormal,
+ },
+ },
+ expectWithoutImageRevision: map[string]SidecarSetUpgradeSpec{
+ "test-sidecarset": {
+ SidecarSetHash: "without-image-aaa",
+ },
+ },
+ expectError: false,
+ },
+ {
+ name: "sidecarSet not found in annotation",
+ getPod: func() *corev1.Pod {
+ pod := &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{
+ SidecarSetHashAnnotation: `{"another-sidecarset":{"state":"Updating"}}`,
+ },
+ },
+ }
+ return pod
+ },
+ getSidecarSet: func() *appsv1alpha1.SidecarSet {
+ return &appsv1alpha1.SidecarSet{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-sidecarset",
+ },
+ }
+ },
+ getState: SidecarSetUpgradeState("Normal"),
+ expectState: nil,
+ expectError: true,
+ },
+ {
+ name: "invalid JSON in annotation",
+ getPod: func() *corev1.Pod {
+ pod := &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{
+ SidecarSetHashAnnotation: `invalid-json`,
+ },
+ },
+ }
+ return pod
+ },
+ getSidecarSet: func() *appsv1alpha1.SidecarSet {
+ return &appsv1alpha1.SidecarSet{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-sidecarset",
+ },
+ }
+ },
+ getState: SidecarSetUpgradeState("Normal"),
+ expectState: nil,
+ expectError: true,
+ },
+ }
+
+ for _, cs := range cases {
+ t.Run(cs.name, func(t *testing.T) {
+ podInput := cs.getPod()
+ sidecarSetInput := cs.getSidecarSet()
+ hashUpdatedStateAnnotation, err := GetPodSidecarSetHashUpdatedState(podInput, sidecarSetInput, cs.getState)
+ if cs.expectError {
+ assert.Error(t, err)
+ } else {
+ assert.NoError(t, err)
+ sidecarSetHash := make(map[string]SidecarSetUpgradeSpec)
+ if err = json.Unmarshal([]byte(hashUpdatedStateAnnotation), &sidecarSetHash); err != nil {
+ t.Fatalf("parse hashUpdatedStateAnnotation failed: %s", err.Error())
+ }
+ for k, o := range sidecarSetHash {
+ eo := cs.expectState[k]
+ if o.State != eo.State {
+ t.Fatalf("expect sidecar container %s state %s, but get state %s", k, eo.SidecarSetHash, o.SidecarSetHash)
+ }
+ }
+ if len(cs.expectWithoutImageRevision) == 0 {
+ return
+ }
+ // without image sidecarSet hash
+ sidecarSetHash = make(map[string]SidecarSetUpgradeSpec)
+ err = json.Unmarshal([]byte(podInput.Annotations[SidecarSetHashWithoutImageAnnotation]), &sidecarSetHash)
+ if err != nil {
+ t.Fatalf("parse pod sidecarSet hash failed: %s", err.Error())
+ }
+ for k, o := range sidecarSetHash {
+ eo := cs.expectWithoutImageRevision[k]
+ if o.SidecarSetHash != eo.SidecarSetHash {
+ t.Fatalf("except sidecar container %s revision %s, but get revision %s", k, eo.SidecarSetHash, o.SidecarSetHash)
+ }
+ }
+ }
+ })
+ }
+}
+
 func TestConvertDownwardAPIFieldLabel(t *testing.T) {
  testCases := []struct {
  version string

pkg/controller/sidecarset/sidecarset_processor.go

@@ -87,11 +87,15 @@ func (p *Processor) UpdateSidecarSet(sidecarSet *appsv1alpha1.SidecarSet) (recon
  }
 
  // 2. calculate SidecarSet status based on pod and revision information
- status := calculateStatus(control, pods, latestRevision, collisionCount)
+ status, podsAwaitingPatchNormal := calculateStatus(control, pods, latestRevision, collisionCount)
  //update sidecarSet status in store
  if err := p.updateSidecarSetStatus(sidecarSet, status); err != nil {
  return reconcile.Result{}, err
  }
+ if err := p.updateSidecarSetUpgradeState(sidecarSet, podsAwaitingPatchNormal, sidecarcontrol.SidecarSetUpgradeStateNormal); err != nil {
+ return reconcile.Result{}, err
+ }
+
  sidecarSet.Status = *status
 
  // in case of informer cache latency
@@ -191,6 +195,27 @@ func (p *Processor) updatePods(control sidecarcontrol.SidecarControl, pods []*co
  return nil
 }
 
+func (p *Processor) updateSidecarSetUpgradeState(sidecarSet *appsv1alpha1.SidecarSet, pods []*corev1.Pod,
+ sideCarSetUpgradeState sidecarcontrol.SidecarSetUpgradeState) (ret error) {
+ for _, pod := range pods {
+ podClone := pod.DeepCopy()
+ updatedAnnotation, err := sidecarcontrol.GetPodSidecarSetHashUpdatedState(podClone, sidecarSet, sideCarSetUpgradeState)
+ if err != nil {
+ klog.ErrorS(err, "SidecarSet got pod SidecarSetUpgradeState annotation failed", "sidecarSet", klog.KObj(sidecarSet), "pod", klog.KObj(podClone))
+ ret = err
+ continue
+ }
+ body := fmt.Sprintf(`{"metadata":%s}`, updatedAnnotation)
+ if err = p.Client.Patch(context.TODO(), podClone, client.RawPatch(types.MergePatchType, []byte(body))); err != nil {
+ klog.ErrorS(err, "SidecarSet patched SidecarSetUpgradeState annotation failed", "sidecarSet", klog.KObj(sidecarSet), "pod", klog.KObj(podClone))
+ ret = err
+ continue
+ }
+ klog.InfoS("SidecarSet updated pod SidecarSetUpgradeState success", "sidecarSet", klog.KObj(sidecarSet), "pod", klog.KObj(podClone))
+ }
+ return
+}
+
 func (p *Processor) updatePodSidecarAndHash(control sidecarcontrol.SidecarControl, pod *corev1.Pod) error {
  podClone := &corev1.Pod{}
  sidecarSet := control.GetSidecarset()
@@ -516,10 +541,11 @@ func replaceRevision(revisions []*apps.ControllerRevision, oldOne, newOne *apps.
 // UpdatedReadyPods: updated and ready pods number
 // UnavailablePods: MatchedPods - UpdatedReadyPods
 func calculateStatus(control sidecarcontrol.SidecarControl, pods []*corev1.Pod, latestRevision *apps.ControllerRevision, collisionCount int32,
-) *appsv1alpha1.SidecarSetStatus {
+) (*appsv1alpha1.SidecarSetStatus, []*corev1.Pod) {
  sidecarset := control.GetSidecarset()
  var matchedPods, updatedPods, readyPods, updatedAndReady int32
  matchedPods = int32(len(pods))
+ var podsAwaitingPatchNormal []*corev1.Pod
  for _, pod := range pods {
  updated := sidecarcontrol.IsPodSidecarUpdated(sidecarset, pod)
  if updated {
@@ -529,6 +555,9 @@ func calculateStatus(control sidecarcontrol.SidecarControl, pods []*corev1.Pod,
  readyPods++
  if updated {
  updatedAndReady++
+ if sidecarcontrol.IsPodSidecarSetUpgradeStateAwaitingPatchNormal(sidecarset, pod) {
+ podsAwaitingPatchNormal = append(podsAwaitingPatchNormal, pod)
+ }
  }
  }
  }
@@ -540,7 +569,7 @@ func calculateStatus(control sidecarcontrol.SidecarControl, pods []*corev1.Pod,
  UpdatedReadyPods: updatedAndReady,
  LatestRevision: latestRevision.Name,
  CollisionCount: pointer.Int32Ptr(collisionCount),
- }
+ }, podsAwaitingPatchNormal
 }
 
 func isSidecarSetNotUpdate(s *appsv1alpha1.SidecarSet) bool {
@@ -612,7 +641,7 @@ func updatePodSidecarContainer(control sidecarcontrol.SidecarControl, pod *corev
  }
  }
  // update sidecarSet hash in pod annotations[kruise.io/sidecarset-hash]
- sidecarcontrol.UpdatePodSidecarSetHash(pod, sidecarSet)
+ sidecarcontrol.UpdatePodSidecarSetHash(pod, sidecarSet, sidecarcontrol.SidecarSetUpgradeStateUpdating)
  // update pod information in upgrade
  // UpdatePodAnnotationsInUpgrade needs to be called when Update Container, including hot-upgrade reset empty image.
  // However, reset empty image should not update pod sidecarSet hash annotation, so UpdatePodSidecarSetHash needs to be called additionally

pkg/controller/sidecarset/sidecarset_strategy_test.go

@@ -93,7 +93,7 @@ func factoryPodsCommon(count, upgraded int, sidecarSet *appsv1alpha1.SidecarSet)
  }
  for i := 0; i < upgraded; i++ {
  pods[i].Spec.Containers[1].Image = "test-image:v2"
- sidecarcontrol.UpdatePodSidecarSetHash(pods[i], control.GetSidecarset())
+ sidecarcontrol.UpdatePodSidecarSetHash(pods[i], control.GetSidecarset(), sidecarcontrol.SidecarSetUpgradeStateUpdating)
  control.UpdatePodAnnotationsInUpgrade([]string{"test-sidecar"}, pods[i])
  }
  return pods

pkg/webhook/pod/mutating/sidecarset.go

@@ -358,6 +358,7 @@ func buildSidecars(isUpdated bool, pod *corev1.Pod, oldPod *corev1.Pod, matchedS
  SidecarSetHash: sidecarcontrol.GetSidecarSetRevision(sidecarSet),
  SidecarSetName: sidecarSet.Name,
  SidecarSetControllerRevision: sidecarSet.Status.LatestRevision,
+ State: sidecarcontrol.SidecarSetUpgradeStateNormal,
  }
  setUpgrade2 := sidecarcontrol.SidecarSetUpgradeSpec{
  UpdateTimestamp: metav1.Now(),

pkg/webhook/pod/mutating/sidecarset_test.go

@@ -1202,12 +1202,14 @@ func TestInjectInitContainer(t *testing.T) {
  SidecarSetHash: "sidecarset-1-hash",
  SidecarSetName: "sidecarset-1",
  SidecarList: []string{"init-1"},
+ State: sidecarcontrol.SidecarSetUpgradeStateNormal,
  }
  sidecarSetHash["sidecarset-2"] = sidecarcontrol.SidecarSetUpgradeSpec{
  UpdateTimestamp: metav1.Now(),
  SidecarSetHash: "sidecarset-2-hash",
  SidecarSetName: "sidecarset-2",
  SidecarList: []string{"hot-init"},
+ State: sidecarcontrol.SidecarSetUpgradeStateNormal,
  }
  sidecarSetHashWithoutImage["sidecarset-1"] = sidecarcontrol.SidecarSetUpgradeSpec{
  UpdateTimestamp: metav1.Now(),