From 0f82afcb9eb36ba10519a30efc1d0286cb30e715 Mon Sep 17 00:00:00 2001 From: Leonardo Cecchi Date: Wed, 9 Oct 2024 10:57:29 +0200 Subject: [PATCH] fix: avoid concurrent updates on the designed primary (#5755) This patch fixes a condition where a demoted replica cluster is stuck waiting for the primary to be demoted to a designed primary, due to a concurrent update on the Cluster CR by the operator and the instance manager. The update by the instance manager is now using the update method instead of a patch, relying on the Kubernetes optimistic locking mechanism. Closes: #5754 Signed-off-by: Leonardo Cecchi Co-authored-by: Armando Ruocco --- .../controller/instance_controller.go | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/internal/management/controller/instance_controller.go b/internal/management/controller/instance_controller.go index 358a6e20ad..c3e65197b7 100644 --- a/internal/management/controller/instance_controller.go +++ b/internal/management/controller/instance_controller.go @@ -36,6 +36,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/util/retry" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -1281,13 +1282,25 @@ func (r *InstanceReconciler) reconcileDesignatedPrimary( // I'm the primary, need to inform the operator log.FromContext(ctx).Info("Setting myself as the current designated primary") - oldCluster := cluster.DeepCopy() - cluster.Status.CurrentPrimary = r.instance.GetPodName() - cluster.Status.CurrentPrimaryTimestamp = pgTime.GetCurrentTimestamp() - if r.instance.RequiresDesignatedPrimaryTransition { - externalcluster.SetDesignatedPrimaryTransitionCompleted(cluster) - } - return changed, r.client.Status().Patch(ctx, cluster, client.MergeFrom(oldCluster)) + return changed, retry.RetryOnConflict(retry.DefaultBackoff, func() error { + var livingCluster apiv1.Cluster + + err := r.client.Get(ctx, client.ObjectKeyFromObject(cluster), &livingCluster) + if err != nil { + return err + } + + updatedCluster := livingCluster.DeepCopy() + updatedCluster.Status.CurrentPrimary = r.instance.GetPodName() + updatedCluster.Status.CurrentPrimaryTimestamp = pgTime.GetCurrentTimestamp() + if r.instance.RequiresDesignatedPrimaryTransition { + externalcluster.SetDesignatedPrimaryTransitionCompleted(updatedCluster) + } + + cluster.Status = updatedCluster.Status + + return r.client.Status().Update(ctx, updatedCluster) + }) } // waitForWalReceiverDown wait until the wal receiver is down, and it's used