Skip to content

Commit

Permalink
handle edge cases
Browse files Browse the repository at this point in the history
  • Loading branch information
@czeslavo
czeslavo committed Oct 2, 2024
1 parent cc41e00 commit 943586e
Show file tree
Hide file tree
Showing 7 changed files with 689 additions and 288 deletions.
25 changes: 25 additions & 0 deletions controller/konnect/errors.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,3 +111,28 @@ type ReferencedKongCertificateDoesNotExist struct {
func (e ReferencedKongCertificateDoesNotExist) Error() string {
return fmt.Sprintf("referenced Kong Certificate %s does not exist: %v", e.Reference, e.Err)
}

// ReferencedKongKeySetDoesNotExist is an error type that is returned when
// a Konnect entity references a KongKeySet which does not exist.
type ReferencedKongKeySetDoesNotExist struct {
Reference types.NamespacedName
Err error
}

// Error implements the error interface.
func (e ReferencedKongKeySetDoesNotExist) Error() string {
return fmt.Sprintf("referenced KongKeySet %s does not exist: %v", e.Reference, e.Err)
}

// ReferencedKongKeySetIsBeingDeleted is an error type that is returned when
// a Konnect entity references a KongKeySet which is being deleted.
type ReferencedKongKeySetIsBeingDeleted struct {
Reference types.NamespacedName
DeletionTimestamp time.Time
}

// Error implements the error interface.
func (e ReferencedKongKeySetIsBeingDeleted) Error() string {
return fmt.Sprintf("referenced KongKeySet %s is being deleted (deletion timestamp: %s)",
e.Reference, e.DeletionTimestamp)
}
325 changes: 41 additions & 284 deletions controller/konnect/reconciler_generic.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -291,6 +291,38 @@ func (r *KonnectEntityReconciler[T, TEnt]) Reconcile(
// If a type has a KongKeySet ref, handle it.
res, err = handleKongKeySetRef(ctx, r.Client, ent)
if err != nil || !res.IsZero() {
// If the referenced KongKeySet is being deleted and the object
// is not being deleted yet then requeue until it will
// get the deletion timestamp set due to having the owner set to KongKeySet.
if errDel := (&ReferencedKongKeySetIsBeingDeleted{}); errors.As(err, errDel) &&
ent.GetDeletionTimestamp().IsZero() {
return ctrl.Result{
RequeueAfter: time.Until(errDel.DeletionTimestamp),
}, nil
}

// If the referenced KongKeySet is not found or is being deleted
// and the object is being deleted, remove the finalizer and let the
// deletion proceed without trying to delete the entity from Konnect
// as the KongKeySet deletion will take care of it on the Konnect side.
if errors.As(err, &ReferencedKongKeySetIsBeingDeleted{}) ||
errors.As(err, &ReferencedKongKeySetDoesNotExist{}) {
if !ent.GetDeletionTimestamp().IsZero() {
if controllerutil.RemoveFinalizer(ent, KonnectCleanupFinalizer) {
if err := r.Client.Update(ctx, ent); err != nil {
if k8serrors.IsConflict(err) {
return ctrl.Result{Requeue: true}, nil
}
return ctrl.Result{}, fmt.Errorf("failed to remove finalizer %s: %w", KonnectCleanupFinalizer, err)
}
log.Debug(logger, "finalizer removed as the owning KongKeySet is being deleted or is already gone", ent,
"finalizer", KonnectCleanupFinalizer,
)
return ctrl.Result{}, nil
}
}
}

return res, err
}

Expand Down Expand Up @@ -731,189 +763,6 @@ func getConsumerRef[T constraints.SupportedKonnectEntityType, TEnt constraints.E
}
}

func getServiceRef[T constraints.SupportedKonnectEntityType, TEnt constraints.EntityType[T]](
e TEnt,
) mo.Option[configurationv1alpha1.ServiceRef] {
switch e := any(e).(type) {
case *configurationv1alpha1.KongRoute:
if e.Spec.ServiceRef == nil {
return mo.None[configurationv1alpha1.ServiceRef]()
}
return mo.Some(*e.Spec.ServiceRef)
default:
return mo.None[configurationv1alpha1.ServiceRef]()
}
}

func getKeySetRef[T constraints.SupportedKonnectEntityType, TEnt constraints.EntityType[T]](
e TEnt,
) mo.Option[configurationv1alpha1.KeySetRef] {
switch e := any(e).(type) {
case *configurationv1alpha1.KongKey:
if e.Spec.KeySetRef == nil {
return mo.None[configurationv1alpha1.KeySetRef]()
}
return mo.Some(*e.Spec.KeySetRef)
default:
return mo.None[configurationv1alpha1.KeySetRef]()
}
}

// handleKongServiceRef handles the ServiceRef for the given entity.
// It sets the owner reference to the referenced KongService and updates the
// status of the entity based on the referenced KongService status.
func handleKongServiceRef[T constraints.SupportedKonnectEntityType, TEnt constraints.EntityType[T]](
ctx context.Context,
cl client.Client,
ent TEnt,
) (ctrl.Result, error) {
kongServiceRef, ok := getServiceRef(ent).Get()
if !ok {
return ctrl.Result{}, nil
}
switch kongServiceRef.Type {
case configurationv1alpha1.ServiceRefNamespacedRef:
svc := configurationv1alpha1.KongService{}
nn := types.NamespacedName{
Name: kongServiceRef.NamespacedRef.Name,
// TODO: handle cross namespace refs
Namespace: ent.GetNamespace(),
}

if err := cl.Get(ctx, nn, &svc); err != nil {
if res, errStatus := updateStatusWithCondition(
ctx, cl, ent,
conditions.KongServiceRefValidConditionType,
metav1.ConditionFalse,
conditions.KongServiceRefReasonInvalid,
err.Error(),
); errStatus != nil || res.Requeue {
return res, errStatus
}

return ctrl.Result{}, fmt.Errorf("can't get the referenced KongService %s: %w", nn, err)
}

// If referenced KongService is being deleted, return an error so that we
// can remove the entity from Konnect first.
if delTimestamp := svc.GetDeletionTimestamp(); !delTimestamp.IsZero() {
return ctrl.Result{}, ReferencedKongServiceIsBeingDeleted{
Reference: nn,
}
}

cond, ok := k8sutils.GetCondition(conditions.KonnectEntityProgrammedConditionType, &svc)
if !ok || cond.Status != metav1.ConditionTrue {
ent.SetKonnectID("")
if res, err := updateStatusWithCondition(
ctx, cl, ent,
conditions.KongServiceRefValidConditionType,
metav1.ConditionFalse,
conditions.KongServiceRefReasonInvalid,
fmt.Sprintf("Referenced KongService %s is not programmed yet", nn),
); err != nil || res.Requeue {
return ctrl.Result{}, err
}
return ctrl.Result{Requeue: true}, nil
}

old := ent.DeepCopyObject().(TEnt)
if err := controllerutil.SetOwnerReference(&svc, ent, cl.Scheme(), controllerutil.WithBlockOwnerDeletion(true)); err != nil {
return ctrl.Result{}, fmt.Errorf("failed to set owner reference: %w", err)
}
if err := cl.Patch(ctx, ent, client.MergeFrom(old)); err != nil {
if k8serrors.IsConflict(err) {
return ctrl.Result{Requeue: true}, nil
}
return ctrl.Result{}, fmt.Errorf("failed to update status: %w", err)
}

// TODO(pmalek): make this generic.
// Service ID is not stored in KonnectEntityStatus because not all entities
// have a ServiceRef, hence the type constraints in the reconciler can't be used.
if route, ok := any(ent).(*configurationv1alpha1.KongRoute); ok {
if route.Status.Konnect == nil {
route.Status.Konnect = &konnectv1alpha1.KonnectEntityStatusWithControlPlaneAndServiceRefs{}
}
route.Status.Konnect.ServiceID = svc.Status.Konnect.GetKonnectID()
}

if res, errStatus := updateStatusWithCondition(
ctx, cl, ent,
conditions.KongServiceRefValidConditionType,
metav1.ConditionTrue,
conditions.KongServiceRefReasonValid,
fmt.Sprintf("Referenced KongService %s programmed", nn),
); errStatus != nil || res.Requeue {
return res, errStatus
}

cpRef, ok := getControlPlaneRef(&svc).Get()
if !ok {
return ctrl.Result{}, fmt.Errorf(
"KongRoute references a KongService %s which does not have a ControlPlane ref",
client.ObjectKeyFromObject(&svc),
)
}
cp, err := getCPForRef(ctx, cl, cpRef, ent.GetNamespace())
if err != nil {
if res, errStatus := updateStatusWithCondition(
ctx, cl, ent,
conditions.ControlPlaneRefValidConditionType,
metav1.ConditionFalse,
conditions.ControlPlaneRefReasonInvalid,
err.Error(),
); errStatus != nil || res.Requeue {
return res, errStatus
}
if k8serrors.IsNotFound(err) {
return ctrl.Result{}, ReferencedControlPlaneDoesNotExistError{
Reference: nn,
Err: err,
}
}
return ctrl.Result{}, err
}

cond, ok = k8sutils.GetCondition(conditions.KonnectEntityProgrammedConditionType, cp)
if !ok || cond.Status != metav1.ConditionTrue || cond.ObservedGeneration != cp.GetGeneration() {
if res, errStatus := updateStatusWithCondition(
ctx, cl, ent,
conditions.ControlPlaneRefValidConditionType,
metav1.ConditionFalse,
conditions.ControlPlaneRefReasonInvalid,
fmt.Sprintf("Referenced ControlPlane %s is not programmed yet", nn),
); errStatus != nil || res.Requeue {
return res, errStatus
}

return ctrl.Result{Requeue: true}, nil
}

// TODO(pmalek): make this generic.
// CP ID is not stored in KonnectEntityStatus because not all entities
// have a ControlPlaneRef, hence the type constraints in the reconciler can't be used.
if resource, ok := any(ent).(EntityWithControlPlaneRef); ok {
resource.SetControlPlaneID(cp.Status.ID)
}

if res, errStatus := updateStatusWithCondition(
ctx, cl, ent,
conditions.ControlPlaneRefValidConditionType,
metav1.ConditionTrue,
conditions.ControlPlaneRefReasonValid,
fmt.Sprintf("Referenced ControlPlane %s is programmed", nn),
); errStatus != nil || res.Requeue {
return res, errStatus
}

default:
return ctrl.Result{}, fmt.Errorf("unimplemented KongService ref type %q", kongServiceRef.Type)
}

return ctrl.Result{}, nil
}

// handleKongConsumerRef handles the ConsumerRef for the given entity.
// It sets the owner reference to the referenced KongConsumer and updates the
// status of the entity based on the referenced KongConsumer status.
Expand Down Expand Up @@ -1205,11 +1054,18 @@ func handleControlPlaneRef[T constraints.SupportedKonnectEntityType, TEnt constr
return ctrl.Result{Requeue: true}, nil
}

old := ent.DeepCopyObject().(TEnt)
if ent.GetNamespace() != "" {
var (
old = ent.DeepCopyObject().(TEnt)

// A cluster scoped object cannot set a namespaced object as its owner, and also we cannot set cross namespaced owner reference.
// So we skip setting owner reference for cluster scoped resources (KongVault).
// TODO: handle cross namespace refs
isNamespaceScoped = ent.GetNamespace() != ""

// If an entity has another owner, we should not set the owner reference as that would prevent the entity from being deleted.
hasNoOwners = len(ent.GetOwnerReferences()) == 0
)
if isNamespaceScoped && hasNoOwners {
if err := controllerutil.SetOwnerReference(&cp, ent, cl.Scheme(), controllerutil.WithBlockOwnerDeletion(true)); err != nil {
return ctrl.Result{}, fmt.Errorf("failed to set owner reference: %w", err)
}
Expand Down Expand Up @@ -1245,105 +1101,6 @@ func handleControlPlaneRef[T constraints.SupportedKonnectEntityType, TEnt constr
}
}

// handleKongKeySetRef handles the KeySetRef for the given entity.
func handleKongKeySetRef[T constraints.SupportedKonnectEntityType, TEnt constraints.EntityType[T]](
ctx context.Context,
cl client.Client,
ent TEnt,
) (ctrl.Result, error) {
keySetRef, ok := getKeySetRef(ent).Get()
if !ok {
if key, ok := any(ent).(*configurationv1alpha1.KongKey); ok {
// If the entity has a resolved reference, but the spec has changed, we need to adjust the status.
if key.Status.Konnect != nil && key.Status.Konnect.GetKeySetID() != "" {
key.Status.Konnect.KeySetID = ""
if res, err := updateStatusWithCondition(ctx, cl, key,
conditions.KeySetRefValidConditionType,
metav1.ConditionTrue,
conditions.KeySetRefReasonValid,
"KeySetRef is nil",
); err != nil || !res.IsZero() {
return res, err
}
}
}
return ctrl.Result{}, nil
}

if keySetRef.Type != configurationv1alpha1.KeySetRefNamespacedRef {
return ctrl.Result{}, fmt.Errorf("unsupported KeySet ref type %q", keySetRef.Type)
}

keySet := configurationv1alpha1.KongKeySet{}
nn := types.NamespacedName{
Name: keySetRef.NamespacedRef.Name,
Namespace: ent.GetNamespace(),
}

if err := cl.Get(ctx, nn, &keySet); err != nil {
if res, errStatus := updateStatusWithCondition(
ctx, cl, ent,
conditions.KeySetRefValidConditionType,
metav1.ConditionFalse,
conditions.KeySetRefReasonInvalid,
err.Error(),
); errStatus != nil || res.Requeue {
return res, errStatus
}

// If it was not found, let's requeue.
if k8serrors.IsNotFound(err) {
return ctrl.Result{Requeue: true}, nil
}

return ctrl.Result{}, fmt.Errorf("failed getting KongKeySet %s: %w", nn, err)
}

// If referenced KongKeySet is being deleted, requeue.
if delTimestamp := keySet.GetDeletionTimestamp(); !delTimestamp.IsZero() {
return ctrl.Result{
RequeueAfter: time.Until(delTimestamp.Time),
}, nil
}

// Verify that the KongKeySet is programmed.
cond, ok := k8sutils.GetCondition(conditions.KonnectEntityProgrammedConditionType, &keySet)
if !ok || cond.Status != metav1.ConditionTrue {
if res, err := updateStatusWithCondition(
ctx, cl, ent,
conditions.KeySetRefValidConditionType,
metav1.ConditionFalse,
conditions.KeySetRefReasonInvalid,
fmt.Sprintf("Referenced KongKeySet %s is not programmed yet", nn),
); err != nil || res.Requeue {
return ctrl.Result{}, err
}
return ctrl.Result{Requeue: true}, nil
}

// TODO: make this generic.
// KongKeySet ID is not stored in KonnectEntityStatus because not all entities
// have a KeySetRef, hence the type constraints in the reconciler can't be used.
if key, ok := any(ent).(*configurationv1alpha1.KongKey); ok {
if key.Status.Konnect == nil {
key.Status.Konnect = &konnectv1alpha1.KonnectEntityStatusWithControlPlaneAndKeySetRef{}
}
key.Status.Konnect.KeySetID = keySet.Status.Konnect.GetKonnectID()
}

if res, errStatus := updateStatusWithCondition(
ctx, cl, ent,
conditions.KeySetRefValidConditionType,
metav1.ConditionTrue,
conditions.KeySetRefReasonValid,
fmt.Sprintf("Referenced KongKeySet %s programmed", nn),
); errStatus != nil || res.Requeue {
return res, errStatus
}

return ctrl.Result{}, nil
}

func conditionMessageReferenceKonnectAPIAuthConfigurationInvalid(apiAuthRef types.NamespacedName) string {
return fmt.Sprintf("referenced KonnectAPIAuthConfiguration %s is invalid", apiAuthRef)
}
Expand Down
Loading

0 comments on commit 943586e

Please sign in to comment.