Skip to content

Commit

Permalink
fix: prevent duplicate process for pending when qos is enabled
Browse files Browse the repository at this point in the history
  • Loading branch information
shreddedbacon committed Sep 18, 2023
1 parent 6956170 commit 139dedf
Show file tree
Hide file tree
Showing 2 changed files with 25 additions and 13 deletions.
36 changes: 23 additions & 13 deletions controllers/v1beta1/podmonitor_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ type LagoonMonitorReconciler struct {
RandomNamespacePrefix bool
EnableDebug bool
LagoonTargetName string
LFFQoSEnabled bool
BuildQoS BuildQoS
}

// slice of the different failure states of pods that we care about
Expand Down Expand Up @@ -144,19 +146,27 @@ func (r *LagoonMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Reques
// we check all `LagoonBuild` in the requested namespace
// if there are no running jobs, we check for any pending jobs
// sorted by their creation timestamp and set the first to running
opLog.Info(fmt.Sprintf("Checking for any pending builds."))
runningBuilds := &lagoonv1beta1.LagoonBuildList{}
listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{
client.InNamespace(req.Namespace),
client.MatchingLabels(map[string]string{"lagoon.sh/buildStatus": lagoonv1beta1.BuildStatusRunning.String()}),
})
// list all builds in the namespace that have the running buildstatus
if err := r.List(ctx, runningBuilds, listOption); err != nil {
return ctrl.Result{}, fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err)
}
// if we have no running builds, then check for any pending builds
if len(runningBuilds.Items) == 0 {
return ctrl.Result{}, helpers.CancelExtraBuilds(ctx, r.Client, opLog, req.Namespace, "Running")
if !r.LFFQoSEnabled {
// if qos is not enabled, then handle the check for pending builds here
opLog.Info(fmt.Sprintf("Checking for any pending builds."))
runningBuilds := &lagoonv1beta1.LagoonBuildList{}
listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{
client.InNamespace(req.Namespace),
client.MatchingLabels(map[string]string{"lagoon.sh/buildStatus": lagoonv1beta1.BuildStatusRunning.String()}),
})
// list all builds in the namespace that have the running buildstatus
if err := r.List(ctx, runningBuilds, listOption); err != nil {
return ctrl.Result{}, fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err)
}
// if we have no running builds, then check for any pending builds
if len(runningBuilds.Items) == 0 {
return ctrl.Result{}, helpers.CancelExtraBuilds(ctx, r.Client, opLog, req.Namespace, "Running")
}
} else {
// since qos handles pending build checks as part of its own operations, we can skip the running pod check step with no-op
if r.EnableDebug {
opLog.Info(fmt.Sprintf("No pending build check in namespaces when QoS is enabled"))
}
}
}
return ctrl.Result{}, nil
Expand Down
2 changes: 2 additions & 0 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -810,6 +810,8 @@ func main() {
RandomNamespacePrefix: randomPrefix,
EnableDebug: enableDebug,
LagoonTargetName: lagoonTargetName,
LFFQoSEnabled: lffQoSEnabled,
BuildQoS: buildQoSConfig,
}).SetupWithManager(mgr); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "LagoonMonitor")
os.Exit(1)
Expand Down

0 comments on commit 139dedf

Please sign in to comment.