From 2e4b8501600ae6007521e662ba3833f3c3987d57 Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Tue, 8 Aug 2023 12:34:58 +1000 Subject: [PATCH 1/6] refactor: split queue process out --- controllers/v1beta1/build_qoshandler.go | 26 ++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/controllers/v1beta1/build_qoshandler.go b/controllers/v1beta1/build_qoshandler.go index 1d9957a0..041528ea 100644 --- a/controllers/v1beta1/build_qoshandler.go +++ b/controllers/v1beta1/build_qoshandler.go @@ -52,18 +52,37 @@ func (r *LagoonBuildReconciler) whichBuildNext(ctx context.Context, opLog logr.L if err := r.List(ctx, runningBuilds, listOption); err != nil { return fmt.Errorf("Unable to list builds in the cluster, there may be none or something went wrong: %v", err) } + buildsToStart := r.BuildQoS.MaxBuilds - len(runningBuilds.Items) if len(runningBuilds.Items) >= r.BuildQoS.MaxBuilds { // if the maximum number of builds is hit, then drop out and try again next time if r.EnableDebug { opLog.Info(fmt.Sprintf("Currently %v running builds, no room for new builds to be started", len(runningBuilds.Items))) } + go r.processQueue(ctx, opLog, buildsToStart, true) return nil } - buildsToStart := r.BuildQoS.MaxBuilds - len(runningBuilds.Items) if buildsToStart > 0 { opLog.Info(fmt.Sprintf("Currently %v running builds, room for %v builds to be started", len(runningBuilds.Items), buildsToStart)) // if there are any free slots to start a build, do that here - listOption = (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + go r.processQueue(ctx, opLog, buildsToStart, false) + } + return nil +} + +var runningProcessQueue bool + +func (r *LagoonBuildReconciler) processQueue(ctx context.Context, opLog logr.Logger, buildsToStart int, limitHit bool) error { + // this should only ever be able to run one instance of at a time within a single controller + // this is because this process is quite heavy when it goes to submit the queue messages to the api + // the downside of this is that there can be delays with the messages it sends to the actual + // status of the builds, but build complete/fail/cancel will always win out on the lagoon-core side + // so this isn't that much of an issue if there are some delays in the messages + if !runningProcessQueue { + runningProcessQueue = true + if r.EnableDebug { + opLog.Info(fmt.Sprintf("Processing queue")) + } + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ client.MatchingLabels(map[string]string{ "lagoon.sh/buildStatus": lagoonv1beta1.BuildStatusPending.String(), "lagoon.sh/controller": r.ControllerNamespace, @@ -98,7 +117,7 @@ func (r *LagoonBuildReconciler) whichBuildNext(ctx context.Context, opLog logr.L return pendingBuilds.Items[i].ObjectMeta.CreationTimestamp.Before(&pendingBuilds.Items[j].ObjectMeta.CreationTimestamp) }) for idx, pBuild := range pendingBuilds.Items { - if idx <= buildsToStart { + if idx <= buildsToStart && !limitHit { if r.EnableDebug { opLog.Info(fmt.Sprintf("Checking if build %s can be started", pBuild.ObjectMeta.Name)) } @@ -146,6 +165,7 @@ func (r *LagoonBuildReconciler) whichBuildNext(ctx context.Context, opLog logr.L r.updateQueuedBuild(ctx, pBuild, fmt.Sprintf("This build is currently queued in position %v/%v", (idx+1), len(pendingBuilds.Items)), opLog) } } + runningProcessQueue = false } return nil } From 134963bb779e8dbaff4e2c708a80d9683d50e2d7 Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Thu, 14 Sep 2023 10:38:30 +1000 Subject: [PATCH 2/6] chore: add some additional notes about the queued processor --- controllers/v1beta1/build_qoshandler.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/controllers/v1beta1/build_qoshandler.go b/controllers/v1beta1/build_qoshandler.go index 7c2a2f7c..028fd1c4 100644 --- a/controllers/v1beta1/build_qoshandler.go +++ b/controllers/v1beta1/build_qoshandler.go @@ -71,6 +71,10 @@ func (r *LagoonBuildReconciler) whichBuildNext(ctx context.Context, opLog logr.L var runningProcessQueue bool +// this is a processor for any builds that are currently `queued` status. all normal build activity will still be performed +// this just allows the controller to update any builds that are in the queue periodically +// if this ran on every single event, it would flood the queue with messages, so it is restricted using `runningProcessQueue` global +// to only run the process at any one time til it is complete func (r *LagoonBuildReconciler) processQueue(ctx context.Context, opLog logr.Logger, buildsToStart int, limitHit bool) error { // this should only ever be able to run one instance of at a time within a single controller // this is because this process is quite heavy when it goes to submit the queue messages to the api From 6956170c9f404c87afc1727a0a63fd9bb47117f7 Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Mon, 18 Sep 2023 13:23:56 +1000 Subject: [PATCH 3/6] fix: sorting for priority and time --- controllers/v1beta1/build_qoshandler.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/controllers/v1beta1/build_qoshandler.go b/controllers/v1beta1/build_qoshandler.go index 028fd1c4..111040fd 100644 --- a/controllers/v1beta1/build_qoshandler.go +++ b/controllers/v1beta1/build_qoshandler.go @@ -81,6 +81,7 @@ func (r *LagoonBuildReconciler) processQueue(ctx context.Context, opLog logr.Log // the downside of this is that there can be delays with the messages it sends to the actual // status of the builds, but build complete/fail/cancel will always win out on the lagoon-core side // so this isn't that much of an issue if there are some delays in the messages + opLog = opLog.WithName("QueueProcessor") if !runningProcessQueue { runningProcessQueue = true if r.EnableDebug { @@ -112,16 +113,17 @@ func (r *LagoonBuildReconciler) processQueue(ctx context.Context, opLog logr.Log if ok := pendingBuilds.Items[j].Spec.Build.Priority; ok != nil { jPriority = *pendingBuilds.Items[j].Spec.Build.Priority } - if iPriority < jPriority { - return false + // better sorting based on priority then creation timestamp + switch { + case iPriority != jPriority: + return iPriority < jPriority + default: + return pendingBuilds.Items[i].ObjectMeta.CreationTimestamp.Before(&pendingBuilds.Items[j].ObjectMeta.CreationTimestamp) } - if iPriority > jPriority { - return true - } - return pendingBuilds.Items[i].ObjectMeta.CreationTimestamp.Before(&pendingBuilds.Items[j].ObjectMeta.CreationTimestamp) }) for idx, pBuild := range pendingBuilds.Items { - if idx <= buildsToStart && !limitHit { + // need to +1 to index because 0 + if idx+1 <= buildsToStart && !limitHit { if r.EnableDebug { opLog.Info(fmt.Sprintf("Checking if build %s can be started", pBuild.ObjectMeta.Name)) } @@ -145,6 +147,8 @@ func (r *LagoonBuildReconciler) processQueue(ctx context.Context, opLog logr.Log // continue on otherwise to allow the queued status updater to run return err } + // don't handle the queued process for this build, continue to next in the list + continue } // The object is not being deleted, so if it does not have our finalizer, // then lets add the finalizer and update the object. This is equivalent From 139dedf61a627cc3dfac144c93998f7d9187c8bc Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Mon, 18 Sep 2023 13:24:20 +1000 Subject: [PATCH 4/6] fix: prevent duplicate process for pending when qos is enabled --- controllers/v1beta1/podmonitor_controller.go | 36 +++++++++++++------- main.go | 2 ++ 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/controllers/v1beta1/podmonitor_controller.go b/controllers/v1beta1/podmonitor_controller.go index 3057846c..260a313c 100644 --- a/controllers/v1beta1/podmonitor_controller.go +++ b/controllers/v1beta1/podmonitor_controller.go @@ -48,6 +48,8 @@ type LagoonMonitorReconciler struct { RandomNamespacePrefix bool EnableDebug bool LagoonTargetName string + LFFQoSEnabled bool + BuildQoS BuildQoS } // slice of the different failure states of pods that we care about @@ -144,19 +146,27 @@ func (r *LagoonMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Reques // we check all `LagoonBuild` in the requested namespace // if there are no running jobs, we check for any pending jobs // sorted by their creation timestamp and set the first to running - opLog.Info(fmt.Sprintf("Checking for any pending builds.")) - runningBuilds := &lagoonv1beta1.LagoonBuildList{} - listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ - client.InNamespace(req.Namespace), - client.MatchingLabels(map[string]string{"lagoon.sh/buildStatus": lagoonv1beta1.BuildStatusRunning.String()}), - }) - // list all builds in the namespace that have the running buildstatus - if err := r.List(ctx, runningBuilds, listOption); err != nil { - return ctrl.Result{}, fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err) - } - // if we have no running builds, then check for any pending builds - if len(runningBuilds.Items) == 0 { - return ctrl.Result{}, helpers.CancelExtraBuilds(ctx, r.Client, opLog, req.Namespace, "Running") + if !r.LFFQoSEnabled { + // if qos is not enabled, then handle the check for pending builds here + opLog.Info(fmt.Sprintf("Checking for any pending builds.")) + runningBuilds := &lagoonv1beta1.LagoonBuildList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(req.Namespace), + client.MatchingLabels(map[string]string{"lagoon.sh/buildStatus": lagoonv1beta1.BuildStatusRunning.String()}), + }) + // list all builds in the namespace that have the running buildstatus + if err := r.List(ctx, runningBuilds, listOption); err != nil { + return ctrl.Result{}, fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err) + } + // if we have no running builds, then check for any pending builds + if len(runningBuilds.Items) == 0 { + return ctrl.Result{}, helpers.CancelExtraBuilds(ctx, r.Client, opLog, req.Namespace, "Running") + } + } else { + // since qos handles pending build checks as part of its own operations, we can skip the running pod check step with no-op + if r.EnableDebug { + opLog.Info(fmt.Sprintf("No pending build check in namespaces when QoS is enabled")) + } } } return ctrl.Result{}, nil diff --git a/main.go b/main.go index 8958525c..04ac3694 100644 --- a/main.go +++ b/main.go @@ -810,6 +810,8 @@ func main() { RandomNamespacePrefix: randomPrefix, EnableDebug: enableDebug, LagoonTargetName: lagoonTargetName, + LFFQoSEnabled: lffQoSEnabled, + BuildQoS: buildQoSConfig, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "LagoonMonitor") os.Exit(1) From 5d82335088df8dcb59731b057b093d44f362649f Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Mon, 18 Sep 2023 17:31:22 +1000 Subject: [PATCH 5/6] test: add test for sorting builds --- controllers/v1beta1/build_helpers.go | 25 ++- controllers/v1beta1/build_helpers_test.go | 233 ++++++++++++++++++++++ controllers/v1beta1/build_qoshandler.go | 20 +- controllers/v1beta1/task_controller.go | 6 +- internal/helpers/helpers.go | 10 +- 5 files changed, 269 insertions(+), 25 deletions(-) create mode 100644 controllers/v1beta1/build_helpers_test.go diff --git a/controllers/v1beta1/build_helpers.go b/controllers/v1beta1/build_helpers.go index d91dc3fb..50703ccb 100644 --- a/controllers/v1beta1/build_helpers.go +++ b/controllers/v1beta1/build_helpers.go @@ -731,7 +731,7 @@ func (r *LagoonBuildReconciler) processBuild(ctx context.Context, opLog logr.Log VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: "lagoon-sshkey", - DefaultMode: helpers.IntPtr(420), + DefaultMode: helpers.Int32Ptr(420), }, }, }, @@ -751,7 +751,7 @@ func (r *LagoonBuildReconciler) processBuild(ctx context.Context, opLog logr.Log VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: serviceaccountTokenSecret, - DefaultMode: helpers.IntPtr(420), + DefaultMode: helpers.Int32Ptr(420), }, }, }) @@ -1015,3 +1015,24 @@ func (r *LagoonBuildReconciler) cancelExtraBuilds(ctx context.Context, opLog log } return nil } + +func sortBuilds(defaultPriority int, pendingBuilds *lagoonv1beta1.LagoonBuildList) { + sort.Slice(pendingBuilds.Items, func(i, j int) bool { + // sort by priority, then creation timestamp + iPriority := defaultPriority + jPriority := defaultPriority + if ok := pendingBuilds.Items[i].Spec.Build.Priority; ok != nil { + iPriority = *pendingBuilds.Items[i].Spec.Build.Priority + } + if ok := pendingBuilds.Items[j].Spec.Build.Priority; ok != nil { + jPriority = *pendingBuilds.Items[j].Spec.Build.Priority + } + // better sorting based on priority then creation timestamp + switch { + case iPriority != jPriority: + return iPriority < jPriority + default: + return pendingBuilds.Items[i].ObjectMeta.CreationTimestamp.Before(&pendingBuilds.Items[j].ObjectMeta.CreationTimestamp) + } + }) +} diff --git a/controllers/v1beta1/build_helpers_test.go b/controllers/v1beta1/build_helpers_test.go new file mode 100644 index 00000000..5692de32 --- /dev/null +++ b/controllers/v1beta1/build_helpers_test.go @@ -0,0 +1,233 @@ +package v1beta1 + +import ( + "testing" + "time" + + "github.com/google/go-cmp/cmp" + lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" + "github.com/uselagoon/remote-controller/internal/helpers" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func timeFromString(s string) time.Time { + time, _ := time.Parse("2006-01-02T15:04:05.000Z", s) + return time +} + +func Test_sortBuilds(t *testing.T) { + type args struct { + defaultPriority int + pendingBuilds *lagoonv1beta1.LagoonBuildList + } + tests := []struct { + name string + args args + wantBuilds *lagoonv1beta1.LagoonBuildList + }{ + { + name: "test1 - 5 and 6 same time order by priority", + args: args{ + defaultPriority: 5, + pendingBuilds: &lagoonv1beta1.LagoonBuildList{ + Items: []lagoonv1beta1.LagoonBuild{ + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-abcdefg", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:45:00.000Z")), + }, + Spec: lagoonv1beta1.LagoonBuildSpec{ + Build: lagoonv1beta1.Build{ + Priority: helpers.IntPtr(5), + }, + }, + }, + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-1234567", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:45:00.000Z")), + }, + Spec: lagoonv1beta1.LagoonBuildSpec{ + Build: lagoonv1beta1.Build{ + Priority: helpers.IntPtr(6), + }, + }, + }, + }, + }, + }, + wantBuilds: &lagoonv1beta1.LagoonBuildList{ + Items: []lagoonv1beta1.LagoonBuild{ + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-abcdefg", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:45:00.000Z")), + }, + Spec: lagoonv1beta1.LagoonBuildSpec{ + Build: lagoonv1beta1.Build{ + Priority: helpers.IntPtr(5), + }, + }, + }, + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-1234567", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:45:00.000Z")), + }, + Spec: lagoonv1beta1.LagoonBuildSpec{ + Build: lagoonv1beta1.Build{ + Priority: helpers.IntPtr(6), + }, + }, + }, + }, + }, + }, + { + name: "test2 - 2x5 sorted by time", + args: args{ + defaultPriority: 5, + pendingBuilds: &lagoonv1beta1.LagoonBuildList{ + Items: []lagoonv1beta1.LagoonBuild{ + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-abcdefg", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:50:00.000Z")), + }, + Spec: lagoonv1beta1.LagoonBuildSpec{ + Build: lagoonv1beta1.Build{ + Priority: helpers.IntPtr(5), + }, + }, + }, + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-1234567", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:45:00.000Z")), + }, + Spec: lagoonv1beta1.LagoonBuildSpec{ + Build: lagoonv1beta1.Build{ + Priority: helpers.IntPtr(5), + }, + }, + }, + }, + }, + }, + wantBuilds: &lagoonv1beta1.LagoonBuildList{ + Items: []lagoonv1beta1.LagoonBuild{ + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-1234567", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:45:00.000Z")), + }, + Spec: lagoonv1beta1.LagoonBuildSpec{ + Build: lagoonv1beta1.Build{ + Priority: helpers.IntPtr(5), + }, + }, + }, + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-abcdefg", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:50:00.000Z")), + }, + Spec: lagoonv1beta1.LagoonBuildSpec{ + Build: lagoonv1beta1.Build{ + Priority: helpers.IntPtr(5), + }, + }, + }, + }, + }, + }, + { + name: "test3 - 2x5 and 1x6 sorted by priority then time", + args: args{ + defaultPriority: 5, + pendingBuilds: &lagoonv1beta1.LagoonBuildList{ + Items: []lagoonv1beta1.LagoonBuild{ + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-abcdefg", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:50:00.000Z")), + }, + Spec: lagoonv1beta1.LagoonBuildSpec{ + Build: lagoonv1beta1.Build{ + Priority: helpers.IntPtr(5), + }, + }, + }, + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-abc1234", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:46:00.000Z")), + }, + Spec: lagoonv1beta1.LagoonBuildSpec{ + Build: lagoonv1beta1.Build{ + Priority: helpers.IntPtr(6), + }, + }, + }, + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-1234567", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:45:00.000Z")), + }, + Spec: lagoonv1beta1.LagoonBuildSpec{ + Build: lagoonv1beta1.Build{ + Priority: helpers.IntPtr(5), + }, + }, + }, + }, + }, + }, + wantBuilds: &lagoonv1beta1.LagoonBuildList{ + Items: []lagoonv1beta1.LagoonBuild{ + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-1234567", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:45:00.000Z")), + }, + Spec: lagoonv1beta1.LagoonBuildSpec{ + Build: lagoonv1beta1.Build{ + Priority: helpers.IntPtr(5), + }, + }, + }, + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-abcdefg", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:50:00.000Z")), + }, + Spec: lagoonv1beta1.LagoonBuildSpec{ + Build: lagoonv1beta1.Build{ + Priority: helpers.IntPtr(5), + }, + }, + }, + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-abc1234", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:46:00.000Z")), + }, + Spec: lagoonv1beta1.LagoonBuildSpec{ + Build: lagoonv1beta1.Build{ + Priority: helpers.IntPtr(6), + }, + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sortBuilds(tt.args.defaultPriority, tt.args.pendingBuilds) + if !cmp.Equal(tt.args.pendingBuilds, tt.wantBuilds) { + t.Errorf("sortBuilds() = %v, want %v", tt.args.pendingBuilds, tt.wantBuilds) + } + }) + } +} diff --git a/controllers/v1beta1/build_qoshandler.go b/controllers/v1beta1/build_qoshandler.go index 111040fd..8b17b645 100644 --- a/controllers/v1beta1/build_qoshandler.go +++ b/controllers/v1beta1/build_qoshandler.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "sort" "github.com/go-logr/logr" lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" @@ -103,24 +102,7 @@ func (r *LagoonBuildReconciler) processQueue(ctx context.Context, opLog logr.Log } // if we have any pending builds, then grab the latest one and make it running // if there are any other pending builds, cancel them so only the latest one runs - sort.Slice(pendingBuilds.Items, func(i, j int) bool { - // sort by priority, then creation timestamp - iPriority := r.BuildQoS.DefaultValue - jPriority := r.BuildQoS.DefaultValue - if ok := pendingBuilds.Items[i].Spec.Build.Priority; ok != nil { - iPriority = *pendingBuilds.Items[i].Spec.Build.Priority - } - if ok := pendingBuilds.Items[j].Spec.Build.Priority; ok != nil { - jPriority = *pendingBuilds.Items[j].Spec.Build.Priority - } - // better sorting based on priority then creation timestamp - switch { - case iPriority != jPriority: - return iPriority < jPriority - default: - return pendingBuilds.Items[i].ObjectMeta.CreationTimestamp.Before(&pendingBuilds.Items[j].ObjectMeta.CreationTimestamp) - } - }) + sortBuilds(r.BuildQoS.DefaultValue, pendingBuilds) for idx, pBuild := range pendingBuilds.Items { // need to +1 to index because 0 if idx+1 <= buildsToStart && !limitHit { diff --git a/controllers/v1beta1/task_controller.go b/controllers/v1beta1/task_controller.go index c66b461f..dae8387c 100644 --- a/controllers/v1beta1/task_controller.go +++ b/controllers/v1beta1/task_controller.go @@ -386,7 +386,7 @@ func (r *LagoonTaskReconciler) createAdvancedTask(ctx context.Context, lagoonTas VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: "lagoon-sshkey", - DefaultMode: helpers.IntPtr(420), + DefaultMode: helpers.Int32Ptr(420), }, }, } @@ -423,7 +423,7 @@ func (r *LagoonTaskReconciler) createAdvancedTask(ctx context.Context, lagoonTas VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: serviceaccountTokenSecret, - DefaultMode: helpers.IntPtr(420), + DefaultMode: helpers.Int32Ptr(420), }, }, }) @@ -587,7 +587,7 @@ func (r *LagoonTaskReconciler) createAdvancedTask(ctx context.Context, lagoonTas VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: secret.Name, - DefaultMode: helpers.IntPtr(444), + DefaultMode: helpers.Int32Ptr(444), }, }, } diff --git a/internal/helpers/helpers.go b/internal/helpers/helpers.go index 0c39bb51..809eef78 100644 --- a/internal/helpers/helpers.go +++ b/internal/helpers/helpers.go @@ -105,7 +105,15 @@ func TaskContainsStatus(slice []lagoonv1beta1.LagoonTaskConditions, s lagoonv1be } // IntPtr . -func IntPtr(i int32) *int32 { +func IntPtr(i int) *int { + var iPtr *int + iPtr = new(int) + *iPtr = i + return iPtr +} + +// Int32Ptr . +func Int32Ptr(i int32) *int32 { var iPtr *int32 iPtr = new(int32) *iPtr = i From 108c490e27e064bc3193d289ff292781dc5f6d56 Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Tue, 19 Sep 2023 12:01:40 +1000 Subject: [PATCH 6/6] fix: missing false resets --- controllers/v1beta1/build_qoshandler.go | 11 ++++++++++- go.mod | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/controllers/v1beta1/build_qoshandler.go b/controllers/v1beta1/build_qoshandler.go index 8b17b645..51467ae2 100644 --- a/controllers/v1beta1/build_qoshandler.go +++ b/controllers/v1beta1/build_qoshandler.go @@ -74,6 +74,8 @@ var runningProcessQueue bool // this just allows the controller to update any builds that are in the queue periodically // if this ran on every single event, it would flood the queue with messages, so it is restricted using `runningProcessQueue` global // to only run the process at any one time til it is complete +// buildsToStart is the number of builds that can be started at the time the process is called +// limitHit is used to determine if the build limit has been hit, this is used to prevent new builds from being started inside this process func (r *LagoonBuildReconciler) processQueue(ctx context.Context, opLog logr.Logger, buildsToStart int, limitHit bool) error { // this should only ever be able to run one instance of at a time within a single controller // this is because this process is quite heavy when it goes to submit the queue messages to the api @@ -94,6 +96,7 @@ func (r *LagoonBuildReconciler) processQueue(ctx context.Context, opLog logr.Log }) pendingBuilds := &lagoonv1beta1.LagoonBuildList{} if err := r.List(ctx, pendingBuilds, listOption); err != nil { + runningProcessQueue = false return fmt.Errorf("Unable to list builds in the cluster, there may be none or something went wrong: %v", err) } if len(pendingBuilds.Items) > 0 { @@ -120,6 +123,7 @@ func (r *LagoonBuildReconciler) processQueue(ctx context.Context, opLog logr.Log }) // list any builds that are running if err := r.List(ctx, runningNSBuilds, listOption); err != nil { + runningProcessQueue = false return fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err) } // if there are no running builds, check if there are any pending builds that can be started @@ -127,6 +131,7 @@ func (r *LagoonBuildReconciler) processQueue(ctx context.Context, opLog logr.Log if err := helpers.CancelExtraBuilds(ctx, r.Client, opLog, pBuild.ObjectMeta.Namespace, "Running"); err != nil { // only return if there is an error doing this operation // continue on otherwise to allow the queued status updater to run + runningProcessQueue = false return err } // don't handle the queued process for this build, continue to next in the list @@ -144,6 +149,7 @@ func (r *LagoonBuildReconciler) processQueue(ctx context.Context, opLog logr.Log }, }) if err := r.Patch(ctx, &pBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { + runningProcessQueue = false return err } } @@ -151,7 +157,10 @@ func (r *LagoonBuildReconciler) processQueue(ctx context.Context, opLog logr.Log // update the build to be queued, and add a log message with the build log with the current position in the queue // this position will update as builds are created/processed, so the position of a build could change depending on // higher or lower priority builds being created - r.updateQueuedBuild(ctx, pBuild, (idx + 1), len(pendingBuilds.Items), opLog) + if err := r.updateQueuedBuild(ctx, pBuild, (idx + 1), len(pendingBuilds.Items), opLog); err != nil { + runningProcessQueue = false + return nil + } } } runningProcessQueue = false diff --git a/go.mod b/go.mod index 8bfaf27f..46085b36 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/cheshir/go-mq/v2 v2.0.1 github.com/coreos/go-semver v0.3.1 github.com/go-logr/logr v1.2.4 + github.com/google/go-cmp v0.5.9 github.com/hashicorp/go-version v1.6.0 github.com/k8up-io/k8up/v2 v2.7.1 github.com/mittwald/goharbor-client/v3 v3.3.0 @@ -51,7 +52,6 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/gnostic v0.5.7-v3refs // indirect - github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/uuid v1.3.0 // indirect github.com/imdario/mergo v0.3.13 // indirect