diff --git a/controllers/v1beta1/build_controller.go b/controllers/v1beta1/build_controller.go index 47c4b201..03ebf734 100644 --- a/controllers/v1beta1/build_controller.go +++ b/controllers/v1beta1/build_controller.go @@ -259,9 +259,58 @@ func (r *LagoonBuildReconciler) createNamespaceBuild(ctx context.Context, // if everything is all good controller will handle the new build resource that gets created as it will have // the `lagoon.sh/buildStatus = Pending` now - // so end this reconcile process - pendingBuilds := &lagoonv1beta1.LagoonBuildList{} - return ctrl.Result{}, helpers.CancelExtraBuilds(ctx, r.Client, opLog, pendingBuilds, namespace.ObjectMeta.Name, lagoonv1beta1.BuildStatusPending.String()) + err = helpers.CancelExtraBuilds(ctx, r.Client, opLog, namespace.ObjectMeta.Name, lagoonv1beta1.BuildStatusPending.String()) + if err != nil { + return ctrl.Result{}, err + } + + // as this is a new build coming through, check if there are any running builds in the namespace + // if there are, then check the status of that build. if the build pod is missing then the build running will block + // if the pod exists, attempt to get the status of it (only if its complete or failed) and ship the status + runningBuilds := &lagoonv1beta1.LagoonBuildList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(namespace.ObjectMeta.Name), + client.MatchingLabels(map[string]string{"lagoon.sh/buildStatus": lagoonv1beta1.BuildStatusRunning.String()}), + }) + // list all builds in the namespace that have the running buildstatus + if err := r.List(ctx, runningBuilds, listOption); err != nil { + return ctrl.Result{}, fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err) + } + // if there are running builds still, check if the pod exists or if the pod is complete/failed and attempt to get the status + for _, rBuild := range runningBuilds.Items { + runningBuild := rBuild.DeepCopy() + lagoonBuildPod := corev1.Pod{} + err := r.Get(ctx, types.NamespacedName{ + Namespace: rBuild.ObjectMeta.Namespace, + Name: rBuild.ObjectMeta.Name, + }, &lagoonBuildPod) + buildCondition := lagoonv1beta1.BuildStatusCancelled + if err != nil { + // cancel the build as there is no pod available + opLog.Info(fmt.Sprintf("Setting build %s as cancelled", runningBuild.ObjectMeta.Name)) + runningBuild.Labels["lagoon.sh/buildStatus"] = buildCondition.String() + } else { + // get the status from the pod and update the build + if lagoonBuildPod.Status.Phase == corev1.PodFailed || lagoonBuildPod.Status.Phase == corev1.PodSucceeded { + buildCondition = helpers.GetBuildConditionFromPod(lagoonBuildPod.Status.Phase) + opLog.Info(fmt.Sprintf("Setting build %s as %s", runningBuild.ObjectMeta.Name, buildCondition.String())) + runningBuild.Labels["lagoon.sh/buildStatus"] = buildCondition.String() + } else { + // drop out, don't do anything else + continue + } + } + if err := r.Update(ctx, runningBuild); err != nil { + // log the error and drop out + opLog.Error(err, fmt.Sprintf("Error setting build %s as cancelled", runningBuild.ObjectMeta.Name)) + continue + } + // send the status change to lagoon + r.updateDeploymentAndEnvironmentTask(ctx, opLog, runningBuild, nil, buildCondition, "cancelled") + continue + } + // handle processing running but no pod/failed pod builds + return ctrl.Result{}, nil } // getOrCreateBuildResource will deepcopy the lagoon build into a new resource and push it to the new namespace diff --git a/controllers/v1beta1/build_deletionhandlers.go b/controllers/v1beta1/build_deletionhandlers.go index a99a46df..fba4c0e2 100644 --- a/controllers/v1beta1/build_deletionhandlers.go +++ b/controllers/v1beta1/build_deletionhandlers.go @@ -214,7 +214,7 @@ Build cancelled // send any messages to lagoon message queues // update the deployment with the status of cancelled in lagoon r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusCancelled, "cancelled") - r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusCancelled) + r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusCancelled, "cancelled") r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, allContainerLogs, lagoonv1beta1.BuildStatusCancelled) } return nil @@ -279,6 +279,7 @@ func (r *LagoonBuildReconciler) updateDeploymentAndEnvironmentTask(ctx context.C lagoonBuild *lagoonv1beta1.LagoonBuild, lagoonEnv *corev1.ConfigMap, buildCondition lagoonv1beta1.BuildStatusType, + buildStep string, ) { namespace := helpers.GenerateNamespaceName( lagoonBuild.Spec.Project.NamespacePattern, // the namespace pattern or `openshiftProjectPattern` from Lagoon is never received by the controller @@ -296,6 +297,7 @@ func (r *LagoonBuildReconciler) updateDeploymentAndEnvironmentTask(ctx context.C Environment: lagoonBuild.Spec.Project.Environment, Project: lagoonBuild.Spec.Project.Name, BuildPhase: buildCondition.ToLower(), + BuildStep: buildStep, BuildName: lagoonBuild.ObjectMeta.Name, LogLink: lagoonBuild.Spec.Project.UILink, RemoteID: string(lagoonBuild.ObjectMeta.UID), diff --git a/controllers/v1beta1/build_helpers.go b/controllers/v1beta1/build_helpers.go index 40b28487..d91dc3fb 100644 --- a/controllers/v1beta1/build_helpers.go +++ b/controllers/v1beta1/build_helpers.go @@ -901,11 +901,11 @@ func (r *LagoonBuildReconciler) updateQueuedBuild( // update the deployment with the status, lagoon v2.12.0 supports queued status, otherwise use pending if helpers.CheckLagoonVersion(&lagoonBuild, "2.12.0") { r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusQueued, fmt.Sprintf("queued %v/%v", queuePosition, queueLength)) - r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusQueued) + r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusQueued, fmt.Sprintf("queued %v/%v", queuePosition, queueLength)) r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, allContainerLogs, lagoonv1beta1.BuildStatusQueued) } else { r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusPending, fmt.Sprintf("queued %v/%v", queuePosition, queueLength)) - r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusPending) + r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusPending, fmt.Sprintf("queued %v/%v", queuePosition, queueLength)) r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, allContainerLogs, lagoonv1beta1.BuildStatusPending) } @@ -961,7 +961,7 @@ Build cancelled // send any messages to lagoon message queues // update the deployment with the status of cancelled in lagoon r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusCancelled, "cancelled") - r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusCancelled) + r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusCancelled, "cancelled") if cancelled { r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, allContainerLogs, lagoonv1beta1.BuildStatusCancelled) } diff --git a/controllers/v1beta1/build_qoshandler.go b/controllers/v1beta1/build_qoshandler.go index b2f4e236..dfe1a70b 100644 --- a/controllers/v1beta1/build_qoshandler.go +++ b/controllers/v1beta1/build_qoshandler.go @@ -117,8 +117,7 @@ func (r *LagoonBuildReconciler) whichBuildNext(ctx context.Context, opLog logr.L } // if there are no running builds, check if there are any pending builds that can be started if len(runningNSBuilds.Items) == 0 { - pendingNSBuilds := &lagoonv1beta1.LagoonBuildList{} - if err := helpers.CancelExtraBuilds(ctx, r.Client, opLog, pendingNSBuilds, pBuild.ObjectMeta.Namespace, "Running"); err != nil { + if err := helpers.CancelExtraBuilds(ctx, r.Client, opLog, pBuild.ObjectMeta.Namespace, "Running"); err != nil { // only return if there is an error doing this operation // continue on otherwise to allow the queued status updater to run return err diff --git a/controllers/v1beta1/build_standardhandler.go b/controllers/v1beta1/build_standardhandler.go index b1edb4dc..95bd74bf 100644 --- a/controllers/v1beta1/build_standardhandler.go +++ b/controllers/v1beta1/build_standardhandler.go @@ -52,8 +52,7 @@ func (r *LagoonBuildReconciler) standardBuildProcessor(ctx context.Context, // if there are no running builds, check if there are any pending builds that can be started if len(runningBuilds.Items) == 0 { - pendingBuilds := &lagoonv1beta1.LagoonBuildList{} - return ctrl.Result{}, helpers.CancelExtraBuilds(ctx, r.Client, opLog, pendingBuilds, req.Namespace, "Running") + return ctrl.Result{}, helpers.CancelExtraBuilds(ctx, r.Client, opLog, req.Namespace, "Running") } // The object is not being deleted, so if it does not have our finalizer, // then lets add the finalizer and update the object. This is equivalent diff --git a/controllers/v1beta1/podmonitor_buildhandlers.go b/controllers/v1beta1/podmonitor_buildhandlers.go index 218dab5a..54017566 100644 --- a/controllers/v1beta1/podmonitor_buildhandlers.go +++ b/controllers/v1beta1/podmonitor_buildhandlers.go @@ -460,17 +460,7 @@ func (r *LagoonMonitorReconciler) updateDeploymentWithLogs( cancel bool, ) error { opLog := r.Log.WithValues("lagoonmonitor", req.NamespacedName) - var buildCondition lagoonv1beta1.BuildStatusType - switch jobPod.Status.Phase { - case corev1.PodFailed: - buildCondition = lagoonv1beta1.BuildStatusFailed - case corev1.PodSucceeded: - buildCondition = lagoonv1beta1.BuildStatusComplete - case corev1.PodPending: - buildCondition = lagoonv1beta1.BuildStatusPending - case corev1.PodRunning: - buildCondition = lagoonv1beta1.BuildStatusRunning - } + buildCondition := helpers.GetBuildConditionFromPod(jobPod.Status.Phase) collectLogs := true if cancel { // only set the status to cancelled if the pod is running/pending/queued diff --git a/controllers/v1beta1/podmonitor_controller.go b/controllers/v1beta1/podmonitor_controller.go index 64ce4c39..3057846c 100644 --- a/controllers/v1beta1/podmonitor_controller.go +++ b/controllers/v1beta1/podmonitor_controller.go @@ -145,7 +145,6 @@ func (r *LagoonMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Reques // if there are no running jobs, we check for any pending jobs // sorted by their creation timestamp and set the first to running opLog.Info(fmt.Sprintf("Checking for any pending builds.")) - pendingBuilds := &lagoonv1beta1.LagoonBuildList{} runningBuilds := &lagoonv1beta1.LagoonBuildList{} listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ client.InNamespace(req.Namespace), @@ -157,7 +156,7 @@ func (r *LagoonMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Reques } // if we have no running builds, then check for any pending builds if len(runningBuilds.Items) == 0 { - return ctrl.Result{}, helpers.CancelExtraBuilds(ctx, r.Client, opLog, pendingBuilds, req.Namespace, "Running") + return ctrl.Result{}, helpers.CancelExtraBuilds(ctx, r.Client, opLog, req.Namespace, "Running") } } return ctrl.Result{}, nil diff --git a/controllers/v1beta1/podmonitor_taskhandlers.go b/controllers/v1beta1/podmonitor_taskhandlers.go index 74bc529c..5f105045 100644 --- a/controllers/v1beta1/podmonitor_taskhandlers.go +++ b/controllers/v1beta1/podmonitor_taskhandlers.go @@ -267,17 +267,7 @@ func (r *LagoonMonitorReconciler) updateTaskWithLogs( cancel bool, ) error { opLog := r.Log.WithValues("lagoonmonitor", req.NamespacedName) - var taskCondition lagoonv1beta1.TaskStatusType - switch jobPod.Status.Phase { - case corev1.PodFailed: - taskCondition = lagoonv1beta1.TaskStatusFailed - case corev1.PodSucceeded: - taskCondition = lagoonv1beta1.TaskStatusComplete - case corev1.PodPending: - taskCondition = lagoonv1beta1.TaskStatusPending - case corev1.PodRunning: - taskCondition = lagoonv1beta1.TaskStatusRunning - } + taskCondition := helpers.GetTaskConditionFromPod(jobPod.Status.Phase) collectLogs := true if cancel { taskCondition = lagoonv1beta1.TaskStatusCancelled diff --git a/internal/helpers/helpers.go b/internal/helpers/helpers.go index 6d9571e2..0c39bb51 100644 --- a/internal/helpers/helpers.go +++ b/internal/helpers/helpers.go @@ -18,6 +18,7 @@ import ( "github.com/go-logr/logr" "github.com/hashicorp/go-version" lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -265,7 +266,8 @@ func CheckLagoonVersion(build *lagoonv1beta1.LagoonBuild, checkVersion string) b } // CancelExtraBuilds cancels extra builds. -func CancelExtraBuilds(ctx context.Context, r client.Client, opLog logr.Logger, pendingBuilds *lagoonv1beta1.LagoonBuildList, ns string, status string) error { +func CancelExtraBuilds(ctx context.Context, r client.Client, opLog logr.Logger, ns string, status string) error { + pendingBuilds := &lagoonv1beta1.LagoonBuildList{} listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ client.InNamespace(ns), client.MatchingLabels(map[string]string{"lagoon.sh/buildStatus": lagoonv1beta1.BuildStatusPending.String()}), @@ -298,6 +300,36 @@ func CancelExtraBuilds(ctx context.Context, r client.Client, opLog logr.Logger, return nil } +func GetBuildConditionFromPod(phase corev1.PodPhase) lagoonv1beta1.BuildStatusType { + var buildCondition lagoonv1beta1.BuildStatusType + switch phase { + case corev1.PodFailed: + buildCondition = lagoonv1beta1.BuildStatusFailed + case corev1.PodSucceeded: + buildCondition = lagoonv1beta1.BuildStatusComplete + case corev1.PodPending: + buildCondition = lagoonv1beta1.BuildStatusPending + case corev1.PodRunning: + buildCondition = lagoonv1beta1.BuildStatusRunning + } + return buildCondition +} + +func GetTaskConditionFromPod(phase corev1.PodPhase) lagoonv1beta1.TaskStatusType { + var taskCondition lagoonv1beta1.TaskStatusType + switch phase { + case corev1.PodFailed: + taskCondition = lagoonv1beta1.TaskStatusFailed + case corev1.PodSucceeded: + taskCondition = lagoonv1beta1.TaskStatusComplete + case corev1.PodPending: + taskCondition = lagoonv1beta1.TaskStatusPending + case corev1.PodRunning: + taskCondition = lagoonv1beta1.TaskStatusRunning + } + return taskCondition +} + // GenerateNamespaceName handles the generation of the namespace name from environment and project name with prefixes and patterns func GenerateNamespaceName(pattern, environmentName, projectname, prefix, controllerNamespace string, randomPrefix bool) string { nsPattern := pattern