Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: when a new build comes in, check the namespace for running builds to determine state #224

Merged
merged 4 commits into from
Sep 13, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 52 additions & 3 deletions controllers/v1beta1/build_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -259,9 +259,58 @@ func (r *LagoonBuildReconciler) createNamespaceBuild(ctx context.Context,

// if everything is all good controller will handle the new build resource that gets created as it will have
// the `lagoon.sh/buildStatus = Pending` now
// so end this reconcile process
pendingBuilds := &lagoonv1beta1.LagoonBuildList{}
return ctrl.Result{}, helpers.CancelExtraBuilds(ctx, r.Client, opLog, pendingBuilds, namespace.ObjectMeta.Name, lagoonv1beta1.BuildStatusPending.String())
err = helpers.CancelExtraBuilds(ctx, r.Client, opLog, namespace.ObjectMeta.Name, lagoonv1beta1.BuildStatusPending.String())
if err != nil {
return ctrl.Result{}, err
}

// as this is a new build coming through, check if there are any running builds in the namespace
// if there are, then check the status of that build. if the build pod is missing then the build running will block
// if the pod exists, attempt to get the status of it (only if its complete or failed) and ship the status
runningBuilds := &lagoonv1beta1.LagoonBuildList{}
listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{
client.InNamespace(namespace.ObjectMeta.Name),
client.MatchingLabels(map[string]string{"lagoon.sh/buildStatus": lagoonv1beta1.BuildStatusRunning.String()}),
})
// list all builds in the namespace that have the running buildstatus
if err := r.List(ctx, runningBuilds, listOption); err != nil {
return ctrl.Result{}, fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err)
}
// if there are running builds still, check if the pod exists or if the pod is complete/failed and attempt to get the status
for _, rBuild := range runningBuilds.Items {
runningBuild := rBuild.DeepCopy()
lagoonBuildPod := corev1.Pod{}
err := r.Get(ctx, types.NamespacedName{
Namespace: rBuild.ObjectMeta.Namespace,
Name: rBuild.ObjectMeta.Name,
}, &lagoonBuildPod)
buildCondition := lagoonv1beta1.BuildStatusCancelled
if err != nil {
// cancel the build as there is no pod available
opLog.Info(fmt.Sprintf("Setting build %s as cancelled", runningBuild.ObjectMeta.Name))
runningBuild.Labels["lagoon.sh/buildStatus"] = buildCondition.String()
} else {
// get the status from the pod and update the build
if lagoonBuildPod.Status.Phase == corev1.PodFailed || lagoonBuildPod.Status.Phase == corev1.PodSucceeded {
buildCondition = helpers.GetBuildConditionFromPod(lagoonBuildPod.Status.Phase)
opLog.Info(fmt.Sprintf("Setting build %s as %s", runningBuild.ObjectMeta.Name, buildCondition.String()))
runningBuild.Labels["lagoon.sh/buildStatus"] = buildCondition.String()
} else {
// drop out, don't do anything else
continue
}
}
if err := r.Update(ctx, runningBuild); err != nil {
// log the error and drop out
opLog.Error(err, fmt.Sprintf("Error setting build %s as cancelled", runningBuild.ObjectMeta.Name))
continue
}
// send the status change to lagoon
r.updateDeploymentAndEnvironmentTask(ctx, opLog, runningBuild, nil, buildCondition, "cancelled")
continue
}
// handle processing running but no pod/failed pod builds
return ctrl.Result{}, nil
}

// getOrCreateBuildResource will deepcopy the lagoon build into a new resource and push it to the new namespace
Expand Down
4 changes: 3 additions & 1 deletion controllers/v1beta1/build_deletionhandlers.go
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ Build cancelled
// send any messages to lagoon message queues
// update the deployment with the status of cancelled in lagoon
r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusCancelled, "cancelled")
r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusCancelled)
r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusCancelled, "cancelled")
r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, allContainerLogs, lagoonv1beta1.BuildStatusCancelled)
}
return nil
Expand Down Expand Up @@ -279,6 +279,7 @@ func (r *LagoonBuildReconciler) updateDeploymentAndEnvironmentTask(ctx context.C
lagoonBuild *lagoonv1beta1.LagoonBuild,
lagoonEnv *corev1.ConfigMap,
buildCondition lagoonv1beta1.BuildStatusType,
buildStep string,
) {
namespace := helpers.GenerateNamespaceName(
lagoonBuild.Spec.Project.NamespacePattern, // the namespace pattern or `openshiftProjectPattern` from Lagoon is never received by the controller
Expand All @@ -296,6 +297,7 @@ func (r *LagoonBuildReconciler) updateDeploymentAndEnvironmentTask(ctx context.C
Environment: lagoonBuild.Spec.Project.Environment,
Project: lagoonBuild.Spec.Project.Name,
BuildPhase: buildCondition.ToLower(),
BuildStep: buildStep,
BuildName: lagoonBuild.ObjectMeta.Name,
LogLink: lagoonBuild.Spec.Project.UILink,
RemoteID: string(lagoonBuild.ObjectMeta.UID),
Expand Down
6 changes: 3 additions & 3 deletions controllers/v1beta1/build_helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -901,11 +901,11 @@ func (r *LagoonBuildReconciler) updateQueuedBuild(
// update the deployment with the status, lagoon v2.12.0 supports queued status, otherwise use pending
if helpers.CheckLagoonVersion(&lagoonBuild, "2.12.0") {
r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusQueued, fmt.Sprintf("queued %v/%v", queuePosition, queueLength))
r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusQueued)
r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusQueued, fmt.Sprintf("queued %v/%v", queuePosition, queueLength))
r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, allContainerLogs, lagoonv1beta1.BuildStatusQueued)
} else {
r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusPending, fmt.Sprintf("queued %v/%v", queuePosition, queueLength))
r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusPending)
r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusPending, fmt.Sprintf("queued %v/%v", queuePosition, queueLength))
r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, allContainerLogs, lagoonv1beta1.BuildStatusPending)

}
Expand Down Expand Up @@ -961,7 +961,7 @@ Build cancelled
// send any messages to lagoon message queues
// update the deployment with the status of cancelled in lagoon
r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusCancelled, "cancelled")
r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusCancelled)
r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusCancelled, "cancelled")
if cancelled {
r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, allContainerLogs, lagoonv1beta1.BuildStatusCancelled)
}
Expand Down
3 changes: 1 addition & 2 deletions controllers/v1beta1/build_qoshandler.go
Original file line number Diff line number Diff line change
Expand Up @@ -117,8 +117,7 @@ func (r *LagoonBuildReconciler) whichBuildNext(ctx context.Context, opLog logr.L
}
// if there are no running builds, check if there are any pending builds that can be started
if len(runningNSBuilds.Items) == 0 {
pendingNSBuilds := &lagoonv1beta1.LagoonBuildList{}
if err := helpers.CancelExtraBuilds(ctx, r.Client, opLog, pendingNSBuilds, pBuild.ObjectMeta.Namespace, "Running"); err != nil {
if err := helpers.CancelExtraBuilds(ctx, r.Client, opLog, pBuild.ObjectMeta.Namespace, "Running"); err != nil {
// only return if there is an error doing this operation
// continue on otherwise to allow the queued status updater to run
return err
Expand Down
3 changes: 1 addition & 2 deletions controllers/v1beta1/build_standardhandler.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,7 @@ func (r *LagoonBuildReconciler) standardBuildProcessor(ctx context.Context,

// if there are no running builds, check if there are any pending builds that can be started
if len(runningBuilds.Items) == 0 {
pendingBuilds := &lagoonv1beta1.LagoonBuildList{}
return ctrl.Result{}, helpers.CancelExtraBuilds(ctx, r.Client, opLog, pendingBuilds, req.Namespace, "Running")
return ctrl.Result{}, helpers.CancelExtraBuilds(ctx, r.Client, opLog, req.Namespace, "Running")
}
// The object is not being deleted, so if it does not have our finalizer,
// then lets add the finalizer and update the object. This is equivalent
Expand Down
12 changes: 1 addition & 11 deletions controllers/v1beta1/podmonitor_buildhandlers.go
Original file line number Diff line number Diff line change
Expand Up @@ -460,17 +460,7 @@ func (r *LagoonMonitorReconciler) updateDeploymentWithLogs(
cancel bool,
) error {
opLog := r.Log.WithValues("lagoonmonitor", req.NamespacedName)
var buildCondition lagoonv1beta1.BuildStatusType
switch jobPod.Status.Phase {
case corev1.PodFailed:
buildCondition = lagoonv1beta1.BuildStatusFailed
case corev1.PodSucceeded:
buildCondition = lagoonv1beta1.BuildStatusComplete
case corev1.PodPending:
buildCondition = lagoonv1beta1.BuildStatusPending
case corev1.PodRunning:
buildCondition = lagoonv1beta1.BuildStatusRunning
}
buildCondition := helpers.GetBuildConditionFromPod(jobPod.Status.Phase)
collectLogs := true
if cancel {
// only set the status to cancelled if the pod is running/pending/queued
Expand Down
3 changes: 1 addition & 2 deletions controllers/v1beta1/podmonitor_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,6 @@ func (r *LagoonMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Reques
// if there are no running jobs, we check for any pending jobs
// sorted by their creation timestamp and set the first to running
opLog.Info(fmt.Sprintf("Checking for any pending builds."))
pendingBuilds := &lagoonv1beta1.LagoonBuildList{}
runningBuilds := &lagoonv1beta1.LagoonBuildList{}
listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{
client.InNamespace(req.Namespace),
Expand All @@ -157,7 +156,7 @@ func (r *LagoonMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Reques
}
// if we have no running builds, then check for any pending builds
if len(runningBuilds.Items) == 0 {
return ctrl.Result{}, helpers.CancelExtraBuilds(ctx, r.Client, opLog, pendingBuilds, req.Namespace, "Running")
return ctrl.Result{}, helpers.CancelExtraBuilds(ctx, r.Client, opLog, req.Namespace, "Running")
}
}
return ctrl.Result{}, nil
Expand Down
12 changes: 1 addition & 11 deletions controllers/v1beta1/podmonitor_taskhandlers.go
Original file line number Diff line number Diff line change
Expand Up @@ -267,17 +267,7 @@ func (r *LagoonMonitorReconciler) updateTaskWithLogs(
cancel bool,
) error {
opLog := r.Log.WithValues("lagoonmonitor", req.NamespacedName)
var taskCondition lagoonv1beta1.TaskStatusType
switch jobPod.Status.Phase {
case corev1.PodFailed:
taskCondition = lagoonv1beta1.TaskStatusFailed
case corev1.PodSucceeded:
taskCondition = lagoonv1beta1.TaskStatusComplete
case corev1.PodPending:
taskCondition = lagoonv1beta1.TaskStatusPending
case corev1.PodRunning:
taskCondition = lagoonv1beta1.TaskStatusRunning
}
taskCondition := helpers.GetTaskConditionFromPod(jobPod.Status.Phase)
collectLogs := true
if cancel {
taskCondition = lagoonv1beta1.TaskStatusCancelled
Expand Down
34 changes: 33 additions & 1 deletion internal/helpers/helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"github.com/go-logr/logr"
"github.com/hashicorp/go-version"
lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"sigs.k8s.io/controller-runtime/pkg/client"
)
Expand Down Expand Up @@ -265,7 +266,8 @@ func CheckLagoonVersion(build *lagoonv1beta1.LagoonBuild, checkVersion string) b
}

// CancelExtraBuilds cancels extra builds.
func CancelExtraBuilds(ctx context.Context, r client.Client, opLog logr.Logger, pendingBuilds *lagoonv1beta1.LagoonBuildList, ns string, status string) error {
func CancelExtraBuilds(ctx context.Context, r client.Client, opLog logr.Logger, ns string, status string) error {
pendingBuilds := &lagoonv1beta1.LagoonBuildList{}
listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{
client.InNamespace(ns),
client.MatchingLabels(map[string]string{"lagoon.sh/buildStatus": lagoonv1beta1.BuildStatusPending.String()}),
Expand Down Expand Up @@ -298,6 +300,36 @@ func CancelExtraBuilds(ctx context.Context, r client.Client, opLog logr.Logger,
return nil
}

func GetBuildConditionFromPod(phase corev1.PodPhase) lagoonv1beta1.BuildStatusType {
var buildCondition lagoonv1beta1.BuildStatusType
switch phase {
case corev1.PodFailed:
buildCondition = lagoonv1beta1.BuildStatusFailed
case corev1.PodSucceeded:
buildCondition = lagoonv1beta1.BuildStatusComplete
case corev1.PodPending:
buildCondition = lagoonv1beta1.BuildStatusPending
case corev1.PodRunning:
buildCondition = lagoonv1beta1.BuildStatusRunning
}
return buildCondition
}

func GetTaskConditionFromPod(phase corev1.PodPhase) lagoonv1beta1.TaskStatusType {
var taskCondition lagoonv1beta1.TaskStatusType
switch phase {
case corev1.PodFailed:
taskCondition = lagoonv1beta1.TaskStatusFailed
case corev1.PodSucceeded:
taskCondition = lagoonv1beta1.TaskStatusComplete
case corev1.PodPending:
taskCondition = lagoonv1beta1.TaskStatusPending
case corev1.PodRunning:
taskCondition = lagoonv1beta1.TaskStatusRunning
}
return taskCondition
}

// GenerateNamespaceName handles the generation of the namespace name from environment and project name with prefixes and patterns
func GenerateNamespaceName(pattern, environmentName, projectname, prefix, controllerNamespace string, randomPrefix bool) string {
nsPattern := pattern
Expand Down
Loading