diff --git a/apis/lagoon/v1beta1/helpers_test.go b/apis/lagoon/v1beta1/helpers_test.go new file mode 100644 index 00000000..69c2a02b --- /dev/null +++ b/apis/lagoon/v1beta1/helpers_test.go @@ -0,0 +1,105 @@ +package v1beta1 + +import ( + "testing" +) + +func TestCheckLagoonVersion(t *testing.T) { + type args struct { + build *LagoonBuild + checkVersion string + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "test1", + args: args{ + build: &LagoonBuild{ + Spec: LagoonBuildSpec{ + Project: Project{ + Variables: LagoonVariables{ + Project: []byte(`[{"name":"LAGOON_SYSTEM_CORE_VERSION","value":"v2.12.0","scope":"internal_system"}]`), + }, + }, + }, + }, + checkVersion: "2.12.0", + }, + want: true, + }, + { + name: "test2", + args: args{ + build: &LagoonBuild{ + Spec: LagoonBuildSpec{ + Project: Project{ + Variables: LagoonVariables{ + Project: []byte(`[{"name":"LAGOON_SYSTEM_CORE_VERSION","value":"v2.11.0","scope":"internal_system"}]`), + }, + }, + }, + }, + checkVersion: "2.12.0", + }, + want: false, + }, + { + name: "test3", + args: args{ + build: &LagoonBuild{ + Spec: LagoonBuildSpec{ + Project: Project{ + Variables: LagoonVariables{ + Project: []byte(`[]`), + }, + }, + }, + }, + checkVersion: "2.12.0", + }, + want: false, + }, + { + name: "test4", + args: args{ + build: &LagoonBuild{ + Spec: LagoonBuildSpec{ + Project: Project{ + Variables: LagoonVariables{ + Project: []byte(`[{"name":"LAGOON_SYSTEM_CORE_VERSION","value":"v2.12.0","scope":"internal_system"}]`), + }, + }, + }, + }, + checkVersion: "v2.12.0", + }, + want: true, + }, + { + name: "test5", + args: args{ + build: &LagoonBuild{ + Spec: LagoonBuildSpec{ + Project: Project{ + Variables: LagoonVariables{ + Project: []byte(`[{"name":"LAGOON_SYSTEM_CORE_VERSION","value":"v2.11.0","scope":"internal_system"}]`), + }, + }, + }, + }, + checkVersion: "v2.12.0", + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := CheckLagoonVersion(tt.args.build, tt.args.checkVersion); got != tt.want { + t.Errorf("CheckLagoonVersion() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/apis/lagoon/v1beta1/lagoonbuild_helpers.go b/apis/lagoon/v1beta1/lagoonbuild_helpers.go new file mode 100644 index 00000000..f32a45ad --- /dev/null +++ b/apis/lagoon/v1beta1/lagoonbuild_helpers.go @@ -0,0 +1,444 @@ +package v1beta1 + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + "github.com/go-logr/logr" + "github.com/hashicorp/go-version" + "github.com/uselagoon/machinery/api/schema" + "github.com/uselagoon/remote-controller/internal/helpers" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var ( + // BuildRunningPendingStatus . + BuildRunningPendingStatus = []string{ + BuildStatusPending.String(), + BuildStatusQueued.String(), + BuildStatusRunning.String(), + } + // BuildCompletedCancelledFailedStatus . + BuildCompletedCancelledFailedStatus = []string{ + BuildStatusFailed.String(), + BuildStatusComplete.String(), + BuildStatusCancelled.String(), + } +) + +// BuildContainsStatus . +func BuildContainsStatus(slice []LagoonBuildConditions, s LagoonBuildConditions) bool { + for _, item := range slice { + if item == s { + return true + } + } + return false +} + +// RemoveBuild remove a LagoonBuild from a slice of LagoonBuilds +func RemoveBuild(slice []LagoonBuild, s LagoonBuild) []LagoonBuild { + result := []LagoonBuild{} + for _, item := range slice { + if item.ObjectMeta.Name == s.ObjectMeta.Name { + continue + } + result = append(result, item) + } + return result +} + +// Check if the version of lagoon provided in the internal_system scope variable is greater than or equal to the checked version +func CheckLagoonVersion(build *LagoonBuild, checkVersion string) bool { + lagoonProjectVariables := &[]helpers.LagoonEnvironmentVariable{} + json.Unmarshal(build.Spec.Project.Variables.Project, lagoonProjectVariables) + lagoonVersion, err := helpers.GetLagoonVariable("LAGOON_SYSTEM_CORE_VERSION", []string{"internal_system"}, *lagoonProjectVariables) + if err != nil { + return false + } + aVer, err := version.NewSemver(lagoonVersion.Value) + if err != nil { + return false + } + bVer, err := version.NewSemver(checkVersion) + if err != nil { + return false + } + return aVer.GreaterThanOrEqual(bVer) +} + +// CancelExtraBuilds cancels extra builds. +func CancelExtraBuilds(ctx context.Context, r client.Client, opLog logr.Logger, ns string, status string) error { + pendingBuilds := &LagoonBuildList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(ns), + client.MatchingLabels(map[string]string{"lagoon.sh/buildStatus": BuildStatusPending.String()}), + }) + if err := r.List(ctx, pendingBuilds, listOption); err != nil { + return fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err) + } + if len(pendingBuilds.Items) > 0 { + // opLog.Info(fmt.Sprintf("There are %v pending builds", len(pendingBuilds.Items))) + // if we have any pending builds, then grab the latest one and make it running + // if there are any other pending builds, cancel them so only the latest one runs + sort.Slice(pendingBuilds.Items, func(i, j int) bool { + return pendingBuilds.Items[i].ObjectMeta.CreationTimestamp.After(pendingBuilds.Items[j].ObjectMeta.CreationTimestamp.Time) + }) + for idx, pBuild := range pendingBuilds.Items { + pendingBuild := pBuild.DeepCopy() + if idx == 0 { + pendingBuild.Labels["lagoon.sh/buildStatus"] = status + } else { + // cancel any other pending builds + opLog.Info(fmt.Sprintf("Setting build %s as cancelled", pendingBuild.ObjectMeta.Name)) + pendingBuild.Labels["lagoon.sh/buildStatus"] = BuildStatusCancelled.String() + pendingBuild.Labels["lagoon.sh/cancelledByNewBuild"] = "true" + } + if err := r.Update(ctx, pendingBuild); err != nil { + return err + } + } + } + return nil +} + +func GetBuildConditionFromPod(phase corev1.PodPhase) BuildStatusType { + var buildCondition BuildStatusType + switch phase { + case corev1.PodFailed: + buildCondition = BuildStatusFailed + case corev1.PodSucceeded: + buildCondition = BuildStatusComplete + case corev1.PodPending: + buildCondition = BuildStatusPending + case corev1.PodRunning: + buildCondition = BuildStatusRunning + } + return buildCondition +} + +func GetTaskConditionFromPod(phase corev1.PodPhase) TaskStatusType { + var taskCondition TaskStatusType + switch phase { + case corev1.PodFailed: + taskCondition = TaskStatusFailed + case corev1.PodSucceeded: + taskCondition = TaskStatusComplete + case corev1.PodPending: + taskCondition = TaskStatusPending + case corev1.PodRunning: + taskCondition = TaskStatusRunning + } + return taskCondition +} + +func CheckRunningBuilds(ctx context.Context, cns string, opLog logr.Logger, cl client.Client, ns corev1.Namespace) bool { + lagoonBuilds := &LagoonBuildList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(ns.ObjectMeta.Name), + client.MatchingLabels(map[string]string{ + "lagoon.sh/controller": cns, // created by this controller + }), + }) + if err := cl.List(context.Background(), lagoonBuilds, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list Lagoon build pods, there may be none or something went wrong")) + return false + } + runningBuilds := false + sort.Slice(lagoonBuilds.Items, func(i, j int) bool { + return lagoonBuilds.Items[i].ObjectMeta.CreationTimestamp.After(lagoonBuilds.Items[j].ObjectMeta.CreationTimestamp.Time) + }) + // if there are any builds pending or running, don't try and refresh the credentials as this + // could break the build + if len(lagoonBuilds.Items) > 0 { + if helpers.ContainsString( + BuildRunningPendingStatus, + lagoonBuilds.Items[0].Labels["lagoon.sh/buildStatus"], + ) { + runningBuilds = true + } + } + return runningBuilds +} + +// DeleteLagoonBuilds will delete any lagoon builds from the namespace. +func DeleteLagoonBuilds(ctx context.Context, opLog logr.Logger, cl client.Client, ns, project, environment string) bool { + lagoonBuilds := &LagoonBuildList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(ns), + }) + if err := cl.List(ctx, lagoonBuilds, listOption); err != nil { + opLog.Error(err, + fmt.Sprintf( + "Unable to list lagoon build in namespace %s for project %s, environment %s", + ns, + project, + environment, + ), + ) + return false + } + for _, lagoonBuild := range lagoonBuilds.Items { + if err := cl.Delete(ctx, &lagoonBuild); helpers.IgnoreNotFound(err) != nil { + opLog.Error(err, + fmt.Sprintf( + "Unable to delete lagoon build %s in %s for project %s, environment %s", + lagoonBuild.ObjectMeta.Name, + ns, + project, + environment, + ), + ) + return false + } + opLog.Info( + fmt.Sprintf( + "Deleted lagoon build %s in %s for project %s, environment %s", + lagoonBuild.ObjectMeta.Name, + ns, + project, + environment, + ), + ) + } + return true +} + +func LagoonBuildPruner(ctx context.Context, cl client.Client, cns string, buildsToKeep int) { + opLog := ctrl.Log.WithName("utilities").WithName("LagoonBuildPruner") + namespaces := &corev1.NamespaceList{} + labelRequirements, _ := labels.NewRequirement("lagoon.sh/environmentType", selection.Exists, nil) + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.MatchingLabelsSelector{ + Selector: labels.NewSelector().Add(*labelRequirements), + }, + }) + if err := cl.List(ctx, namespaces, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list namespaces created by Lagoon, there may be none or something went wrong")) + return + } + for _, ns := range namespaces.Items { + if ns.Status.Phase == corev1.NamespaceTerminating { + // if the namespace is terminating, don't try to renew the robot credentials + opLog.Info(fmt.Sprintf("Namespace %s is being terminated, aborting build pruner", ns.ObjectMeta.Name)) + continue + } + opLog.Info(fmt.Sprintf("Checking LagoonBuilds in namespace %s", ns.ObjectMeta.Name)) + lagoonBuilds := &LagoonBuildList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(ns.ObjectMeta.Name), + client.MatchingLabels(map[string]string{ + "lagoon.sh/controller": cns, // created by this controller + }), + }) + if err := cl.List(ctx, lagoonBuilds, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list LagoonBuild resources, there may be none or something went wrong")) + continue + } + // sort the build pods by creation timestamp + sort.Slice(lagoonBuilds.Items, func(i, j int) bool { + return lagoonBuilds.Items[i].ObjectMeta.CreationTimestamp.After(lagoonBuilds.Items[j].ObjectMeta.CreationTimestamp.Time) + }) + if len(lagoonBuilds.Items) > buildsToKeep { + for idx, lagoonBuild := range lagoonBuilds.Items { + if idx >= buildsToKeep { + if helpers.ContainsString( + BuildCompletedCancelledFailedStatus, + lagoonBuild.ObjectMeta.Labels["lagoon.sh/buildStatus"], + ) { + opLog.Info(fmt.Sprintf("Cleaning up LagoonBuild %s", lagoonBuild.ObjectMeta.Name)) + if err := cl.Delete(ctx, &lagoonBuild); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to update status condition")) + break + } + } + } + } + } + } + return +} + +// BuildPodPruner will prune any build pods that are hanging around. +func BuildPodPruner(ctx context.Context, cl client.Client, cns string, buildPodsToKeep int) { + opLog := ctrl.Log.WithName("utilities").WithName("BuildPodPruner") + namespaces := &corev1.NamespaceList{} + labelRequirements, _ := labels.NewRequirement("lagoon.sh/environmentType", selection.Exists, nil) + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.MatchingLabelsSelector{ + Selector: labels.NewSelector().Add(*labelRequirements), + }, + }) + if err := cl.List(ctx, namespaces, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list namespaces created by Lagoon, there may be none or something went wrong")) + return + } + for _, ns := range namespaces.Items { + if ns.Status.Phase == corev1.NamespaceTerminating { + // if the namespace is terminating, don't try to renew the robot credentials + opLog.Info(fmt.Sprintf("Namespace %s is being terminated, aborting build pod pruner", ns.ObjectMeta.Name)) + return + } + opLog.Info(fmt.Sprintf("Checking Lagoon build pods in namespace %s", ns.ObjectMeta.Name)) + buildPods := &corev1.PodList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(ns.ObjectMeta.Name), + client.MatchingLabels(map[string]string{ + "lagoon.sh/jobType": "build", + "lagoon.sh/controller": cns, // created by this controller + }), + }) + if err := cl.List(ctx, buildPods, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list Lagoon build pods, there may be none or something went wrong")) + return + } + // sort the build pods by creation timestamp + sort.Slice(buildPods.Items, func(i, j int) bool { + return buildPods.Items[i].ObjectMeta.CreationTimestamp.After(buildPods.Items[j].ObjectMeta.CreationTimestamp.Time) + }) + if len(buildPods.Items) > buildPodsToKeep { + for idx, pod := range buildPods.Items { + if idx >= buildPodsToKeep { + if pod.Status.Phase == corev1.PodFailed || + pod.Status.Phase == corev1.PodSucceeded { + opLog.Info(fmt.Sprintf("Cleaning up pod %s", pod.ObjectMeta.Name)) + if err := cl.Delete(ctx, &pod); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to update status condition")) + break + } + } + } + } + } + } + return +} + +func updateLagoonBuild(opLog logr.Logger, namespace string, jobSpec LagoonTaskSpec, lagoonBuild *LagoonBuild) ([]byte, error) { + // if the build isn't found by the controller + // then publish a response back to controllerhandler to tell it to update the build to cancelled + // this allows us to update builds in the API that may have gone stale or not updated from `New`, `Pending`, or `Running` status + buildCondition := "cancelled" + if lagoonBuild != nil { + if val, ok := lagoonBuild.ObjectMeta.Labels["lagoon.sh/buildStatus"]; ok { + // if the build isnt running,pending,queued, then set the buildcondition to the value failed/complete/cancelled + if !helpers.ContainsString(BuildRunningPendingStatus, val) { + buildCondition = strings.ToLower(val) + } + } + } + msg := schema.LagoonMessage{ + Type: "build", + Namespace: namespace, + Meta: &schema.LagoonLogMeta{ + Environment: jobSpec.Environment.Name, + Project: jobSpec.Project.Name, + BuildStatus: buildCondition, + BuildName: jobSpec.Misc.Name, + }, + } + // set the start/end time to be now as the default + // to stop the duration counter in the ui + msg.Meta.StartTime = time.Now().UTC().Format("2006-01-02 15:04:05") + msg.Meta.EndTime = time.Now().UTC().Format("2006-01-02 15:04:05") + + // if possible, get the start and end times from the build resource, these will be sent back to lagoon to update the api + if lagoonBuild != nil && lagoonBuild.Status.Conditions != nil { + conditions := lagoonBuild.Status.Conditions + // sort the build conditions by time so the first and last can be extracted + sort.Slice(conditions, func(i, j int) bool { + iTime, _ := time.Parse("2006-01-02T15:04:05Z", conditions[i].LastTransitionTime) + jTime, _ := time.Parse("2006-01-02T15:04:05Z", conditions[j].LastTransitionTime) + return iTime.Before(jTime) + }) + // get the starting time, or fallback to default + sTime, err := time.Parse("2006-01-02T15:04:05Z", conditions[0].LastTransitionTime) + if err == nil { + msg.Meta.StartTime = sTime.Format("2006-01-02 15:04:05") + } + // get the ending time, or fallback to default + eTime, err := time.Parse("2006-01-02T15:04:05Z", conditions[len(conditions)-1].LastTransitionTime) + if err == nil { + msg.Meta.EndTime = eTime.Format("2006-01-02 15:04:05") + } + } + msgBytes, err := json.Marshal(msg) + if err != nil { + return nil, fmt.Errorf("Unable to encode message as JSON: %v", err) + } + return msgBytes, nil +} + +// CancelBuild handles cancelling builds or handling if a build no longer exists. +func CancelBuild(ctx context.Context, cl client.Client, namespace string, body []byte) (bool, []byte, error) { + opLog := ctrl.Log.WithName("handlers").WithName("LagoonTasks") + jobSpec := &LagoonTaskSpec{} + json.Unmarshal(body, jobSpec) + var jobPod corev1.Pod + if err := cl.Get(ctx, types.NamespacedName{ + Name: jobSpec.Misc.Name, + Namespace: namespace, + }, &jobPod); err != nil { + opLog.Info(fmt.Sprintf( + "Unable to find build pod %s to cancel it. Checking to see if LagoonBuild exists.", + jobSpec.Misc.Name, + )) + // since there was no build pod, check for the lagoon build resource + var lagoonBuild LagoonBuild + if err := cl.Get(ctx, types.NamespacedName{ + Name: jobSpec.Misc.Name, + Namespace: namespace, + }, &lagoonBuild); err != nil { + opLog.Info(fmt.Sprintf( + "Unable to find build %s to cancel it. Sending response to Lagoon to update the build to cancelled.", + jobSpec.Misc.Name, + )) + // if there is no pod or build, update the build in Lagoon to cancelled, assume completely cancelled with no other information + // and then send the response back to lagoon to say it was cancelled. + b, err := updateLagoonBuild(opLog, namespace, *jobSpec, nil) + return false, b, err + } + // as there is no build pod, but there is a lagoon build resource + // update it to cancelled so that the controller doesn't try to run it + // check if the build has existing status or not though to consume it + if helpers.ContainsString( + BuildRunningPendingStatus, + lagoonBuild.ObjectMeta.Labels["lagoon.sh/buildStatus"], + ) { + lagoonBuild.ObjectMeta.Labels["lagoon.sh/buildStatus"] = BuildStatusCancelled.String() + } + lagoonBuild.ObjectMeta.Labels["lagoon.sh/cancelBuildNoPod"] = "true" + if err := cl.Update(ctx, &lagoonBuild); err != nil { + opLog.Error(err, + fmt.Sprintf( + "Unable to update build %s to cancel it.", + jobSpec.Misc.Name, + ), + ) + return false, nil, err + } + // and then send the response back to lagoon to say it was cancelled. + b, err := updateLagoonBuild(opLog, namespace, *jobSpec, &lagoonBuild) + return true, b, err + } + jobPod.ObjectMeta.Labels["lagoon.sh/cancelBuild"] = "true" + if err := cl.Update(ctx, &jobPod); err != nil { + opLog.Error(err, + fmt.Sprintf( + "Unable to update build %s to cancel it.", + jobSpec.Misc.Name, + ), + ) + return false, nil, err + } + return false, nil, nil +} diff --git a/apis/lagoon/v1beta1/lagoonmessaging_types.go b/apis/lagoon/v1beta1/lagoonmessaging_types.go index f9fe6ecd..5480b387 100644 --- a/apis/lagoon/v1beta1/lagoonmessaging_types.go +++ b/apis/lagoon/v1beta1/lagoonmessaging_types.go @@ -1,59 +1,13 @@ package v1beta1 -// LagoonMessaging - -// LagoonLog is used to sendToLagoonLogs messaging queue -// this is general logging information -type LagoonLog struct { - Severity string `json:"severity,omitempty"` - Project string `json:"project,omitempty"` - UUID string `json:"uuid,omitempty"` - Event string `json:"event,omitempty"` - Meta *LagoonLogMeta `json:"meta,omitempty"` - Message string `json:"message,omitempty"` -} - -// LagoonLogMeta is the metadata that is used by logging in Lagoon. -type LagoonLogMeta struct { - BranchName string `json:"branchName,omitempty"` - BuildName string `json:"buildName,omitempty"` - BuildPhase string `json:"buildPhase,omitempty"` // @TODO: deprecate once controller-handler is fixed - BuildStatus string `json:"buildStatus,omitempty"` - BuildStep string `json:"buildStep,omitempty"` - EndTime string `json:"endTime,omitempty"` - Environment string `json:"environment,omitempty"` - EnvironmentID *uint `json:"environmentId,omitempty"` - JobName string `json:"jobName,omitempty"` // used by tasks/jobs - JobStatus string `json:"jobStatus,omitempty"` // used by tasks/jobs - JobStep string `json:"jobStep,omitempty"` // used by tasks/jobs - LogLink string `json:"logLink,omitempty"` - Project string `json:"project,omitempty"` - ProjectID *uint `json:"projectId,omitempty"` - ProjectName string `json:"projectName,omitempty"` - RemoteID string `json:"remoteId,omitempty"` - Route string `json:"route,omitempty"` - Routes []string `json:"routes,omitempty"` - StartTime string `json:"startTime,omitempty"` - Services []string `json:"services,omitempty"` - Task *LagoonTaskInfo `json:"task,omitempty"` - Key string `json:"key,omitempty"` - AdvancedData string `json:"advancedData,omitempty"` - Cluster string `json:"clusterName,omitempty"` -} - -// LagoonMessage is used for sending build info back to Lagoon -// messaging queue to update the environment or deployment -type LagoonMessage struct { - Type string `json:"type,omitempty"` - Namespace string `json:"namespace,omitempty"` - Meta *LagoonLogMeta `json:"meta,omitempty"` - // BuildInfo *LagoonBuildInfo `json:"buildInfo,omitempty"` -} +import "github.com/uselagoon/machinery/api/schema" // LagoonStatusMessages is where unsent messages are stored for re-sending. type LagoonStatusMessages struct { - StatusMessage *LagoonLog `json:"statusMessage,omitempty"` - BuildLogMessage *LagoonLog `json:"buildLogMessage,omitempty"` - TaskLogMessage *LagoonLog `json:"taskLogMessage,omitempty"` - EnvironmentMessage *LagoonMessage `json:"environmentMessage,omitempty"` + StatusMessage *schema.LagoonLog `json:"statusMessage,omitempty"` + BuildLogMessage *schema.LagoonLog `json:"buildLogMessage,omitempty"` + TaskLogMessage *schema.LagoonLog `json:"taskLogMessage,omitempty"` + // LagoonMessage is used for sending build info back to Lagoon + // messaging queue to update the environment or deployment + EnvironmentMessage *schema.LagoonMessage `json:"environmentMessage,omitempty"` } diff --git a/apis/lagoon/v1beta1/lagoontask_helpers.go b/apis/lagoon/v1beta1/lagoontask_helpers.go new file mode 100644 index 00000000..ba30f3c6 --- /dev/null +++ b/apis/lagoon/v1beta1/lagoontask_helpers.go @@ -0,0 +1,292 @@ +package v1beta1 + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "time" + + "github.com/go-logr/logr" + "github.com/uselagoon/machinery/api/schema" + "github.com/uselagoon/remote-controller/internal/helpers" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var ( + // TaskRunningPendingStatus . + TaskRunningPendingStatus = []string{ + TaskStatusPending.String(), + TaskStatusQueued.String(), + TaskStatusRunning.String(), + } + // TaskCompletedCancelledFailedStatus . + TaskCompletedCancelledFailedStatus = []string{ + TaskStatusFailed.String(), + TaskStatusComplete.String(), + TaskStatusCancelled.String(), + } +) + +// TaskContainsStatus . +func TaskContainsStatus(slice []LagoonTaskConditions, s LagoonTaskConditions) bool { + for _, item := range slice { + if item == s { + return true + } + } + return false +} + +// DeleteLagoonTasks will delete any lagoon tasks from the namespace. +func DeleteLagoonTasks(ctx context.Context, opLog logr.Logger, cl client.Client, ns, project, environment string) bool { + lagoonTasks := &LagoonTaskList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(ns), + }) + if err := cl.List(ctx, lagoonTasks, listOption); err != nil { + opLog.Error(err, + fmt.Sprintf( + "Unable to list lagoon task in namespace %s for project %s, environment %s", + ns, + project, + environment, + ), + ) + return false + } + for _, lagoonTask := range lagoonTasks.Items { + if err := cl.Delete(ctx, &lagoonTask); helpers.IgnoreNotFound(err) != nil { + opLog.Error(err, + fmt.Sprintf( + "Unable to delete lagoon task %s in %s for project %s, environment %s", + lagoonTask.ObjectMeta.Name, + ns, + project, + environment, + ), + ) + return false + } + opLog.Info( + fmt.Sprintf( + "Deleted lagoon task %s in %s for project %s, environment %s", + lagoonTask.ObjectMeta.Name, + ns, + project, + environment, + ), + ) + } + return true +} + +// LagoonTaskPruner will prune any build crds that are hanging around. +func LagoonTaskPruner(ctx context.Context, cl client.Client, cns string, tasksToKeep int) { + opLog := ctrl.Log.WithName("utilities").WithName("LagoonTaskPruner") + namespaces := &corev1.NamespaceList{} + labelRequirements, _ := labels.NewRequirement("lagoon.sh/environmentType", selection.Exists, nil) + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.MatchingLabelsSelector{ + Selector: labels.NewSelector().Add(*labelRequirements), + }, + }) + if err := cl.List(ctx, namespaces, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list namespaces created by Lagoon, there may be none or something went wrong")) + return + } + for _, ns := range namespaces.Items { + if ns.Status.Phase == corev1.NamespaceTerminating { + // if the namespace is terminating, don't try to renew the robot credentials + opLog.Info(fmt.Sprintf("Namespace %s is being terminated, aborting task pruner", ns.ObjectMeta.Name)) + continue + } + opLog.Info(fmt.Sprintf("Checking LagoonTasks in namespace %s", ns.ObjectMeta.Name)) + lagoonTasks := &LagoonTaskList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(ns.ObjectMeta.Name), + client.MatchingLabels(map[string]string{ + "lagoon.sh/controller": cns, // created by this controller + }), + }) + if err := cl.List(ctx, lagoonTasks, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list LagoonTask resources, there may be none or something went wrong")) + continue + } + // sort the build pods by creation timestamp + sort.Slice(lagoonTasks.Items, func(i, j int) bool { + return lagoonTasks.Items[i].ObjectMeta.CreationTimestamp.After(lagoonTasks.Items[j].ObjectMeta.CreationTimestamp.Time) + }) + if len(lagoonTasks.Items) > tasksToKeep { + for idx, lagoonTask := range lagoonTasks.Items { + if idx >= tasksToKeep { + if helpers.ContainsString( + TaskCompletedCancelledFailedStatus, + lagoonTask.ObjectMeta.Labels["lagoon.sh/taskStatus"], + ) { + opLog.Info(fmt.Sprintf("Cleaning up LagoonTask %s", lagoonTask.ObjectMeta.Name)) + if err := cl.Delete(ctx, &lagoonTask); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to update status condition")) + break + } + } + } + } + } + } + return +} + +// TaskPodPruner will prune any task pods that are hanging around. +func TaskPodPruner(ctx context.Context, cl client.Client, cns string, taskPodsToKeep int) { + opLog := ctrl.Log.WithName("utilities").WithName("TaskPodPruner") + namespaces := &corev1.NamespaceList{} + labelRequirements, _ := labels.NewRequirement("lagoon.sh/environmentType", selection.Exists, nil) + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.MatchingLabelsSelector{ + Selector: labels.NewSelector().Add(*labelRequirements), + }, + }) + if err := cl.List(ctx, namespaces, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list namespaces created by Lagoon, there may be none or something went wrong")) + return + } + for _, ns := range namespaces.Items { + if ns.Status.Phase == corev1.NamespaceTerminating { + // if the namespace is terminating, don't try to renew the robot credentials + opLog.Info(fmt.Sprintf("Namespace %s is being terminated, aborting task pod pruner", ns.ObjectMeta.Name)) + return + } + opLog.Info(fmt.Sprintf("Checking Lagoon task pods in namespace %s", ns.ObjectMeta.Name)) + taskPods := &corev1.PodList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(ns.ObjectMeta.Name), + client.MatchingLabels(map[string]string{ + "lagoon.sh/jobType": "task", + "lagoon.sh/controller": cns, // created by this controller + }), + }) + if err := cl.List(ctx, taskPods, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list Lagoon task pods, there may be none or something went wrong")) + return + } + // sort the build pods by creation timestamp + sort.Slice(taskPods.Items, func(i, j int) bool { + return taskPods.Items[i].ObjectMeta.CreationTimestamp.After(taskPods.Items[j].ObjectMeta.CreationTimestamp.Time) + }) + if len(taskPods.Items) > taskPodsToKeep { + for idx, pod := range taskPods.Items { + if idx >= taskPodsToKeep { + if pod.Status.Phase == corev1.PodFailed || + pod.Status.Phase == corev1.PodSucceeded { + opLog.Info(fmt.Sprintf("Cleaning up pod %s", pod.ObjectMeta.Name)) + if err := cl.Delete(ctx, &pod); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to delete pod")) + break + } + } + } + } + } + } + return +} + +func updateLagoonTask(opLog logr.Logger, namespace string, taskSpec LagoonTaskSpec) ([]byte, error) { + //@TODO: use `taskName` in the future only + taskName := fmt.Sprintf("lagoon-task-%s-%s", taskSpec.Task.ID, helpers.HashString(taskSpec.Task.ID)[0:6]) + if taskSpec.Task.TaskName != "" { + taskName = taskSpec.Task.TaskName + } + // if the task isn't found by the controller + // then publish a response back to controllerhandler to tell it to update the task to cancelled + // this allows us to update tasks in the API that may have gone stale or not updated from `New`, `Pending`, or `Running` status + msg := schema.LagoonMessage{ + Type: "task", + Namespace: namespace, + Meta: &schema.LagoonLogMeta{ + Environment: taskSpec.Environment.Name, + Project: taskSpec.Project.Name, + JobName: taskName, + JobStatus: "cancelled", + Task: &schema.LagoonTaskInfo{ + TaskName: taskSpec.Task.TaskName, + ID: taskSpec.Task.ID, + Name: taskSpec.Task.Name, + Service: taskSpec.Task.Service, + }, + }, + } + // if the task isn't found at all, then set the start/end time to be now + // to stop the duration counter in the ui + msg.Meta.StartTime = time.Now().UTC().Format("2006-01-02 15:04:05") + msg.Meta.EndTime = time.Now().UTC().Format("2006-01-02 15:04:05") + msgBytes, err := json.Marshal(msg) + if err != nil { + return nil, fmt.Errorf("Unable to encode message as JSON: %v", err) + } + return msgBytes, nil +} + +// CancelTask handles cancelling tasks or handling if a tasks no longer exists. +func CancelTask(ctx context.Context, cl client.Client, namespace string, body []byte) (bool, []byte, error) { + opLog := ctrl.Log.WithName("handlers").WithName("LagoonTasks") + jobSpec := &LagoonTaskSpec{} + json.Unmarshal(body, jobSpec) + var jobPod corev1.Pod + //@TODO: use `taskName` in the future only + taskName := fmt.Sprintf("lagoon-task-%s-%s", jobSpec.Task.ID, helpers.HashString(jobSpec.Task.ID)[0:6]) + if jobSpec.Task.TaskName != "" { + taskName = jobSpec.Task.TaskName + } + if err := cl.Get(ctx, types.NamespacedName{ + Name: taskName, + Namespace: namespace, + }, &jobPod); err != nil { + // since there was no task pod, check for the lagoon task resource + var lagoonTask LagoonTask + if err := cl.Get(ctx, types.NamespacedName{ + Name: taskName, + Namespace: namespace, + }, &lagoonTask); err != nil { + opLog.Info(fmt.Sprintf( + "Unable to find task %s to cancel it. Sending response to Lagoon to update the task to cancelled.", + taskName, + )) + // if there is no pod or task, update the task in Lagoon to cancelled + b, err := updateLagoonTask(opLog, namespace, *jobSpec) + return false, b, err + } + // as there is no task pod, but there is a lagoon task resource + // update it to cancelled so that the controller doesn't try to run it + lagoonTask.ObjectMeta.Labels["lagoon.sh/taskStatus"] = TaskStatusCancelled.String() + if err := cl.Update(ctx, &lagoonTask); err != nil { + opLog.Error(err, + fmt.Sprintf( + "Unable to update task %s to cancel it.", + taskName, + ), + ) + return false, nil, err + } + // and then send the response back to lagoon to say it was cancelled. + b, err := updateLagoonTask(opLog, namespace, *jobSpec) + return true, b, err + } + jobPod.ObjectMeta.Labels["lagoon.sh/cancelTask"] = "true" + if err := cl.Update(ctx, &jobPod); err != nil { + opLog.Error(err, + fmt.Sprintf( + "Unable to update task %s to cancel it.", + jobSpec.Misc.Name, + ), + ) + return false, nil, err + } + return false, nil, nil +} diff --git a/apis/lagoon/v1beta1/lagoontask_types.go b/apis/lagoon/v1beta1/lagoontask_types.go index 3231e530..bb48276d 100644 --- a/apis/lagoon/v1beta1/lagoontask_types.go +++ b/apis/lagoon/v1beta1/lagoontask_types.go @@ -22,6 +22,7 @@ import ( "strconv" "strings" + "github.com/uselagoon/machinery/api/schema" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -76,7 +77,7 @@ type LagoonTaskSpec struct { // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster // Important: Run "make" to regenerate code after modifying this file Key string `json:"key,omitempty"` - Task LagoonTaskInfo `json:"task,omitempty"` + Task schema.LagoonTaskInfo `json:"task,omitempty"` Project LagoonTaskProject `json:"project,omitempty"` Environment LagoonTaskEnvironment `json:"environment,omitempty"` Misc *LagoonMiscInfo `json:"misc,omitempty"` diff --git a/apis/lagoon/v1beta1/zz_generated.deepcopy.go b/apis/lagoon/v1beta1/zz_generated.deepcopy.go index 842af614..c11c785c 100644 --- a/apis/lagoon/v1beta1/zz_generated.deepcopy.go +++ b/apis/lagoon/v1beta1/zz_generated.deepcopy.go @@ -198,86 +198,6 @@ func (in *LagoonBuildStatus) DeepCopy() *LagoonBuildStatus { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *LagoonLog) DeepCopyInto(out *LagoonLog) { - *out = *in - if in.Meta != nil { - in, out := &in.Meta, &out.Meta - *out = new(LagoonLogMeta) - (*in).DeepCopyInto(*out) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonLog. -func (in *LagoonLog) DeepCopy() *LagoonLog { - if in == nil { - return nil - } - out := new(LagoonLog) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *LagoonLogMeta) DeepCopyInto(out *LagoonLogMeta) { - *out = *in - if in.EnvironmentID != nil { - in, out := &in.EnvironmentID, &out.EnvironmentID - *out = new(uint) - **out = **in - } - if in.ProjectID != nil { - in, out := &in.ProjectID, &out.ProjectID - *out = new(uint) - **out = **in - } - if in.Routes != nil { - in, out := &in.Routes, &out.Routes - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.Services != nil { - in, out := &in.Services, &out.Services - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.Task != nil { - in, out := &in.Task, &out.Task - *out = new(LagoonTaskInfo) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonLogMeta. -func (in *LagoonLogMeta) DeepCopy() *LagoonLogMeta { - if in == nil { - return nil - } - out := new(LagoonLogMeta) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *LagoonMessage) DeepCopyInto(out *LagoonMessage) { - *out = *in - if in.Meta != nil { - in, out := &in.Meta, &out.Meta - *out = new(LagoonLogMeta) - (*in).DeepCopyInto(*out) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonMessage. -func (in *LagoonMessage) DeepCopy() *LagoonMessage { - if in == nil { - return nil - } - out := new(LagoonMessage) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LagoonMiscBackupInfo) DeepCopyInto(out *LagoonMiscBackupInfo) { *out = *in @@ -323,23 +243,19 @@ func (in *LagoonStatusMessages) DeepCopyInto(out *LagoonStatusMessages) { *out = *in if in.StatusMessage != nil { in, out := &in.StatusMessage, &out.StatusMessage - *out = new(LagoonLog) - (*in).DeepCopyInto(*out) + *out = (*in).DeepCopy() } if in.BuildLogMessage != nil { in, out := &in.BuildLogMessage, &out.BuildLogMessage - *out = new(LagoonLog) - (*in).DeepCopyInto(*out) + *out = (*in).DeepCopy() } if in.TaskLogMessage != nil { in, out := &in.TaskLogMessage, &out.TaskLogMessage - *out = new(LagoonLog) - (*in).DeepCopyInto(*out) + *out = (*in).DeepCopy() } if in.EnvironmentMessage != nil { in, out := &in.EnvironmentMessage, &out.EnvironmentMessage - *out = new(LagoonMessage) - (*in).DeepCopyInto(*out) + *out = (*in).DeepCopy() } } diff --git a/controllers/v1beta1/harborcredential_controller.go b/controllers/harbor/harborcredential_controller.go similarity index 100% rename from controllers/v1beta1/harborcredential_controller.go rename to controllers/harbor/harborcredential_controller.go diff --git a/controllers/harbor/predicates.go b/controllers/harbor/predicates.go new file mode 100644 index 00000000..dab932dd --- /dev/null +++ b/controllers/harbor/predicates.go @@ -0,0 +1,61 @@ +package v1beta1 + +// contains all the event watch conditions for secret and ingresses + +import ( + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +// SecretPredicates defines the funcs for predicates +type SecretPredicates struct { + predicate.Funcs + ControllerNamespace string +} + +// Create is used when a creation event is received by the controller. +func (n SecretPredicates) Create(e event.CreateEvent) bool { + if controller, ok := e.Object.GetLabels()["lagoon.sh/controller"]; ok { + if controller == n.ControllerNamespace { + if val, ok := e.Object.GetLabels()["lagoon.sh/harbor-credential"]; ok { + if val == "true" { + return true + } + } + } + } + return false +} + +// Delete is used when a deletion event is received by the controller. +func (n SecretPredicates) Delete(e event.DeleteEvent) bool { + return false +} + +// Update is used when an update event is received by the controller. +func (n SecretPredicates) Update(e event.UpdateEvent) bool { + if controller, ok := e.ObjectOld.GetLabels()["lagoon.sh/controller"]; ok { + if controller == n.ControllerNamespace { + if val, ok := e.ObjectOld.GetLabels()["lagoon.sh/harbor-credential"]; ok { + if val == "true" { + return true + } + } + } + } + return false +} + +// Generic is used when any other event is received by the controller. +func (n SecretPredicates) Generic(e event.GenericEvent) bool { + if controller, ok := e.Object.GetLabels()["lagoon.sh/controller"]; ok { + if controller == n.ControllerNamespace { + if val, ok := e.Object.GetLabels()["lagoon.sh/harbor-credential"]; ok { + if val == "true" { + return true + } + } + } + } + return false +} diff --git a/controllers/v1beta1/build_controller.go b/controllers/v1beta1/build_controller.go index 421cdac3..db1f608c 100644 --- a/controllers/v1beta1/build_controller.go +++ b/controllers/v1beta1/build_controller.go @@ -259,7 +259,7 @@ func (r *LagoonBuildReconciler) createNamespaceBuild(ctx context.Context, // if everything is all good controller will handle the new build resource that gets created as it will have // the `lagoon.sh/buildStatus = Pending` now - err = helpers.CancelExtraBuilds(ctx, r.Client, opLog, namespace.ObjectMeta.Name, lagoonv1beta1.BuildStatusPending.String()) + err = lagoonv1beta1.CancelExtraBuilds(ctx, r.Client, opLog, namespace.ObjectMeta.Name, lagoonv1beta1.BuildStatusPending.String()) if err != nil { return ctrl.Result{}, err } @@ -292,7 +292,7 @@ func (r *LagoonBuildReconciler) createNamespaceBuild(ctx context.Context, } else { // get the status from the pod and update the build if lagoonBuildPod.Status.Phase == corev1.PodFailed || lagoonBuildPod.Status.Phase == corev1.PodSucceeded { - buildCondition = helpers.GetBuildConditionFromPod(lagoonBuildPod.Status.Phase) + buildCondition = lagoonv1beta1.GetBuildConditionFromPod(lagoonBuildPod.Status.Phase) opLog.Info(fmt.Sprintf("Setting build %s as %s", runningBuild.ObjectMeta.Name, buildCondition.String())) runningBuild.Labels["lagoon.sh/buildStatus"] = buildCondition.String() } else { diff --git a/controllers/v1beta1/build_deletionhandlers.go b/controllers/v1beta1/build_deletionhandlers.go index fba4c0e2..9af24062 100644 --- a/controllers/v1beta1/build_deletionhandlers.go +++ b/controllers/v1beta1/build_deletionhandlers.go @@ -11,6 +11,7 @@ import ( "time" "github.com/go-logr/logr" + "github.com/uselagoon/machinery/api/schema" lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" "github.com/uselagoon/remote-controller/internal/helpers" "gopkg.in/matryer/try.v1" @@ -102,7 +103,7 @@ func (r *LagoonBuildReconciler) deleteExternalResources( // if there are any running builds, check if it is the one currently being deleted if lagoonBuild.ObjectMeta.Name == runningBuild.ObjectMeta.Name { // if the one being deleted is a running one, remove it from the list of running builds - newRunningBuilds = helpers.RemoveBuild(newRunningBuilds, runningBuild) + newRunningBuilds = lagoonv1beta1.RemoveBuild(newRunningBuilds, runningBuild) } } // if the number of runningBuilds is 0 (excluding the one being deleted) @@ -125,7 +126,7 @@ func (r *LagoonBuildReconciler) deleteExternalResources( // if there are any pending builds, check if it is the one currently being deleted if lagoonBuild.ObjectMeta.Name == pendingBuild.ObjectMeta.Name { // if the one being deleted a the pending one, remove it from the list of pending builds - newPendingBuilds = helpers.RemoveBuild(newPendingBuilds, pendingBuild) + newPendingBuilds = lagoonv1beta1.RemoveBuild(newPendingBuilds, pendingBuild) } } // sort the pending builds by creation timestamp @@ -165,7 +166,7 @@ func (r *LagoonBuildReconciler) updateCancelledDeploymentWithLogs( // if it was already Failed or Completed, lagoon probably already knows // so we don't have to do anything else. if helpers.ContainsString( - helpers.BuildRunningPendingStatus, + lagoonv1beta1.BuildRunningPendingStatus, lagoonBuild.Labels["lagoon.sh/buildStatus"], ) { opLog.Info( @@ -234,11 +235,11 @@ func (r *LagoonBuildReconciler) buildLogsToLagoonLogs(ctx context.Context, if condition == lagoonv1beta1.BuildStatusCancelled { buildStep = "cancelled" } - msg := lagoonv1beta1.LagoonLog{ + msg := schema.LagoonLog{ Severity: "info", Project: lagoonBuild.Spec.Project.Name, Event: "build-logs:builddeploy-kubernetes:" + lagoonBuild.ObjectMeta.Name, - Meta: &lagoonv1beta1.LagoonLogMeta{ + Meta: &schema.LagoonLogMeta{ JobName: lagoonBuild.ObjectMeta.Name, // @TODO: remove once lagoon is corrected in controller-handler BuildName: lagoonBuild.ObjectMeta.Name, BuildPhase: buildCondition.ToLower(), // @TODO: same as buildstatus label, remove once lagoon is corrected in controller-handler @@ -290,10 +291,10 @@ func (r *LagoonBuildReconciler) updateDeploymentAndEnvironmentTask(ctx context.C r.RandomNamespacePrefix, ) if r.EnableMQ { - msg := lagoonv1beta1.LagoonMessage{ + msg := schema.LagoonMessage{ Type: "build", Namespace: namespace, - Meta: &lagoonv1beta1.LagoonLogMeta{ + Meta: &schema.LagoonLogMeta{ Environment: lagoonBuild.Spec.Project.Environment, Project: lagoonBuild.Spec.Project.Name, BuildPhase: buildCondition.ToLower(), @@ -372,11 +373,11 @@ func (r *LagoonBuildReconciler) buildStatusLogsToLagoonLogs(ctx context.Context, buildStep string, ) { if r.EnableMQ { - msg := lagoonv1beta1.LagoonLog{ + msg := schema.LagoonLog{ Severity: "info", Project: lagoonBuild.Spec.Project.Name, Event: "task:builddeploy-kubernetes:" + buildCondition.ToLower(), //@TODO: this probably needs to be changed to a new task event for the controller - Meta: &lagoonv1beta1.LagoonLogMeta{ + Meta: &schema.LagoonLogMeta{ ProjectName: lagoonBuild.Spec.Project.Name, BranchName: lagoonBuild.Spec.Project.Environment, BuildPhase: buildCondition.ToLower(), @@ -426,7 +427,7 @@ func (r *LagoonBuildReconciler) buildStatusLogsToLagoonLogs(ctx context.Context, // updateEnvironmentMessage this is called if the message queue is unavailable, it stores the message that would be sent in the lagoon build func (r *LagoonBuildReconciler) updateEnvironmentMessage(ctx context.Context, lagoonBuild *lagoonv1beta1.LagoonBuild, - envMessage lagoonv1beta1.LagoonMessage, + envMessage schema.LagoonMessage, ) error { // set the transition time mergePatch, _ := json.Marshal(map[string]interface{}{ @@ -448,7 +449,7 @@ func (r *LagoonBuildReconciler) updateEnvironmentMessage(ctx context.Context, // updateBuildStatusMessage this is called if the message queue is unavailable, it stores the message that would be sent in the lagoon build func (r *LagoonBuildReconciler) updateBuildStatusMessage(ctx context.Context, lagoonBuild *lagoonv1beta1.LagoonBuild, - statusMessage lagoonv1beta1.LagoonLog, + statusMessage schema.LagoonLog, ) error { // set the transition time mergePatch, _ := json.Marshal(map[string]interface{}{ @@ -494,7 +495,7 @@ func (r *LagoonBuildReconciler) removeBuildPendingMessageStatus(ctx context.Cont // updateBuildLogMessage this is called if the message queue is unavailable, it stores the message that would be sent in the lagoon build func (r *LagoonBuildReconciler) updateBuildLogMessage(ctx context.Context, lagoonBuild *lagoonv1beta1.LagoonBuild, - buildMessage lagoonv1beta1.LagoonLog, + buildMessage schema.LagoonLog, ) error { // set the transition time mergePatch, _ := json.Marshal(map[string]interface{}{ diff --git a/controllers/v1beta1/build_helpers.go b/controllers/v1beta1/build_helpers.go index 81831ab1..0ff88268 100644 --- a/controllers/v1beta1/build_helpers.go +++ b/controllers/v1beta1/build_helpers.go @@ -922,7 +922,7 @@ func (r *LagoonBuildReconciler) updateQueuedBuild( } // send any messages to lagoon message queues // update the deployment with the status, lagoon v2.12.0 supports queued status, otherwise use pending - if helpers.CheckLagoonVersion(&lagoonBuild, "2.12.0") { + if lagoonv1beta1.CheckLagoonVersion(&lagoonBuild, "2.12.0") { r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusQueued, fmt.Sprintf("queued %v/%v", queuePosition, queueLength)) r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusQueued, fmt.Sprintf("queued %v/%v", queuePosition, queueLength)) r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, allContainerLogs, lagoonv1beta1.BuildStatusQueued) diff --git a/controllers/v1beta1/build_qoshandler.go b/controllers/v1beta1/build_qoshandler.go index 51467ae2..f28cda4a 100644 --- a/controllers/v1beta1/build_qoshandler.go +++ b/controllers/v1beta1/build_qoshandler.go @@ -128,7 +128,7 @@ func (r *LagoonBuildReconciler) processQueue(ctx context.Context, opLog logr.Log } // if there are no running builds, check if there are any pending builds that can be started if len(runningNSBuilds.Items) == 0 { - if err := helpers.CancelExtraBuilds(ctx, r.Client, opLog, pBuild.ObjectMeta.Namespace, "Running"); err != nil { + if err := lagoonv1beta1.CancelExtraBuilds(ctx, r.Client, opLog, pBuild.ObjectMeta.Namespace, "Running"); err != nil { // only return if there is an error doing this operation // continue on otherwise to allow the queued status updater to run runningProcessQueue = false diff --git a/controllers/v1beta1/build_standardhandler.go b/controllers/v1beta1/build_standardhandler.go index 95bd74bf..6ce5fb41 100644 --- a/controllers/v1beta1/build_standardhandler.go +++ b/controllers/v1beta1/build_standardhandler.go @@ -52,7 +52,7 @@ func (r *LagoonBuildReconciler) standardBuildProcessor(ctx context.Context, // if there are no running builds, check if there are any pending builds that can be started if len(runningBuilds.Items) == 0 { - return ctrl.Result{}, helpers.CancelExtraBuilds(ctx, r.Client, opLog, req.Namespace, "Running") + return ctrl.Result{}, lagoonv1beta1.CancelExtraBuilds(ctx, r.Client, opLog, req.Namespace, "Running") } // The object is not being deleted, so if it does not have our finalizer, // then lets add the finalizer and update the object. This is equivalent diff --git a/controllers/v1beta1/podmonitor_buildhandlers.go b/controllers/v1beta1/podmonitor_buildhandlers.go index 0159cb65..8ddac801 100644 --- a/controllers/v1beta1/podmonitor_buildhandlers.go +++ b/controllers/v1beta1/podmonitor_buildhandlers.go @@ -12,6 +12,7 @@ import ( "github.com/go-logr/logr" "github.com/prometheus/client_golang/prometheus" + "github.com/uselagoon/machinery/api/schema" lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" "github.com/uselagoon/remote-controller/internal/helpers" appsv1 "k8s.io/api/apps/v1" @@ -143,7 +144,7 @@ func (r *LagoonMonitorReconciler) buildLogsToLagoonLogs(ctx context.Context, namespace *corev1.Namespace, condition string, logs []byte, -) (bool, lagoonv1beta1.LagoonLog) { +) (bool, schema.LagoonLog) { if r.EnableMQ { buildStep := "running" if condition == "failed" || condition == "complete" || condition == "cancelled" { @@ -170,11 +171,11 @@ func (r *LagoonMonitorReconciler) buildLogsToLagoonLogs(ctx context.Context, if value, ok := jobPod.Labels["lagoon.sh/buildRemoteID"]; ok { remoteId = value } - msg := lagoonv1beta1.LagoonLog{ + msg := schema.LagoonLog{ Severity: "info", Project: projectName, Event: "build-logs:builddeploy-kubernetes:" + jobPod.ObjectMeta.Name, - Meta: &lagoonv1beta1.LagoonLogMeta{ + Meta: &schema.LagoonLogMeta{ EnvironmentID: envID, ProjectID: projectID, BuildName: jobPod.ObjectMeta.Name, @@ -222,7 +223,7 @@ Logs on pod %s, assigned to cluster %s // if we are able to publish the message, then we need to remove any pending messages from the resource // and make sure we don't try and publish again } - return false, lagoonv1beta1.LagoonLog{} + return false, schema.LagoonLog{} } // updateDeploymentAndEnvironmentTask sends the status of the build and deployment to the controllerhandler message queue in lagoon, @@ -234,7 +235,7 @@ func (r *LagoonMonitorReconciler) updateDeploymentAndEnvironmentTask(ctx context lagoonEnv *corev1.ConfigMap, namespace *corev1.Namespace, condition string, -) (bool, lagoonv1beta1.LagoonMessage) { +) (bool, schema.LagoonMessage) { if r.EnableMQ { buildStep := "running" if condition == "failed" || condition == "complete" || condition == "cancelled" { @@ -270,10 +271,10 @@ func (r *LagoonMonitorReconciler) updateDeploymentAndEnvironmentTask(ctx context if value, ok := jobPod.Labels["lagoon.sh/buildRemoteID"]; ok { remoteId = value } - msg := lagoonv1beta1.LagoonMessage{ + msg := schema.LagoonMessage{ Type: "build", Namespace: namespace.ObjectMeta.Name, - Meta: &lagoonv1beta1.LagoonLogMeta{ + Meta: &schema.LagoonLogMeta{ Environment: envName, EnvironmentID: envID, Project: projectName, @@ -359,7 +360,7 @@ func (r *LagoonMonitorReconciler) updateDeploymentAndEnvironmentTask(ctx context // if we are able to publish the message, then we need to remove any pending messages from the resource // and make sure we don't try and publish again } - return false, lagoonv1beta1.LagoonMessage{} + return false, schema.LagoonMessage{} } // buildStatusLogsToLagoonLogs sends the logs to lagoon-logs message queue, used for general messaging @@ -370,7 +371,7 @@ func (r *LagoonMonitorReconciler) buildStatusLogsToLagoonLogs(ctx context.Contex lagoonEnv *corev1.ConfigMap, namespace *corev1.Namespace, condition string, -) (bool, lagoonv1beta1.LagoonLog) { +) (bool, schema.LagoonLog) { if r.EnableMQ { buildStep := "running" if condition == "failed" || condition == "complete" || condition == "cancelled" { @@ -393,11 +394,11 @@ func (r *LagoonMonitorReconciler) buildStatusLogsToLagoonLogs(ctx context.Contex pID, _ := strconv.Atoi(namespace.ObjectMeta.Labels["lagoon.sh/environment"]) projectID = helpers.UintPtr(uint(pID)) } - msg := lagoonv1beta1.LagoonLog{ + msg := schema.LagoonLog{ Severity: "info", Project: projectName, Event: "task:builddeploy-kubernetes:" + condition, //@TODO: this probably needs to be changed to a new task event for the controller - Meta: &lagoonv1beta1.LagoonLogMeta{ + Meta: &schema.LagoonLogMeta{ EnvironmentID: envID, ProjectID: projectID, ProjectName: projectName, @@ -455,7 +456,7 @@ func (r *LagoonMonitorReconciler) buildStatusLogsToLagoonLogs(ctx context.Contex // if we are able to publish the message, then we need to remove any pending messages from the resource // and make sure we don't try and publish again } - return false, lagoonv1beta1.LagoonLog{} + return false, schema.LagoonLog{} } // updateDeploymentWithLogs collects logs from the build containers and ships or stores them @@ -468,13 +469,13 @@ func (r *LagoonMonitorReconciler) updateDeploymentWithLogs( cancel bool, ) error { opLog := r.Log.WithValues("lagoonmonitor", req.NamespacedName) - buildCondition := helpers.GetBuildConditionFromPod(jobPod.Status.Phase) + buildCondition := lagoonv1beta1.GetBuildConditionFromPod(jobPod.Status.Phase) collectLogs := true if cancel { // only set the status to cancelled if the pod is running/pending/queued // otherwise send the existing status of complete/failed/cancelled if helpers.ContainsString( - helpers.BuildRunningPendingStatus, + lagoonv1beta1.BuildRunningPendingStatus, lagoonBuild.Labels["lagoon.sh/buildStatus"], ) { buildCondition = lagoonv1beta1.BuildStatusCancelled @@ -497,7 +498,7 @@ func (r *LagoonMonitorReconciler) updateDeploymentWithLogs( // if the buildstatus is pending or running, or the cancel flag is provided // send the update status to lagoon if helpers.ContainsString( - helpers.BuildRunningPendingStatus, + lagoonv1beta1.BuildRunningPendingStatus, lagoonBuild.Labels["lagoon.sh/buildStatus"], ) || cancel { opLog.Info( @@ -550,7 +551,7 @@ Build %s Status: corev1.ConditionTrue, LastTransitionTime: time.Now().UTC().Format(time.RFC3339), } - if !helpers.BuildContainsStatus(lagoonBuild.Status.Conditions, condition) { + if !lagoonv1beta1.BuildContainsStatus(lagoonBuild.Status.Conditions, condition) { lagoonBuild.Status.Conditions = append(lagoonBuild.Status.Conditions, condition) mergeMap["status"] = map[string]interface{}{ "conditions": lagoonBuild.Status.Conditions, @@ -577,7 +578,7 @@ Build %s pendingStatus, pendingStatusMessage := r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &jobPod, &lagoonEnv, namespace, buildCondition.ToLower()) pendingEnvironment, pendingEnvironmentMessage := r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &jobPod, &lagoonEnv, namespace, buildCondition.ToLower()) var pendingBuildLog bool - var pendingBuildLogMessage lagoonv1beta1.LagoonLog + var pendingBuildLogMessage schema.LagoonLog // if the container logs can't be retrieved, we don't want to send any build logs back, as this will nuke // any previously received logs if !strings.Contains(string(allContainerLogs), "unable to retrieve container logs for containerd") { diff --git a/controllers/v1beta1/podmonitor_controller.go b/controllers/v1beta1/podmonitor_controller.go index 185be749..a1ea2406 100644 --- a/controllers/v1beta1/podmonitor_controller.go +++ b/controllers/v1beta1/podmonitor_controller.go @@ -125,7 +125,7 @@ func (r *LagoonMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Reques } } else { if helpers.ContainsString( - helpers.BuildRunningPendingStatus, + lagoonv1beta1.BuildRunningPendingStatus, lagoonBuild.Labels["lagoon.sh/buildStatus"], ) { opLog.Info(fmt.Sprintf("Attempting to update the LagoonBuild with cancellation if required.")) @@ -162,7 +162,7 @@ func (r *LagoonMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Reques } // if we have no running builds, then check for any pending builds if len(runningBuilds.Items) == 0 { - return ctrl.Result{}, helpers.CancelExtraBuilds(ctx, r.Client, opLog, req.Namespace, "Running") + return ctrl.Result{}, lagoonv1beta1.CancelExtraBuilds(ctx, r.Client, opLog, req.Namespace, "Running") } } else { // since qos handles pending build checks as part of its own operations, we can skip the running pod check step with no-op diff --git a/controllers/v1beta1/podmonitor_taskhandlers.go b/controllers/v1beta1/podmonitor_taskhandlers.go index 5ffe9305..c5e34c1f 100644 --- a/controllers/v1beta1/podmonitor_taskhandlers.go +++ b/controllers/v1beta1/podmonitor_taskhandlers.go @@ -12,6 +12,7 @@ import ( "github.com/go-logr/logr" "github.com/prometheus/client_golang/prometheus" + "github.com/uselagoon/machinery/api/schema" lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" "github.com/uselagoon/remote-controller/internal/helpers" corev1 "k8s.io/api/core/v1" @@ -112,13 +113,13 @@ func (r *LagoonMonitorReconciler) taskLogsToLagoonLogs(opLog logr.Logger, jobPod *corev1.Pod, condition string, logs []byte, -) (bool, lagoonv1beta1.LagoonLog) { +) (bool, schema.LagoonLog) { if r.EnableMQ && lagoonTask != nil { - msg := lagoonv1beta1.LagoonLog{ + msg := schema.LagoonLog{ Severity: "info", Project: lagoonTask.Spec.Project.Name, Event: "task-logs:job-kubernetes:" + lagoonTask.ObjectMeta.Name, - Meta: &lagoonv1beta1.LagoonLogMeta{ + Meta: &schema.LagoonLogMeta{ Task: &lagoonTask.Spec.Task, Environment: lagoonTask.Spec.Environment.Name, JobName: lagoonTask.ObjectMeta.Name, @@ -152,7 +153,7 @@ Logs on pod %s, assigned to cluster %s // if we are able to publish the message, then we need to remove any pending messages from the resource // and make sure we don't try and publish again } - return false, lagoonv1beta1.LagoonLog{} + return false, schema.LagoonLog{} } // updateLagoonTask sends the status of the task and deployment to the controllerhandler message queue in lagoon, @@ -161,7 +162,7 @@ func (r *LagoonMonitorReconciler) updateLagoonTask(ctx context.Context, opLog lo lagoonTask *lagoonv1beta1.LagoonTask, jobPod *corev1.Pod, condition string, -) (bool, lagoonv1beta1.LagoonMessage) { +) (bool, schema.LagoonMessage) { if r.EnableMQ && lagoonTask != nil { if condition == "failed" || condition == "complete" || condition == "cancelled" { time.AfterFunc(31*time.Second, func() { @@ -172,10 +173,10 @@ func (r *LagoonMonitorReconciler) updateLagoonTask(ctx context.Context, opLog lo }) time.Sleep(2 * time.Second) // smol sleep to reduce race of final messages with previous messages } - msg := lagoonv1beta1.LagoonMessage{ + msg := schema.LagoonMessage{ Type: "task", Namespace: lagoonTask.ObjectMeta.Namespace, - Meta: &lagoonv1beta1.LagoonLogMeta{ + Meta: &schema.LagoonLogMeta{ Task: &lagoonTask.Spec.Task, Environment: lagoonTask.Spec.Environment.Name, Project: lagoonTask.Spec.Project.Name, @@ -217,7 +218,7 @@ func (r *LagoonMonitorReconciler) updateLagoonTask(ctx context.Context, opLog lo // if we are able to publish the message, then we need to remove any pending messages from the resource // and make sure we don't try and publish again } - return false, lagoonv1beta1.LagoonMessage{} + return false, schema.LagoonMessage{} } // taskStatusLogsToLagoonLogs sends the logs to lagoon-logs message queue, used for general messaging @@ -225,13 +226,13 @@ func (r *LagoonMonitorReconciler) taskStatusLogsToLagoonLogs(opLog logr.Logger, lagoonTask *lagoonv1beta1.LagoonTask, jobPod *corev1.Pod, condition string, -) (bool, lagoonv1beta1.LagoonLog) { +) (bool, schema.LagoonLog) { if r.EnableMQ && lagoonTask != nil { - msg := lagoonv1beta1.LagoonLog{ + msg := schema.LagoonLog{ Severity: "info", Project: lagoonTask.Spec.Project.Name, Event: "task:job-kubernetes:" + condition, //@TODO: this probably needs to be changed to a new task event for the controller - Meta: &lagoonv1beta1.LagoonLogMeta{ + Meta: &schema.LagoonLogMeta{ Task: &lagoonTask.Spec.Task, ProjectName: lagoonTask.Spec.Project.Name, Environment: lagoonTask.Spec.Environment.Name, @@ -263,7 +264,7 @@ func (r *LagoonMonitorReconciler) taskStatusLogsToLagoonLogs(opLog logr.Logger, // if we are able to publish the message, then we need to remove any pending messages from the resource // and make sure we don't try and publish again } - return false, lagoonv1beta1.LagoonLog{} + return false, schema.LagoonLog{} } // updateTaskWithLogs collects logs from the task containers and ships or stores them @@ -276,7 +277,7 @@ func (r *LagoonMonitorReconciler) updateTaskWithLogs( cancel bool, ) error { opLog := r.Log.WithValues("lagoonmonitor", req.NamespacedName) - taskCondition := helpers.GetTaskConditionFromPod(jobPod.Status.Phase) + taskCondition := lagoonv1beta1.GetTaskConditionFromPod(jobPod.Status.Phase) collectLogs := true if cancel { taskCondition = lagoonv1beta1.TaskStatusCancelled @@ -286,7 +287,7 @@ func (r *LagoonMonitorReconciler) updateTaskWithLogs( // then update the task to reflect the current pod status // we do this so we don't update the status of the task again if helpers.ContainsString( - helpers.TaskRunningPendingStatus, + lagoonv1beta1.TaskRunningPendingStatus, lagoonTask.Labels["lagoon.sh/taskStatus"], ) || cancel { opLog.Info( @@ -340,7 +341,7 @@ Task %s Status: corev1.ConditionTrue, LastTransitionTime: time.Now().UTC().Format(time.RFC3339), } - if !helpers.TaskContainsStatus(lagoonTask.Status.Conditions, condition) { + if !lagoonv1beta1.TaskContainsStatus(lagoonTask.Status.Conditions, condition) { lagoonTask.Status.Conditions = append(lagoonTask.Status.Conditions, condition) mergeMap["status"] = map[string]interface{}{ "conditions": lagoonTask.Status.Conditions, @@ -353,7 +354,7 @@ Task %s pendingStatus, pendingStatusMessage := r.taskStatusLogsToLagoonLogs(opLog, &lagoonTask, &jobPod, taskCondition.ToLower()) pendingEnvironment, pendingEnvironmentMessage := r.updateLagoonTask(ctx, opLog, &lagoonTask, &jobPod, taskCondition.ToLower()) var pendingTaskLog bool - var pendingTaskLogMessage lagoonv1beta1.LagoonLog + var pendingTaskLogMessage schema.LagoonLog // if the container logs can't be retrieved, we don't want to send any task logs back, as this will nuke // any previously received logs if !strings.Contains(string(allContainerLogs), "unable to retrieve container logs for containerd") { diff --git a/controllers/v1beta1/predicates.go b/controllers/v1beta1/predicates.go index 77aa324a..afb99a03 100644 --- a/controllers/v1beta1/predicates.go +++ b/controllers/v1beta1/predicates.go @@ -225,56 +225,3 @@ func (t TaskPredicates) Generic(e event.GenericEvent) bool { } return false } - -// SecretPredicates defines the funcs for predicates -type SecretPredicates struct { - predicate.Funcs - ControllerNamespace string -} - -// Create is used when a creation event is received by the controller. -func (n SecretPredicates) Create(e event.CreateEvent) bool { - if controller, ok := e.Object.GetLabels()["lagoon.sh/controller"]; ok { - if controller == n.ControllerNamespace { - if val, ok := e.Object.GetLabels()["lagoon.sh/harbor-credential"]; ok { - if val == "true" { - return true - } - } - } - } - return false -} - -// Delete is used when a deletion event is received by the controller. -func (n SecretPredicates) Delete(e event.DeleteEvent) bool { - return false -} - -// Update is used when an update event is received by the controller. -func (n SecretPredicates) Update(e event.UpdateEvent) bool { - if controller, ok := e.ObjectOld.GetLabels()["lagoon.sh/controller"]; ok { - if controller == n.ControllerNamespace { - if val, ok := e.ObjectOld.GetLabels()["lagoon.sh/harbor-credential"]; ok { - if val == "true" { - return true - } - } - } - } - return false -} - -// Generic is used when any other event is received by the controller. -func (n SecretPredicates) Generic(e event.GenericEvent) bool { - if controller, ok := e.Object.GetLabels()["lagoon.sh/controller"]; ok { - if controller == n.ControllerNamespace { - if val, ok := e.Object.GetLabels()["lagoon.sh/harbor-credential"]; ok { - if val == "true" { - return true - } - } - } - } - return false -} diff --git a/go.mod b/go.mod index 88770eda..c38f7648 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.27.7 github.com/prometheus/client_golang v1.15.1 + github.com/uselagoon/machinery v0.0.17-0.20240109062854-3b567fb41003 github.com/vshn/k8up v1.99.99 github.com/xhit/go-str2duration/v2 v2.0.0 gopkg.in/matryer/try.v1 v1.0.0-20150601225556-312d2599e12e @@ -55,6 +56,7 @@ require ( github.com/google/gnostic v0.5.7-v3refs // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/uuid v1.3.0 // indirect + github.com/guregu/null v4.0.0+incompatible // indirect github.com/imdario/mergo v0.3.13 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -83,8 +85,8 @@ require ( go.uber.org/zap v1.24.0 // indirect golang.org/x/net v0.10.0 // indirect golang.org/x/oauth2 v0.5.0 // indirect - golang.org/x/sys v0.8.0 // indirect - golang.org/x/term v0.8.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/term v0.15.0 // indirect golang.org/x/text v0.9.0 // indirect golang.org/x/time v0.3.0 // indirect gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect diff --git a/go.sum b/go.sum index 53473ad0..d7d22f0b 100644 --- a/go.sum +++ b/go.sum @@ -747,6 +747,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.3.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpg github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/guregu/null v4.0.0+incompatible h1:4zw0ckM7ECd6FNNddc3Fu4aty9nTlpkkzH7dPn4/4Gw= +github.com/guregu/null v4.0.0+incompatible/go.mod h1:ePGpQaN9cw0tj45IR5E5ehMvsFlLlQZAkkOXZurJ3NM= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -1240,6 +1242,10 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +github.com/uselagoon/machinery v0.0.17-0.20240109062106-ccc88289f3ca h1:Opwha2XIEUFIUbAmYy6xdxtbnk0DOjjYY4MkUSArzGI= +github.com/uselagoon/machinery v0.0.17-0.20240109062106-ccc88289f3ca/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU= +github.com/uselagoon/machinery v0.0.17-0.20240109062854-3b567fb41003 h1:fLhncKjshd8f0DBFB45u7OWsF1H6OgNmpkswJYqwCA8= +github.com/uselagoon/machinery v0.0.17-0.20240109062854-3b567fb41003/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU= github.com/uudashr/gocognit v0.0.0-20190926065955-1655d0de0517/go.mod h1:j44Ayx2KW4+oB6SWMv8KsmHzZrOInQav7D3cQMJ5JUM= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s= @@ -1654,16 +1660,16 @@ golang.org/x/sys v0.0.0-20220405210540-1e041c57c461/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/internal/harbor/harbor_credentialrotation.go b/internal/harbor/harbor_credentialrotation.go index 5256e316..7a3bc4e7 100644 --- a/internal/harbor/harbor_credentialrotation.go +++ b/internal/harbor/harbor_credentialrotation.go @@ -2,7 +2,6 @@ package harbor import ( "fmt" - "sort" "context" "time" @@ -45,32 +44,8 @@ func (h *Harbor) RotateRobotCredentials(ctx context.Context, cl client.Client) { } opLog.Info(fmt.Sprintf("Checking if %s needs robot credentials rotated", ns.ObjectMeta.Name)) // check for running builds! - lagoonBuilds := &lagoonv1beta1.LagoonBuildList{} - listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ - client.InNamespace(ns.ObjectMeta.Name), - client.MatchingLabels(map[string]string{ - "lagoon.sh/controller": h.ControllerNamespace, // created by this controller - }), - }) - if err := cl.List(context.Background(), lagoonBuilds, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list Lagoon build pods, there may be none or something went wrong")) - continue - } - runningBuilds := false - sort.Slice(lagoonBuilds.Items, func(i, j int) bool { - return lagoonBuilds.Items[i].ObjectMeta.CreationTimestamp.After(lagoonBuilds.Items[j].ObjectMeta.CreationTimestamp.Time) - }) - // if there are any builds pending or running, don't try and refresh the credentials as this - // could break the build - if len(lagoonBuilds.Items) > 0 { - if helpers.ContainsString( - helpers.BuildRunningPendingStatus, - lagoonBuilds.Items[0].Labels["lagoon.sh/buildStatus"], - ) { - runningBuilds = true - } - } - if !runningBuilds { + runningBuildsv1beta1 := lagoonv1beta1.CheckRunningBuilds(ctx, h.ControllerNamespace, opLog, cl, ns) + if !runningBuildsv1beta1 { rotated, err := h.RotateRobotCredential(ctx, cl, ns, false) if err != nil { opLog.Error(err, "error") diff --git a/internal/helpers/helpers.go b/internal/helpers/helpers.go index 809eef78..680141d4 100644 --- a/internal/helpers/helpers.go +++ b/internal/helpers/helpers.go @@ -1,53 +1,18 @@ package helpers import ( - "context" "crypto/sha1" "crypto/sha256" "encoding/base32" - "encoding/json" "fmt" "math/rand" "os" "regexp" - "sort" "strconv" "strings" "time" - "github.com/go-logr/logr" - "github.com/hashicorp/go-version" - lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" - corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -var ( - // BuildRunningPendingStatus . - BuildRunningPendingStatus = []string{ - lagoonv1beta1.BuildStatusPending.String(), - lagoonv1beta1.BuildStatusQueued.String(), - lagoonv1beta1.BuildStatusRunning.String(), - } - // BuildCompletedCancelledFailedStatus . - BuildCompletedCancelledFailedStatus = []string{ - lagoonv1beta1.BuildStatusFailed.String(), - lagoonv1beta1.BuildStatusComplete.String(), - lagoonv1beta1.BuildStatusCancelled.String(), - } - // TaskRunningPendingStatus . - TaskRunningPendingStatus = []string{ - lagoonv1beta1.TaskStatusPending.String(), - lagoonv1beta1.TaskStatusQueued.String(), - lagoonv1beta1.TaskStatusRunning.String(), - } - // TaskCompletedCancelledFailedStatus . - TaskCompletedCancelledFailedStatus = []string{ - lagoonv1beta1.TaskStatusFailed.String(), - lagoonv1beta1.TaskStatusComplete.String(), - lagoonv1beta1.TaskStatusCancelled.String(), - } ) const ( @@ -84,26 +49,6 @@ func RemoveString(slice []string, s string) (result []string) { return } -// BuildContainsStatus . -func BuildContainsStatus(slice []lagoonv1beta1.LagoonBuildConditions, s lagoonv1beta1.LagoonBuildConditions) bool { - for _, item := range slice { - if item == s { - return true - } - } - return false -} - -// TaskContainsStatus . -func TaskContainsStatus(slice []lagoonv1beta1.LagoonTaskConditions, s lagoonv1beta1.LagoonTaskConditions) bool { - for _, item := range slice { - if item == s { - return true - } - } - return false -} - // IntPtr . func IntPtr(i int) *int { var iPtr *int @@ -166,18 +111,6 @@ func HashString(s string) string { return fmt.Sprintf("%x", bs) } -// RemoveBuild remove a LagoonBuild from a slice of LagoonBuilds -func RemoveBuild(slice []lagoonv1beta1.LagoonBuild, s lagoonv1beta1.LagoonBuild) []lagoonv1beta1.LagoonBuild { - result := []lagoonv1beta1.LagoonBuild{} - for _, item := range slice { - if item.ObjectMeta.Name == s.ObjectMeta.Name { - continue - } - result = append(result, item) - } - return result -} - var lowerAlNum = regexp.MustCompile("[^a-z0-9]+") // ShortName returns a deterministic random short name of 8 lowercase @@ -254,90 +187,6 @@ func containsStr(s []string, str string) bool { return false } -// Check if the version of lagoon provided in the internal_system scope variable is greater than or equal to the checked version -func CheckLagoonVersion(build *lagoonv1beta1.LagoonBuild, checkVersion string) bool { - lagoonProjectVariables := &[]LagoonEnvironmentVariable{} - json.Unmarshal(build.Spec.Project.Variables.Project, lagoonProjectVariables) - lagoonVersion, err := GetLagoonVariable("LAGOON_SYSTEM_CORE_VERSION", []string{"internal_system"}, *lagoonProjectVariables) - if err != nil { - return false - } - aVer, err := version.NewSemver(lagoonVersion.Value) - if err != nil { - return false - } - bVer, err := version.NewSemver(checkVersion) - if err != nil { - return false - } - return aVer.GreaterThanOrEqual(bVer) -} - -// CancelExtraBuilds cancels extra builds. -func CancelExtraBuilds(ctx context.Context, r client.Client, opLog logr.Logger, ns string, status string) error { - pendingBuilds := &lagoonv1beta1.LagoonBuildList{} - listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ - client.InNamespace(ns), - client.MatchingLabels(map[string]string{"lagoon.sh/buildStatus": lagoonv1beta1.BuildStatusPending.String()}), - }) - if err := r.List(ctx, pendingBuilds, listOption); err != nil { - return fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err) - } - if len(pendingBuilds.Items) > 0 { - // opLog.Info(fmt.Sprintf("There are %v pending builds", len(pendingBuilds.Items))) - // if we have any pending builds, then grab the latest one and make it running - // if there are any other pending builds, cancel them so only the latest one runs - sort.Slice(pendingBuilds.Items, func(i, j int) bool { - return pendingBuilds.Items[i].ObjectMeta.CreationTimestamp.After(pendingBuilds.Items[j].ObjectMeta.CreationTimestamp.Time) - }) - for idx, pBuild := range pendingBuilds.Items { - pendingBuild := pBuild.DeepCopy() - if idx == 0 { - pendingBuild.Labels["lagoon.sh/buildStatus"] = status - } else { - // cancel any other pending builds - opLog.Info(fmt.Sprintf("Setting build %s as cancelled", pendingBuild.ObjectMeta.Name)) - pendingBuild.Labels["lagoon.sh/buildStatus"] = lagoonv1beta1.BuildStatusCancelled.String() - pendingBuild.Labels["lagoon.sh/cancelledByNewBuild"] = "true" - } - if err := r.Update(ctx, pendingBuild); err != nil { - return err - } - } - } - return nil -} - -func GetBuildConditionFromPod(phase corev1.PodPhase) lagoonv1beta1.BuildStatusType { - var buildCondition lagoonv1beta1.BuildStatusType - switch phase { - case corev1.PodFailed: - buildCondition = lagoonv1beta1.BuildStatusFailed - case corev1.PodSucceeded: - buildCondition = lagoonv1beta1.BuildStatusComplete - case corev1.PodPending: - buildCondition = lagoonv1beta1.BuildStatusPending - case corev1.PodRunning: - buildCondition = lagoonv1beta1.BuildStatusRunning - } - return buildCondition -} - -func GetTaskConditionFromPod(phase corev1.PodPhase) lagoonv1beta1.TaskStatusType { - var taskCondition lagoonv1beta1.TaskStatusType - switch phase { - case corev1.PodFailed: - taskCondition = lagoonv1beta1.TaskStatusFailed - case corev1.PodSucceeded: - taskCondition = lagoonv1beta1.TaskStatusComplete - case corev1.PodPending: - taskCondition = lagoonv1beta1.TaskStatusPending - case corev1.PodRunning: - taskCondition = lagoonv1beta1.TaskStatusRunning - } - return taskCondition -} - // GenerateNamespaceName handles the generation of the namespace name from environment and project name with prefixes and patterns func GenerateNamespaceName(pattern, environmentName, projectname, prefix, controllerNamespace string, randomPrefix bool) string { nsPattern := pattern diff --git a/internal/helpers/helpers_test.go b/internal/helpers/helpers_test.go index b0004c9b..e1181b7d 100644 --- a/internal/helpers/helpers_test.go +++ b/internal/helpers/helpers_test.go @@ -4,8 +4,6 @@ import ( "os" "reflect" "testing" - - lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" ) func TestShortName(t *testing.T) { @@ -258,103 +256,3 @@ func TestGetLagoonFeatureFlags(t *testing.T) { }) } } - -func TestCheckLagoonVersion(t *testing.T) { - type args struct { - build *lagoonv1beta1.LagoonBuild - checkVersion string - } - tests := []struct { - name string - args args - want bool - }{ - { - name: "test1", - args: args{ - build: &lagoonv1beta1.LagoonBuild{ - Spec: lagoonv1beta1.LagoonBuildSpec{ - Project: lagoonv1beta1.Project{ - Variables: lagoonv1beta1.LagoonVariables{ - Project: []byte(`[{"name":"LAGOON_SYSTEM_CORE_VERSION","value":"v2.12.0","scope":"internal_system"}]`), - }, - }, - }, - }, - checkVersion: "2.12.0", - }, - want: true, - }, - { - name: "test2", - args: args{ - build: &lagoonv1beta1.LagoonBuild{ - Spec: lagoonv1beta1.LagoonBuildSpec{ - Project: lagoonv1beta1.Project{ - Variables: lagoonv1beta1.LagoonVariables{ - Project: []byte(`[{"name":"LAGOON_SYSTEM_CORE_VERSION","value":"v2.11.0","scope":"internal_system"}]`), - }, - }, - }, - }, - checkVersion: "2.12.0", - }, - want: false, - }, - { - name: "test3", - args: args{ - build: &lagoonv1beta1.LagoonBuild{ - Spec: lagoonv1beta1.LagoonBuildSpec{ - Project: lagoonv1beta1.Project{ - Variables: lagoonv1beta1.LagoonVariables{ - Project: []byte(`[]`), - }, - }, - }, - }, - checkVersion: "2.12.0", - }, - want: false, - }, - { - name: "test4", - args: args{ - build: &lagoonv1beta1.LagoonBuild{ - Spec: lagoonv1beta1.LagoonBuildSpec{ - Project: lagoonv1beta1.Project{ - Variables: lagoonv1beta1.LagoonVariables{ - Project: []byte(`[{"name":"LAGOON_SYSTEM_CORE_VERSION","value":"v2.12.0","scope":"internal_system"}]`), - }, - }, - }, - }, - checkVersion: "v2.12.0", - }, - want: true, - }, - { - name: "test5", - args: args{ - build: &lagoonv1beta1.LagoonBuild{ - Spec: lagoonv1beta1.LagoonBuildSpec{ - Project: lagoonv1beta1.Project{ - Variables: lagoonv1beta1.LagoonVariables{ - Project: []byte(`[{"name":"LAGOON_SYSTEM_CORE_VERSION","value":"v2.11.0","scope":"internal_system"}]`), - }, - }, - }, - }, - checkVersion: "v2.12.0", - }, - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := CheckLagoonVersion(tt.args.build, tt.args.checkVersion); got != tt.want { - t.Errorf("CheckLagoonVersion() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/internal/messenger/consumer.go b/internal/messenger/consumer.go index a0b90853..51173b37 100644 --- a/internal/messenger/consumer.go +++ b/internal/messenger/consumer.go @@ -9,6 +9,7 @@ import ( "time" "github.com/cheshir/go-mq/v2" + "github.com/uselagoon/machinery/api/schema" lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" "github.com/uselagoon/remote-controller/internal/helpers" "gopkg.in/matryer/try.v1" @@ -150,10 +151,10 @@ func (m *Messenger) Consumer(targetName string) { //error { branch, ), ) - msg := lagoonv1beta1.LagoonMessage{ + msg := schema.LagoonMessage{ Type: "remove", Namespace: ns, - Meta: &lagoonv1beta1.LagoonLogMeta{ + Meta: &schema.LagoonLogMeta{ Project: project, Environment: branch, }, @@ -183,10 +184,10 @@ func (m *Messenger) Consumer(targetName string) { //error { go func() { err := m.DeletionHandler.ProcessDeletion(ctx, opLog, namespace) if err == nil { - msg := lagoonv1beta1.LagoonMessage{ + msg := schema.LagoonMessage{ Type: "remove", Namespace: namespace.ObjectMeta.Name, - Meta: &lagoonv1beta1.LagoonLogMeta{ + Meta: &schema.LagoonLogMeta{ Project: project, Environment: branch, }, @@ -313,12 +314,20 @@ func (m *Messenger) Consumer(targetName string) { //error { ), ) m.Cache.Add(jobSpec.Misc.Name, jobSpec.Project.Name) - err := m.CancelBuild(namespace, jobSpec) + _, v1beta1Bytes, err := lagoonv1beta1.CancelBuild(ctx, m.Client, namespace, message.Body()) if err != nil { - //@TODO: send msg back to lagoon and update task to failed? + //@TODO: send msg back to lagoon and update build to failed? message.Ack(false) // ack to remove from queue return } + if v1beta1Bytes != nil { + // if v1beta1 has a build, send its response + if err := m.Publish("lagoon-tasks:controller", v1beta1Bytes); err != nil { + opLog.Error(err, "Unable to publish message.") + message.Ack(false) // ack to remove from queue + return + } + } case "deploytarget:task:cancel", "kubernetes:task:cancel": opLog.Info( fmt.Sprintf( @@ -329,12 +338,19 @@ func (m *Messenger) Consumer(targetName string) { //error { ), ) m.Cache.Add(jobSpec.Task.TaskName, jobSpec.Project.Name) - err := m.CancelTask(namespace, jobSpec) + _, v1beta1Bytes, err := lagoonv1beta1.CancelTask(ctx, m.Client, namespace, message.Body()) if err != nil { //@TODO: send msg back to lagoon and update task to failed? message.Ack(false) // ack to remove from queue return } + if v1beta1Bytes != nil { + if err := m.Publish("lagoon-tasks:controller", v1beta1Bytes); err != nil { + opLog.Error(err, "Unable to publish message.") + message.Ack(false) // ack to remove from queue + return + } + } case "deploytarget:restic:backup:restore", "kubernetes:restic:backup:restore": opLog.Info( fmt.Sprintf( diff --git a/internal/messenger/pending_messages.go b/internal/messenger/pending_messages.go deleted file mode 100644 index 26658df6..00000000 --- a/internal/messenger/pending_messages.go +++ /dev/null @@ -1,151 +0,0 @@ -package messenger - -import ( - "context" - "encoding/json" - "fmt" - - "github.com/go-logr/logr" - lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" - "k8s.io/apimachinery/pkg/types" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -// GetPendingMessages will get any pending messages from the queue and attempt to publish them if possible -func (m *Messenger) GetPendingMessages() { - opLog := ctrl.Log.WithName("handlers").WithName("PendingMessages") - ctx := context.Background() - opLog.Info(fmt.Sprintf("Checking pending build messages across all namespaces")) - m.pendingBuildLogMessages(ctx, opLog) - opLog.Info(fmt.Sprintf("Checking pending task messages across all namespaces")) - m.pendingTaskLogMessages(ctx, opLog) -} - -func (m *Messenger) pendingBuildLogMessages(ctx context.Context, opLog logr.Logger) { - pendingMsgs := &lagoonv1beta1.LagoonBuildList{} - listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ - client.MatchingLabels(map[string]string{ - "lagoon.sh/pendingMessages": "true", - "lagoon.sh/controller": m.ControllerNamespace, - }), - }) - if err := m.Client.List(ctx, pendingMsgs, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list LagoonBuilds, there may be none or something went wrong")) - return - } - for _, build := range pendingMsgs.Items { - // get the latest resource in case it has been updated since the loop started - if err := m.Client.Get(ctx, types.NamespacedName{ - Name: build.ObjectMeta.Name, - Namespace: build.ObjectMeta.Namespace, - }, &build); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to get LagoonBuild, something went wrong")) - break - } - opLog.Info(fmt.Sprintf("LagoonBuild %s has pending messages, attempting to re-send", build.ObjectMeta.Name)) - - // try to re-publish message or break and try the next build with pending message - if build.StatusMessages.StatusMessage != nil { - statusBytes, _ := json.Marshal(build.StatusMessages.StatusMessage) - if err := m.Publish("lagoon-logs", statusBytes); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to publush message")) - break - } - } - if build.StatusMessages.BuildLogMessage != nil { - logBytes, _ := json.Marshal(build.StatusMessages.BuildLogMessage) - if err := m.Publish("lagoon-logs", logBytes); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to publush message")) - break - } - } - if build.StatusMessages.EnvironmentMessage != nil { - envBytes, _ := json.Marshal(build.StatusMessages.EnvironmentMessage) - if err := m.Publish("lagoon-tasks:controller", envBytes); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to publush message")) - break - } - } - // if we managed to send all the pending messages, then update the resource to remove the pending state - // so we don't send the same message multiple times - opLog.Info(fmt.Sprintf("Sent pending messages for LagoonBuild %s", build.ObjectMeta.Name)) - mergePatch, _ := json.Marshal(map[string]interface{}{ - "metadata": map[string]interface{}{ - "labels": map[string]interface{}{ - "lagoon.sh/pendingMessages": "false", - }, - }, - "statusMessages": nil, - }) - if err := m.Client.Patch(ctx, &build, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to update status condition")) - break - } - } - return -} - -func (m *Messenger) pendingTaskLogMessages(ctx context.Context, opLog logr.Logger) { - pendingMsgs := &lagoonv1beta1.LagoonTaskList{} - listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ - client.MatchingLabels(map[string]string{ - "lagoon.sh/pendingMessages": "true", - "lagoon.sh/controller": m.ControllerNamespace, - }), - }) - if err := m.Client.List(ctx, pendingMsgs, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list LagoonBuilds, there may be none or something went wrong")) - return - } - for _, task := range pendingMsgs.Items { - // get the latest resource in case it has been updated since the loop started - if err := m.Client.Get(ctx, types.NamespacedName{ - Name: task.ObjectMeta.Name, - Namespace: task.ObjectMeta.Namespace, - }, &task); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to get LagoonBuild, something went wrong")) - break - } - opLog.Info(fmt.Sprintf("LagoonTasl %s has pending messages, attempting to re-send", task.ObjectMeta.Name)) - - // try to re-publish message or break and try the next build with pending message - if task.StatusMessages.StatusMessage != nil { - statusBytes, _ := json.Marshal(task.StatusMessages.StatusMessage) - if err := m.Publish("lagoon-logs", statusBytes); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to publush message")) - break - } - } - if task.StatusMessages.TaskLogMessage != nil { - taskLogBytes, _ := json.Marshal(task.StatusMessages.TaskLogMessage) - if err := m.Publish("lagoon-logs", taskLogBytes); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to publush message")) - break - } - } - if task.StatusMessages.EnvironmentMessage != nil { - envBytes, _ := json.Marshal(task.StatusMessages.EnvironmentMessage) - if err := m.Publish("lagoon-tasks:controller", envBytes); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to publush message")) - break - } - } - // if we managed to send all the pending messages, then update the resource to remove the pending state - // so we don't send the same message multiple times - opLog.Info(fmt.Sprintf("Sent pending messages for LagoonTask %s", task.ObjectMeta.Name)) - mergePatch, _ := json.Marshal(map[string]interface{}{ - "metadata": map[string]interface{}{ - "labels": map[string]interface{}{ - "lagoon.sh/pendingMessages": "false", - }, - }, - "statusMessages": nil, - }) - if err := m.Client.Patch(ctx, &task, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to update status condition")) - break - } - } - return -} diff --git a/internal/messenger/tasks_handler.go b/internal/messenger/tasks_handler.go index a5ff074a..2b8b68a0 100644 --- a/internal/messenger/tasks_handler.go +++ b/internal/messenger/tasks_handler.go @@ -5,16 +5,10 @@ import ( "encoding/base64" "encoding/json" "fmt" - "sort" - "strings" - "time" - "github.com/go-logr/logr" lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" "github.com/uselagoon/remote-controller/internal/helpers" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" ) @@ -23,221 +17,6 @@ type ActiveStandbyPayload struct { DestinationNamespace string `json:"destinationNamespace"` } -// CancelBuild handles cancelling builds or handling if a build no longer exists. -func (m *Messenger) CancelBuild(namespace string, jobSpec *lagoonv1beta1.LagoonTaskSpec) error { - opLog := ctrl.Log.WithName("handlers").WithName("LagoonTasks") - var jobPod corev1.Pod - if err := m.Client.Get(context.Background(), types.NamespacedName{ - Name: jobSpec.Misc.Name, - Namespace: namespace, - }, &jobPod); err != nil { - opLog.Info(fmt.Sprintf( - "Unable to find build pod %s to cancel it. Checking to see if LagoonBuild exists.", - jobSpec.Misc.Name, - )) - // since there was no build pod, check for the lagoon build resource - var lagoonBuild lagoonv1beta1.LagoonBuild - if err := m.Client.Get(context.Background(), types.NamespacedName{ - Name: jobSpec.Misc.Name, - Namespace: namespace, - }, &lagoonBuild); err != nil { - opLog.Info(fmt.Sprintf( - "Unable to find build %s to cancel it. Sending response to Lagoon to update the build to cancelled.", - jobSpec.Misc.Name, - )) - // if there is no pod or build, update the build in Lagoon to cancelled, assume completely cancelled with no other information - m.updateLagoonBuild(opLog, namespace, *jobSpec, nil) - return nil - } - // as there is no build pod, but there is a lagoon build resource - // update it to cancelled so that the controller doesn't try to run it - // check if the build has existing status or not though to consume it - if helpers.ContainsString( - helpers.BuildRunningPendingStatus, - lagoonBuild.ObjectMeta.Labels["lagoon.sh/buildStatus"], - ) { - lagoonBuild.ObjectMeta.Labels["lagoon.sh/buildStatus"] = lagoonv1beta1.BuildStatusCancelled.String() - } - lagoonBuild.ObjectMeta.Labels["lagoon.sh/cancelBuildNoPod"] = "true" - if err := m.Client.Update(context.Background(), &lagoonBuild); err != nil { - opLog.Error(err, - fmt.Sprintf( - "Unable to update build %s to cancel it.", - jobSpec.Misc.Name, - ), - ) - return err - } - // and then send the response back to lagoon to say it was cancelled. - m.updateLagoonBuild(opLog, namespace, *jobSpec, &lagoonBuild) - return nil - } - jobPod.ObjectMeta.Labels["lagoon.sh/cancelBuild"] = "true" - if err := m.Client.Update(context.Background(), &jobPod); err != nil { - opLog.Error(err, - fmt.Sprintf( - "Unable to update build %s to cancel it.", - jobSpec.Misc.Name, - ), - ) - return err - } - return nil -} - -// CancelTask handles cancelling tasks or handling if a tasks no longer exists. -func (m *Messenger) CancelTask(namespace string, jobSpec *lagoonv1beta1.LagoonTaskSpec) error { - opLog := ctrl.Log.WithName("handlers").WithName("LagoonTasks") - var jobPod corev1.Pod - //@TODO: use `taskName` in the future only - taskName := fmt.Sprintf("lagoon-task-%s-%s", jobSpec.Task.ID, helpers.HashString(jobSpec.Task.ID)[0:6]) - if jobSpec.Task.TaskName != "" { - taskName = jobSpec.Task.TaskName - } - if err := m.Client.Get(context.Background(), types.NamespacedName{ - Name: taskName, - Namespace: namespace, - }, &jobPod); err != nil { - // since there was no task pod, check for the lagoon task resource - var lagoonTask lagoonv1beta1.LagoonTask - if err := m.Client.Get(context.Background(), types.NamespacedName{ - Name: taskName, - Namespace: namespace, - }, &lagoonTask); err != nil { - opLog.Info(fmt.Sprintf( - "Unable to find task %s to cancel it. Sending response to Lagoon to update the task to cancelled.", - taskName, - )) - // if there is no pod or task, update the task in Lagoon to cancelled - m.updateLagoonTask(opLog, namespace, *jobSpec) - return nil - } - // as there is no task pod, but there is a lagoon task resource - // update it to cancelled so that the controller doesn't try to run it - lagoonTask.ObjectMeta.Labels["lagoon.sh/taskStatus"] = lagoonv1beta1.TaskStatusCancelled.String() - if err := m.Client.Update(context.Background(), &lagoonTask); err != nil { - opLog.Error(err, - fmt.Sprintf( - "Unable to update task %s to cancel it.", - taskName, - ), - ) - return err - } - // and then send the response back to lagoon to say it was cancelled. - m.updateLagoonTask(opLog, namespace, *jobSpec) - return nil - } - jobPod.ObjectMeta.Labels["lagoon.sh/cancelTask"] = "true" - if err := m.Client.Update(context.Background(), &jobPod); err != nil { - opLog.Error(err, - fmt.Sprintf( - "Unable to update task %s to cancel it.", - jobSpec.Misc.Name, - ), - ) - return err - } - return nil -} - -func (m *Messenger) updateLagoonBuild(opLog logr.Logger, namespace string, jobSpec lagoonv1beta1.LagoonTaskSpec, lagoonBuild *lagoonv1beta1.LagoonBuild) { - // if the build isn't found by the controller - // then publish a response back to controllerhandler to tell it to update the build to cancelled - // this allows us to update builds in the API that may have gone stale or not updated from `New`, `Pending`, or `Running` status - buildCondition := "cancelled" - if lagoonBuild != nil { - if val, ok := lagoonBuild.ObjectMeta.Labels["lagoon.sh/buildStatus"]; ok { - // if the build isnt running,pending,queued, then set the buildcondition to the value failed/complete/cancelled - if !helpers.ContainsString(helpers.BuildRunningPendingStatus, val) { - buildCondition = strings.ToLower(val) - } - } - } - msg := lagoonv1beta1.LagoonMessage{ - Type: "build", - Namespace: namespace, - Meta: &lagoonv1beta1.LagoonLogMeta{ - Environment: jobSpec.Environment.Name, - Project: jobSpec.Project.Name, - BuildPhase: buildCondition, - BuildName: jobSpec.Misc.Name, - }, - } - // set the start/end time to be now as the default - // to stop the duration counter in the ui - msg.Meta.StartTime = time.Now().UTC().Format("2006-01-02 15:04:05") - msg.Meta.EndTime = time.Now().UTC().Format("2006-01-02 15:04:05") - - // if possible, get the start and end times from the build resource, these will be sent back to lagoon to update the api - if lagoonBuild != nil && lagoonBuild.Status.Conditions != nil { - conditions := lagoonBuild.Status.Conditions - // sort the build conditions by time so the first and last can be extracted - sort.Slice(conditions, func(i, j int) bool { - iTime, _ := time.Parse("2006-01-02T15:04:05Z", conditions[i].LastTransitionTime) - jTime, _ := time.Parse("2006-01-02T15:04:05Z", conditions[j].LastTransitionTime) - return iTime.Before(jTime) - }) - // get the starting time, or fallback to default - sTime, err := time.Parse("2006-01-02T15:04:05Z", conditions[0].LastTransitionTime) - if err == nil { - msg.Meta.StartTime = sTime.Format("2006-01-02 15:04:05") - } - // get the ending time, or fallback to default - eTime, err := time.Parse("2006-01-02T15:04:05Z", conditions[len(conditions)-1].LastTransitionTime) - if err == nil { - msg.Meta.EndTime = eTime.Format("2006-01-02 15:04:05") - } - } - msgBytes, err := json.Marshal(msg) - if err != nil { - opLog.Error(err, "Unable to encode message as JSON") - } - // publish the cancellation result back to lagoon - if err := m.Publish("lagoon-tasks:controller", msgBytes); err != nil { - opLog.Error(err, "Unable to publish message.") - } -} - -func (m *Messenger) updateLagoonTask(opLog logr.Logger, namespace string, jobSpec lagoonv1beta1.LagoonTaskSpec) { - //@TODO: use `taskName` in the future only - taskName := fmt.Sprintf("lagoon-task-%s-%s", jobSpec.Task.ID, helpers.HashString(jobSpec.Task.ID)[0:6]) - if jobSpec.Task.TaskName != "" { - taskName = jobSpec.Task.TaskName - } - // if the task isn't found by the controller - // then publish a response back to controllerhandler to tell it to update the task to cancelled - // this allows us to update tasks in the API that may have gone stale or not updated from `New`, `Pending`, or `Running` status - msg := lagoonv1beta1.LagoonMessage{ - Type: "task", - Namespace: namespace, - Meta: &lagoonv1beta1.LagoonLogMeta{ - Environment: jobSpec.Environment.Name, - Project: jobSpec.Project.Name, - JobName: taskName, - JobStatus: "cancelled", - Task: &lagoonv1beta1.LagoonTaskInfo{ - TaskName: jobSpec.Task.TaskName, - ID: jobSpec.Task.ID, - Name: jobSpec.Task.Name, - Service: jobSpec.Task.Service, - }, - }, - } - // if the task isn't found at all, then set the start/end time to be now - // to stop the duration counter in the ui - msg.Meta.StartTime = time.Now().UTC().Format("2006-01-02 15:04:05") - msg.Meta.EndTime = time.Now().UTC().Format("2006-01-02 15:04:05") - msgBytes, err := json.Marshal(msg) - if err != nil { - opLog.Error(err, "Unable to encode message as JSON") - } - // publish the cancellation result back to lagoon - if err := m.Publish("lagoon-tasks:controller", msgBytes); err != nil { - opLog.Error(err, "Unable to publish message.") - } -} - // IngressRouteMigration handles running the ingress migrations. func (m *Messenger) IngressRouteMigration(namespace string, jobSpec *lagoonv1beta1.LagoonTaskSpec) error { // always set these to true for ingress migration tasks diff --git a/internal/utilities/deletions/lagoon.go b/internal/utilities/deletions/lagoon.go deleted file mode 100644 index d894e877..00000000 --- a/internal/utilities/deletions/lagoon.go +++ /dev/null @@ -1,98 +0,0 @@ -package deletions - -import ( - "context" - "fmt" - - "github.com/go-logr/logr" - lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/uselagoon/remote-controller/internal/helpers" -) - -// DeleteLagoonBuilds will delete any lagoon builds from the namespace. -func (d *Deletions) DeleteLagoonBuilds(ctx context.Context, opLog logr.Logger, ns, project, environment string) bool { - lagoonBuilds := &lagoonv1beta1.LagoonBuildList{} - listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ - client.InNamespace(ns), - }) - if err := d.Client.List(ctx, lagoonBuilds, listOption); err != nil { - opLog.Error(err, - fmt.Sprintf( - "Unable to list lagoon build in namespace %s for project %s, environment %s", - ns, - project, - environment, - ), - ) - return false - } - for _, lagoonBuild := range lagoonBuilds.Items { - if err := d.Client.Delete(ctx, &lagoonBuild); helpers.IgnoreNotFound(err) != nil { - opLog.Error(err, - fmt.Sprintf( - "Unable to delete lagoon build %s in %s for project %s, environment %s", - lagoonBuild.ObjectMeta.Name, - ns, - project, - environment, - ), - ) - return false - } - opLog.Info( - fmt.Sprintf( - "Deleted lagoon build %s in %s for project %s, environment %s", - lagoonBuild.ObjectMeta.Name, - ns, - project, - environment, - ), - ) - } - return true -} - -// DeleteLagoonTasks will delete any lagoon tasks from the namespace. -func (d *Deletions) DeleteLagoonTasks(ctx context.Context, opLog logr.Logger, ns, project, environment string) bool { - lagoonTasks := &lagoonv1beta1.LagoonTaskList{} - listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ - client.InNamespace(ns), - }) - if err := d.Client.List(ctx, lagoonTasks, listOption); err != nil { - opLog.Error(err, - fmt.Sprintf( - "Unable to list lagoon task in namespace %s for project %s, environment %s", - ns, - project, - environment, - ), - ) - return false - } - for _, lagoonTask := range lagoonTasks.Items { - if err := d.Client.Delete(ctx, &lagoonTask); helpers.IgnoreNotFound(err) != nil { - opLog.Error(err, - fmt.Sprintf( - "Unable to delete lagoon task %s in %s for project %s, environment %s", - lagoonTask.ObjectMeta.Name, - ns, - project, - environment, - ), - ) - return false - } - opLog.Info( - fmt.Sprintf( - "Deleted lagoon task %s in %s for project %s, environment %s", - lagoonTask.ObjectMeta.Name, - ns, - project, - environment, - ), - ) - } - return true -} diff --git a/internal/utilities/deletions/process.go b/internal/utilities/deletions/process.go index 53acc365..c8521a3a 100644 --- a/internal/utilities/deletions/process.go +++ b/internal/utilities/deletions/process.go @@ -9,6 +9,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" "github.com/uselagoon/remote-controller/internal/harbor" ) @@ -97,10 +98,10 @@ func (d *Deletions) ProcessDeletion(ctx context.Context, opLog logr.Logger, name get any deployments/statefulsets/daemonsets then delete them */ - if del := d.DeleteLagoonTasks(ctx, opLog.WithName("DeleteLagoonTasks"), namespace.ObjectMeta.Name, project, environment); del == false { + if del := lagoonv1beta1.DeleteLagoonTasks(ctx, opLog.WithName("DeleteLagoonTasks"), d.Client, namespace.ObjectMeta.Name, project, environment); del == false { return fmt.Errorf("error deleting tasks") } - if del := d.DeleteLagoonBuilds(ctx, opLog.WithName("DeleteLagoonBuilds"), namespace.ObjectMeta.Name, project, environment); del == false { + if del := lagoonv1beta1.DeleteLagoonBuilds(ctx, opLog.WithName("DeleteLagoonBuilds"), d.Client, namespace.ObjectMeta.Name, project, environment); del == false { return fmt.Errorf("error deleting builds") } if del := d.DeleteDeployments(ctx, opLog.WithName("DeleteDeployments"), namespace.ObjectMeta.Name, project, environment); del == false { diff --git a/internal/utilities/pruner/build_pruner.go b/internal/utilities/pruner/build_pruner.go deleted file mode 100644 index 5568077c..00000000 --- a/internal/utilities/pruner/build_pruner.go +++ /dev/null @@ -1,127 +0,0 @@ -package pruner - -import ( - "context" - "fmt" - "sort" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/selection" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - - lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" - "github.com/uselagoon/remote-controller/internal/helpers" -) - -// LagoonBuildPruner will prune any build crds that are hanging around. -func (p *Pruner) LagoonBuildPruner() { - opLog := ctrl.Log.WithName("utilities").WithName("LagoonBuildPruner") - namespaces := &corev1.NamespaceList{} - labelRequirements, _ := labels.NewRequirement("lagoon.sh/environmentType", selection.Exists, nil) - listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ - client.MatchingLabelsSelector{ - Selector: labels.NewSelector().Add(*labelRequirements), - }, - }) - if err := p.Client.List(context.Background(), namespaces, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list namespaces created by Lagoon, there may be none or something went wrong")) - return - } - for _, ns := range namespaces.Items { - if ns.Status.Phase == corev1.NamespaceTerminating { - // if the namespace is terminating, don't try to renew the robot credentials - opLog.Info(fmt.Sprintf("Namespace %s is being terminated, aborting build pruner", ns.ObjectMeta.Name)) - continue - } - opLog.Info(fmt.Sprintf("Checking LagoonBuilds in namespace %s", ns.ObjectMeta.Name)) - lagoonBuilds := &lagoonv1beta1.LagoonBuildList{} - listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ - client.InNamespace(ns.ObjectMeta.Name), - client.MatchingLabels(map[string]string{ - "lagoon.sh/controller": p.ControllerNamespace, // created by this controller - }), - }) - if err := p.Client.List(context.Background(), lagoonBuilds, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list LagoonBuild resources, there may be none or something went wrong")) - continue - } - // sort the build pods by creation timestamp - sort.Slice(lagoonBuilds.Items, func(i, j int) bool { - return lagoonBuilds.Items[i].ObjectMeta.CreationTimestamp.After(lagoonBuilds.Items[j].ObjectMeta.CreationTimestamp.Time) - }) - if len(lagoonBuilds.Items) > p.BuildsToKeep { - for idx, lagoonBuild := range lagoonBuilds.Items { - if idx >= p.BuildsToKeep { - if helpers.ContainsString( - helpers.BuildCompletedCancelledFailedStatus, - lagoonBuild.ObjectMeta.Labels["lagoon.sh/buildStatus"], - ) { - opLog.Info(fmt.Sprintf("Cleaning up LagoonBuild %s", lagoonBuild.ObjectMeta.Name)) - if err := p.Client.Delete(context.Background(), &lagoonBuild); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to update status condition")) - break - } - } - } - } - } - } - return -} - -// BuildPodPruner will prune any build pods that are hanging around. -func (p *Pruner) BuildPodPruner() { - opLog := ctrl.Log.WithName("utilities").WithName("BuildPodPruner") - namespaces := &corev1.NamespaceList{} - labelRequirements, _ := labels.NewRequirement("lagoon.sh/environmentType", selection.Exists, nil) - listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ - client.MatchingLabelsSelector{ - Selector: labels.NewSelector().Add(*labelRequirements), - }, - }) - if err := p.Client.List(context.Background(), namespaces, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list namespaces created by Lagoon, there may be none or something went wrong")) - return - } - for _, ns := range namespaces.Items { - if ns.Status.Phase == corev1.NamespaceTerminating { - // if the namespace is terminating, don't try to renew the robot credentials - opLog.Info(fmt.Sprintf("Namespace %s is being terminated, aborting build pod pruner", ns.ObjectMeta.Name)) - return - } - opLog.Info(fmt.Sprintf("Checking Lagoon build pods in namespace %s", ns.ObjectMeta.Name)) - buildPods := &corev1.PodList{} - listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ - client.InNamespace(ns.ObjectMeta.Name), - client.MatchingLabels(map[string]string{ - "lagoon.sh/jobType": "build", - "lagoon.sh/controller": p.ControllerNamespace, // created by this controller - }), - }) - if err := p.Client.List(context.Background(), buildPods, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list Lagoon build pods, there may be none or something went wrong")) - return - } - // sort the build pods by creation timestamp - sort.Slice(buildPods.Items, func(i, j int) bool { - return buildPods.Items[i].ObjectMeta.CreationTimestamp.After(buildPods.Items[j].ObjectMeta.CreationTimestamp.Time) - }) - if len(buildPods.Items) > p.BuildPodsToKeep { - for idx, pod := range buildPods.Items { - if idx >= p.BuildPodsToKeep { - if pod.Status.Phase == corev1.PodFailed || - pod.Status.Phase == corev1.PodSucceeded { - opLog.Info(fmt.Sprintf("Cleaning up pod %s", pod.ObjectMeta.Name)) - if err := p.Client.Delete(context.Background(), &pod); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to update status condition")) - break - } - } - } - } - } - } - return -} diff --git a/internal/utilities/pruner/old_process_pruner.go b/internal/utilities/pruner/old_process_pruner.go index 4b74506d..8e43db04 100644 --- a/internal/utilities/pruner/old_process_pruner.go +++ b/internal/utilities/pruner/old_process_pruner.go @@ -4,16 +4,15 @@ import ( "context" "errors" "fmt" + "strconv" + "time" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/selection" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "strconv" - "time" - //lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" - //"github.com/uselagoon/remote-controller/internal/helpers" ) // LagoonOldProcPruner will identify and remove any long running builds or tasks. diff --git a/internal/utilities/pruner/task_pruner.go b/internal/utilities/pruner/task_pruner.go deleted file mode 100644 index 0dea5280..00000000 --- a/internal/utilities/pruner/task_pruner.go +++ /dev/null @@ -1,127 +0,0 @@ -package pruner - -import ( - "context" - "fmt" - "sort" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/selection" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - - lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" - "github.com/uselagoon/remote-controller/internal/helpers" -) - -// LagoonTaskPruner will prune any build crds that are hanging around. -func (p *Pruner) LagoonTaskPruner() { - opLog := ctrl.Log.WithName("utilities").WithName("LagoonTaskPruner") - namespaces := &corev1.NamespaceList{} - labelRequirements, _ := labels.NewRequirement("lagoon.sh/environmentType", selection.Exists, nil) - listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ - client.MatchingLabelsSelector{ - Selector: labels.NewSelector().Add(*labelRequirements), - }, - }) - if err := p.Client.List(context.Background(), namespaces, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list namespaces created by Lagoon, there may be none or something went wrong")) - return - } - for _, ns := range namespaces.Items { - if ns.Status.Phase == corev1.NamespaceTerminating { - // if the namespace is terminating, don't try to renew the robot credentials - opLog.Info(fmt.Sprintf("Namespace %s is being terminated, aborting task pruner", ns.ObjectMeta.Name)) - continue - } - opLog.Info(fmt.Sprintf("Checking LagoonTasks in namespace %s", ns.ObjectMeta.Name)) - lagoonTasks := &lagoonv1beta1.LagoonTaskList{} - listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ - client.InNamespace(ns.ObjectMeta.Name), - client.MatchingLabels(map[string]string{ - "lagoon.sh/controller": p.ControllerNamespace, // created by this controller - }), - }) - if err := p.Client.List(context.Background(), lagoonTasks, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list LagoonTask resources, there may be none or something went wrong")) - continue - } - // sort the build pods by creation timestamp - sort.Slice(lagoonTasks.Items, func(i, j int) bool { - return lagoonTasks.Items[i].ObjectMeta.CreationTimestamp.After(lagoonTasks.Items[j].ObjectMeta.CreationTimestamp.Time) - }) - if len(lagoonTasks.Items) > p.TasksToKeep { - for idx, lagoonTask := range lagoonTasks.Items { - if idx >= p.TasksToKeep { - if helpers.ContainsString( - helpers.TaskCompletedCancelledFailedStatus, - lagoonTask.ObjectMeta.Labels["lagoon.sh/taskStatus"], - ) { - opLog.Info(fmt.Sprintf("Cleaning up LagoonTask %s", lagoonTask.ObjectMeta.Name)) - if err := p.Client.Delete(context.Background(), &lagoonTask); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to update status condition")) - break - } - } - } - } - } - } - return -} - -// TaskPodPruner will prune any task pods that are hanging around. -func (p *Pruner) TaskPodPruner() { - opLog := ctrl.Log.WithName("utilities").WithName("TaskPodPruner") - namespaces := &corev1.NamespaceList{} - labelRequirements, _ := labels.NewRequirement("lagoon.sh/environmentType", selection.Exists, nil) - listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ - client.MatchingLabelsSelector{ - Selector: labels.NewSelector().Add(*labelRequirements), - }, - }) - if err := p.Client.List(context.Background(), namespaces, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list namespaces created by Lagoon, there may be none or something went wrong")) - return - } - for _, ns := range namespaces.Items { - if ns.Status.Phase == corev1.NamespaceTerminating { - // if the namespace is terminating, don't try to renew the robot credentials - opLog.Info(fmt.Sprintf("Namespace %s is being terminated, aborting task pod pruner", ns.ObjectMeta.Name)) - return - } - opLog.Info(fmt.Sprintf("Checking Lagoon task pods in namespace %s", ns.ObjectMeta.Name)) - taskPods := &corev1.PodList{} - listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ - client.InNamespace(ns.ObjectMeta.Name), - client.MatchingLabels(map[string]string{ - "lagoon.sh/jobType": "task", - "lagoon.sh/controller": p.ControllerNamespace, // created by this controller - }), - }) - if err := p.Client.List(context.Background(), taskPods, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list Lagoon task pods, there may be none or something went wrong")) - return - } - // sort the build pods by creation timestamp - sort.Slice(taskPods.Items, func(i, j int) bool { - return taskPods.Items[i].ObjectMeta.CreationTimestamp.After(taskPods.Items[j].ObjectMeta.CreationTimestamp.Time) - }) - if len(taskPods.Items) > p.TaskPodsToKeep { - for idx, pod := range taskPods.Items { - if idx >= p.TaskPodsToKeep { - if pod.Status.Phase == corev1.PodFailed || - pod.Status.Phase == corev1.PodSucceeded { - opLog.Info(fmt.Sprintf("Cleaning up pod %s", pod.ObjectMeta.Name)) - if err := p.Client.Delete(context.Background(), &pod); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to delete pod")) - break - } - } - } - } - } - } - return -} diff --git a/main.go b/main.go index 7238b1ab..9aef681d 100644 --- a/main.go +++ b/main.go @@ -45,6 +45,7 @@ import ( "github.com/hashicorp/golang-lru/v2/expirable" k8upv1 "github.com/k8up-io/k8up/v2/api/v1" lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" + harborctrl "github.com/uselagoon/remote-controller/controllers/harbor" lagoonv1beta1ctrl "github.com/uselagoon/remote-controller/controllers/v1beta1" "github.com/uselagoon/remote-controller/internal/messenger" k8upv1alpha1 "github.com/vshn/k8up/api/v1alpha1" @@ -85,7 +86,6 @@ func main() { var enableLeaderElection bool var enableMQ bool var leaderElectionID string - var pendingMessageCron string var mqWorkers int var rabbitRetryInterval int var startupConnectionAttempts int @@ -191,8 +191,8 @@ func main() { "The retry interval for rabbitmq.") flag.StringVar(&leaderElectionID, "leader-election-id", "lagoon-builddeploy-leader-election-helper", "The ID to use for leader election.") - flag.StringVar(&pendingMessageCron, "pending-message-cron", "15,45 * * * *", - "The cron definition for pending messages.") + flag.String("pending-message-cron", "", + "This feature has been deprecated, this flag will be removed in a future version.") flag.IntVar(&startupConnectionAttempts, "startup-connection-attempts", 10, "The number of startup attempts before exiting.") flag.IntVar(&startupConnectionInterval, "startup-connection-interval-seconds", 30, @@ -375,7 +375,6 @@ func main() { mqPass = helpers.GetEnv("RABBITMQ_PASSWORD", mqPass) mqHost = helpers.GetEnv("RABBITMQ_HOSTNAME", mqHost) lagoonTargetName = helpers.GetEnv("LAGOON_TARGET_NAME", lagoonTargetName) - pendingMessageCron = helpers.GetEnv("PENDING_MESSAGE_CRON", pendingMessageCron) overrideBuildDeployImage = helpers.GetEnv("OVERRIDE_BUILD_DEPLOY_DIND_IMAGE", overrideBuildDeployImage) namespacePrefix = helpers.GetEnv("NAMESPACE_PREFIX", namespacePrefix) if len(namespacePrefix) > 8 { @@ -657,16 +656,9 @@ func main() { if enableMQ { setupLog.Info("starting messaging handler") go messaging.Consumer(lagoonTargetName) - - // use cron to run a pending message task - // this will check any `LagoonBuild` resources for the pendingMessages label - // and attempt to re-publish them - c.AddFunc(pendingMessageCron, func() { - messaging.GetPendingMessages() - }) } - buildQoSConfig := lagoonv1beta1ctrl.BuildQoS{ + buildQoSConfigv1beta1 := lagoonv1beta1ctrl.BuildQoS{ MaxBuilds: qosMaxBuilds, DefaultValue: qosDefaultValue, } @@ -688,7 +680,7 @@ func main() { // use cron to run a lagoonbuild cleanup task // this will check any Lagoon builds and attempt to delete them c.AddFunc(buildsCleanUpCron, func() { - resourceCleanup.LagoonBuildPruner() + lagoonv1beta1.LagoonBuildPruner(context.Background(), mgr.GetClient(), controllerNamespace, buildsToKeep) }) } // if the build pod cleanup is enabled, add the cronjob for it @@ -697,7 +689,7 @@ func main() { // use cron to run a build pod cleanup task // this will check any Lagoon build pods and attempt to delete them c.AddFunc(buildPodCleanUpCron, func() { - resourceCleanup.BuildPodPruner() + lagoonv1beta1.BuildPodPruner(context.Background(), mgr.GetClient(), controllerNamespace, buildPodsToKeep) }) } // if the lagoontask cleanup is enabled, add the cronjob for it @@ -706,7 +698,7 @@ func main() { // use cron to run a lagoontask cleanup task // this will check any Lagoon tasks and attempt to delete them c.AddFunc(taskCleanUpCron, func() { - resourceCleanup.LagoonTaskPruner() + lagoonv1beta1.LagoonTaskPruner(context.Background(), mgr.GetClient(), controllerNamespace, tasksToKeep) }) } // if the task pod cleanup is enabled, add the cronjob for it @@ -715,7 +707,7 @@ func main() { // use cron to run a task pod cleanup task // this will check any Lagoon task pods and attempt to delete them c.AddFunc(taskPodCleanUpCron, func() { - resourceCleanup.TaskPodPruner() + lagoonv1beta1.TaskPodPruner(context.Background(), mgr.GetClient(), controllerNamespace, taskPodsToKeep) }) } // if harbor is enabled, add the cronjob for credential rotation @@ -748,6 +740,7 @@ func main() { setupLog.Info("starting controllers") + // v1beta1 is deprecated, these controllers will eventually be removed if err = (&lagoonv1beta1ctrl.LagoonBuildReconciler{ Client: mgr.GetClient(), Log: ctrl.Log.WithName("v1beta1").WithName("LagoonBuild"), @@ -789,7 +782,7 @@ func main() { LFFHarborEnabled: lffHarborEnabled, Harbor: harborConfig, LFFQoSEnabled: lffQoSEnabled, - BuildQoS: buildQoSConfig, + BuildQoS: buildQoSConfigv1beta1, NativeCronPodMinFrequency: nativeCronPodMinFrequency, LagoonTargetName: lagoonTargetName, LagoonFeatureFlags: helpers.GetLagoonFeatureFlags(), @@ -821,7 +814,7 @@ func main() { EnableDebug: enableDebug, LagoonTargetName: lagoonTargetName, LFFQoSEnabled: lffQoSEnabled, - BuildQoS: buildQoSConfig, + BuildQoS: buildQoSConfigv1beta1, Cache: cache, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "LagoonMonitor") @@ -855,9 +848,9 @@ func main() { // for now the namespace reconciler only needs to run if harbor is enabled so that we can watch the namespace for rotation label events if lffHarborEnabled { - if err = (&lagoonv1beta1ctrl.HarborCredentialReconciler{ + if err = (&harborctrl.HarborCredentialReconciler{ Client: mgr.GetClient(), - Log: ctrl.Log.WithName("v1beta1").WithName("HarborCredentialReconciler"), + Log: ctrl.Log.WithName("harbor").WithName("HarborCredentialReconciler"), Scheme: mgr.GetScheme(), LFFHarborEnabled: lffHarborEnabled, ControllerNamespace: controllerNamespace,