From 0c44796c00939eb338d0b8d093c00651f1aed99d Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Tue, 9 Jan 2024 19:33:02 +1100 Subject: [PATCH] feat: add v1beta2 api --- PROJECT | 6 + apis/lagoon/v1beta2/groupversion_info.go | 35 + apis/lagoon/v1beta2/helpers_test.go | 105 ++ apis/lagoon/v1beta2/lagoonbuild_helpers.go | 444 +++++++ apis/lagoon/v1beta2/lagoonbuild_types.go | 181 +++ apis/lagoon/v1beta2/lagoontask_helpers.go | 292 +++++ apis/lagoon/v1beta2/lagoontask_types.go | 204 ++++ apis/lagoon/v1beta2/zz_generated.deepcopy.go | 539 +++++++++ .../crd/bases/crd.lagoon.sh_lagoonbuilds.yaml | 190 ++- .../crd/bases/crd.lagoon.sh_lagoontasks.yaml | 172 ++- controller-test.sh | 10 + controllers/v1beta2/build_controller.go | 342 ++++++ controllers/v1beta2/build_deletionhandlers.go | 514 ++++++++ controllers/v1beta2/build_helpers.go | 1061 +++++++++++++++++ controllers/v1beta2/build_helpers_test.go | 233 ++++ controllers/v1beta2/build_qoshandler.go | 169 +++ controllers/v1beta2/build_standardhandler.go | 73 ++ controllers/v1beta2/metrics.go | 92 ++ .../v1beta2/podmonitor_buildhandlers.go | 623 ++++++++++ controllers/v1beta2/podmonitor_controller.go | 225 ++++ controllers/v1beta2/podmonitor_metrics.go | 51 + .../v1beta2/podmonitor_taskhandlers.go | 401 +++++++ controllers/v1beta2/predicates.go | 227 ++++ controllers/v1beta2/suite_test.go | 80 ++ controllers/v1beta2/task_controller.go | 678 +++++++++++ controllers/v1beta2/task_helpers.go | 59 + internal/harbor/harbor_credentialrotation.go | 4 +- internal/messenger/consumer.go | 90 +- internal/messenger/tasks_handler.go | 18 +- internal/messenger/tasks_restore.go | 8 +- internal/utilities/deletions/process.go | 7 + main.go | 118 ++ .../dynamic-secret-in-task-project1.yaml | 2 +- test-resources/example-project1.yaml | 2 +- test-resources/example-project2.yaml | 2 +- test-resources/example-project3.yaml | 32 + 36 files changed, 7251 insertions(+), 38 deletions(-) create mode 100644 apis/lagoon/v1beta2/groupversion_info.go create mode 100644 apis/lagoon/v1beta2/helpers_test.go create mode 100644 apis/lagoon/v1beta2/lagoonbuild_helpers.go create mode 100644 apis/lagoon/v1beta2/lagoonbuild_types.go create mode 100644 apis/lagoon/v1beta2/lagoontask_helpers.go create mode 100644 apis/lagoon/v1beta2/lagoontask_types.go create mode 100644 apis/lagoon/v1beta2/zz_generated.deepcopy.go create mode 100644 controllers/v1beta2/build_controller.go create mode 100644 controllers/v1beta2/build_deletionhandlers.go create mode 100644 controllers/v1beta2/build_helpers.go create mode 100644 controllers/v1beta2/build_helpers_test.go create mode 100644 controllers/v1beta2/build_qoshandler.go create mode 100644 controllers/v1beta2/build_standardhandler.go create mode 100644 controllers/v1beta2/metrics.go create mode 100644 controllers/v1beta2/podmonitor_buildhandlers.go create mode 100644 controllers/v1beta2/podmonitor_controller.go create mode 100644 controllers/v1beta2/podmonitor_metrics.go create mode 100644 controllers/v1beta2/podmonitor_taskhandlers.go create mode 100644 controllers/v1beta2/predicates.go create mode 100644 controllers/v1beta2/suite_test.go create mode 100644 controllers/v1beta2/task_controller.go create mode 100644 controllers/v1beta2/task_helpers.go create mode 100644 test-resources/example-project3.yaml diff --git a/PROJECT b/PROJECT index 7b02f074..6bcbae60 100644 --- a/PROJECT +++ b/PROJECT @@ -2,6 +2,12 @@ domain: lagoon.sh multigroup: true repo: github.com/uselagoon/remote-controller resources: +- group: crd + kind: LagoonBuild + version: v1beta2 +- group: crd + kind: LagoonTask + version: v1beta2 - group: crd kind: LagoonBuild version: v1beta1 diff --git a/apis/lagoon/v1beta2/groupversion_info.go b/apis/lagoon/v1beta2/groupversion_info.go new file mode 100644 index 00000000..19bc86ce --- /dev/null +++ b/apis/lagoon/v1beta2/groupversion_info.go @@ -0,0 +1,35 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1beta1 contains API Schema definitions for the lagoon v1beta1 API group +// +kubebuilder:object:generate=true +// +groupName=crd.lagoon.sh +package v1beta2 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "crd.lagoon.sh", Version: "v1beta2"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/apis/lagoon/v1beta2/helpers_test.go b/apis/lagoon/v1beta2/helpers_test.go new file mode 100644 index 00000000..86a51639 --- /dev/null +++ b/apis/lagoon/v1beta2/helpers_test.go @@ -0,0 +1,105 @@ +package v1beta2 + +import ( + "testing" +) + +func TestCheckLagoonVersion(t *testing.T) { + type args struct { + build *LagoonBuild + checkVersion string + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "test1", + args: args{ + build: &LagoonBuild{ + Spec: LagoonBuildSpec{ + Project: Project{ + Variables: LagoonVariables{ + Project: []byte(`[{"name":"LAGOON_SYSTEM_CORE_VERSION","value":"v2.12.0","scope":"internal_system"}]`), + }, + }, + }, + }, + checkVersion: "2.12.0", + }, + want: true, + }, + { + name: "test2", + args: args{ + build: &LagoonBuild{ + Spec: LagoonBuildSpec{ + Project: Project{ + Variables: LagoonVariables{ + Project: []byte(`[{"name":"LAGOON_SYSTEM_CORE_VERSION","value":"v2.11.0","scope":"internal_system"}]`), + }, + }, + }, + }, + checkVersion: "2.12.0", + }, + want: false, + }, + { + name: "test3", + args: args{ + build: &LagoonBuild{ + Spec: LagoonBuildSpec{ + Project: Project{ + Variables: LagoonVariables{ + Project: []byte(`[]`), + }, + }, + }, + }, + checkVersion: "2.12.0", + }, + want: false, + }, + { + name: "test4", + args: args{ + build: &LagoonBuild{ + Spec: LagoonBuildSpec{ + Project: Project{ + Variables: LagoonVariables{ + Project: []byte(`[{"name":"LAGOON_SYSTEM_CORE_VERSION","value":"v2.12.0","scope":"internal_system"}]`), + }, + }, + }, + }, + checkVersion: "v2.12.0", + }, + want: true, + }, + { + name: "test5", + args: args{ + build: &LagoonBuild{ + Spec: LagoonBuildSpec{ + Project: Project{ + Variables: LagoonVariables{ + Project: []byte(`[{"name":"LAGOON_SYSTEM_CORE_VERSION","value":"v2.11.0","scope":"internal_system"}]`), + }, + }, + }, + }, + checkVersion: "v2.12.0", + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := CheckLagoonVersion(tt.args.build, tt.args.checkVersion); got != tt.want { + t.Errorf("CheckLagoonVersion() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/apis/lagoon/v1beta2/lagoonbuild_helpers.go b/apis/lagoon/v1beta2/lagoonbuild_helpers.go new file mode 100644 index 00000000..7dd7f337 --- /dev/null +++ b/apis/lagoon/v1beta2/lagoonbuild_helpers.go @@ -0,0 +1,444 @@ +package v1beta2 + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + "github.com/go-logr/logr" + "github.com/hashicorp/go-version" + "github.com/uselagoon/machinery/api/schema" + "github.com/uselagoon/remote-controller/internal/helpers" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var ( + // BuildRunningPendingStatus . + BuildRunningPendingStatus = []string{ + BuildStatusPending.String(), + BuildStatusQueued.String(), + BuildStatusRunning.String(), + } + // BuildCompletedCancelledFailedStatus . + BuildCompletedCancelledFailedStatus = []string{ + BuildStatusFailed.String(), + BuildStatusComplete.String(), + BuildStatusCancelled.String(), + } +) + +// BuildContainsStatus . +func BuildContainsStatus(slice []LagoonBuildConditions, s LagoonBuildConditions) bool { + for _, item := range slice { + if item == s { + return true + } + } + return false +} + +// RemoveBuild remove a LagoonBuild from a slice of LagoonBuilds +func RemoveBuild(slice []LagoonBuild, s LagoonBuild) []LagoonBuild { + result := []LagoonBuild{} + for _, item := range slice { + if item.ObjectMeta.Name == s.ObjectMeta.Name { + continue + } + result = append(result, item) + } + return result +} + +// Check if the version of lagoon provided in the internal_system scope variable is greater than or equal to the checked version +func CheckLagoonVersion(build *LagoonBuild, checkVersion string) bool { + lagoonProjectVariables := &[]helpers.LagoonEnvironmentVariable{} + json.Unmarshal(build.Spec.Project.Variables.Project, lagoonProjectVariables) + lagoonVersion, err := helpers.GetLagoonVariable("LAGOON_SYSTEM_CORE_VERSION", []string{"internal_system"}, *lagoonProjectVariables) + if err != nil { + return false + } + aVer, err := version.NewSemver(lagoonVersion.Value) + if err != nil { + return false + } + bVer, err := version.NewSemver(checkVersion) + if err != nil { + return false + } + return aVer.GreaterThanOrEqual(bVer) +} + +// CancelExtraBuilds cancels extra builds. +func CancelExtraBuilds(ctx context.Context, r client.Client, opLog logr.Logger, ns string, status string) error { + pendingBuilds := &LagoonBuildList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(ns), + client.MatchingLabels(map[string]string{"lagoon.sh/buildStatus": BuildStatusPending.String()}), + }) + if err := r.List(ctx, pendingBuilds, listOption); err != nil { + return fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err) + } + if len(pendingBuilds.Items) > 0 { + // opLog.Info(fmt.Sprintf("There are %v pending builds", len(pendingBuilds.Items))) + // if we have any pending builds, then grab the latest one and make it running + // if there are any other pending builds, cancel them so only the latest one runs + sort.Slice(pendingBuilds.Items, func(i, j int) bool { + return pendingBuilds.Items[i].ObjectMeta.CreationTimestamp.After(pendingBuilds.Items[j].ObjectMeta.CreationTimestamp.Time) + }) + for idx, pBuild := range pendingBuilds.Items { + pendingBuild := pBuild.DeepCopy() + if idx == 0 { + pendingBuild.Labels["lagoon.sh/buildStatus"] = status + } else { + // cancel any other pending builds + opLog.Info(fmt.Sprintf("Setting build %s as cancelled", pendingBuild.ObjectMeta.Name)) + pendingBuild.Labels["lagoon.sh/buildStatus"] = BuildStatusCancelled.String() + pendingBuild.Labels["lagoon.sh/cancelledByNewBuild"] = "true" + } + if err := r.Update(ctx, pendingBuild); err != nil { + return err + } + } + } + return nil +} + +func GetBuildConditionFromPod(phase corev1.PodPhase) BuildStatusType { + var buildCondition BuildStatusType + switch phase { + case corev1.PodFailed: + buildCondition = BuildStatusFailed + case corev1.PodSucceeded: + buildCondition = BuildStatusComplete + case corev1.PodPending: + buildCondition = BuildStatusPending + case corev1.PodRunning: + buildCondition = BuildStatusRunning + } + return buildCondition +} + +func GetTaskConditionFromPod(phase corev1.PodPhase) TaskStatusType { + var taskCondition TaskStatusType + switch phase { + case corev1.PodFailed: + taskCondition = TaskStatusFailed + case corev1.PodSucceeded: + taskCondition = TaskStatusComplete + case corev1.PodPending: + taskCondition = TaskStatusPending + case corev1.PodRunning: + taskCondition = TaskStatusRunning + } + return taskCondition +} + +func CheckRunningBuilds(ctx context.Context, cns string, opLog logr.Logger, cl client.Client, ns corev1.Namespace) bool { + lagoonBuilds := &LagoonBuildList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(ns.ObjectMeta.Name), + client.MatchingLabels(map[string]string{ + "lagoon.sh/controller": cns, // created by this controller + }), + }) + if err := cl.List(ctx, lagoonBuilds, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list Lagoon build pods, there may be none or something went wrong")) + return false + } + runningBuilds := false + sort.Slice(lagoonBuilds.Items, func(i, j int) bool { + return lagoonBuilds.Items[i].ObjectMeta.CreationTimestamp.After(lagoonBuilds.Items[j].ObjectMeta.CreationTimestamp.Time) + }) + // if there are any builds pending or running, don't try and refresh the credentials as this + // could break the build + if len(lagoonBuilds.Items) > 0 { + if helpers.ContainsString( + BuildRunningPendingStatus, + lagoonBuilds.Items[0].Labels["lagoon.sh/buildStatus"], + ) { + runningBuilds = true + } + } + return runningBuilds +} + +// DeleteLagoonBuilds will delete any lagoon builds from the namespace. +func DeleteLagoonBuilds(ctx context.Context, opLog logr.Logger, cl client.Client, ns, project, environment string) bool { + lagoonBuilds := &LagoonBuildList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(ns), + }) + if err := cl.List(ctx, lagoonBuilds, listOption); err != nil { + opLog.Error(err, + fmt.Sprintf( + "Unable to list lagoon build in namespace %s for project %s, environment %s", + ns, + project, + environment, + ), + ) + return false + } + for _, lagoonBuild := range lagoonBuilds.Items { + if err := cl.Delete(ctx, &lagoonBuild); helpers.IgnoreNotFound(err) != nil { + opLog.Error(err, + fmt.Sprintf( + "Unable to delete lagoon build %s in %s for project %s, environment %s", + lagoonBuild.ObjectMeta.Name, + ns, + project, + environment, + ), + ) + return false + } + opLog.Info( + fmt.Sprintf( + "Deleted lagoon build %s in %s for project %s, environment %s", + lagoonBuild.ObjectMeta.Name, + ns, + project, + environment, + ), + ) + } + return true +} + +func LagoonBuildPruner(ctx context.Context, cl client.Client, cns string, buildsToKeep int) { + opLog := ctrl.Log.WithName("utilities").WithName("LagoonBuildPruner") + namespaces := &corev1.NamespaceList{} + labelRequirements, _ := labels.NewRequirement("lagoon.sh/environmentType", selection.Exists, nil) + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.MatchingLabelsSelector{ + Selector: labels.NewSelector().Add(*labelRequirements), + }, + }) + if err := cl.List(ctx, namespaces, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list namespaces created by Lagoon, there may be none or something went wrong")) + return + } + for _, ns := range namespaces.Items { + if ns.Status.Phase == corev1.NamespaceTerminating { + // if the namespace is terminating, don't try to renew the robot credentials + opLog.Info(fmt.Sprintf("Namespace %s is being terminated, aborting build pruner", ns.ObjectMeta.Name)) + continue + } + opLog.Info(fmt.Sprintf("Checking LagoonBuilds in namespace %s", ns.ObjectMeta.Name)) + lagoonBuilds := &LagoonBuildList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(ns.ObjectMeta.Name), + client.MatchingLabels(map[string]string{ + "lagoon.sh/controller": cns, // created by this controller + }), + }) + if err := cl.List(ctx, lagoonBuilds, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list LagoonBuild resources, there may be none or something went wrong")) + continue + } + // sort the build pods by creation timestamp + sort.Slice(lagoonBuilds.Items, func(i, j int) bool { + return lagoonBuilds.Items[i].ObjectMeta.CreationTimestamp.After(lagoonBuilds.Items[j].ObjectMeta.CreationTimestamp.Time) + }) + if len(lagoonBuilds.Items) > buildsToKeep { + for idx, lagoonBuild := range lagoonBuilds.Items { + if idx >= buildsToKeep { + if helpers.ContainsString( + BuildCompletedCancelledFailedStatus, + lagoonBuild.ObjectMeta.Labels["lagoon.sh/buildStatus"], + ) { + opLog.Info(fmt.Sprintf("Cleaning up LagoonBuild %s", lagoonBuild.ObjectMeta.Name)) + if err := cl.Delete(ctx, &lagoonBuild); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to update status condition")) + break + } + } + } + } + } + } + return +} + +// BuildPodPruner will prune any build pods that are hanging around. +func BuildPodPruner(ctx context.Context, cl client.Client, cns string, buildPodsToKeep int) { + opLog := ctrl.Log.WithName("utilities").WithName("BuildPodPruner") + namespaces := &corev1.NamespaceList{} + labelRequirements, _ := labels.NewRequirement("lagoon.sh/environmentType", selection.Exists, nil) + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.MatchingLabelsSelector{ + Selector: labels.NewSelector().Add(*labelRequirements), + }, + }) + if err := cl.List(ctx, namespaces, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list namespaces created by Lagoon, there may be none or something went wrong")) + return + } + for _, ns := range namespaces.Items { + if ns.Status.Phase == corev1.NamespaceTerminating { + // if the namespace is terminating, don't try to renew the robot credentials + opLog.Info(fmt.Sprintf("Namespace %s is being terminated, aborting build pod pruner", ns.ObjectMeta.Name)) + return + } + opLog.Info(fmt.Sprintf("Checking Lagoon build pods in namespace %s", ns.ObjectMeta.Name)) + buildPods := &corev1.PodList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(ns.ObjectMeta.Name), + client.MatchingLabels(map[string]string{ + "lagoon.sh/jobType": "build", + "lagoon.sh/controller": cns, // created by this controller + }), + }) + if err := cl.List(ctx, buildPods, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list Lagoon build pods, there may be none or something went wrong")) + return + } + // sort the build pods by creation timestamp + sort.Slice(buildPods.Items, func(i, j int) bool { + return buildPods.Items[i].ObjectMeta.CreationTimestamp.After(buildPods.Items[j].ObjectMeta.CreationTimestamp.Time) + }) + if len(buildPods.Items) > buildPodsToKeep { + for idx, pod := range buildPods.Items { + if idx >= buildPodsToKeep { + if pod.Status.Phase == corev1.PodFailed || + pod.Status.Phase == corev1.PodSucceeded { + opLog.Info(fmt.Sprintf("Cleaning up pod %s", pod.ObjectMeta.Name)) + if err := cl.Delete(ctx, &pod); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to update status condition")) + break + } + } + } + } + } + } + return +} + +func updateLagoonBuild(opLog logr.Logger, namespace string, jobSpec LagoonTaskSpec, lagoonBuild *LagoonBuild) ([]byte, error) { + // if the build isn't found by the controller + // then publish a response back to controllerhandler to tell it to update the build to cancelled + // this allows us to update builds in the API that may have gone stale or not updated from `New`, `Pending`, or `Running` status + buildCondition := "cancelled" + if lagoonBuild != nil { + if val, ok := lagoonBuild.ObjectMeta.Labels["lagoon.sh/buildStatus"]; ok { + // if the build isnt running,pending,queued, then set the buildcondition to the value failed/complete/cancelled + if !helpers.ContainsString(BuildRunningPendingStatus, val) { + buildCondition = strings.ToLower(val) + } + } + } + msg := schema.LagoonMessage{ + Type: "build", + Namespace: namespace, + Meta: &schema.LagoonLogMeta{ + Environment: jobSpec.Environment.Name, + Project: jobSpec.Project.Name, + BuildStatus: buildCondition, + BuildName: jobSpec.Misc.Name, + }, + } + // set the start/end time to be now as the default + // to stop the duration counter in the ui + msg.Meta.StartTime = time.Now().UTC().Format("2006-01-02 15:04:05") + msg.Meta.EndTime = time.Now().UTC().Format("2006-01-02 15:04:05") + + // if possible, get the start and end times from the build resource, these will be sent back to lagoon to update the api + if lagoonBuild != nil && lagoonBuild.Status.Conditions != nil { + conditions := lagoonBuild.Status.Conditions + // sort the build conditions by time so the first and last can be extracted + sort.Slice(conditions, func(i, j int) bool { + iTime, _ := time.Parse("2006-01-02T15:04:05Z", conditions[i].LastTransitionTime) + jTime, _ := time.Parse("2006-01-02T15:04:05Z", conditions[j].LastTransitionTime) + return iTime.Before(jTime) + }) + // get the starting time, or fallback to default + sTime, err := time.Parse("2006-01-02T15:04:05Z", conditions[0].LastTransitionTime) + if err == nil { + msg.Meta.StartTime = sTime.Format("2006-01-02 15:04:05") + } + // get the ending time, or fallback to default + eTime, err := time.Parse("2006-01-02T15:04:05Z", conditions[len(conditions)-1].LastTransitionTime) + if err == nil { + msg.Meta.EndTime = eTime.Format("2006-01-02 15:04:05") + } + } + msgBytes, err := json.Marshal(msg) + if err != nil { + return nil, fmt.Errorf("Unable to encode message as JSON: %v", err) + } + return msgBytes, nil +} + +// CancelBuild handles cancelling builds or handling if a build no longer exists. +func CancelBuild(ctx context.Context, cl client.Client, namespace string, body []byte) (bool, []byte, error) { + opLog := ctrl.Log.WithName("handlers").WithName("LagoonTasks") + jobSpec := &LagoonTaskSpec{} + json.Unmarshal(body, jobSpec) + var jobPod corev1.Pod + if err := cl.Get(ctx, types.NamespacedName{ + Name: jobSpec.Misc.Name, + Namespace: namespace, + }, &jobPod); err != nil { + opLog.Info(fmt.Sprintf( + "Unable to find build pod %s to cancel it. Checking to see if LagoonBuild exists.", + jobSpec.Misc.Name, + )) + // since there was no build pod, check for the lagoon build resource + var lagoonBuild LagoonBuild + if err := cl.Get(ctx, types.NamespacedName{ + Name: jobSpec.Misc.Name, + Namespace: namespace, + }, &lagoonBuild); err != nil { + opLog.Info(fmt.Sprintf( + "Unable to find build %s to cancel it. Sending response to Lagoon to update the build to cancelled.", + jobSpec.Misc.Name, + )) + // if there is no pod or build, update the build in Lagoon to cancelled, assume completely cancelled with no other information + // and then send the response back to lagoon to say it was cancelled. + b, err := updateLagoonBuild(opLog, namespace, *jobSpec, nil) + return false, b, err + } + // as there is no build pod, but there is a lagoon build resource + // update it to cancelled so that the controller doesn't try to run it + // check if the build has existing status or not though to consume it + if helpers.ContainsString( + BuildRunningPendingStatus, + lagoonBuild.ObjectMeta.Labels["lagoon.sh/buildStatus"], + ) { + lagoonBuild.ObjectMeta.Labels["lagoon.sh/buildStatus"] = BuildStatusCancelled.String() + } + lagoonBuild.ObjectMeta.Labels["lagoon.sh/cancelBuildNoPod"] = "true" + if err := cl.Update(ctx, &lagoonBuild); err != nil { + opLog.Error(err, + fmt.Sprintf( + "Unable to update build %s to cancel it.", + jobSpec.Misc.Name, + ), + ) + return false, nil, err + } + // and then send the response back to lagoon to say it was cancelled. + b, err := updateLagoonBuild(opLog, namespace, *jobSpec, &lagoonBuild) + return true, b, err + } + jobPod.ObjectMeta.Labels["lagoon.sh/cancelBuild"] = "true" + if err := cl.Update(ctx, &jobPod); err != nil { + opLog.Error(err, + fmt.Sprintf( + "Unable to update build %s to cancel it.", + jobSpec.Misc.Name, + ), + ) + return false, nil, err + } + return false, nil, nil +} diff --git a/apis/lagoon/v1beta2/lagoonbuild_types.go b/apis/lagoon/v1beta2/lagoonbuild_types.go new file mode 100644 index 00000000..6db883a6 --- /dev/null +++ b/apis/lagoon/v1beta2/lagoonbuild_types.go @@ -0,0 +1,181 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta2 + +import ( + "strings" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// BuildStatusType const for the status type +type BuildStatusType string + +// These are valid conditions of a job. +const ( + // BuildStatusPending means the build is pending. + BuildStatusPending BuildStatusType = "Pending" + // BuildStatusQueued means the build is queued. + BuildStatusQueued BuildStatusType = "Queued" + // BuildStatusRunning means the build is running. + BuildStatusRunning BuildStatusType = "Running" + // BuildStatusComplete means the build has completed its execution. + BuildStatusComplete BuildStatusType = "Complete" + // BuildStatusFailed means the job has failed its execution. + BuildStatusFailed BuildStatusType = "Failed" + // BuildStatusCancelled means the job been cancelled. + BuildStatusCancelled BuildStatusType = "Cancelled" +) + +func (b BuildStatusType) String() string { + return string(b) +} + +func (b BuildStatusType) ToLower() string { + return strings.ToLower(b.String()) +} + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +//+kubebuilder:subresource:status + +// LagoonBuildSpec defines the desired state of LagoonBuild +type LagoonBuildSpec struct { + // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + // Important: Run "make" to regenerate code after modifying this file + + Build Build `json:"build"` + Project Project `json:"project"` + Branch Branch `json:"branch,omitempty"` + Pullrequest Pullrequest `json:"pullrequest,omitempty"` + Promote Promote `json:"promote,omitempty"` + GitReference string `json:"gitReference"` +} + +// LagoonBuildStatus defines the observed state of LagoonBuild +type LagoonBuildStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file + Conditions []LagoonBuildConditions `json:"conditions,omitempty"` + Log []byte `json:"log,omitempty"` +} + +// LagoonBuildConditions defines the observed conditions of build pods. +type LagoonBuildConditions struct { + LastTransitionTime string `json:"lastTransitionTime"` + Status corev1.ConditionStatus `json:"status"` + Type BuildStatusType `json:"type"` + // Condition string `json:"condition"` +} + +// +kubebuilder:object:root=true + +// LagoonBuild is the Schema for the lagoonbuilds API +type LagoonBuild struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec LagoonBuildSpec `json:"spec,omitempty"` + Status LagoonBuildStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// LagoonBuildList contains a list of LagoonBuild +type LagoonBuildList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []LagoonBuild `json:"items"` +} + +func init() { + SchemeBuilder.Register(&LagoonBuild{}, &LagoonBuildList{}) +} + +// Build contains the type of build, and the image to use for the builder. +type Build struct { + CI string `json:"ci,omitempty"` + Image string `json:"image,omitempty"` + Type string `json:"type"` + Priority *int `json:"priority,omitempty"` + BulkID string `json:"bulkId,omitempty"` +} + +// Project contains the project information from lagoon. +type Project struct { + ID *uint `json:"id,omitempty"` + Name string `json:"name"` + Environment string `json:"environment"` + EnvironmentID *uint `json:"environmentId,omitempty"` + UILink string `json:"uiLink,omitempty"` + GitURL string `json:"gitUrl"` + NamespacePattern string `json:"namespacePattern,omitempty"` + RouterPattern string `json:"routerPattern,omitempty"` + EnvironmentType string `json:"environmentType"` + ProductionEnvironment string `json:"productionEnvironment"` + StandbyEnvironment string `json:"standbyEnvironment"` + DeployTarget string `json:"deployTarget"` + ProjectSecret string `json:"projectSecret"` + SubFolder string `json:"subfolder,omitempty"` + Key []byte `json:"key"` + Monitoring Monitoring `json:"monitoring"` + Variables LagoonVariables `json:"variables"` + Registry string `json:"registry,omitempty"` + EnvironmentIdling *int `json:"environmentIdling,omitempty"` + ProjectIdling *int `json:"projectIdling,omitempty"` + StorageCalculator *int `json:"storageCalculator,omitempty"` + Organization *Organization `json:"organization,omitempty"` +} + +type Organization struct { + ID *uint `json:"id,omitempty"` + Name string `json:"name,omitempty"` +} + +// Variables contains the project and environment variables from lagoon. +type LagoonVariables struct { + Project []byte `json:"project,omitempty"` + Environment []byte `json:"environment,omitempty"` +} + +// Branch contains the branch name used for a branch deployment. +type Branch struct { + Name string `json:"name,omitempty"` +} + +// Pullrequest contains the information for a pullrequest deployment. +type Pullrequest struct { + HeadBranch string `json:"headBranch,omitempty"` + HeadSha string `json:"headSha,omitempty"` + BaseBranch string `json:"baseBranch,omitempty"` + BaseSha string `json:"baseSha,omitempty"` + Title string `json:"title,omitempty"` + Number string `json:"number,omitempty"` +} + +// Promote contains the information for a promote deployment. +type Promote struct { + SourceEnvironment string `json:"sourceEnvironment,omitempty"` + SourceProject string `json:"sourceProject,omitempty"` +} + +// Monitoring contains the monitoring information for the project in Lagoon. +type Monitoring struct { + Contact string `json:"contact,omitempty"` + StatuspageID string `json:"statuspageID,omitempty"` +} diff --git a/apis/lagoon/v1beta2/lagoontask_helpers.go b/apis/lagoon/v1beta2/lagoontask_helpers.go new file mode 100644 index 00000000..b8964272 --- /dev/null +++ b/apis/lagoon/v1beta2/lagoontask_helpers.go @@ -0,0 +1,292 @@ +package v1beta2 + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "time" + + "github.com/go-logr/logr" + "github.com/uselagoon/machinery/api/schema" + "github.com/uselagoon/remote-controller/internal/helpers" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var ( + // TaskRunningPendingStatus . + TaskRunningPendingStatus = []string{ + TaskStatusPending.String(), + TaskStatusQueued.String(), + TaskStatusRunning.String(), + } + // TaskCompletedCancelledFailedStatus . + TaskCompletedCancelledFailedStatus = []string{ + TaskStatusFailed.String(), + TaskStatusComplete.String(), + TaskStatusCancelled.String(), + } +) + +// TaskContainsStatus . +func TaskContainsStatus(slice []LagoonTaskConditions, s LagoonTaskConditions) bool { + for _, item := range slice { + if item == s { + return true + } + } + return false +} + +// DeleteLagoonTasks will delete any lagoon tasks from the namespace. +func DeleteLagoonTasks(ctx context.Context, opLog logr.Logger, cl client.Client, ns, project, environment string) bool { + lagoonTasks := &LagoonTaskList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(ns), + }) + if err := cl.List(ctx, lagoonTasks, listOption); err != nil { + opLog.Error(err, + fmt.Sprintf( + "Unable to list lagoon task in namespace %s for project %s, environment %s", + ns, + project, + environment, + ), + ) + return false + } + for _, lagoonTask := range lagoonTasks.Items { + if err := cl.Delete(ctx, &lagoonTask); helpers.IgnoreNotFound(err) != nil { + opLog.Error(err, + fmt.Sprintf( + "Unable to delete lagoon task %s in %s for project %s, environment %s", + lagoonTask.ObjectMeta.Name, + ns, + project, + environment, + ), + ) + return false + } + opLog.Info( + fmt.Sprintf( + "Deleted lagoon task %s in %s for project %s, environment %s", + lagoonTask.ObjectMeta.Name, + ns, + project, + environment, + ), + ) + } + return true +} + +// LagoonTaskPruner will prune any build crds that are hanging around. +func LagoonTaskPruner(ctx context.Context, cl client.Client, cns string, tasksToKeep int) { + opLog := ctrl.Log.WithName("utilities").WithName("LagoonTaskPruner") + namespaces := &corev1.NamespaceList{} + labelRequirements, _ := labels.NewRequirement("lagoon.sh/environmentType", selection.Exists, nil) + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.MatchingLabelsSelector{ + Selector: labels.NewSelector().Add(*labelRequirements), + }, + }) + if err := cl.List(ctx, namespaces, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list namespaces created by Lagoon, there may be none or something went wrong")) + return + } + for _, ns := range namespaces.Items { + if ns.Status.Phase == corev1.NamespaceTerminating { + // if the namespace is terminating, don't try to renew the robot credentials + opLog.Info(fmt.Sprintf("Namespace %s is being terminated, aborting task pruner", ns.ObjectMeta.Name)) + continue + } + opLog.Info(fmt.Sprintf("Checking LagoonTasks in namespace %s", ns.ObjectMeta.Name)) + lagoonTasks := &LagoonTaskList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(ns.ObjectMeta.Name), + client.MatchingLabels(map[string]string{ + "lagoon.sh/controller": cns, // created by this controller + }), + }) + if err := cl.List(ctx, lagoonTasks, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list LagoonTask resources, there may be none or something went wrong")) + continue + } + // sort the build pods by creation timestamp + sort.Slice(lagoonTasks.Items, func(i, j int) bool { + return lagoonTasks.Items[i].ObjectMeta.CreationTimestamp.After(lagoonTasks.Items[j].ObjectMeta.CreationTimestamp.Time) + }) + if len(lagoonTasks.Items) > tasksToKeep { + for idx, lagoonTask := range lagoonTasks.Items { + if idx >= tasksToKeep { + if helpers.ContainsString( + TaskCompletedCancelledFailedStatus, + lagoonTask.ObjectMeta.Labels["lagoon.sh/taskStatus"], + ) { + opLog.Info(fmt.Sprintf("Cleaning up LagoonTask %s", lagoonTask.ObjectMeta.Name)) + if err := cl.Delete(ctx, &lagoonTask); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to update status condition")) + break + } + } + } + } + } + } + return +} + +// TaskPodPruner will prune any task pods that are hanging around. +func TaskPodPruner(ctx context.Context, cl client.Client, cns string, taskPodsToKeep int) { + opLog := ctrl.Log.WithName("utilities").WithName("TaskPodPruner") + namespaces := &corev1.NamespaceList{} + labelRequirements, _ := labels.NewRequirement("lagoon.sh/environmentType", selection.Exists, nil) + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.MatchingLabelsSelector{ + Selector: labels.NewSelector().Add(*labelRequirements), + }, + }) + if err := cl.List(ctx, namespaces, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list namespaces created by Lagoon, there may be none or something went wrong")) + return + } + for _, ns := range namespaces.Items { + if ns.Status.Phase == corev1.NamespaceTerminating { + // if the namespace is terminating, don't try to renew the robot credentials + opLog.Info(fmt.Sprintf("Namespace %s is being terminated, aborting task pod pruner", ns.ObjectMeta.Name)) + return + } + opLog.Info(fmt.Sprintf("Checking Lagoon task pods in namespace %s", ns.ObjectMeta.Name)) + taskPods := &corev1.PodList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(ns.ObjectMeta.Name), + client.MatchingLabels(map[string]string{ + "lagoon.sh/jobType": "task", + "lagoon.sh/controller": cns, // created by this controller + }), + }) + if err := cl.List(ctx, taskPods, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list Lagoon task pods, there may be none or something went wrong")) + return + } + // sort the build pods by creation timestamp + sort.Slice(taskPods.Items, func(i, j int) bool { + return taskPods.Items[i].ObjectMeta.CreationTimestamp.After(taskPods.Items[j].ObjectMeta.CreationTimestamp.Time) + }) + if len(taskPods.Items) > taskPodsToKeep { + for idx, pod := range taskPods.Items { + if idx >= taskPodsToKeep { + if pod.Status.Phase == corev1.PodFailed || + pod.Status.Phase == corev1.PodSucceeded { + opLog.Info(fmt.Sprintf("Cleaning up pod %s", pod.ObjectMeta.Name)) + if err := cl.Delete(ctx, &pod); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to delete pod")) + break + } + } + } + } + } + } + return +} + +func updateLagoonTask(opLog logr.Logger, namespace string, taskSpec LagoonTaskSpec) ([]byte, error) { + //@TODO: use `taskName` in the future only + taskName := fmt.Sprintf("lagoon-task-%s-%s", taskSpec.Task.ID, helpers.HashString(taskSpec.Task.ID)[0:6]) + if taskSpec.Task.TaskName != "" { + taskName = taskSpec.Task.TaskName + } + // if the task isn't found by the controller + // then publish a response back to controllerhandler to tell it to update the task to cancelled + // this allows us to update tasks in the API that may have gone stale or not updated from `New`, `Pending`, or `Running` status + msg := schema.LagoonMessage{ + Type: "task", + Namespace: namespace, + Meta: &schema.LagoonLogMeta{ + Environment: taskSpec.Environment.Name, + Project: taskSpec.Project.Name, + JobName: taskName, + JobStatus: "cancelled", + Task: &schema.LagoonTaskInfo{ + TaskName: taskSpec.Task.TaskName, + ID: taskSpec.Task.ID, + Name: taskSpec.Task.Name, + Service: taskSpec.Task.Service, + }, + }, + } + // if the task isn't found at all, then set the start/end time to be now + // to stop the duration counter in the ui + msg.Meta.StartTime = time.Now().UTC().Format("2006-01-02 15:04:05") + msg.Meta.EndTime = time.Now().UTC().Format("2006-01-02 15:04:05") + msgBytes, err := json.Marshal(msg) + if err != nil { + return nil, fmt.Errorf("Unable to encode message as JSON: %v", err) + } + return msgBytes, nil +} + +// CancelTask handles cancelling tasks or handling if a tasks no longer exists. +func CancelTask(ctx context.Context, cl client.Client, namespace string, body []byte) (bool, []byte, error) { + opLog := ctrl.Log.WithName("handlers").WithName("LagoonTasks") + jobSpec := &LagoonTaskSpec{} + json.Unmarshal(body, jobSpec) + var jobPod corev1.Pod + //@TODO: use `taskName` in the future only + taskName := fmt.Sprintf("lagoon-task-%s-%s", jobSpec.Task.ID, helpers.HashString(jobSpec.Task.ID)[0:6]) + if jobSpec.Task.TaskName != "" { + taskName = jobSpec.Task.TaskName + } + if err := cl.Get(ctx, types.NamespacedName{ + Name: taskName, + Namespace: namespace, + }, &jobPod); err != nil { + // since there was no task pod, check for the lagoon task resource + var lagoonTask LagoonTask + if err := cl.Get(ctx, types.NamespacedName{ + Name: taskName, + Namespace: namespace, + }, &lagoonTask); err != nil { + opLog.Info(fmt.Sprintf( + "Unable to find task %s to cancel it. Sending response to Lagoon to update the task to cancelled.", + taskName, + )) + // if there is no pod or task, update the task in Lagoon to cancelled + b, err := updateLagoonTask(opLog, namespace, *jobSpec) + return false, b, err + } + // as there is no task pod, but there is a lagoon task resource + // update it to cancelled so that the controller doesn't try to run it + lagoonTask.ObjectMeta.Labels["lagoon.sh/taskStatus"] = TaskStatusCancelled.String() + if err := cl.Update(ctx, &lagoonTask); err != nil { + opLog.Error(err, + fmt.Sprintf( + "Unable to update task %s to cancel it.", + taskName, + ), + ) + return false, nil, err + } + // and then send the response back to lagoon to say it was cancelled. + b, err := updateLagoonTask(opLog, namespace, *jobSpec) + return true, b, err + } + jobPod.ObjectMeta.Labels["lagoon.sh/cancelTask"] = "true" + if err := cl.Update(ctx, &jobPod); err != nil { + opLog.Error(err, + fmt.Sprintf( + "Unable to update task %s to cancel it.", + jobSpec.Misc.Name, + ), + ) + return false, nil, err + } + return false, nil, nil +} diff --git a/apis/lagoon/v1beta2/lagoontask_types.go b/apis/lagoon/v1beta2/lagoontask_types.go new file mode 100644 index 00000000..19e45ebc --- /dev/null +++ b/apis/lagoon/v1beta2/lagoontask_types.go @@ -0,0 +1,204 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta2 + +import ( + "encoding/json" + "fmt" + "reflect" + "strconv" + "strings" + + "github.com/uselagoon/machinery/api/schema" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// TaskStatusType const for the status type +type TaskStatusType string + +// These are valid conditions of a job. +const ( + // TaskStatusPending means the job is pending. + TaskStatusPending TaskStatusType = "Pending" + // TaskStatusQueued means the job is queued. + TaskStatusQueued TaskStatusType = "Queued" + // TaskStatusRunning means the job is running. + TaskStatusRunning TaskStatusType = "Running" + // TaskStatusComplete means the job has completed its execution. + TaskStatusComplete TaskStatusType = "Complete" + // TaskStatusFailed means the job has failed its execution. + TaskStatusFailed TaskStatusType = "Failed" + // TaskStatusCancelled means the job been cancelled. + TaskStatusCancelled TaskStatusType = "Cancelled" +) + +func (b TaskStatusType) String() string { + return string(b) +} + +func (b TaskStatusType) ToLower() string { + return strings.ToLower(b.String()) +} + +// TaskType const for the status type +type TaskType string + +// These are valid conditions of a job. +const ( + // TaskTypeStandard means the task is a standard task. + TaskTypeStandard TaskType = "standard" + // TaskTypeAdvanced means the task is an advanced task. + TaskTypeAdvanced TaskType = "advanced" +) + +func (b TaskType) String() string { + return string(b) +} + +// LagoonTaskSpec defines the desired state of LagoonTask +type LagoonTaskSpec struct { + // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + // Important: Run "make" to regenerate code after modifying this file + Key string `json:"key,omitempty"` + Task schema.LagoonTaskInfo `json:"task,omitempty"` + Project LagoonTaskProject `json:"project,omitempty"` + Environment LagoonTaskEnvironment `json:"environment,omitempty"` + Misc *LagoonMiscInfo `json:"misc,omitempty"` + AdvancedTask *LagoonAdvancedTaskInfo `json:"advancedTask,omitempty"` +} + +// LagoonAdvancedTaskInfo defines what an advanced task can use for the creation of the pod. +type LagoonAdvancedTaskInfo struct { + RunnerImage string `json:"runnerImage,omitempty"` + JSONPayload string `json:"JSONPayload,omitempty"` + DeployerToken bool `json:"deployerToken,omitempty"` + SSHKey bool `json:"sshKey,omitempty"` +} + +// LagoonMiscInfo defines the resource or backup information for a misc task. +type LagoonMiscInfo struct { + ID string `json:"id"` // should be int, but the api sends it as a string :\ + Name string `json:"name,omitempty"` + Backup *LagoonMiscBackupInfo `json:"backup,omitempty"` + MiscResource []byte `json:"miscResource,omitempty"` +} + +// LagoonMiscBackupInfo defines the information for a backup. +type LagoonMiscBackupInfo struct { + ID string `json:"id"` // should be int, but the api sends it as a string :\ + Source string `json:"source"` + BackupID string `json:"backupId"` +} + +// LagoonTaskProject defines the lagoon project information. +type LagoonTaskProject struct { + ID string `json:"id"` // should be int, but the api sends it as a string :\ + Name string `json:"name"` + NamespacePattern string `json:"namespacePattern,omitempty"` + Variables LagoonVariables `json:"variables,omitempty"` + Organization *Organization `json:"organization,omitempty"` +} + +// LagoonTaskEnvironment defines the lagoon environment information. +type LagoonTaskEnvironment struct { + ID string `json:"id"` // should be int, but the api sends it as a string :\ + Name string `json:"name"` + Project string `json:"project"` // should be int, but the api sends it as a string :\ + EnvironmentType string `json:"environmentType"` +} + +// LagoonTaskStatus defines the observed state of LagoonTask +type LagoonTaskStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file + Conditions []LagoonTaskConditions `json:"conditions,omitempty"` + Log []byte `json:"log,omitempty"` +} + +// LagoonTaskConditions defines the observed conditions of task pods. +type LagoonTaskConditions struct { + LastTransitionTime string `json:"lastTransitionTime"` + Status corev1.ConditionStatus `json:"status"` + Type TaskStatusType `json:"type"` + // Condition string `json:"condition"` +} + +// +kubebuilder:object:root=true + +// LagoonTask is the Schema for the lagoontasks API +type LagoonTask struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec LagoonTaskSpec `json:"spec,omitempty"` + Status LagoonTaskStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// LagoonTaskList contains a list of LagoonTask +type LagoonTaskList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []LagoonTask `json:"items"` +} + +func init() { + SchemeBuilder.Register(&LagoonTask{}, &LagoonTaskList{}) +} + +// this is a custom unmarshal function that will check deployerToken and sshKey which come from Lagoon as `1|0` booleans because javascript +// this converts them from floats to bools +func (a *LagoonAdvancedTaskInfo) UnmarshalJSON(data []byte) error { + tmpMap := map[string]interface{}{} + json.Unmarshal(data, &tmpMap) + if value, ok := tmpMap["deployerToken"]; ok { + if reflect.TypeOf(value).Kind() == reflect.Float64 { + vBool, err := strconv.ParseBool(fmt.Sprintf("%v", value)) + if err == nil { + a.DeployerToken = vBool + } + } + if reflect.TypeOf(value).Kind() == reflect.Bool { + a.DeployerToken = value.(bool) + } + } + if value, ok := tmpMap["sshKey"]; ok { + if reflect.TypeOf(value).Kind() == reflect.Float64 { + vBool, err := strconv.ParseBool(fmt.Sprintf("%v", value)) + if err == nil { + a.SSHKey = vBool + } + } + if reflect.TypeOf(value).Kind() == reflect.Bool { + a.SSHKey = value.(bool) + } + } + if value, ok := tmpMap["RunnerImage"]; ok { + a.RunnerImage = value.(string) + } + if value, ok := tmpMap["runnerImage"]; ok { + a.RunnerImage = value.(string) + } + if value, ok := tmpMap["JSONPayload"]; ok { + a.JSONPayload = value.(string) + } + return nil +} diff --git a/apis/lagoon/v1beta2/zz_generated.deepcopy.go b/apis/lagoon/v1beta2/zz_generated.deepcopy.go new file mode 100644 index 00000000..54401dc6 --- /dev/null +++ b/apis/lagoon/v1beta2/zz_generated.deepcopy.go @@ -0,0 +1,539 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1beta2 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Branch) DeepCopyInto(out *Branch) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Branch. +func (in *Branch) DeepCopy() *Branch { + if in == nil { + return nil + } + out := new(Branch) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Build) DeepCopyInto(out *Build) { + *out = *in + if in.Priority != nil { + in, out := &in.Priority, &out.Priority + *out = new(int) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Build. +func (in *Build) DeepCopy() *Build { + if in == nil { + return nil + } + out := new(Build) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LagoonAdvancedTaskInfo) DeepCopyInto(out *LagoonAdvancedTaskInfo) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonAdvancedTaskInfo. +func (in *LagoonAdvancedTaskInfo) DeepCopy() *LagoonAdvancedTaskInfo { + if in == nil { + return nil + } + out := new(LagoonAdvancedTaskInfo) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LagoonBuild) DeepCopyInto(out *LagoonBuild) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonBuild. +func (in *LagoonBuild) DeepCopy() *LagoonBuild { + if in == nil { + return nil + } + out := new(LagoonBuild) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *LagoonBuild) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LagoonBuildConditions) DeepCopyInto(out *LagoonBuildConditions) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonBuildConditions. +func (in *LagoonBuildConditions) DeepCopy() *LagoonBuildConditions { + if in == nil { + return nil + } + out := new(LagoonBuildConditions) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LagoonBuildList) DeepCopyInto(out *LagoonBuildList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]LagoonBuild, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonBuildList. +func (in *LagoonBuildList) DeepCopy() *LagoonBuildList { + if in == nil { + return nil + } + out := new(LagoonBuildList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *LagoonBuildList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LagoonBuildSpec) DeepCopyInto(out *LagoonBuildSpec) { + *out = *in + in.Build.DeepCopyInto(&out.Build) + in.Project.DeepCopyInto(&out.Project) + out.Branch = in.Branch + out.Pullrequest = in.Pullrequest + out.Promote = in.Promote +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonBuildSpec. +func (in *LagoonBuildSpec) DeepCopy() *LagoonBuildSpec { + if in == nil { + return nil + } + out := new(LagoonBuildSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LagoonBuildStatus) DeepCopyInto(out *LagoonBuildStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]LagoonBuildConditions, len(*in)) + copy(*out, *in) + } + if in.Log != nil { + in, out := &in.Log, &out.Log + *out = make([]byte, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonBuildStatus. +func (in *LagoonBuildStatus) DeepCopy() *LagoonBuildStatus { + if in == nil { + return nil + } + out := new(LagoonBuildStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LagoonMiscBackupInfo) DeepCopyInto(out *LagoonMiscBackupInfo) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonMiscBackupInfo. +func (in *LagoonMiscBackupInfo) DeepCopy() *LagoonMiscBackupInfo { + if in == nil { + return nil + } + out := new(LagoonMiscBackupInfo) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LagoonMiscInfo) DeepCopyInto(out *LagoonMiscInfo) { + *out = *in + if in.Backup != nil { + in, out := &in.Backup, &out.Backup + *out = new(LagoonMiscBackupInfo) + **out = **in + } + if in.MiscResource != nil { + in, out := &in.MiscResource, &out.MiscResource + *out = make([]byte, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonMiscInfo. +func (in *LagoonMiscInfo) DeepCopy() *LagoonMiscInfo { + if in == nil { + return nil + } + out := new(LagoonMiscInfo) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LagoonTask) DeepCopyInto(out *LagoonTask) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonTask. +func (in *LagoonTask) DeepCopy() *LagoonTask { + if in == nil { + return nil + } + out := new(LagoonTask) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *LagoonTask) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LagoonTaskConditions) DeepCopyInto(out *LagoonTaskConditions) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonTaskConditions. +func (in *LagoonTaskConditions) DeepCopy() *LagoonTaskConditions { + if in == nil { + return nil + } + out := new(LagoonTaskConditions) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LagoonTaskEnvironment) DeepCopyInto(out *LagoonTaskEnvironment) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonTaskEnvironment. +func (in *LagoonTaskEnvironment) DeepCopy() *LagoonTaskEnvironment { + if in == nil { + return nil + } + out := new(LagoonTaskEnvironment) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LagoonTaskList) DeepCopyInto(out *LagoonTaskList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]LagoonTask, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonTaskList. +func (in *LagoonTaskList) DeepCopy() *LagoonTaskList { + if in == nil { + return nil + } + out := new(LagoonTaskList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *LagoonTaskList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LagoonTaskProject) DeepCopyInto(out *LagoonTaskProject) { + *out = *in + in.Variables.DeepCopyInto(&out.Variables) + if in.Organization != nil { + in, out := &in.Organization, &out.Organization + *out = new(Organization) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonTaskProject. +func (in *LagoonTaskProject) DeepCopy() *LagoonTaskProject { + if in == nil { + return nil + } + out := new(LagoonTaskProject) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LagoonTaskSpec) DeepCopyInto(out *LagoonTaskSpec) { + *out = *in + out.Task = in.Task + in.Project.DeepCopyInto(&out.Project) + out.Environment = in.Environment + if in.Misc != nil { + in, out := &in.Misc, &out.Misc + *out = new(LagoonMiscInfo) + (*in).DeepCopyInto(*out) + } + if in.AdvancedTask != nil { + in, out := &in.AdvancedTask, &out.AdvancedTask + *out = new(LagoonAdvancedTaskInfo) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonTaskSpec. +func (in *LagoonTaskSpec) DeepCopy() *LagoonTaskSpec { + if in == nil { + return nil + } + out := new(LagoonTaskSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LagoonTaskStatus) DeepCopyInto(out *LagoonTaskStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]LagoonTaskConditions, len(*in)) + copy(*out, *in) + } + if in.Log != nil { + in, out := &in.Log, &out.Log + *out = make([]byte, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonTaskStatus. +func (in *LagoonTaskStatus) DeepCopy() *LagoonTaskStatus { + if in == nil { + return nil + } + out := new(LagoonTaskStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LagoonVariables) DeepCopyInto(out *LagoonVariables) { + *out = *in + if in.Project != nil { + in, out := &in.Project, &out.Project + *out = make([]byte, len(*in)) + copy(*out, *in) + } + if in.Environment != nil { + in, out := &in.Environment, &out.Environment + *out = make([]byte, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonVariables. +func (in *LagoonVariables) DeepCopy() *LagoonVariables { + if in == nil { + return nil + } + out := new(LagoonVariables) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Monitoring) DeepCopyInto(out *Monitoring) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Monitoring. +func (in *Monitoring) DeepCopy() *Monitoring { + if in == nil { + return nil + } + out := new(Monitoring) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Organization) DeepCopyInto(out *Organization) { + *out = *in + if in.ID != nil { + in, out := &in.ID, &out.ID + *out = new(uint) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Organization. +func (in *Organization) DeepCopy() *Organization { + if in == nil { + return nil + } + out := new(Organization) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Project) DeepCopyInto(out *Project) { + *out = *in + if in.ID != nil { + in, out := &in.ID, &out.ID + *out = new(uint) + **out = **in + } + if in.EnvironmentID != nil { + in, out := &in.EnvironmentID, &out.EnvironmentID + *out = new(uint) + **out = **in + } + if in.Key != nil { + in, out := &in.Key, &out.Key + *out = make([]byte, len(*in)) + copy(*out, *in) + } + out.Monitoring = in.Monitoring + in.Variables.DeepCopyInto(&out.Variables) + if in.EnvironmentIdling != nil { + in, out := &in.EnvironmentIdling, &out.EnvironmentIdling + *out = new(int) + **out = **in + } + if in.ProjectIdling != nil { + in, out := &in.ProjectIdling, &out.ProjectIdling + *out = new(int) + **out = **in + } + if in.StorageCalculator != nil { + in, out := &in.StorageCalculator, &out.StorageCalculator + *out = new(int) + **out = **in + } + if in.Organization != nil { + in, out := &in.Organization, &out.Organization + *out = new(Organization) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Project. +func (in *Project) DeepCopy() *Project { + if in == nil { + return nil + } + out := new(Project) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Promote) DeepCopyInto(out *Promote) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Promote. +func (in *Promote) DeepCopy() *Promote { + if in == nil { + return nil + } + out := new(Promote) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Pullrequest) DeepCopyInto(out *Pullrequest) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Pullrequest. +func (in *Pullrequest) DeepCopy() *Pullrequest { + if in == nil { + return nil + } + out := new(Pullrequest) + in.DeepCopyInto(out) + return out +} diff --git a/config/crd/bases/crd.lagoon.sh_lagoonbuilds.yaml b/config/crd/bases/crd.lagoon.sh_lagoonbuilds.yaml index bbc2dc2d..9337223b 100644 --- a/config/crd/bases/crd.lagoon.sh_lagoonbuilds.yaml +++ b/config/crd/bases/crd.lagoon.sh_lagoonbuilds.yaml @@ -574,7 +574,195 @@ spec: type: object type: object served: true - storage: true + storage: false + - name: v1beta2 + schema: + openAPIV3Schema: + description: LagoonBuild is the Schema for the lagoonbuilds API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: LagoonBuildSpec defines the desired state of LagoonBuild + properties: + branch: + description: Branch contains the branch name used for a branch deployment. + properties: + name: + type: string + type: object + build: + description: Build contains the type of build, and the image to use + for the builder. + properties: + bulkId: + type: string + ci: + type: string + image: + type: string + priority: + type: integer + type: + type: string + required: + - type + type: object + gitReference: + type: string + project: + description: Project contains the project information from lagoon. + properties: + deployTarget: + type: string + environment: + type: string + environmentId: + type: integer + environmentIdling: + type: integer + environmentType: + type: string + gitUrl: + type: string + id: + type: integer + key: + format: byte + type: string + monitoring: + description: Monitoring contains the monitoring information for + the project in Lagoon. + properties: + contact: + type: string + statuspageID: + type: string + type: object + name: + type: string + namespacePattern: + type: string + organization: + properties: + id: + type: integer + name: + type: string + type: object + productionEnvironment: + type: string + projectIdling: + type: integer + projectSecret: + type: string + registry: + type: string + routerPattern: + type: string + standbyEnvironment: + type: string + storageCalculator: + type: integer + subfolder: + type: string + uiLink: + type: string + variables: + description: Variables contains the project and environment variables + from lagoon. + properties: + environment: + format: byte + type: string + project: + format: byte + type: string + type: object + required: + - deployTarget + - environment + - environmentType + - gitUrl + - key + - monitoring + - name + - productionEnvironment + - projectSecret + - standbyEnvironment + - variables + type: object + promote: + description: Promote contains the information for a promote deployment. + properties: + sourceEnvironment: + type: string + sourceProject: + type: string + type: object + pullrequest: + description: Pullrequest contains the information for a pullrequest + deployment. + properties: + baseBranch: + type: string + baseSha: + type: string + headBranch: + type: string + headSha: + type: string + number: + type: string + title: + type: string + type: object + required: + - build + - gitReference + - project + type: object + status: + description: LagoonBuildStatus defines the observed state of LagoonBuild + properties: + conditions: + description: 'INSERT ADDITIONAL STATUS FIELD - define observed state + of cluster Important: Run "make" to regenerate code after modifying + this file' + items: + description: LagoonBuildConditions defines the observed conditions + of build pods. + properties: + lastTransitionTime: + type: string + status: + type: string + type: + description: BuildStatusType const for the status type + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + log: + format: byte + type: string + type: object + type: object + served: true + storage: false status: acceptedNames: kind: "" diff --git a/config/crd/bases/crd.lagoon.sh_lagoontasks.yaml b/config/crd/bases/crd.lagoon.sh_lagoontasks.yaml index f1394f91..82cc0acb 100644 --- a/config/crd/bases/crd.lagoon.sh_lagoontasks.yaml +++ b/config/crd/bases/crd.lagoon.sh_lagoontasks.yaml @@ -556,7 +556,177 @@ spec: type: object type: object served: true - storage: true + storage: false + - name: v1beta2 + schema: + openAPIV3Schema: + description: LagoonTask is the Schema for the lagoontasks API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: LagoonTaskSpec defines the desired state of LagoonTask + properties: + advancedTask: + description: LagoonAdvancedTaskInfo defines what an advanced task + can use for the creation of the pod. + properties: + JSONPayload: + type: string + deployerToken: + type: boolean + runnerImage: + type: string + sshKey: + type: boolean + type: object + environment: + description: LagoonTaskEnvironment defines the lagoon environment + information. + properties: + environmentType: + type: string + id: + type: string + name: + type: string + project: + type: string + required: + - environmentType + - id + - name + - project + type: object + key: + description: 'INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + Important: Run "make" to regenerate code after modifying this file' + type: string + misc: + description: LagoonMiscInfo defines the resource or backup information + for a misc task. + properties: + backup: + description: LagoonMiscBackupInfo defines the information for + a backup. + properties: + backupId: + type: string + id: + type: string + source: + type: string + required: + - backupId + - id + - source + type: object + id: + type: string + miscResource: + format: byte + type: string + name: + type: string + required: + - id + type: object + project: + description: LagoonTaskProject defines the lagoon project information. + properties: + id: + type: string + name: + type: string + namespacePattern: + type: string + organization: + properties: + id: + type: integer + name: + type: string + type: object + variables: + description: Variables contains the project and environment variables + from lagoon. + properties: + environment: + format: byte + type: string + project: + format: byte + type: string + type: object + required: + - id + - name + type: object + task: + description: LagoonTaskInfo defines what a task can use to communicate + with Lagoon via SSH/API. + properties: + apiHost: + type: string + command: + type: string + id: + type: string + name: + type: string + service: + type: string + sshHost: + type: string + sshPort: + type: string + taskName: + type: string + required: + - id + type: object + type: object + status: + description: LagoonTaskStatus defines the observed state of LagoonTask + properties: + conditions: + description: 'INSERT ADDITIONAL STATUS FIELD - define observed state + of cluster Important: Run "make" to regenerate code after modifying + this file' + items: + description: LagoonTaskConditions defines the observed conditions + of task pods. + properties: + lastTransitionTime: + type: string + status: + type: string + type: + description: TaskStatusType const for the status type + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + log: + format: byte + type: string + type: object + type: object + served: true + storage: false status: acceptedNames: kind: "" diff --git a/controller-test.sh b/controller-test.sh index 014247c9..7b8b1c23 100755 --- a/controller-test.sh +++ b/controller-test.sh @@ -18,6 +18,7 @@ NS=nginx-example-main LBUILD=7m5zypx LBUILD2=8m5zypx LBUILD3=9m5zypx +LBUILD4=1m5zypx HARBOR_VERSION=${HARBOR_VERSION:-1.6.4} @@ -233,6 +234,15 @@ else echo "===> label exists" fi +echo "==> deprecated v1beta1 api: Trigger a lagoon build using kubectl apply" +kubectl -n $CONTROLLER_NAMESPACE apply -f test-resources/example-project3.yaml +# patch the resource with the controller namespace +kubectl -n $CONTROLLER_NAMESPACE patch lagoonbuilds.v1beta1.crd.lagoon.sh lagoon-build-${LBUILD4} --type=merge --patch '{"metadata":{"labels":{"lagoon.sh/controller":"'$CONTROLLER_NAMESPACE'"}}}' +# patch the resource with a random label to bump the controller event filter +kubectl -n $CONTROLLER_NAMESPACE patch lagoonbuilds.v1beta1.crd.lagoon.sh lagoon-build-${LBUILD4} --type=merge --patch '{"metadata":{"labels":{"bump":"bump"}}}' +sleep 10 +check_lagoon_build lagoon-build-${LBUILD4} + echo "==> Trigger a Task using kubectl apply to test dynamic secret mounting" kubectl -n $NS apply -f test-resources/dynamic-secret-in-task-project1-secret.yaml diff --git a/controllers/v1beta2/build_controller.go b/controllers/v1beta2/build_controller.go new file mode 100644 index 00000000..a5dc9481 --- /dev/null +++ b/controllers/v1beta2/build_controller.go @@ -0,0 +1,342 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta2 + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + lagooncrd "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" + "github.com/uselagoon/remote-controller/internal/harbor" + "github.com/uselagoon/remote-controller/internal/helpers" + "github.com/uselagoon/remote-controller/internal/messenger" +) + +// LagoonBuildReconciler reconciles a LagoonBuild object +type LagoonBuildReconciler struct { + client.Client + Log logr.Logger + Scheme *runtime.Scheme + EnableMQ bool + Messaging *messenger.Messenger + BuildImage string + NamespacePrefix string + RandomNamespacePrefix bool + ControllerNamespace string + EnableDebug bool + FastlyServiceID string + FastlyWatchStatus bool + // BuildPodRunAsUser sets the build pod securityContext.runAsUser value. + BuildPodRunAsUser int64 + // BuildPodRunAsGroup sets the build pod securityContext.runAsGroup value. + BuildPodRunAsGroup int64 + // BuildPodFSGroup sets the build pod securityContext.fsGroup value. + BuildPodFSGroup int64 + // Lagoon feature flags + LFFForceRootlessWorkload string + LFFDefaultRootlessWorkload string + LFFForceIsolationNetworkPolicy string + LFFDefaultIsolationNetworkPolicy string + LFFForceInsights string + LFFDefaultInsights string + LFFForceRWX2RWO string + LFFDefaultRWX2RWO string + LFFBackupWeeklyRandom bool + LFFRouterURL bool + LFFHarborEnabled bool + BackupConfig BackupConfig + Harbor harbor.Harbor + LFFQoSEnabled bool + BuildQoS BuildQoS + NativeCronPodMinFrequency int + LagoonTargetName string + LagoonFeatureFlags map[string]string + LagoonAPIConfiguration helpers.LagoonAPIConfiguration + ProxyConfig ProxyConfig +} + +// BackupConfig holds all the backup configuration settings +type BackupConfig struct { + BackupDefaultSchedule string + BackupDefaultMonthlyRetention int + BackupDefaultWeeklyRetention int + BackupDefaultDailyRetention int + BackupDefaultHourlyRetention int + + BackupDefaultDevelopmentSchedule string + BackupDefaultPullrequestSchedule string + BackupDefaultDevelopmentRetention string + BackupDefaultPullrequestRetention string +} + +// ProxyConfig is used for proxy configuration. +type ProxyConfig struct { + HTTPProxy string + HTTPSProxy string + NoProxy string +} + +var ( + buildFinalizer = "finalizer.lagoonbuild.crd.lagoon.sh/v1beta2" +) + +// +kubebuilder:rbac:groups=crd.lagoon.sh,resources=lagoonbuilds,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=crd.lagoon.sh,resources=lagoonbuilds/status,verbs=get;update;patch + +// @TODO: all the things for now, review later +// +kubebuilder:rbac:groups="*",resources="*",verbs="*" + +// Reconcile runs when a request comes through +func (r *LagoonBuildReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + opLog := r.Log.WithValues("lagoonbuild", req.NamespacedName) + + // your logic here + var lagoonBuild lagooncrd.LagoonBuild + if err := r.Get(ctx, req.NamespacedName, &lagoonBuild); err != nil { + return ctrl.Result{}, helpers.IgnoreNotFound(err) + } + + // examine DeletionTimestamp to determine if object is under deletion + if lagoonBuild.ObjectMeta.DeletionTimestamp.IsZero() { + // if the build isn't being deleted, but the status is cancelled + // then clean up the undeployable build + if value, ok := lagoonBuild.ObjectMeta.Labels["lagoon.sh/buildStatus"]; ok { + if value == lagooncrd.BuildStatusCancelled.String() { + if value, ok := lagoonBuild.ObjectMeta.Labels["lagoon.sh/cancelledByNewBuild"]; ok { + if value == "true" { + opLog.Info(fmt.Sprintf("Cleaning up build %s as cancelled by new build", lagoonBuild.ObjectMeta.Name)) + r.cleanUpUndeployableBuild(ctx, lagoonBuild, "This build was cancelled as a newer build was triggered.", opLog, true) + } else { + opLog.Info(fmt.Sprintf("Cleaning up build %s as cancelled", lagoonBuild.ObjectMeta.Name)) + r.cleanUpUndeployableBuild(ctx, lagoonBuild, "", opLog, false) + } + } + } + } + if r.LFFQoSEnabled { + // handle QoS builds here + // if we do have a `lagoon.sh/buildStatus` set as running, then process it + runningNSBuilds := &lagooncrd.LagoonBuildList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(req.Namespace), + client.MatchingLabels(map[string]string{ + "lagoon.sh/buildStatus": lagooncrd.BuildStatusRunning.String(), + "lagoon.sh/controller": r.ControllerNamespace, + }), + }) + // list any builds that are running + if err := r.List(ctx, runningNSBuilds, listOption); err != nil { + return ctrl.Result{}, fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err) + } + for _, runningBuild := range runningNSBuilds.Items { + // if the running build is the one from this request then process it + if lagoonBuild.ObjectMeta.Name == runningBuild.ObjectMeta.Name { + // actually process the build here + if _, ok := lagoonBuild.ObjectMeta.Labels["lagoon.sh/buildStarted"]; !ok { + if err := r.processBuild(ctx, opLog, lagoonBuild); err != nil { + return ctrl.Result{}, err + } + } + } // end check if running build is current LagoonBuild + } // end loop for running builds + // once running builds are processed, run the qos handler + return r.qosBuildProcessor(ctx, opLog, lagoonBuild, req) + } + // if qos is not enabled, just process it as a standard build + return r.standardBuildProcessor(ctx, opLog, lagoonBuild, req) + } + // The object is being deleted + if helpers.ContainsString(lagoonBuild.ObjectMeta.Finalizers, buildFinalizer) { + // our finalizer is present, so lets handle any external dependency + // first deleteExternalResources will try and check for any pending builds that it can + // can change to running to kick off the next pending build + if err := r.deleteExternalResources(ctx, + opLog, + &lagoonBuild, + req, + ); err != nil { + // if fail to delete the external dependency here, return with error + // so that it can be retried + opLog.Error(err, fmt.Sprintf("Unable to delete external resources")) + return ctrl.Result{}, err + } + // remove our finalizer from the list and update it. + lagoonBuild.ObjectMeta.Finalizers = helpers.RemoveString(lagoonBuild.ObjectMeta.Finalizers, buildFinalizer) + // use patches to avoid update errors + mergePatch, _ := json.Marshal(map[string]interface{}{ + "metadata": map[string]interface{}{ + "finalizers": lagoonBuild.ObjectMeta.Finalizers, + }, + }) + if err := r.Patch(ctx, &lagoonBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { + return ctrl.Result{}, helpers.IgnoreNotFound(err) + } + } + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the given manager +// and we set it to watch LagoonBuilds +func (r *LagoonBuildReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&lagooncrd.LagoonBuild{}). + WithEventFilter(BuildPredicates{ + ControllerNamespace: r.ControllerNamespace, + }). + Complete(r) +} + +func (r *LagoonBuildReconciler) createNamespaceBuild(ctx context.Context, + opLog logr.Logger, + lagoonBuild lagooncrd.LagoonBuild) (ctrl.Result, error) { + + namespace := &corev1.Namespace{} + if r.EnableDebug { + opLog.Info(fmt.Sprintf("Checking Namespace exists for: %s", lagoonBuild.ObjectMeta.Name)) + } + err := r.getOrCreateNamespace(ctx, namespace, lagoonBuild, opLog) + if err != nil { + return ctrl.Result{}, err + } + // create the `lagoon-deployer` ServiceAccount + if r.EnableDebug { + opLog.Info(fmt.Sprintf("Checking `lagoon-deployer` ServiceAccount exists: %s", lagoonBuild.ObjectMeta.Name)) + } + serviceAccount := &corev1.ServiceAccount{} + err = r.getOrCreateServiceAccount(ctx, serviceAccount, namespace.ObjectMeta.Name) + if err != nil { + return ctrl.Result{}, err + } + // ServiceAccount RoleBinding creation + if r.EnableDebug { + opLog.Info(fmt.Sprintf("Checking `lagoon-deployer-admin` RoleBinding exists: %s", lagoonBuild.ObjectMeta.Name)) + } + saRoleBinding := &rbacv1.RoleBinding{} + err = r.getOrCreateSARoleBinding(ctx, saRoleBinding, namespace.ObjectMeta.Name) + if err != nil { + return ctrl.Result{}, err + } + + // Get or create the lagoon-env configmap + lagoonEnvConfigMap := &corev1.ConfigMap{} + if r.EnableDebug { + opLog.Info("Checking `lagoon-env` configMap exists - creating if not") + } + err = r.getOrCreateConfigMap(ctx, "lagoon-env", lagoonEnvConfigMap, namespace.ObjectMeta.Name) + if err != nil { + return ctrl.Result{}, err + } + + // copy the build resource into a new resource and set the status to pending + // create the new resource and the controller will handle it via queue + opLog.Info(fmt.Sprintf("Creating LagoonBuild in Pending status: %s", lagoonBuild.ObjectMeta.Name)) + err = r.getOrCreateBuildResource(ctx, &lagoonBuild, namespace.ObjectMeta.Name) + if err != nil { + return ctrl.Result{}, err + } + + // if everything is all good controller will handle the new build resource that gets created as it will have + // the `lagoon.sh/buildStatus = Pending` now + err = lagooncrd.CancelExtraBuilds(ctx, r.Client, opLog, namespace.ObjectMeta.Name, lagooncrd.BuildStatusPending.String()) + if err != nil { + return ctrl.Result{}, err + } + + // as this is a new build coming through, check if there are any running builds in the namespace + // if there are, then check the status of that build. if the build pod is missing then the build running will block + // if the pod exists, attempt to get the status of it (only if its complete or failed) and ship the status + runningBuilds := &lagooncrd.LagoonBuildList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(namespace.ObjectMeta.Name), + client.MatchingLabels(map[string]string{"lagoon.sh/buildStatus": lagooncrd.BuildStatusRunning.String()}), + }) + // list all builds in the namespace that have the running buildstatus + if err := r.List(ctx, runningBuilds, listOption); err != nil { + return ctrl.Result{}, fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err) + } + // if there are running builds still, check if the pod exists or if the pod is complete/failed and attempt to get the status + for _, rBuild := range runningBuilds.Items { + runningBuild := rBuild.DeepCopy() + lagoonBuildPod := corev1.Pod{} + err := r.Get(ctx, types.NamespacedName{ + Namespace: rBuild.ObjectMeta.Namespace, + Name: rBuild.ObjectMeta.Name, + }, &lagoonBuildPod) + buildCondition := lagooncrd.BuildStatusCancelled + if err != nil { + // cancel the build as there is no pod available + opLog.Info(fmt.Sprintf("Setting build %s as cancelled", runningBuild.ObjectMeta.Name)) + runningBuild.Labels["lagoon.sh/buildStatus"] = buildCondition.String() + } else { + // get the status from the pod and update the build + if lagoonBuildPod.Status.Phase == corev1.PodFailed || lagoonBuildPod.Status.Phase == corev1.PodSucceeded { + buildCondition = lagooncrd.GetBuildConditionFromPod(lagoonBuildPod.Status.Phase) + opLog.Info(fmt.Sprintf("Setting build %s as %s", runningBuild.ObjectMeta.Name, buildCondition.String())) + runningBuild.Labels["lagoon.sh/buildStatus"] = buildCondition.String() + } else { + // drop out, don't do anything else + continue + } + } + if err := r.Update(ctx, runningBuild); err != nil { + // log the error and drop out + opLog.Error(err, fmt.Sprintf("Error setting build %s as cancelled", runningBuild.ObjectMeta.Name)) + continue + } + // send the status change to lagoon + r.updateDeploymentAndEnvironmentTask(ctx, opLog, runningBuild, nil, buildCondition, "cancelled") + continue + } + // handle processing running but no pod/failed pod builds + return ctrl.Result{}, nil +} + +// getOrCreateBuildResource will deepcopy the lagoon build into a new resource and push it to the new namespace +// then clean up the old one. +func (r *LagoonBuildReconciler) getOrCreateBuildResource(ctx context.Context, lagoonBuild *lagooncrd.LagoonBuild, ns string) error { + newBuild := lagoonBuild.DeepCopy() + newBuild.SetNamespace(ns) + newBuild.SetResourceVersion("") + newBuild.SetLabels( + map[string]string{ + "lagoon.sh/buildStatus": lagooncrd.BuildStatusPending.String(), + "lagoon.sh/controller": r.ControllerNamespace, + }, + ) + err := r.Get(ctx, types.NamespacedName{ + Namespace: ns, + Name: newBuild.ObjectMeta.Name, + }, newBuild) + if err != nil { + if err := r.Create(ctx, newBuild); err != nil { + return err + } + } + err = r.Delete(ctx, lagoonBuild) + if err != nil { + return err + } + return nil +} diff --git a/controllers/v1beta2/build_deletionhandlers.go b/controllers/v1beta2/build_deletionhandlers.go new file mode 100644 index 00000000..bfe90ccb --- /dev/null +++ b/controllers/v1beta2/build_deletionhandlers.go @@ -0,0 +1,514 @@ +package v1beta2 + +// this file is used by the `lagoonbuild` controller + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + "github.com/go-logr/logr" + "github.com/uselagoon/machinery/api/schema" + lagooncrd "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" + "github.com/uselagoon/remote-controller/internal/helpers" + "gopkg.in/matryer/try.v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// handle deleting any external resources here +func (r *LagoonBuildReconciler) deleteExternalResources( + ctx context.Context, + opLog logr.Logger, + lagoonBuild *lagooncrd.LagoonBuild, + req ctrl.Request, +) error { + // get any running pods that this build may have already created + lagoonBuildPod := corev1.Pod{} + err := r.Get(ctx, types.NamespacedName{ + Namespace: lagoonBuild.ObjectMeta.Namespace, + Name: lagoonBuild.ObjectMeta.Name, + }, &lagoonBuildPod) + if err != nil { + opLog.Info(fmt.Sprintf("Unable to find a build pod for %s, continuing to process build deletion", lagoonBuild.ObjectMeta.Name)) + // handle updating lagoon for a deleted build with no running pod + // only do it if the build status is Pending or Running though + err = r.updateCancelledDeploymentWithLogs(ctx, req, *lagoonBuild) + if err != nil { + opLog.Error(err, fmt.Sprintf("Unable to update the lagoon with LagoonBuild result")) + } + } else { + opLog.Info(fmt.Sprintf("Found build pod for %s, deleting it", lagoonBuild.ObjectMeta.Name)) + // handle updating lagoon for a deleted build with a running pod + // only do it if the build status is Pending or Running though + // delete the pod, let the pod deletion handler deal with the cleanup there + if err := r.Delete(ctx, &lagoonBuildPod); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to delete the the LagoonBuild pod %s", lagoonBuild.ObjectMeta.Name)) + } + // check that the pod is deleted before continuing, this allows the pod deletion to happen + // and the pod deletion process in the LagoonMonitor controller to be able to send what it needs back to lagoon + // this 1 minute timeout will just hold up the deletion of `LagoonBuild` resources only if a build pod exists + // if the 1 minute timeout is reached the worst that happens is a deployment will show as running + // but cancelling the deployment in lagoon will force it to go to a cancelling state in the lagoon api + // @TODO: we could use finalizers on the build pods, but to avoid holding up other processes we can just give up after waiting for a minute + try.MaxRetries = 12 + err = try.Do(func(attempt int) (bool, error) { + var podErr error + err := r.Get(ctx, types.NamespacedName{ + Namespace: lagoonBuild.ObjectMeta.Namespace, + Name: lagoonBuild.ObjectMeta.Name, + }, &lagoonBuildPod) + if err != nil { + // the pod doesn't exist anymore, so exit the retry + podErr = nil + opLog.Info(fmt.Sprintf("Pod %s deleted", lagoonBuild.ObjectMeta.Name)) + } else { + // if the pod still exists wait 5 seconds before trying again + time.Sleep(5 * time.Second) + podErr = fmt.Errorf("pod %s still exists", lagoonBuild.ObjectMeta.Name) + opLog.Info(fmt.Sprintf("Pod %s still exists", lagoonBuild.ObjectMeta.Name)) + } + return attempt < 12, podErr + }) + if err != nil { + return err + } + } + + // if the LagoonBuild is deleted, then check if the only running build is the one being deleted + // or if there are any pending builds that can be started + runningBuilds := &lagooncrd.LagoonBuildList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(lagoonBuild.ObjectMeta.Namespace), + client.MatchingLabels(map[string]string{ + "lagoon.sh/buildStatus": lagooncrd.BuildStatusRunning.String(), + "lagoon.sh/controller": r.ControllerNamespace, + }), + }) + // list any builds that are running + if err := r.List(ctx, runningBuilds, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list builds in the namespace, there may be none or something went wrong")) + // just return nil so the deletion of the resource isn't held up + return nil + } + newRunningBuilds := runningBuilds.Items + for _, runningBuild := range runningBuilds.Items { + // if there are any running builds, check if it is the one currently being deleted + if lagoonBuild.ObjectMeta.Name == runningBuild.ObjectMeta.Name { + // if the one being deleted is a running one, remove it from the list of running builds + newRunningBuilds = lagooncrd.RemoveBuild(newRunningBuilds, runningBuild) + } + } + // if the number of runningBuilds is 0 (excluding the one being deleted) + if len(newRunningBuilds) == 0 { + pendingBuilds := &lagooncrd.LagoonBuildList{} + listOption = (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(lagoonBuild.ObjectMeta.Namespace), + client.MatchingLabels(map[string]string{ + "lagoon.sh/buildStatus": lagooncrd.BuildStatusPending.String(), + "lagoon.sh/controller": r.ControllerNamespace, + }), + }) + if err := r.List(ctx, pendingBuilds, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list builds in the namespace, there may be none or something went wrong")) + // just return nil so the deletion of the resource isn't held up + return nil + } + newPendingBuilds := pendingBuilds.Items + for _, pendingBuild := range pendingBuilds.Items { + // if there are any pending builds, check if it is the one currently being deleted + if lagoonBuild.ObjectMeta.Name == pendingBuild.ObjectMeta.Name { + // if the one being deleted a the pending one, remove it from the list of pending builds + newPendingBuilds = lagooncrd.RemoveBuild(newPendingBuilds, pendingBuild) + } + } + // sort the pending builds by creation timestamp + sort.Slice(newPendingBuilds, func(i, j int) bool { + return newPendingBuilds[i].ObjectMeta.CreationTimestamp.Before(&newPendingBuilds[j].ObjectMeta.CreationTimestamp) + }) + // if there are more than 1 pending builds (excluding the one being deleted), update the oldest one to running + if len(newPendingBuilds) > 0 { + pendingBuild := pendingBuilds.Items[0].DeepCopy() + mergePatch, _ := json.Marshal(map[string]interface{}{ + "metadata": map[string]interface{}{ + "labels": map[string]interface{}{ + "lagoon.sh/buildStatus": lagooncrd.BuildStatusRunning.String(), + }, + }, + }) + if err := r.Patch(ctx, pendingBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to update pending build to running status")) + return nil + } + } else { + opLog.Info(fmt.Sprintf("No pending builds")) + } + } + return nil +} + +func (r *LagoonBuildReconciler) updateCancelledDeploymentWithLogs( + ctx context.Context, + req ctrl.Request, + lagoonBuild lagooncrd.LagoonBuild, +) error { + opLog := r.Log.WithValues("lagoonbuild", req.NamespacedName) + // if the build status is Pending or Running, + // then the buildCondition will be set to cancelled when we tell lagoon + // this is because we are deleting it, so we are basically cancelling it + // if it was already Failed or Completed, lagoon probably already knows + // so we don't have to do anything else. + if helpers.ContainsString( + lagooncrd.BuildRunningPendingStatus, + lagoonBuild.Labels["lagoon.sh/buildStatus"], + ) { + opLog.Info( + fmt.Sprintf( + "Updating build status for %s to %v", + lagoonBuild.ObjectMeta.Name, + lagoonBuild.Labels["lagoon.sh/buildStatus"], + ), + ) + + var allContainerLogs []byte + // if we get this handler, then it is likely that the build was in a pending or running state with no actual running pod + // so just set the logs to be cancellation message + allContainerLogs = []byte(fmt.Sprintf(` +======================================== +Build cancelled +========================================`)) + var buildCondition lagooncrd.BuildStatusType + buildCondition = lagooncrd.BuildStatusCancelled + lagoonBuild.Labels["lagoon.sh/buildStatus"] = buildCondition.String() + mergePatch, _ := json.Marshal(map[string]interface{}{ + "metadata": map[string]interface{}{ + "labels": map[string]interface{}{ + "lagoon.sh/buildStatus": buildCondition.String(), + }, + }, + }) + if err := r.Patch(ctx, &lagoonBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to update build status")) + } + // get the configmap for lagoon-env so we can use it for updating the deployment in lagoon + var lagoonEnv corev1.ConfigMap + err := r.Get(ctx, types.NamespacedName{ + Namespace: lagoonBuild.ObjectMeta.Namespace, + Name: "lagoon-env", + }, + &lagoonEnv, + ) + if err != nil { + // if there isn't a configmap, just info it and move on + // the updatedeployment function will see it as nil and not bother doing the bits that require the configmap + if r.EnableDebug { + opLog.Info(fmt.Sprintf("There is no configmap %s in namespace %s ", "lagoon-env", lagoonBuild.ObjectMeta.Namespace)) + } + } + // send any messages to lagoon message queues + // update the deployment with the status of cancelled in lagoon + r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &lagoonEnv, lagooncrd.BuildStatusCancelled, "cancelled") + r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagooncrd.BuildStatusCancelled, "cancelled") + r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, allContainerLogs, lagooncrd.BuildStatusCancelled) + } + return nil +} + +// buildLogsToLagoonLogs sends the build logs to the lagoon-logs message queue +// it contains the actual pod log output that is sent to elasticsearch, it is what eventually is displayed in the UI +func (r *LagoonBuildReconciler) buildLogsToLagoonLogs(ctx context.Context, + opLog logr.Logger, + lagoonBuild *lagooncrd.LagoonBuild, + logs []byte, + buildCondition lagooncrd.BuildStatusType, +) { + if r.EnableMQ { + condition := buildCondition + buildStep := "queued" + if condition == lagooncrd.BuildStatusCancelled { + buildStep = "cancelled" + } + msg := schema.LagoonLog{ + Severity: "info", + Project: lagoonBuild.Spec.Project.Name, + Event: "build-logs:builddeploy-kubernetes:" + lagoonBuild.ObjectMeta.Name, + Meta: &schema.LagoonLogMeta{ + JobName: lagoonBuild.ObjectMeta.Name, // @TODO: remove once lagoon is corrected in controller-handler + BuildName: lagoonBuild.ObjectMeta.Name, + BuildStatus: buildCondition.ToLower(), // same as buildstatus label + BuildStep: buildStep, + BranchName: lagoonBuild.Spec.Project.Environment, + RemoteID: string(lagoonBuild.ObjectMeta.UID), + LogLink: lagoonBuild.Spec.Project.UILink, + Cluster: r.LagoonTargetName, + }, + } + // add the actual build log message + msg.Message = fmt.Sprintf("%s", logs) + msgBytes, err := json.Marshal(msg) + if err != nil { + opLog.Error(err, "Unable to encode message as JSON") + } + // @TODO: if we can't publish the message because we are deleting the resource, then should we even + // bother to patch the resource?? + // leave it for now cause the resource will just be deleted anyway + if err := r.Messaging.Publish("lagoon-logs", msgBytes); err != nil { + // if we can't publish the message, set it as a pending message + // overwrite whatever is there as these are just current state messages so it doesn't + // really matter if we don't smootly transition in what we send back to lagoon + r.updateBuildLogMessage(ctx, lagoonBuild, msg) + return + } + // if we are able to publish the message, then we need to remove any pending messages from the resource + // and make sure we don't try and publish again + r.removeBuildPendingMessageStatus(ctx, lagoonBuild) + } +} + +// updateDeploymentAndEnvironmentTask sends the status of the build and deployment to the controllerhandler message queue in lagoon, +// this is for the handler in lagoon to process. +func (r *LagoonBuildReconciler) updateDeploymentAndEnvironmentTask(ctx context.Context, + opLog logr.Logger, + lagoonBuild *lagooncrd.LagoonBuild, + lagoonEnv *corev1.ConfigMap, + buildCondition lagooncrd.BuildStatusType, + buildStep string, +) { + namespace := helpers.GenerateNamespaceName( + lagoonBuild.Spec.Project.NamespacePattern, // the namespace pattern or `openshiftProjectPattern` from Lagoon is never received by the controller + lagoonBuild.Spec.Project.Environment, + lagoonBuild.Spec.Project.Name, + r.NamespacePrefix, + r.ControllerNamespace, + r.RandomNamespacePrefix, + ) + if r.EnableMQ { + msg := schema.LagoonMessage{ + Type: "build", + Namespace: namespace, + Meta: &schema.LagoonLogMeta{ + Environment: lagoonBuild.Spec.Project.Environment, + Project: lagoonBuild.Spec.Project.Name, + BuildStatus: buildCondition.ToLower(), + BuildStep: buildStep, + BuildName: lagoonBuild.ObjectMeta.Name, + LogLink: lagoonBuild.Spec.Project.UILink, + RemoteID: string(lagoonBuild.ObjectMeta.UID), + Cluster: r.LagoonTargetName, + }, + } + labelRequirements1, _ := labels.NewRequirement("lagoon.sh/service", selection.NotIn, []string{"faketest"}) + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(lagoonBuild.ObjectMeta.Namespace), + client.MatchingLabelsSelector{ + Selector: labels.NewSelector().Add(*labelRequirements1), + }, + }) + podList := &corev1.PodList{} + serviceNames := []string{} + if err := r.List(context.TODO(), podList, listOption); err == nil { + // generate the list of services to add to the environment + for _, pod := range podList.Items { + if _, ok := pod.ObjectMeta.Labels["lagoon.sh/service"]; ok { + for _, container := range pod.Spec.Containers { + serviceNames = append(serviceNames, container.Name) + } + } + if _, ok := pod.ObjectMeta.Labels["service"]; ok { + for _, container := range pod.Spec.Containers { + serviceNames = append(serviceNames, container.Name) + } + } + } + msg.Meta.Services = serviceNames + } + // if we aren't being provided the lagoon config, we can skip adding the routes etc + if lagoonEnv != nil { + msg.Meta.Route = "" + if route, ok := lagoonEnv.Data["LAGOON_ROUTE"]; ok { + msg.Meta.Route = route + } + msg.Meta.Routes = []string{} + if routes, ok := lagoonEnv.Data["LAGOON_ROUTES"]; ok { + msg.Meta.Routes = strings.Split(routes, ",") + } + } + if buildCondition.ToLower() == "failed" || buildCondition.ToLower() == "complete" || buildCondition.ToLower() == "cancelled" { + msg.Meta.EndTime = time.Now().UTC().Format("2006-01-02 15:04:05") + } + msgBytes, err := json.Marshal(msg) + if err != nil { + opLog.Error(err, "Unable to encode message as JSON") + } + // @TODO: if we can't publish the message because we are deleting the resource, then should we even + // bother to patch the resource?? + // leave it for now cause the resource will just be deleted anyway + if err := r.Messaging.Publish("lagoon-tasks:controller", msgBytes); err != nil { + // if we can't publish the message, set it as a pending message + // overwrite whatever is there as these are just current state messages so it doesn't + // really matter if we don't smootly transition in what we send back to lagoon + r.updateEnvironmentMessage(ctx, lagoonBuild, msg) + return + } + // if we are able to publish the message, then we need to remove any pending messages from the resource + // and make sure we don't try and publish again + r.removeBuildPendingMessageStatus(ctx, lagoonBuild) + } +} + +// buildStatusLogsToLagoonLogs sends the logs to lagoon-logs message queue, used for general messaging +func (r *LagoonBuildReconciler) buildStatusLogsToLagoonLogs(ctx context.Context, + opLog logr.Logger, + lagoonBuild *lagooncrd.LagoonBuild, + lagoonEnv *corev1.ConfigMap, + buildCondition lagooncrd.BuildStatusType, + buildStep string, +) { + if r.EnableMQ { + msg := schema.LagoonLog{ + Severity: "info", + Project: lagoonBuild.Spec.Project.Name, + Event: "task:builddeploy-kubernetes:" + buildCondition.ToLower(), //@TODO: this probably needs to be changed to a new task event for the controller + Meta: &schema.LagoonLogMeta{ + ProjectName: lagoonBuild.Spec.Project.Name, + BranchName: lagoonBuild.Spec.Project.Environment, + BuildStatus: buildCondition.ToLower(), // same as buildstatus label + BuildName: lagoonBuild.ObjectMeta.Name, + BuildStep: buildStep, + LogLink: lagoonBuild.Spec.Project.UILink, + Cluster: r.LagoonTargetName, + }, + Message: fmt.Sprintf("*[%s]* %s Build `%s` %s", + lagoonBuild.Spec.Project.Name, + lagoonBuild.Spec.Project.Environment, + lagoonBuild.ObjectMeta.Name, + buildCondition.ToLower(), + ), + } + // if we aren't being provided the lagoon config, we can skip adding the routes etc + if lagoonEnv != nil { + msg.Meta.Route = "" + if route, ok := lagoonEnv.Data["LAGOON_ROUTE"]; ok { + msg.Meta.Route = route + } + msg.Meta.Routes = []string{} + if routes, ok := lagoonEnv.Data["LAGOON_ROUTES"]; ok { + msg.Meta.Routes = strings.Split(routes, ",") + } + } + msgBytes, err := json.Marshal(msg) + if err != nil { + opLog.Error(err, "Unable to encode message as JSON") + } + // @TODO: if we can't publish the message because we are deleting the resource, then should we even + // bother to patch the resource?? + // leave it for now cause the resource will just be deleted anyway + if err := r.Messaging.Publish("lagoon-logs", msgBytes); err != nil { + // if we can't publish the message, set it as a pending message + // overwrite whatever is there as these are just current state messages so it doesn't + // really matter if we don't smootly transition in what we send back to lagoon + r.updateBuildStatusMessage(ctx, lagoonBuild, msg) + return + } + // if we are able to publish the message, then we need to remove any pending messages from the resource + // and make sure we don't try and publish again + r.removeBuildPendingMessageStatus(ctx, lagoonBuild) + } +} + +// updateEnvironmentMessage this is called if the message queue is unavailable, it stores the message that would be sent in the lagoon build +func (r *LagoonBuildReconciler) updateEnvironmentMessage(ctx context.Context, + lagoonBuild *lagooncrd.LagoonBuild, + envMessage schema.LagoonMessage, +) error { + // set the transition time + mergePatch, _ := json.Marshal(map[string]interface{}{ + "metadata": map[string]interface{}{ + "labels": map[string]interface{}{ + "lagoon.sh/pendingMessages": "true", + }, + }, + "statusMessages": map[string]interface{}{ + "environmentMessage": envMessage, + }, + }) + if err := r.Patch(ctx, lagoonBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { + return fmt.Errorf("Unable to update status condition: %v", err) + } + return nil +} + +// updateBuildStatusMessage this is called if the message queue is unavailable, it stores the message that would be sent in the lagoon build +func (r *LagoonBuildReconciler) updateBuildStatusMessage(ctx context.Context, + lagoonBuild *lagooncrd.LagoonBuild, + statusMessage schema.LagoonLog, +) error { + // set the transition time + mergePatch, _ := json.Marshal(map[string]interface{}{ + "metadata": map[string]interface{}{ + "labels": map[string]interface{}{ + "lagoon.sh/pendingMessages": "true", + }, + }, + "statusMessages": map[string]interface{}{ + "statusMessage": statusMessage, + }, + }) + if err := r.Patch(ctx, lagoonBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { + return fmt.Errorf("Unable to update status condition: %v", err) + } + return nil +} + +// removeBuildPendingMessageStatus purges the status messages from the resource once they are successfully re-sent +func (r *LagoonBuildReconciler) removeBuildPendingMessageStatus(ctx context.Context, + lagoonBuild *lagooncrd.LagoonBuild, +) error { + // if we have the pending messages label as true, then we want to remove this label and any pending statusmessages + // so we can avoid double handling, or an old pending message from being sent after a new pending message + if val, ok := lagoonBuild.ObjectMeta.Labels["lagoon.sh/pendingMessages"]; !ok { + if val == "true" { + mergePatch, _ := json.Marshal(map[string]interface{}{ + "metadata": map[string]interface{}{ + "labels": map[string]interface{}{ + "lagoon.sh/pendingMessages": "false", + }, + }, + "statusMessages": nil, + }) + if err := r.Patch(ctx, lagoonBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { + return fmt.Errorf("Unable to update status condition: %v", err) + } + } + } + return nil +} + +// updateBuildLogMessage this is called if the message queue is unavailable, it stores the message that would be sent in the lagoon build +func (r *LagoonBuildReconciler) updateBuildLogMessage(ctx context.Context, + lagoonBuild *lagooncrd.LagoonBuild, + buildMessage schema.LagoonLog, +) error { + // set the transition time + mergePatch, _ := json.Marshal(map[string]interface{}{ + "metadata": map[string]interface{}{ + "labels": map[string]interface{}{ + "lagoon.sh/pendingMessages": "true", + }, + }, + "statusMessages": map[string]interface{}{ + "buildLogMessage": buildMessage, + }, + }) + if err := r.Patch(ctx, lagoonBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { + return fmt.Errorf("Unable to update status condition: %v", err) + } + return nil +} diff --git a/controllers/v1beta2/build_helpers.go b/controllers/v1beta2/build_helpers.go new file mode 100644 index 00000000..74b0e6f4 --- /dev/null +++ b/controllers/v1beta2/build_helpers.go @@ -0,0 +1,1061 @@ +package v1beta2 + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "regexp" + "sort" + "strconv" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/go-logr/logr" + "github.com/prometheus/client_golang/prometheus" + lagooncrd "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" + "github.com/uselagoon/remote-controller/internal/harbor" + "github.com/uselagoon/remote-controller/internal/helpers" +) + +var ( + crdVersion string = "v1beta2" +) + +const ( + // NotOwnedByControllerMessage is used to describe an error where the controller was unable to start the build because + // the `lagoon.sh/controller` label does not match this controllers name + NotOwnedByControllerMessage = `Build was cancelled due to an issue with the build controller. +This issue is related to the deployment system, not the repository or code base changes. +Contact your Lagoon support team for help` + // MissingLabelsMessage is used to describe an error where the controller was unable to start the build because + // the `lagoon.sh/controller` label is missing + MissingLabelsMessage = `"Build was cancelled due to namespace configuration issue. A label or labels are missing on the namespace. +This issue is related to the deployment system, not the repository or code base changes. +Contact your Lagoon support team for help` +) + +// getOrCreateServiceAccount will create the lagoon-deployer service account if it doesn't exist. +func (r *LagoonBuildReconciler) getOrCreateServiceAccount(ctx context.Context, serviceAccount *corev1.ServiceAccount, ns string) error { + serviceAccount.ObjectMeta = metav1.ObjectMeta{ + Name: "lagoon-deployer", + Namespace: ns, + } + err := r.Get(ctx, types.NamespacedName{ + Namespace: ns, + Name: "lagoon-deployer", + }, serviceAccount) + if err != nil { + if err := r.Create(ctx, serviceAccount); err != nil { + return fmt.Errorf("There was an error creating the lagoon-deployer servicea account. Error was: %v", err) + } + } + return nil +} + +// getOrCreateSARoleBinding will create the rolebinding for the lagoon-deployer if it doesn't exist. +func (r *LagoonBuildReconciler) getOrCreateSARoleBinding(ctx context.Context, saRoleBinding *rbacv1.RoleBinding, ns string) error { + saRoleBinding.ObjectMeta = metav1.ObjectMeta{ + Name: "lagoon-deployer-admin", + Namespace: ns, + } + saRoleBinding.RoleRef = rbacv1.RoleRef{ + Name: "admin", + Kind: "ClusterRole", + APIGroup: "rbac.authorization.k8s.io", + } + saRoleBinding.Subjects = []rbacv1.Subject{ + { + Name: "lagoon-deployer", + Kind: "ServiceAccount", + Namespace: ns, + }, + } + err := r.Get(ctx, types.NamespacedName{ + Namespace: ns, + Name: "lagoon-deployer-admin", + }, saRoleBinding) + if err != nil { + if err := r.Create(ctx, saRoleBinding); err != nil { + return fmt.Errorf("There was an error creating the lagoon-deployer-admin role binding. Error was: %v", err) + } + } + return nil +} + +// getOrCreateNamespace will create the namespace if it doesn't exist. +func (r *LagoonBuildReconciler) getOrCreateNamespace(ctx context.Context, namespace *corev1.Namespace, lagoonBuild lagooncrd.LagoonBuild, opLog logr.Logger) error { + // parse the project/env through the project pattern, or use the default + ns := helpers.GenerateNamespaceName( + lagoonBuild.Spec.Project.NamespacePattern, // the namespace pattern or `openshiftProjectPattern` from Lagoon is never received by the controller + lagoonBuild.Spec.Project.Environment, + lagoonBuild.Spec.Project.Name, + r.NamespacePrefix, + r.ControllerNamespace, + r.RandomNamespacePrefix, + ) + nsLabels := map[string]string{ + "lagoon.sh/project": lagoonBuild.Spec.Project.Name, + "lagoon.sh/environment": lagoonBuild.Spec.Project.Environment, + "lagoon.sh/environmentType": lagoonBuild.Spec.Project.EnvironmentType, + "lagoon.sh/controller": r.ControllerNamespace, + } + if lagoonBuild.Spec.Project.Organization != nil { + nsLabels["organization.lagoon.sh/id"] = fmt.Sprintf("%d", *lagoonBuild.Spec.Project.Organization.ID) + nsLabels["organization.lagoon.sh/name"] = lagoonBuild.Spec.Project.Organization.Name + } + if lagoonBuild.Spec.Project.ID != nil { + nsLabels["lagoon.sh/projectId"] = fmt.Sprintf("%d", *lagoonBuild.Spec.Project.ID) + } + if lagoonBuild.Spec.Project.EnvironmentID != nil { + nsLabels["lagoon.sh/environmentId"] = fmt.Sprintf("%d", *lagoonBuild.Spec.Project.EnvironmentID) + } + // set the auto idling values if they are defined + if lagoonBuild.Spec.Project.EnvironmentIdling != nil { + // eventually deprecate 'lagoon.sh/environmentAutoIdle' for 'lagoon.sh/environmentIdlingEnabled' + nsLabels["lagoon.sh/environmentAutoIdle"] = fmt.Sprintf("%d", *lagoonBuild.Spec.Project.EnvironmentIdling) + if *lagoonBuild.Spec.Project.EnvironmentIdling == 1 { + nsLabels["lagoon.sh/environmentIdlingEnabled"] = "true" + } else { + nsLabels["lagoon.sh/environmentIdlingEnabled"] = "false" + } + } + if lagoonBuild.Spec.Project.ProjectIdling != nil { + // eventually deprecate 'lagoon.sh/projectAutoIdle' for 'lagoon.sh/projectIdlingEnabled' + nsLabels["lagoon.sh/projectAutoIdle"] = fmt.Sprintf("%d", *lagoonBuild.Spec.Project.ProjectIdling) + if *lagoonBuild.Spec.Project.ProjectIdling == 1 { + nsLabels["lagoon.sh/projectIdlingEnabled"] = "true" + } else { + nsLabels["lagoon.sh/projectIdlingEnabled"] = "false" + } + } + if lagoonBuild.Spec.Project.StorageCalculator != nil { + if *lagoonBuild.Spec.Project.StorageCalculator == 1 { + nsLabels["lagoon.sh/storageCalculatorEnabled"] = "true" + } else { + nsLabels["lagoon.sh/storageCalculatorEnabled"] = "false" + } + } + // add the required lagoon labels to the namespace when creating + namespace.ObjectMeta = metav1.ObjectMeta{ + Name: ns, + Labels: nsLabels, + } + + if err := r.Get(ctx, types.NamespacedName{Name: ns}, namespace); err != nil { + if helpers.IgnoreNotFound(err) != nil { + return fmt.Errorf("There was an error getting the namespace. Error was: %v", err) + } + } + if namespace.Status.Phase == corev1.NamespaceTerminating { + opLog.Info(fmt.Sprintf("Cleaning up build %s as cancelled, the namespace is stuck in terminating state", lagoonBuild.ObjectMeta.Name)) + r.cleanUpUndeployableBuild(ctx, lagoonBuild, "Namespace is currently in terminating status - contact your Lagoon support team for help", opLog, true) + return fmt.Errorf("%s is currently terminating, aborting build", ns) + } + + // if the namespace exists, check that the controller label exists and matches this controllers namespace name + if namespace.Status.Phase == corev1.NamespaceActive { + if value, ok := namespace.ObjectMeta.Labels["lagoon.sh/controller"]; ok { + if value != r.ControllerNamespace { + // if the namespace is deployed by a different controller, fail the build + opLog.Info(fmt.Sprintf("Cleaning up build %s as cancelled, the namespace is owned by a different remote-controller", lagoonBuild.ObjectMeta.Name)) + r.cleanUpUndeployableBuild(ctx, lagoonBuild, NotOwnedByControllerMessage, opLog, true) + return fmt.Errorf("%s is owned by a different remote-controller, aborting build", ns) + } + } else { + // if the label doesn't exist at all, fail the build + opLog.Info(fmt.Sprintf("Cleaning up build %s as cancelled, the namespace is not a Lagoon project/environment", lagoonBuild.ObjectMeta.Name)) + r.cleanUpUndeployableBuild(ctx, lagoonBuild, MissingLabelsMessage, opLog, true) + return fmt.Errorf("%s is not a Lagoon project/environment, aborting build", ns) + } + } + + // if kubernetes, just create it if it doesn't exist + if err := r.Get(ctx, types.NamespacedName{Name: ns}, namespace); err != nil { + if err := r.Create(ctx, namespace); err != nil { + return fmt.Errorf("There was an error creating the namespace. Error was: %v", err) + } + } + + // once the namespace exists, then we can patch it with our labels + // this means the labels will always get added or updated if we need to change them or add new labels + // after the namespace has been created + mergePatch, _ := json.Marshal(map[string]interface{}{ + "metadata": map[string]interface{}{ + "labels": nsLabels, + }, + }) + if err := r.Patch(ctx, namespace, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { + return fmt.Errorf("There was an error patching the namespace. Error was: %v", err) + } + if err := r.Get(ctx, types.NamespacedName{Name: ns}, namespace); err != nil { + return fmt.Errorf("There was an error getting the namespace. Error was: %v", err) + } + + // if local/regional harbor is enabled + if r.LFFHarborEnabled { + // create the harbor client + lagoonHarbor, err := harbor.New(r.Harbor) + if err != nil { + return fmt.Errorf("Error creating harbor client, check your harbor configuration. Error was: %v", err) + } + // create the project in harbor + robotCreds := &helpers.RegistryCredentials{} + curVer, err := lagoonHarbor.GetHarborVersion(ctx) + if err != nil { + return fmt.Errorf("Error getting harbor version, check your harbor configuration. Error was: %v", err) + } + if lagoonHarbor.UseV2Functions(curVer) { + hProject, err := lagoonHarbor.CreateProjectV2(ctx, lagoonBuild.Spec.Project.Name) + if err != nil { + return fmt.Errorf("Error creating harbor project: %v", err) + } + // create or refresh the robot credentials + robotCreds, err = lagoonHarbor.CreateOrRefreshRobotV2(ctx, + r.Client, + hProject, + lagoonBuild.Spec.Project.Environment, + ns, + lagoonHarbor.RobotAccountExpiry, + false) + if err != nil { + return fmt.Errorf("Error creating harbor robot account: %v", err) + } + } else { + hProject, err := lagoonHarbor.CreateProject(ctx, lagoonBuild.Spec.Project.Name) + if err != nil { + return fmt.Errorf("Error creating harbor project: %v", err) + } + // create or refresh the robot credentials + robotCreds, err = lagoonHarbor.CreateOrRefreshRobot(ctx, + r.Client, + hProject, + lagoonBuild.Spec.Project.Environment, + ns, + time.Now().Add(lagoonHarbor.RobotAccountExpiry).Unix(), + false) + if err != nil { + return fmt.Errorf("Error creating harbor robot account: %v", err) + } + } + // if we have robotcredentials to create, do that here + _, err = lagoonHarbor.UpsertHarborSecret(ctx, + r.Client, + ns, + "lagoon-internal-registry-secret", + robotCreds) + if err != nil { + return fmt.Errorf("Error upserting harbor robot account secret: %v", err) + } + } + return nil +} + +// getCreateOrUpdateSSHKeySecret will create or update the ssh key. +func (r *LagoonBuildReconciler) getCreateOrUpdateSSHKeySecret(ctx context.Context, + sshKey *corev1.Secret, + spec lagooncrd.LagoonBuildSpec, + ns string) error { + sshKey.ObjectMeta = metav1.ObjectMeta{ + Name: "lagoon-sshkey", + Namespace: ns, + } + sshKey.Type = "kubernetes.io/ssh-auth" + sshKey.Data = map[string][]byte{ + "ssh-privatekey": spec.Project.Key, + } + err := r.Get(ctx, types.NamespacedName{ + Namespace: ns, + Name: "lagoon-sshkey", + }, sshKey) + if err != nil { + if err := r.Create(ctx, sshKey); err != nil { + return fmt.Errorf("There was an error creating the lagoon-sshkey. Error was: %v", err) + } + } + // if the keys are different, then load in the new key from the spec + if bytes.Compare(sshKey.Data["ssh-privatekey"], spec.Project.Key) != 0 { + sshKey.Data = map[string][]byte{ + "ssh-privatekey": spec.Project.Key, + } + if err := r.Update(ctx, sshKey); err != nil { + return fmt.Errorf("There was an error updating the lagoon-sshkey. Error was: %v", err) + } + } + return nil +} + +func (r *LagoonBuildReconciler) getOrCreateConfigMap(ctx context.Context, cmName string, configMap *corev1.ConfigMap, ns string) error { + err := r.Get(ctx, types.NamespacedName{ + Namespace: ns, + Name: cmName, + }, configMap) + if err != nil { + configMap.SetNamespace(ns) + configMap.SetName(cmName) + //we create it + if err = r.Create(ctx, configMap); err != nil { + return fmt.Errorf("There was an error creating the configmap '%v'. Error was: %v", cmName, err) + } + } + return nil +} + +// processBuild will actually process the build. +func (r *LagoonBuildReconciler) processBuild(ctx context.Context, opLog logr.Logger, lagoonBuild lagooncrd.LagoonBuild) error { + // we run these steps again just to be sure that it gets updated/created if it hasn't already + opLog.Info(fmt.Sprintf("Checking and preparing namespace and associated resources for build: %s", lagoonBuild.ObjectMeta.Name)) + // create the lagoon-sshkey secret + sshKey := &corev1.Secret{} + if r.EnableDebug { + opLog.Info(fmt.Sprintf("Checking `lagoon-sshkey` Secret exists: %s", lagoonBuild.ObjectMeta.Name)) + } + err := r.getCreateOrUpdateSSHKeySecret(ctx, sshKey, lagoonBuild.Spec, lagoonBuild.ObjectMeta.Namespace) + if err != nil { + return err + } + + // create the `lagoon-deployer` ServiceAccount + if r.EnableDebug { + opLog.Info(fmt.Sprintf("Checking `lagoon-deployer` ServiceAccount exists: %s", lagoonBuild.ObjectMeta.Name)) + } + serviceAccount := &corev1.ServiceAccount{} + err = r.getOrCreateServiceAccount(ctx, serviceAccount, lagoonBuild.ObjectMeta.Namespace) + if err != nil { + return err + } + + // ServiceAccount RoleBinding creation + if r.EnableDebug { + opLog.Info(fmt.Sprintf("Checking `lagoon-deployer-admin` RoleBinding exists: %s", lagoonBuild.ObjectMeta.Name)) + } + saRoleBinding := &rbacv1.RoleBinding{} + err = r.getOrCreateSARoleBinding(ctx, saRoleBinding, lagoonBuild.ObjectMeta.Namespace) + if err != nil { + return err + } + + if r.EnableDebug { + opLog.Info(fmt.Sprintf("Checking `lagoon-deployer` Token exists: %s", lagoonBuild.ObjectMeta.Name)) + } + + var serviceaccountTokenSecret string + for _, secret := range serviceAccount.Secrets { + match, _ := regexp.MatchString("^lagoon-deployer-token", secret.Name) + if match { + serviceaccountTokenSecret = secret.Name + break + } + } + + // create the Pod that will do the work + podEnvs := []corev1.EnvVar{ + { + Name: "SOURCE_REPOSITORY", + Value: lagoonBuild.Spec.Project.GitURL, + }, + { + Name: "GIT_REF", + Value: lagoonBuild.Spec.GitReference, + }, + { + Name: "SUBFOLDER", + Value: lagoonBuild.Spec.Project.SubFolder, + }, + { + Name: "BRANCH", + Value: lagoonBuild.Spec.Branch.Name, + }, + { + Name: "PROJECT", + Value: lagoonBuild.Spec.Project.Name, + }, + { + Name: "ENVIRONMENT_TYPE", + Value: lagoonBuild.Spec.Project.EnvironmentType, + }, + { + Name: "ACTIVE_ENVIRONMENT", + Value: lagoonBuild.Spec.Project.ProductionEnvironment, + }, + { + Name: "STANDBY_ENVIRONMENT", + Value: lagoonBuild.Spec.Project.StandbyEnvironment, + }, + { + Name: "PROJECT_SECRET", + Value: lagoonBuild.Spec.Project.ProjectSecret, + }, + { + Name: "MONITORING_ALERTCONTACT", + Value: lagoonBuild.Spec.Project.Monitoring.Contact, + }, + { + Name: "DEFAULT_BACKUP_SCHEDULE", + Value: r.BackupConfig.BackupDefaultSchedule, + }, + { + Name: "MONTHLY_BACKUP_DEFAULT_RETENTION", + Value: strconv.Itoa(r.BackupConfig.BackupDefaultMonthlyRetention), + }, + { + Name: "WEEKLY_BACKUP_DEFAULT_RETENTION", + Value: strconv.Itoa(r.BackupConfig.BackupDefaultWeeklyRetention), + }, + { + Name: "DAILY_BACKUP_DEFAULT_RETENTION", + Value: strconv.Itoa(r.BackupConfig.BackupDefaultDailyRetention), + }, + { + Name: "HOURLY_BACKUP_DEFAULT_RETENTION", + Value: strconv.Itoa(r.BackupConfig.BackupDefaultHourlyRetention), + }, + { + Name: "LAGOON_FEATURE_BACKUP_DEV_SCHEDULE", + Value: r.BackupConfig.BackupDefaultDevelopmentSchedule, + }, + { + Name: "LAGOON_FEATURE_BACKUP_PR_SCHEDULE", + Value: r.BackupConfig.BackupDefaultPullrequestSchedule, + }, + { + Name: "LAGOON_FEATURE_BACKUP_DEV_RETENTION", + Value: r.BackupConfig.BackupDefaultDevelopmentRetention, + }, + { + Name: "LAGOON_FEATURE_BACKUP_PR_RETENTION", + Value: r.BackupConfig.BackupDefaultPullrequestRetention, + }, + { + Name: "K8UP_WEEKLY_RANDOM_FEATURE_FLAG", + Value: strconv.FormatBool(r.LFFBackupWeeklyRandom), + }, + { + Name: "NATIVE_CRON_POD_MINIMUM_FREQUENCY", + Value: strconv.Itoa(r.NativeCronPodMinFrequency), + }, + // add the API and SSH endpoint configuration to environments + { + Name: "LAGOON_CONFIG_API_HOST", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "LAGOON_CONFIG_API_HOST"), + }, + { + Name: "LAGOON_CONFIG_TOKEN_HOST", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "LAGOON_CONFIG_TOKEN_HOST"), + }, + { + Name: "LAGOON_CONFIG_TOKEN_PORT", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "LAGOON_CONFIG_TOKEN_PORT"), + }, + { + Name: "LAGOON_CONFIG_SSH_HOST", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "LAGOON_CONFIG_SSH_HOST"), + }, + { + Name: "LAGOON_CONFIG_SSH_PORT", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "LAGOON_CONFIG_SSH_PORT"), + }, + } + // add proxy variables to builds if they are defined + if r.ProxyConfig.HTTPProxy != "" { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "HTTP_PROXY", + Value: r.ProxyConfig.HTTPProxy, + }) + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "http_proxy", + Value: r.ProxyConfig.HTTPProxy, + }) + } + if r.ProxyConfig.HTTPSProxy != "" { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "HTTPS_PROXY", + Value: r.ProxyConfig.HTTPSProxy, + }) + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "https_proxy", + Value: r.ProxyConfig.HTTPSProxy, + }) + } + if r.ProxyConfig.NoProxy != "" { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "NO_PROXY", + Value: r.ProxyConfig.NoProxy, + }) + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "no_proxy", + Value: r.ProxyConfig.NoProxy, + }) + } + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "BUILD_TYPE", + Value: lagoonBuild.Spec.Build.Type, + }) + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "ENVIRONMENT", + Value: lagoonBuild.Spec.Project.Environment, + }) + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "KUBERNETES", + Value: lagoonBuild.Spec.Project.DeployTarget, + }) + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "REGISTRY", + Value: lagoonBuild.Spec.Project.Registry, + }) + // this is enabled by default for now + // eventually will be disabled by default because support for the generation/modification of this will + // be handled by lagoon or the builds themselves + if r.LFFRouterURL { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "ROUTER_URL", + Value: strings.ToLower( + strings.Replace( + strings.Replace( + lagoonBuild.Spec.Project.RouterPattern, + "${environment}", + lagoonBuild.Spec.Project.Environment, + -1, + ), + "${project}", + lagoonBuild.Spec.Project.Name, + -1, + ), + ), + }) + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "SHORT_ROUTER_URL", + Value: strings.ToLower( + strings.Replace( + strings.Replace( + lagoonBuild.Spec.Project.RouterPattern, + "${environment}", + helpers.ShortName(lagoonBuild.Spec.Project.Environment), + -1, + ), + "${project}", + helpers.ShortName(lagoonBuild.Spec.Project.Name), + -1, + ), + ), + }) + } + if lagoonBuild.Spec.Build.CI != "" { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "CI", + Value: lagoonBuild.Spec.Build.CI, + }) + } + if lagoonBuild.Spec.Build.Type == "pullrequest" { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "PR_HEAD_BRANCH", + Value: lagoonBuild.Spec.Pullrequest.HeadBranch, + }) + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "PR_HEAD_SHA", + Value: lagoonBuild.Spec.Pullrequest.HeadSha, + }) + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "PR_BASE_BRANCH", + Value: lagoonBuild.Spec.Pullrequest.BaseBranch, + }) + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "PR_BASE_SHA", + Value: lagoonBuild.Spec.Pullrequest.BaseSha, + }) + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "PR_TITLE", + Value: lagoonBuild.Spec.Pullrequest.Title, + }) + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "PR_NUMBER", + Value: string(lagoonBuild.Spec.Pullrequest.Number), + }) + } + if lagoonBuild.Spec.Build.Type == "promote" { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "PROMOTION_SOURCE_ENVIRONMENT", + Value: lagoonBuild.Spec.Promote.SourceEnvironment, + }) + } + // if local/regional harbor is enabled + if r.LFFHarborEnabled { + // unmarshal the project variables + lagoonProjectVariables := &[]helpers.LagoonEnvironmentVariable{} + lagoonEnvironmentVariables := &[]helpers.LagoonEnvironmentVariable{} + json.Unmarshal(lagoonBuild.Spec.Project.Variables.Project, lagoonProjectVariables) + json.Unmarshal(lagoonBuild.Spec.Project.Variables.Environment, lagoonEnvironmentVariables) + // check if INTERNAL_REGISTRY_SOURCE_LAGOON is defined, and if it isn't true + // if this value is true, then we want to use what is provided by Lagoon + // if it is false, or not set, then we use what is provided by this controller + // this allows us to make it so a specific environment or the project entirely + // can still use whats provided by lagoon + if !helpers.VariableExists(lagoonProjectVariables, "INTERNAL_REGISTRY_SOURCE_LAGOON", "true") || + !helpers.VariableExists(lagoonEnvironmentVariables, "INTERNAL_REGISTRY_SOURCE_LAGOON", "true") { + // source the robot credential, and inject it into the lagoon project variables + // this will overwrite what is provided by lagoon (if lagoon has provided them) + // or it will add them. + robotCredential := &corev1.Secret{} + if err = r.Get(ctx, types.NamespacedName{ + Namespace: lagoonBuild.ObjectMeta.Namespace, + Name: "lagoon-internal-registry-secret", + }, robotCredential); err != nil { + return fmt.Errorf("Could not find Harbor RobotAccount credential") + } + auths := helpers.Auths{} + if secretData, ok := robotCredential.Data[".dockerconfigjson"]; ok { + if err := json.Unmarshal(secretData, &auths); err != nil { + return fmt.Errorf("Could not unmarshal Harbor RobotAccount credential") + } + // if the defined regional harbor key exists using the hostname + if creds, ok := auths.Registries[r.Harbor.URL]; ok { + // use the regional harbor in the build + helpers.ReplaceOrAddVariable(lagoonProjectVariables, "INTERNAL_REGISTRY_URL", r.Harbor.URL, "internal_container_registry") + helpers.ReplaceOrAddVariable(lagoonProjectVariables, "INTERNAL_REGISTRY_USERNAME", creds.Username, "internal_container_registry") + helpers.ReplaceOrAddVariable(lagoonProjectVariables, "INTERNAL_REGISTRY_PASSWORD", creds.Password, "internal_container_registry") + } + if creds, ok := auths.Registries[r.Harbor.Hostname]; ok { + // use the regional harbor in the build + helpers.ReplaceOrAddVariable(lagoonProjectVariables, "INTERNAL_REGISTRY_URL", r.Harbor.Hostname, "internal_container_registry") + helpers.ReplaceOrAddVariable(lagoonProjectVariables, "INTERNAL_REGISTRY_USERNAME", creds.Username, "internal_container_registry") + helpers.ReplaceOrAddVariable(lagoonProjectVariables, "INTERNAL_REGISTRY_PASSWORD", creds.Password, "internal_container_registry") + } + } + // marshal any changes into the project spec on the fly, don't save the spec though + // these values are being overwritten and injected directly into the build pod to be consumed + // by the build pod image + lagoonBuild.Spec.Project.Variables.Project, _ = json.Marshal(lagoonProjectVariables) + } + } + if lagoonBuild.Spec.Project.Variables.Project != nil { + // if this is 2 bytes long, then it means its just an empty json array + // we only want to add it if it is more than 2 bytes + if len(lagoonBuild.Spec.Project.Variables.Project) > 2 { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "LAGOON_PROJECT_VARIABLES", + Value: string(lagoonBuild.Spec.Project.Variables.Project), + }) + } + } + if lagoonBuild.Spec.Project.Variables.Environment != nil { + // if this is 2 bytes long, then it means its just an empty json array + // we only want to add it if it is more than 2 bytes + if len(lagoonBuild.Spec.Project.Variables.Environment) > 2 { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "LAGOON_ENVIRONMENT_VARIABLES", + Value: string(lagoonBuild.Spec.Project.Variables.Environment), + }) + } + } + if lagoonBuild.Spec.Project.Monitoring.StatuspageID != "" { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "MONITORING_STATUSPAGEID", + Value: lagoonBuild.Spec.Project.Monitoring.StatuspageID, + }) + } + // if the fastly watch status is set on the controller, inject the fastly service ID into the build pod to be consumed + // by the build-depoy-dind image + if r.FastlyWatchStatus { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "LAGOON_FASTLY_NOCACHE_SERVICE_ID", + Value: r.FastlyServiceID, + }) + } + // Set any defined Lagoon feature flags in the build environment. + if r.LFFForceRootlessWorkload != "" { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "LAGOON_FEATURE_FLAG_FORCE_ROOTLESS_WORKLOAD", + Value: r.LFFForceRootlessWorkload, + }) + } + if r.LFFDefaultRootlessWorkload != "" { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "LAGOON_FEATURE_FLAG_DEFAULT_ROOTLESS_WORKLOAD", + Value: r.LFFDefaultRootlessWorkload, + }) + } + if r.LFFForceIsolationNetworkPolicy != "" { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "LAGOON_FEATURE_FLAG_FORCE_ISOLATION_NETWORK_POLICY", + Value: r.LFFForceIsolationNetworkPolicy, + }) + } + if r.LFFDefaultIsolationNetworkPolicy != "" { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "LAGOON_FEATURE_FLAG_DEFAULT_ISOLATION_NETWORK_POLICY", + Value: r.LFFDefaultIsolationNetworkPolicy, + }) + } + if r.LFFForceInsights != "" { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "LAGOON_FEATURE_FLAG_FORCE_INSIGHTS", + Value: r.LFFForceInsights, + }) + } + if r.LFFDefaultInsights != "" { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "LAGOON_FEATURE_FLAG_DEFAULT_INSIGHTS", + Value: r.LFFDefaultInsights, + }) + } + if r.LFFForceRWX2RWO != "" { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "LAGOON_FEATURE_FLAG_FORCE_RWX_TO_RWO", + Value: r.LFFForceRWX2RWO, + }) + } + if r.LFFDefaultRWX2RWO != "" { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "LAGOON_FEATURE_FLAG_DEFAULT_RWX_TO_RWO", + Value: r.LFFDefaultRWX2RWO, + }) + } + + // set the organization variables into the build pod so they're available to builds for consumption + if lagoonBuild.Spec.Project.Organization != nil { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "LAGOON_ORGANIZATION_ID", + Value: fmt.Sprintf("%d", *lagoonBuild.Spec.Project.Organization.ID), + }) + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "LAGOON_ORGANIZATION_NAME", + Value: lagoonBuild.Spec.Project.Organization.Name, + }) + } + + // add any LAGOON_FEATURE_FLAG_ variables in the controller into the build pods + for fName, fValue := range r.LagoonFeatureFlags { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: fName, + Value: fValue, + }) + } + // Use the build image in the controller definition + buildImage := r.BuildImage + if lagoonBuild.Spec.Build.Image != "" { + // otherwise if the build spec contains an image definition, use it instead. + buildImage = lagoonBuild.Spec.Build.Image + } + volumes := []corev1.Volume{ + { + Name: "lagoon-sshkey", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "lagoon-sshkey", + DefaultMode: helpers.Int32Ptr(420), + }, + }, + }, + } + volumeMounts := []corev1.VolumeMount{ + { + Name: "lagoon-sshkey", + ReadOnly: true, + MountPath: "/var/run/secrets/lagoon/ssh", + }, + } + + // if the existing token exists, mount it + if serviceaccountTokenSecret != "" { + volumes = append(volumes, corev1.Volume{ + Name: serviceaccountTokenSecret, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: serviceaccountTokenSecret, + DefaultMode: helpers.Int32Ptr(420), + }, + }, + }) + // legacy tokens are mounted /var/run/secrets/lagoon/deployer + // new tokens using volume projection are mounted /var/run/secrets/kubernetes.io/serviceaccount/token + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: serviceaccountTokenSecret, + ReadOnly: true, + MountPath: "/var/run/secrets/lagoon/deployer", + }) + } + newPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: lagoonBuild.ObjectMeta.Name, + Namespace: lagoonBuild.ObjectMeta.Namespace, + Labels: map[string]string{ + "lagoon.sh/jobType": "build", + "lagoon.sh/buildName": lagoonBuild.ObjectMeta.Name, + "lagoon.sh/controller": r.ControllerNamespace, + "lagoon.sh/crdVersion": crdVersion, + "lagoon.sh/buildRemoteID": string(lagoonBuild.ObjectMeta.UID), + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: fmt.Sprintf("%v", lagooncrd.GroupVersion), + Kind: "LagoonBuild", + Name: lagoonBuild.ObjectMeta.Name, + UID: lagoonBuild.UID, + }, + }, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: "lagoon-deployer", + RestartPolicy: "Never", + Volumes: volumes, + Tolerations: []corev1.Toleration{ + { + Key: "lagoon/build", + Effect: "NoSchedule", + Operator: "Exists", + }, + { + Key: "lagoon/build", + Effect: "PreferNoSchedule", + Operator: "Exists", + }, + { + Key: "lagoon.sh/build", + Effect: "NoSchedule", + Operator: "Exists", + }, + { + Key: "lagoon.sh/build", + Effect: "PreferNoSchedule", + Operator: "Exists", + }, + }, + Containers: []corev1.Container{ + { + Name: "lagoon-build", + Image: buildImage, + ImagePullPolicy: "Always", + Env: podEnvs, + VolumeMounts: volumeMounts, + }, + }, + }, + } + + // set the organization labels on build pods + if lagoonBuild.Spec.Project.Organization != nil { + newPod.ObjectMeta.Labels["organization.lagoon.sh/id"] = fmt.Sprintf("%d", *lagoonBuild.Spec.Project.Organization.ID) + newPod.ObjectMeta.Labels["organization.lagoon.sh/name"] = lagoonBuild.Spec.Project.Organization.Name + } + + // set the pod security context, if defined to a non-default value + if r.BuildPodRunAsUser != 0 || r.BuildPodRunAsGroup != 0 || + r.BuildPodFSGroup != 0 { + newPod.Spec.SecurityContext = &corev1.PodSecurityContext{ + RunAsUser: &r.BuildPodRunAsUser, + RunAsGroup: &r.BuildPodRunAsGroup, + FSGroup: &r.BuildPodFSGroup, + } + } + + if r.EnableDebug { + opLog.Info(fmt.Sprintf("Checking build pod for: %s", lagoonBuild.ObjectMeta.Name)) + } + // once the pod spec has been defined, check if it isn't already created + err = r.Get(ctx, types.NamespacedName{ + Namespace: lagoonBuild.ObjectMeta.Namespace, + Name: newPod.ObjectMeta.Name, + }, newPod) + if err != nil { + // if it doesn't exist, then create the build pod + opLog.Info(fmt.Sprintf("Creating build pod for: %s", lagoonBuild.ObjectMeta.Name)) + if err := r.Create(ctx, newPod); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to create build pod")) + // log the error and just exit, don't continue to try and do anything + // @TODO: should update the build to failed + return nil + } + buildRunningStatus.With(prometheus.Labels{ + "build_namespace": lagoonBuild.ObjectMeta.Namespace, + "build_name": lagoonBuild.ObjectMeta.Name, + }).Set(1) + buildStatus.With(prometheus.Labels{ + "build_namespace": lagoonBuild.ObjectMeta.Namespace, + "build_name": lagoonBuild.ObjectMeta.Name, + "build_step": "running", + }).Set(1) + buildsStartedCounter.Inc() + // then break out of the build + } + opLog.Info(fmt.Sprintf("Build pod already running for: %s", lagoonBuild.ObjectMeta.Name)) + return nil +} + +// updateQueuedBuild will update a build if it is queued +func (r *LagoonBuildReconciler) updateQueuedBuild( + ctx context.Context, + lagoonBuild lagooncrd.LagoonBuild, + queuePosition, queueLength int, + opLog logr.Logger, +) error { + if r.EnableDebug { + opLog.Info(fmt.Sprintf("Updating build %s to queued: %s", lagoonBuild.ObjectMeta.Name, fmt.Sprintf("This build is currently queued in position %v/%v", queuePosition, queueLength))) + } + var allContainerLogs []byte + // if we get this handler, then it is likely that the build was in a pending or running state with no actual running pod + // so just set the logs to be cancellation message + allContainerLogs = []byte(fmt.Sprintf(` +======================================== +%s +======================================== +`, fmt.Sprintf("This build is currently queued in position %v/%v", queuePosition, queueLength))) + // get the configmap for lagoon-env so we can use it for updating the deployment in lagoon + var lagoonEnv corev1.ConfigMap + err := r.Get(ctx, types.NamespacedName{ + Namespace: lagoonBuild.ObjectMeta.Namespace, + Name: "lagoon-env", + }, + &lagoonEnv, + ) + if err != nil { + // if there isn't a configmap, just info it and move on + // the updatedeployment function will see it as nil and not bother doing the bits that require the configmap + if r.EnableDebug { + opLog.Info(fmt.Sprintf("There is no configmap %s in namespace %s ", "lagoon-env", lagoonBuild.ObjectMeta.Namespace)) + } + } + // send any messages to lagoon message queues + // update the deployment with the status, lagoon v2.12.0 supports queued status, otherwise use pending + if lagooncrd.CheckLagoonVersion(&lagoonBuild, "2.12.0") { + r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &lagoonEnv, lagooncrd.BuildStatusQueued, fmt.Sprintf("queued %v/%v", queuePosition, queueLength)) + r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagooncrd.BuildStatusQueued, fmt.Sprintf("queued %v/%v", queuePosition, queueLength)) + r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, allContainerLogs, lagooncrd.BuildStatusQueued) + } else { + r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &lagoonEnv, lagooncrd.BuildStatusPending, fmt.Sprintf("queued %v/%v", queuePosition, queueLength)) + r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagooncrd.BuildStatusPending, fmt.Sprintf("queued %v/%v", queuePosition, queueLength)) + r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, allContainerLogs, lagooncrd.BuildStatusPending) + + } + return nil +} + +// cleanUpUndeployableBuild will clean up a build if the namespace is being terminated, or some other reason that it can't deploy (or create the pod, pending in queue) +func (r *LagoonBuildReconciler) cleanUpUndeployableBuild( + ctx context.Context, + lagoonBuild lagooncrd.LagoonBuild, + message string, + opLog logr.Logger, + cancelled bool, +) error { + var allContainerLogs []byte + if cancelled { + // if we get this handler, then it is likely that the build was in a pending or running state with no actual running pod + // so just set the logs to be cancellation message + allContainerLogs = []byte(fmt.Sprintf(` +======================================== +Build cancelled +======================================== +%s`, message)) + var buildCondition lagooncrd.BuildStatusType + buildCondition = lagooncrd.BuildStatusCancelled + lagoonBuild.Labels["lagoon.sh/buildStatus"] = buildCondition.String() + mergePatch, _ := json.Marshal(map[string]interface{}{ + "metadata": map[string]interface{}{ + "labels": map[string]interface{}{ + "lagoon.sh/buildStatus": buildCondition.String(), + }, + }, + }) + if err := r.Patch(ctx, &lagoonBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to update build status")) + } + } + // get the configmap for lagoon-env so we can use it for updating the deployment in lagoon + var lagoonEnv corev1.ConfigMap + err := r.Get(ctx, types.NamespacedName{ + Namespace: lagoonBuild.ObjectMeta.Namespace, + Name: "lagoon-env", + }, + &lagoonEnv, + ) + if err != nil { + // if there isn't a configmap, just info it and move on + // the updatedeployment function will see it as nil and not bother doing the bits that require the configmap + if r.EnableDebug { + opLog.Info(fmt.Sprintf("There is no configmap %s in namespace %s ", "lagoon-env", lagoonBuild.ObjectMeta.Namespace)) + } + } + // send any messages to lagoon message queues + // update the deployment with the status of cancelled in lagoon + r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &lagoonEnv, lagooncrd.BuildStatusCancelled, "cancelled") + r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagooncrd.BuildStatusCancelled, "cancelled") + if cancelled { + r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, allContainerLogs, lagooncrd.BuildStatusCancelled) + } + // delete the build from the lagoon namespace in kubernetes entirely + err = r.Delete(ctx, &lagoonBuild) + if err != nil { + return fmt.Errorf("There was an error deleting the lagoon build. Error was: %v", err) + } + return nil +} + +func (r *LagoonBuildReconciler) cancelExtraBuilds(ctx context.Context, opLog logr.Logger, pendingBuilds *lagooncrd.LagoonBuildList, ns string, status string) error { + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(ns), + client.MatchingLabels(map[string]string{"lagoon.sh/buildStatus": lagooncrd.BuildStatusPending.String()}), + }) + if err := r.List(ctx, pendingBuilds, listOption); err != nil { + return fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err) + } + if len(pendingBuilds.Items) > 0 { + if r.EnableDebug { + opLog.Info(fmt.Sprintf("There are %v pending builds", len(pendingBuilds.Items))) + } + // if we have any pending builds, then grab the latest one and make it running + // if there are any other pending builds, cancel them so only the latest one runs + sort.Slice(pendingBuilds.Items, func(i, j int) bool { + return pendingBuilds.Items[i].ObjectMeta.CreationTimestamp.After(pendingBuilds.Items[j].ObjectMeta.CreationTimestamp.Time) + }) + for idx, pBuild := range pendingBuilds.Items { + pendingBuild := pBuild.DeepCopy() + if idx == 0 { + pendingBuild.Labels["lagoon.sh/buildStatus"] = status + } else { + // cancel any other pending builds + opLog.Info(fmt.Sprintf("Attempting to cancel build %s", pendingBuild.ObjectMeta.Name)) + pendingBuild.Labels["lagoon.sh/buildStatus"] = lagooncrd.BuildStatusCancelled.String() + } + if err := r.Update(ctx, pendingBuild); err != nil { + return fmt.Errorf("There was an error updating the pending build. Error was: %v", err) + } + var lagoonBuild lagooncrd.LagoonBuild + if err := r.Get(ctx, types.NamespacedName{ + Namespace: pendingBuild.ObjectMeta.Namespace, + Name: pendingBuild.ObjectMeta.Name, + }, &lagoonBuild); err != nil { + return helpers.IgnoreNotFound(err) + } + opLog.Info(fmt.Sprintf("Cleaning up build %s as cancelled extra build", lagoonBuild.ObjectMeta.Name)) + r.cleanUpUndeployableBuild(ctx, lagoonBuild, "This build was cancelled as a newer build was triggered.", opLog, true) + } + } + return nil +} + +func sortBuilds(defaultPriority int, pendingBuilds *lagooncrd.LagoonBuildList) { + sort.Slice(pendingBuilds.Items, func(i, j int) bool { + // sort by priority, then creation timestamp + iPriority := defaultPriority + jPriority := defaultPriority + if ok := pendingBuilds.Items[i].Spec.Build.Priority; ok != nil { + iPriority = *pendingBuilds.Items[i].Spec.Build.Priority + } + if ok := pendingBuilds.Items[j].Spec.Build.Priority; ok != nil { + jPriority = *pendingBuilds.Items[j].Spec.Build.Priority + } + // better sorting based on priority then creation timestamp + switch { + case iPriority != jPriority: + return iPriority < jPriority + default: + return pendingBuilds.Items[i].ObjectMeta.CreationTimestamp.Before(&pendingBuilds.Items[j].ObjectMeta.CreationTimestamp) + } + }) +} diff --git a/controllers/v1beta2/build_helpers_test.go b/controllers/v1beta2/build_helpers_test.go new file mode 100644 index 00000000..ac8909f6 --- /dev/null +++ b/controllers/v1beta2/build_helpers_test.go @@ -0,0 +1,233 @@ +package v1beta2 + +import ( + "testing" + "time" + + "github.com/google/go-cmp/cmp" + lagooncrd "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" + "github.com/uselagoon/remote-controller/internal/helpers" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func timeFromString(s string) time.Time { + time, _ := time.Parse("2006-01-02T15:04:05.000Z", s) + return time +} + +func Test_sortBuilds(t *testing.T) { + type args struct { + defaultPriority int + pendingBuilds *lagooncrd.LagoonBuildList + } + tests := []struct { + name string + args args + wantBuilds *lagooncrd.LagoonBuildList + }{ + { + name: "test1 - 5 and 6 same time order by priority", + args: args{ + defaultPriority: 5, + pendingBuilds: &lagooncrd.LagoonBuildList{ + Items: []lagooncrd.LagoonBuild{ + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-abcdefg", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:45:00.000Z")), + }, + Spec: lagooncrd.LagoonBuildSpec{ + Build: lagooncrd.Build{ + Priority: helpers.IntPtr(5), + }, + }, + }, + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-1234567", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:45:00.000Z")), + }, + Spec: lagooncrd.LagoonBuildSpec{ + Build: lagooncrd.Build{ + Priority: helpers.IntPtr(6), + }, + }, + }, + }, + }, + }, + wantBuilds: &lagooncrd.LagoonBuildList{ + Items: []lagooncrd.LagoonBuild{ + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-abcdefg", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:45:00.000Z")), + }, + Spec: lagooncrd.LagoonBuildSpec{ + Build: lagooncrd.Build{ + Priority: helpers.IntPtr(5), + }, + }, + }, + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-1234567", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:45:00.000Z")), + }, + Spec: lagooncrd.LagoonBuildSpec{ + Build: lagooncrd.Build{ + Priority: helpers.IntPtr(6), + }, + }, + }, + }, + }, + }, + { + name: "test2 - 2x5 sorted by time", + args: args{ + defaultPriority: 5, + pendingBuilds: &lagooncrd.LagoonBuildList{ + Items: []lagooncrd.LagoonBuild{ + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-abcdefg", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:50:00.000Z")), + }, + Spec: lagooncrd.LagoonBuildSpec{ + Build: lagooncrd.Build{ + Priority: helpers.IntPtr(5), + }, + }, + }, + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-1234567", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:45:00.000Z")), + }, + Spec: lagooncrd.LagoonBuildSpec{ + Build: lagooncrd.Build{ + Priority: helpers.IntPtr(5), + }, + }, + }, + }, + }, + }, + wantBuilds: &lagooncrd.LagoonBuildList{ + Items: []lagooncrd.LagoonBuild{ + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-1234567", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:45:00.000Z")), + }, + Spec: lagooncrd.LagoonBuildSpec{ + Build: lagooncrd.Build{ + Priority: helpers.IntPtr(5), + }, + }, + }, + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-abcdefg", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:50:00.000Z")), + }, + Spec: lagooncrd.LagoonBuildSpec{ + Build: lagooncrd.Build{ + Priority: helpers.IntPtr(5), + }, + }, + }, + }, + }, + }, + { + name: "test3 - 2x5 and 1x6 sorted by priority then time", + args: args{ + defaultPriority: 5, + pendingBuilds: &lagooncrd.LagoonBuildList{ + Items: []lagooncrd.LagoonBuild{ + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-abcdefg", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:50:00.000Z")), + }, + Spec: lagooncrd.LagoonBuildSpec{ + Build: lagooncrd.Build{ + Priority: helpers.IntPtr(5), + }, + }, + }, + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-abc1234", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:46:00.000Z")), + }, + Spec: lagooncrd.LagoonBuildSpec{ + Build: lagooncrd.Build{ + Priority: helpers.IntPtr(6), + }, + }, + }, + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-1234567", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:45:00.000Z")), + }, + Spec: lagooncrd.LagoonBuildSpec{ + Build: lagooncrd.Build{ + Priority: helpers.IntPtr(5), + }, + }, + }, + }, + }, + }, + wantBuilds: &lagooncrd.LagoonBuildList{ + Items: []lagooncrd.LagoonBuild{ + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-1234567", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:45:00.000Z")), + }, + Spec: lagooncrd.LagoonBuildSpec{ + Build: lagooncrd.Build{ + Priority: helpers.IntPtr(5), + }, + }, + }, + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-abcdefg", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:50:00.000Z")), + }, + Spec: lagooncrd.LagoonBuildSpec{ + Build: lagooncrd.Build{ + Priority: helpers.IntPtr(5), + }, + }, + }, + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-abc1234", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:46:00.000Z")), + }, + Spec: lagooncrd.LagoonBuildSpec{ + Build: lagooncrd.Build{ + Priority: helpers.IntPtr(6), + }, + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sortBuilds(tt.args.defaultPriority, tt.args.pendingBuilds) + if !cmp.Equal(tt.args.pendingBuilds, tt.wantBuilds) { + t.Errorf("sortBuilds() = %v, want %v", tt.args.pendingBuilds, tt.wantBuilds) + } + }) + } +} diff --git a/controllers/v1beta2/build_qoshandler.go b/controllers/v1beta2/build_qoshandler.go new file mode 100644 index 00000000..6cddf05c --- /dev/null +++ b/controllers/v1beta2/build_qoshandler.go @@ -0,0 +1,169 @@ +package v1beta2 + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/go-logr/logr" + lagooncrd "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" + "github.com/uselagoon/remote-controller/internal/helpers" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// BuildQoS is use for the quality of service configuration for lagoon builds. +type BuildQoS struct { + MaxBuilds int + DefaultValue int +} + +func (r *LagoonBuildReconciler) qosBuildProcessor(ctx context.Context, + opLog logr.Logger, + lagoonBuild lagooncrd.LagoonBuild, + req ctrl.Request) (ctrl.Result, error) { + // check if we get a lagoonbuild that hasn't got any buildstatus + // this means it was created by the message queue handler + // so we should do the steps required for a lagoon build and then copy the build + // into the created namespace + if _, ok := lagoonBuild.ObjectMeta.Labels["lagoon.sh/buildStatus"]; !ok { + if r.EnableDebug { + opLog.Info(fmt.Sprintf("Creating new build %s from message queue", lagoonBuild.ObjectMeta.Name)) + } + return r.createNamespaceBuild(ctx, opLog, lagoonBuild) + } + if r.EnableDebug { + opLog.Info(fmt.Sprintf("Checking which build next")) + } + // handle the QoS build process here + return ctrl.Result{}, r.whichBuildNext(ctx, opLog) +} + +func (r *LagoonBuildReconciler) whichBuildNext(ctx context.Context, opLog logr.Logger) error { + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.MatchingLabels(map[string]string{ + "lagoon.sh/buildStatus": lagooncrd.BuildStatusRunning.String(), + "lagoon.sh/controller": r.ControllerNamespace, + }), + }) + runningBuilds := &lagooncrd.LagoonBuildList{} + if err := r.List(ctx, runningBuilds, listOption); err != nil { + return fmt.Errorf("Unable to list builds in the cluster, there may be none or something went wrong: %v", err) + } + buildsToStart := r.BuildQoS.MaxBuilds - len(runningBuilds.Items) + if len(runningBuilds.Items) >= r.BuildQoS.MaxBuilds { + // if the maximum number of builds is hit, then drop out and try again next time + if r.EnableDebug { + opLog.Info(fmt.Sprintf("Currently %v running builds, no room for new builds to be started", len(runningBuilds.Items))) + } + go r.processQueue(ctx, opLog, buildsToStart, true) + return nil + } + if buildsToStart > 0 { + opLog.Info(fmt.Sprintf("Currently %v running builds, room for %v builds to be started", len(runningBuilds.Items), buildsToStart)) + // if there are any free slots to start a build, do that here + go r.processQueue(ctx, opLog, buildsToStart, false) + } + return nil +} + +var runningProcessQueue bool + +// this is a processor for any builds that are currently `queued` status. all normal build activity will still be performed +// this just allows the controller to update any builds that are in the queue periodically +// if this ran on every single event, it would flood the queue with messages, so it is restricted using `runningProcessQueue` global +// to only run the process at any one time til it is complete +// buildsToStart is the number of builds that can be started at the time the process is called +// limitHit is used to determine if the build limit has been hit, this is used to prevent new builds from being started inside this process +func (r *LagoonBuildReconciler) processQueue(ctx context.Context, opLog logr.Logger, buildsToStart int, limitHit bool) error { + // this should only ever be able to run one instance of at a time within a single controller + // this is because this process is quite heavy when it goes to submit the queue messages to the api + // the downside of this is that there can be delays with the messages it sends to the actual + // status of the builds, but build complete/fail/cancel will always win out on the lagoon-core side + // so this isn't that much of an issue if there are some delays in the messages + opLog = opLog.WithName("QueueProcessor") + if !runningProcessQueue { + runningProcessQueue = true + if r.EnableDebug { + opLog.Info(fmt.Sprintf("Processing queue")) + } + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.MatchingLabels(map[string]string{ + "lagoon.sh/buildStatus": lagooncrd.BuildStatusPending.String(), + "lagoon.sh/controller": r.ControllerNamespace, + }), + }) + pendingBuilds := &lagooncrd.LagoonBuildList{} + if err := r.List(ctx, pendingBuilds, listOption); err != nil { + runningProcessQueue = false + return fmt.Errorf("Unable to list builds in the cluster, there may be none or something went wrong: %v", err) + } + if len(pendingBuilds.Items) > 0 { + if r.EnableDebug { + opLog.Info(fmt.Sprintf("There are %v pending builds", len(pendingBuilds.Items))) + } + // if we have any pending builds, then grab the latest one and make it running + // if there are any other pending builds, cancel them so only the latest one runs + sortBuilds(r.BuildQoS.DefaultValue, pendingBuilds) + for idx, pBuild := range pendingBuilds.Items { + // need to +1 to index because 0 + if idx+1 <= buildsToStart && !limitHit { + if r.EnableDebug { + opLog.Info(fmt.Sprintf("Checking if build %s can be started", pBuild.ObjectMeta.Name)) + } + // if we do have a `lagoon.sh/buildStatus` set, then process as normal + runningNSBuilds := &lagooncrd.LagoonBuildList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(pBuild.ObjectMeta.Namespace), + client.MatchingLabels(map[string]string{ + "lagoon.sh/buildStatus": lagooncrd.BuildStatusRunning.String(), + "lagoon.sh/controller": r.ControllerNamespace, + }), + }) + // list any builds that are running + if err := r.List(ctx, runningNSBuilds, listOption); err != nil { + runningProcessQueue = false + return fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err) + } + // if there are no running builds, check if there are any pending builds that can be started + if len(runningNSBuilds.Items) == 0 { + if err := lagooncrd.CancelExtraBuilds(ctx, r.Client, opLog, pBuild.ObjectMeta.Namespace, "Running"); err != nil { + // only return if there is an error doing this operation + // continue on otherwise to allow the queued status updater to run + runningProcessQueue = false + return err + } + // don't handle the queued process for this build, continue to next in the list + continue + } + // The object is not being deleted, so if it does not have our finalizer, + // then lets add the finalizer and update the object. This is equivalent + // registering our finalizer. + if !helpers.ContainsString(pBuild.ObjectMeta.Finalizers, buildFinalizer) { + pBuild.ObjectMeta.Finalizers = append(pBuild.ObjectMeta.Finalizers, buildFinalizer) + // use patches to avoid update errors + mergePatch, _ := json.Marshal(map[string]interface{}{ + "metadata": map[string]interface{}{ + "finalizers": pBuild.ObjectMeta.Finalizers, + }, + }) + if err := r.Patch(ctx, &pBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { + runningProcessQueue = false + return err + } + } + } + // update the build to be queued, and add a log message with the build log with the current position in the queue + // this position will update as builds are created/processed, so the position of a build could change depending on + // higher or lower priority builds being created + if err := r.updateQueuedBuild(ctx, pBuild, (idx + 1), len(pendingBuilds.Items), opLog); err != nil { + runningProcessQueue = false + return nil + } + } + } + runningProcessQueue = false + } + return nil +} diff --git a/controllers/v1beta2/build_standardhandler.go b/controllers/v1beta2/build_standardhandler.go new file mode 100644 index 00000000..5315fd89 --- /dev/null +++ b/controllers/v1beta2/build_standardhandler.go @@ -0,0 +1,73 @@ +package v1beta2 + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/go-logr/logr" + lagooncrd "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" + "github.com/uselagoon/remote-controller/internal/helpers" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func (r *LagoonBuildReconciler) standardBuildProcessor(ctx context.Context, + opLog logr.Logger, + lagoonBuild lagooncrd.LagoonBuild, + req ctrl.Request) (ctrl.Result, error) { + // check if we get a lagoonbuild that hasn't got any buildstatus + // this means it was created by the message queue handler + // so we should do the steps required for a lagoon build and then copy the build + // into the created namespace + if _, ok := lagoonBuild.ObjectMeta.Labels["lagoon.sh/buildStatus"]; !ok { + return r.createNamespaceBuild(ctx, opLog, lagoonBuild) + } + + // if we do have a `lagoon.sh/buildStatus` set, then process as normal + runningBuilds := &lagooncrd.LagoonBuildList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(req.Namespace), + client.MatchingLabels(map[string]string{ + "lagoon.sh/buildStatus": lagooncrd.BuildStatusRunning.String(), + "lagoon.sh/controller": r.ControllerNamespace, + }), + }) + // list any builds that are running + if err := r.List(ctx, runningBuilds, listOption); err != nil { + return ctrl.Result{}, fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err) + } + for _, runningBuild := range runningBuilds.Items { + // if the running build is the one from this request then process it + if lagoonBuild.ObjectMeta.Name == runningBuild.ObjectMeta.Name { + // actually process the build here + if _, ok := lagoonBuild.ObjectMeta.Labels["lagoon.sh/buildStarted"]; !ok { + if err := r.processBuild(ctx, opLog, lagoonBuild); err != nil { + return ctrl.Result{}, err + } + } + } // end check if running build is current LagoonBuild + } // end loop for running builds + + // if there are no running builds, check if there are any pending builds that can be started + if len(runningBuilds.Items) == 0 { + return ctrl.Result{}, lagooncrd.CancelExtraBuilds(ctx, r.Client, opLog, req.Namespace, "Running") + } + // The object is not being deleted, so if it does not have our finalizer, + // then lets add the finalizer and update the object. This is equivalent + // registering our finalizer. + if !helpers.ContainsString(lagoonBuild.ObjectMeta.Finalizers, buildFinalizer) { + lagoonBuild.ObjectMeta.Finalizers = append(lagoonBuild.ObjectMeta.Finalizers, buildFinalizer) + // use patches to avoid update errors + mergePatch, _ := json.Marshal(map[string]interface{}{ + "metadata": map[string]interface{}{ + "finalizers": lagoonBuild.ObjectMeta.Finalizers, + }, + }) + if err := r.Patch(ctx, &lagoonBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { + return ctrl.Result{}, err + } + } + return ctrl.Result{}, nil +} diff --git a/controllers/v1beta2/metrics.go b/controllers/v1beta2/metrics.go new file mode 100644 index 00000000..900ca39b --- /dev/null +++ b/controllers/v1beta2/metrics.go @@ -0,0 +1,92 @@ +package v1beta2 + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var ( + // general counters for builds + buildsRunningGauge = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "lagoon_builds_running_current", + Help: "The total number of Lagoon builds running", + }) + buildsPendingGauge = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "lagoon_builds_pending_current", + Help: "The total number of Lagoon builds pending", + }) + buildsStartedCounter = promauto.NewCounter(prometheus.CounterOpts{ + Name: "lagoon_builds_started_total", + Help: "The total number of Lagoon builds started", + }) + buildsCompletedCounter = promauto.NewCounter(prometheus.CounterOpts{ + Name: "lagoon_builds_completed_total", + Help: "The total number of Lagoon builds completed", + }) + buildsFailedCounter = promauto.NewCounter(prometheus.CounterOpts{ + Name: "lagoon_builds_failed_total", + Help: "The total number of Lagoon builds failed", + }) + buildsCancelledCounter = promauto.NewCounter(prometheus.CounterOpts{ + Name: "lagoon_builds_cancelled_total", + Help: "The total number of Lagoon builds cancelled", + }) + + // general counters for tasks + tasksRunningGauge = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "lagoon_tasks_running_current", + Help: "The total number of Lagoon tasks running", + }) + tasksStartedCounter = promauto.NewCounter(prometheus.CounterOpts{ + Name: "lagoon_tasks_started_total", + Help: "The total number of Lagoon tasks started", + }) + tasksCompletedCounter = promauto.NewCounter(prometheus.CounterOpts{ + Name: "lagoon_tasks_completed_total", + Help: "The total number of Lagoon tasks completed", + }) + tasksFailedCounter = promauto.NewCounter(prometheus.CounterOpts{ + Name: "lagoon_tasks_failed_total", + Help: "The total number of Lagoon tasks failed", + }) + tasksCancelledCounter = promauto.NewCounter(prometheus.CounterOpts{ + Name: "lagoon_tasks_cancelled_total", + Help: "The total number of Lagoon tasks cancelled", + }) + + // buildStatus will count the build transisiton steps + // when the build step changes, the count is removed and the new step metric is created + // this is useful to gauge how long particular steps take in a build + buildStatus = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "lagoon_build_status", + Help: "The status of running Lagoon builds", + }, + []string{ + "build_name", + "build_namespace", + "build_step", + }, + ) + + // RunningStatus will count when a build or task is running + // when the build or task is complete, the count is removed + // this is useful to gauge how long a build or task runs for + buildRunningStatus = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "lagoon_build_running_status", + Help: "The duration of running Lagoon builds", + }, + []string{ + "build_name", + "build_namespace", + }, + ) + taskRunningStatus = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "lagoon_task_running_status", + Help: "The duration of running Lagoon tasks", + }, + []string{ + "task_name", + "task_namespace", + }, + ) +) diff --git a/controllers/v1beta2/podmonitor_buildhandlers.go b/controllers/v1beta2/podmonitor_buildhandlers.go new file mode 100644 index 00000000..0bacdc78 --- /dev/null +++ b/controllers/v1beta2/podmonitor_buildhandlers.go @@ -0,0 +1,623 @@ +package v1beta2 + +// this file is used by the `lagoonmonitor` controller + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "strings" + "time" + + "github.com/go-logr/logr" + "github.com/prometheus/client_golang/prometheus" + "github.com/uselagoon/machinery/api/schema" + lagooncrd "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" + "github.com/uselagoon/remote-controller/internal/helpers" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func (r *LagoonMonitorReconciler) handleBuildMonitor(ctx context.Context, + opLog logr.Logger, + req ctrl.Request, + jobPod corev1.Pod, +) error { + // get the build associated to this pod, we wil need update it at some point + var lagoonBuild lagooncrd.LagoonBuild + err := r.Get(ctx, types.NamespacedName{ + Namespace: jobPod.ObjectMeta.Namespace, + Name: jobPod.ObjectMeta.Labels["lagoon.sh/buildName"], + }, &lagoonBuild) + if err != nil { + return err + } + cancel := false + if cancelBuild, ok := jobPod.ObjectMeta.Labels["lagoon.sh/cancelBuild"]; ok { + cancel, _ = strconv.ParseBool(cancelBuild) + } + _, ok := r.Cache.Get(lagoonBuild.ObjectMeta.Name) + if ok { + opLog.Info(fmt.Sprintf("Cached cancellation exists for: %s", lagoonBuild.ObjectMeta.Name)) + // this object exists in the cache meaning the task has been cancelled, set cancel to true and remove from cache + r.Cache.Remove(lagoonBuild.ObjectMeta.Name) + cancel = true + } + if cancel { + opLog.Info(fmt.Sprintf("Attempting to cancel build %s", lagoonBuild.ObjectMeta.Name)) + return r.updateDeploymentWithLogs(ctx, req, lagoonBuild, jobPod, nil, cancel) + } + // check if the build pod is in pending, a container in the pod could be failed in this state + if jobPod.Status.Phase == corev1.PodPending { + // check each container in the pod + for _, container := range jobPod.Status.ContainerStatuses { + // if the container is a lagoon-build container + // which currently it will be as only one container is spawned in a build + if container.Name == "lagoon-build" { + // check if the state of the pod is one of our failure states + if container.State.Waiting != nil && helpers.ContainsString(failureStates, container.State.Waiting.Reason) { + // if we have a failure state, then fail the build and get the logs from the container + opLog.Info(fmt.Sprintf("Build failed, container exit reason was: %v", container.State.Waiting.Reason)) + lagoonBuild.Labels["lagoon.sh/buildStatus"] = lagooncrd.BuildStatusFailed.String() + if err := r.Update(ctx, &lagoonBuild); err != nil { + return err + } + opLog.Info(fmt.Sprintf("Marked build %s as %s", lagoonBuild.ObjectMeta.Name, lagooncrd.BuildStatusFailed.String())) + if err := r.Delete(ctx, &jobPod); err != nil { + return err + } + opLog.Info(fmt.Sprintf("Deleted failed build pod: %s", jobPod.ObjectMeta.Name)) + // update the status to failed on the deleted pod + // and set the terminate time to now, it is used when we update the deployment and environment + jobPod.Status.Phase = corev1.PodFailed + state := corev1.ContainerStatus{ + State: corev1.ContainerState{ + Terminated: &corev1.ContainerStateTerminated{ + FinishedAt: metav1.Time{Time: time.Now().UTC()}, + }, + }, + } + jobPod.Status.ContainerStatuses[0] = state + + // get the configmap for lagoon-env so we can use it for updating the deployment in lagoon + var lagoonEnv corev1.ConfigMap + err := r.Get(ctx, types.NamespacedName{Namespace: jobPod.ObjectMeta.Namespace, Name: "lagoon-env"}, &lagoonEnv) + if err != nil { + // if there isn't a configmap, just info it and move on + // the updatedeployment function will see it as nil and not bother doing the bits that require the configmap + if r.EnableDebug { + opLog.Info(fmt.Sprintf("There is no configmap %s in namespace %s ", "lagoon-env", jobPod.ObjectMeta.Namespace)) + } + } + // send any messages to lagoon message queues + logMsg := fmt.Sprintf("%v: %v", container.State.Waiting.Reason, container.State.Waiting.Message) + return r.updateDeploymentWithLogs(ctx, req, lagoonBuild, jobPod, []byte(logMsg), false) + } + } + } + return r.updateDeploymentWithLogs(ctx, req, lagoonBuild, jobPod, nil, false) + } else if jobPod.Status.Phase == corev1.PodRunning { + // if the pod is running and detects a change to the pod (eg, detecting an updated lagoon.sh/buildStep label) + // then ship or store the logs + // get the build associated to this pod, the information in the resource is used for shipping the logs + var lagoonBuild lagooncrd.LagoonBuild + err := r.Get(ctx, + types.NamespacedName{ + Namespace: jobPod.ObjectMeta.Namespace, + Name: jobPod.ObjectMeta.Labels["lagoon.sh/buildName"], + }, &lagoonBuild) + if err != nil { + return err + } + } + // if the buildpod status is failed or succeeded + // mark the build accordingly and ship the information back to lagoon + if jobPod.Status.Phase == corev1.PodFailed || jobPod.Status.Phase == corev1.PodSucceeded { + // get the build associated to this pod, we wil need update it at some point + var lagoonBuild lagooncrd.LagoonBuild + err := r.Get(ctx, + types.NamespacedName{ + Namespace: jobPod.ObjectMeta.Namespace, + Name: jobPod.ObjectMeta.Labels["lagoon.sh/buildName"], + }, &lagoonBuild) + if err != nil { + return err + } + } + // send any messages to lagoon message queues + return r.updateDeploymentWithLogs(ctx, req, lagoonBuild, jobPod, nil, false) +} + +// buildLogsToLagoonLogs sends the build logs to the lagoon-logs message queue +// it contains the actual pod log output that is sent to elasticsearch, it is what eventually is displayed in the UI +func (r *LagoonMonitorReconciler) buildLogsToLagoonLogs(ctx context.Context, + opLog logr.Logger, + lagoonBuild *lagooncrd.LagoonBuild, + jobPod *corev1.Pod, + namespace *corev1.Namespace, + condition string, + logs []byte, +) (bool, schema.LagoonLog) { + if r.EnableMQ { + buildStep := "running" + if condition == "failed" || condition == "complete" || condition == "cancelled" { + // set build step to anything other than running if the condition isn't running + buildStep = condition + } + // then check the resource to see if the buildstep exists, this bit we care about so we can see where it maybe failed if its available + if value, ok := jobPod.Labels["lagoon.sh/buildStep"]; ok { + buildStep = value + } + envName := lagoonBuild.Spec.Project.Environment + envID := lagoonBuild.Spec.Project.EnvironmentID + projectName := lagoonBuild.Spec.Project.Name + projectID := lagoonBuild.Spec.Project.ID + if lagoonBuild == nil { + envName = namespace.ObjectMeta.Labels["lagoon.sh/environment"] + eID, _ := strconv.Atoi(namespace.ObjectMeta.Labels["lagoon.sh/environment"]) + envID = helpers.UintPtr(uint(eID)) + projectName = namespace.ObjectMeta.Labels["lagoon.sh/environment"] + pID, _ := strconv.Atoi(namespace.ObjectMeta.Labels["lagoon.sh/environment"]) + projectID = helpers.UintPtr(uint(pID)) + } + remoteId := string(jobPod.ObjectMeta.UID) + if value, ok := jobPod.Labels["lagoon.sh/buildRemoteID"]; ok { + remoteId = value + } + msg := schema.LagoonLog{ + Severity: "info", + Project: projectName, + Event: "build-logs:builddeploy-kubernetes:" + jobPod.ObjectMeta.Name, + Meta: &schema.LagoonLogMeta{ + EnvironmentID: envID, + ProjectID: projectID, + BuildName: jobPod.ObjectMeta.Name, + BranchName: envName, + BuildStatus: condition, // same as buildstatus label + BuildStep: buildStep, + RemoteID: remoteId, + LogLink: lagoonBuild.Spec.Project.UILink, + Cluster: r.LagoonTargetName, + }, + } + // add the actual build log message + if jobPod.Spec.NodeName != "" { + msg.Message = fmt.Sprintf(`================================================================================ +Logs on pod %s, assigned to node %s on cluster %s +================================================================================ +%s`, jobPod.ObjectMeta.Name, jobPod.Spec.NodeName, r.LagoonTargetName, logs) + } else { + msg.Message = fmt.Sprintf(`======================================== +Logs on pod %s, assigned to cluster %s +======================================== +%s`, jobPod.ObjectMeta.Name, r.LagoonTargetName, logs) + } + msgBytes, err := json.Marshal(msg) + if err != nil { + opLog.Error(err, "Unable to encode message as JSON") + } + if err := r.Messaging.Publish("lagoon-logs", msgBytes); err != nil { + // if we can't publish the message, set it as a pending message + // overwrite whatever is there as these are just current state messages so it doesn't + // really matter if we don't smootly transition in what we send back to lagoon + // r.updateBuildLogMessage(ctx, lagoonBuild, msg) + return true, msg + } + if r.EnableDebug { + opLog.Info( + fmt.Sprintf( + "Published event %s for %s to lagoon-logs exchange", + fmt.Sprintf("build-logs:builddeploy-kubernetes:%s", jobPod.ObjectMeta.Name), + jobPod.ObjectMeta.Name, + ), + ) + } + // if we are able to publish the message, then we need to remove any pending messages from the resource + // and make sure we don't try and publish again + } + return false, schema.LagoonLog{} +} + +// updateDeploymentAndEnvironmentTask sends the status of the build and deployment to the controllerhandler message queue in lagoon, +// this is for the handler in lagoon to process. +func (r *LagoonMonitorReconciler) updateDeploymentAndEnvironmentTask(ctx context.Context, + opLog logr.Logger, + lagoonBuild *lagooncrd.LagoonBuild, + jobPod *corev1.Pod, + lagoonEnv *corev1.ConfigMap, + namespace *corev1.Namespace, + condition string, +) (bool, schema.LagoonMessage) { + if r.EnableMQ { + buildStep := "running" + if condition == "failed" || condition == "complete" || condition == "cancelled" { + // set build step to anything other than running if the condition isn't running + buildStep = condition + } + // then check the resource to see if the buildstep exists, this bit we care about so we can see where it maybe failed if its available + if value, ok := jobPod.Labels["lagoon.sh/buildStep"]; ok { + buildStep = value + } + if condition == "failed" || condition == "complete" || condition == "cancelled" { + time.AfterFunc(31*time.Second, func() { + buildRunningStatus.Delete(prometheus.Labels{ + "build_namespace": lagoonBuild.ObjectMeta.Namespace, + "build_name": lagoonBuild.ObjectMeta.Name, + }) + }) + time.Sleep(2 * time.Second) // smol sleep to reduce race of final messages with previous messages + } + envName := lagoonBuild.Spec.Project.Environment + envID := lagoonBuild.Spec.Project.EnvironmentID + projectName := lagoonBuild.Spec.Project.Name + projectID := lagoonBuild.Spec.Project.ID + if lagoonBuild == nil { + envName = namespace.ObjectMeta.Labels["lagoon.sh/environment"] + eID, _ := strconv.Atoi(namespace.ObjectMeta.Labels["lagoon.sh/environment"]) + envID = helpers.UintPtr(uint(eID)) + projectName = namespace.ObjectMeta.Labels["lagoon.sh/environment"] + pID, _ := strconv.Atoi(namespace.ObjectMeta.Labels["lagoon.sh/environment"]) + projectID = helpers.UintPtr(uint(pID)) + } + remoteId := string(jobPod.ObjectMeta.UID) + if value, ok := jobPod.Labels["lagoon.sh/buildRemoteID"]; ok { + remoteId = value + } + msg := schema.LagoonMessage{ + Type: "build", + Namespace: namespace.ObjectMeta.Name, + Meta: &schema.LagoonLogMeta{ + Environment: envName, + EnvironmentID: envID, + Project: projectName, + ProjectID: projectID, + BuildName: jobPod.ObjectMeta.Name, + BuildStatus: condition, // same as buildstatus label + BuildStep: buildStep, + LogLink: lagoonBuild.Spec.Project.UILink, + RemoteID: remoteId, + Cluster: r.LagoonTargetName, + }, + } + labelRequirements1, _ := labels.NewRequirement("lagoon.sh/service", selection.NotIn, []string{"faketest"}) + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(jobPod.ObjectMeta.Namespace), + client.MatchingLabelsSelector{ + Selector: labels.NewSelector().Add(*labelRequirements1), + }, + }) + depList := &appsv1.DeploymentList{} + serviceNames := []string{} + if err := r.List(context.TODO(), depList, listOption); err == nil { + // generate the list of services to add to the environment + for _, deployment := range depList.Items { + if _, ok := deployment.ObjectMeta.Labels["lagoon.sh/service"]; ok { + for _, container := range deployment.Spec.Template.Spec.Containers { + serviceNames = append(serviceNames, container.Name) + } + } + if _, ok := deployment.ObjectMeta.Labels["service"]; ok { + for _, container := range deployment.Spec.Template.Spec.Containers { + serviceNames = append(serviceNames, container.Name) + } + } + } + msg.Meta.Services = serviceNames + } + // if we aren't being provided the lagoon config, we can skip adding the routes etc + if lagoonEnv != nil { + msg.Meta.Route = "" + if route, ok := lagoonEnv.Data["LAGOON_ROUTE"]; ok { + msg.Meta.Route = route + } + msg.Meta.Routes = []string{} + if routes, ok := lagoonEnv.Data["LAGOON_ROUTES"]; ok { + msg.Meta.Routes = strings.Split(routes, ",") + } + } + // we can add the build start time here + if jobPod.Status.StartTime != nil { + msg.Meta.StartTime = jobPod.Status.StartTime.Time.UTC().Format("2006-01-02 15:04:05") + } + if condition == "cancelled" { + // if the build has been canclled, the pod termination time may not exist yet. + // use the current time first, it will get overwritten if there is a pod termination time later. + msg.Meta.EndTime = time.Now().UTC().Format("2006-01-02 15:04:05") + } + // and then once the pod is terminated we can add the terminated time here + if jobPod.Status.ContainerStatuses != nil { + if jobPod.Status.ContainerStatuses[0].State.Terminated != nil { + msg.Meta.EndTime = jobPod.Status.ContainerStatuses[0].State.Terminated.FinishedAt.Time.UTC().Format("2006-01-02 15:04:05") + } + } + msgBytes, err := json.Marshal(msg) + if err != nil { + opLog.Error(err, "Unable to encode message as JSON") + } + if err := r.Messaging.Publish("lagoon-tasks:controller", msgBytes); err != nil { + // if we can't publish the message, set it as a pending message + // overwrite whatever is there as these are just current state messages so it doesn't + // really matter if we don't smootly transition in what we send back to lagoon + return true, msg + } + if r.EnableDebug { + opLog.Info( + fmt.Sprintf( + "Published build update message for %s to lagoon-tasks:controller queue", + jobPod.ObjectMeta.Name, + ), + ) + } + // if we are able to publish the message, then we need to remove any pending messages from the resource + // and make sure we don't try and publish again + } + return false, schema.LagoonMessage{} +} + +// buildStatusLogsToLagoonLogs sends the logs to lagoon-logs message queue, used for general messaging +func (r *LagoonMonitorReconciler) buildStatusLogsToLagoonLogs(ctx context.Context, + opLog logr.Logger, + lagoonBuild *lagooncrd.LagoonBuild, + jobPod *corev1.Pod, + lagoonEnv *corev1.ConfigMap, + namespace *corev1.Namespace, + condition string, +) (bool, schema.LagoonLog) { + if r.EnableMQ { + buildStep := "running" + if condition == "failed" || condition == "complete" || condition == "cancelled" { + // set build step to anything other than running if the condition isn't running + buildStep = condition + } + // then check the resource to see if the buildstep exists, this bit we care about so we can see where it maybe failed if its available + if value, ok := jobPod.Labels["lagoon.sh/buildStep"]; ok { + buildStep = value + } + envName := lagoonBuild.Spec.Project.Environment + envID := lagoonBuild.Spec.Project.EnvironmentID + projectName := lagoonBuild.Spec.Project.Name + projectID := lagoonBuild.Spec.Project.ID + if lagoonBuild == nil { + envName = namespace.ObjectMeta.Labels["lagoon.sh/environment"] + eID, _ := strconv.Atoi(namespace.ObjectMeta.Labels["lagoon.sh/environment"]) + envID = helpers.UintPtr(uint(eID)) + projectName = namespace.ObjectMeta.Labels["lagoon.sh/environment"] + pID, _ := strconv.Atoi(namespace.ObjectMeta.Labels["lagoon.sh/environment"]) + projectID = helpers.UintPtr(uint(pID)) + } + msg := schema.LagoonLog{ + Severity: "info", + Project: projectName, + Event: "task:builddeploy-kubernetes:" + condition, //@TODO: this probably needs to be changed to a new task event for the controller + Meta: &schema.LagoonLogMeta{ + EnvironmentID: envID, + ProjectID: projectID, + ProjectName: projectName, + BranchName: envName, + BuildName: jobPod.ObjectMeta.Name, + BuildStatus: condition, // same as buildstatus label + BuildStep: buildStep, + LogLink: lagoonBuild.Spec.Project.UILink, + Cluster: r.LagoonTargetName, + }, + } + // if we aren't being provided the lagoon config, we can skip adding the routes etc + var addRoute, addRoutes string + if lagoonEnv != nil { + msg.Meta.Route = "" + if route, ok := lagoonEnv.Data["LAGOON_ROUTE"]; ok { + msg.Meta.Route = route + addRoute = fmt.Sprintf("\n%s", route) + } + msg.Meta.Routes = []string{} + if routes, ok := lagoonEnv.Data["LAGOON_ROUTES"]; ok { + msg.Meta.Routes = strings.Split(routes, ",") + addRoutes = fmt.Sprintf("\n%s", strings.Join(strings.Split(routes, ","), "\n")) + } + } + msg.Message = fmt.Sprintf("*[%s]* `%s` Build `%s` %s <%s|Logs>%s%s", + projectName, + envName, + jobPod.ObjectMeta.Name, + string(jobPod.Status.Phase), + lagoonBuild.Spec.Project.UILink, + addRoute, + addRoutes, + ) + msgBytes, err := json.Marshal(msg) + if err != nil { + opLog.Error(err, "Unable to encode message as JSON") + } + if err := r.Messaging.Publish("lagoon-logs", msgBytes); err != nil { + // if we can't publish the message, set it as a pending message + // overwrite whatever is there as these are just current state messages so it doesn't + // really matter if we don't smootly transition in what we send back to lagoon + return true, msg + } + if r.EnableDebug { + opLog.Info( + fmt.Sprintf( + "Published event %s for %s to lagoon-logs exchange", + fmt.Sprintf("task:builddeploy-kubernetes:%s", condition), + jobPod.ObjectMeta.Name, + ), + ) + } + // if we are able to publish the message, then we need to remove any pending messages from the resource + // and make sure we don't try and publish again + } + return false, schema.LagoonLog{} +} + +// updateDeploymentWithLogs collects logs from the build containers and ships or stores them +func (r *LagoonMonitorReconciler) updateDeploymentWithLogs( + ctx context.Context, + req ctrl.Request, + lagoonBuild lagooncrd.LagoonBuild, + jobPod corev1.Pod, + logs []byte, + cancel bool, +) error { + opLog := r.Log.WithValues("lagoonmonitor", req.NamespacedName) + buildCondition := lagooncrd.GetBuildConditionFromPod(jobPod.Status.Phase) + collectLogs := true + if cancel { + // only set the status to cancelled if the pod is running/pending/queued + // otherwise send the existing status of complete/failed/cancelled + if helpers.ContainsString( + lagooncrd.BuildRunningPendingStatus, + lagoonBuild.Labels["lagoon.sh/buildStatus"], + ) { + buildCondition = lagooncrd.BuildStatusCancelled + } + if _, ok := lagoonBuild.ObjectMeta.Labels["lagoon.sh/cancelBuildNoPod"]; ok { + collectLogs = false + } + } + buildStep := "running" + if value, ok := jobPod.Labels["lagoon.sh/buildStep"]; ok { + buildStep = value + } + + namespace := &corev1.Namespace{} + if err := r.Get(ctx, types.NamespacedName{Name: jobPod.ObjectMeta.Namespace}, namespace); err != nil { + if helpers.IgnoreNotFound(err) != nil { + return err + } + } + // if the buildstatus is pending or running, or the cancel flag is provided + // send the update status to lagoon + if helpers.ContainsString( + lagooncrd.BuildRunningPendingStatus, + lagoonBuild.Labels["lagoon.sh/buildStatus"], + ) || cancel { + opLog.Info( + fmt.Sprintf( + "Updating build status for %s to %v/%v", + jobPod.ObjectMeta.Labels["lagoon.sh/buildName"], + buildCondition, + buildStep, + ), + ) + var allContainerLogs []byte + var err error + if logs == nil { + if collectLogs { + allContainerLogs, err = r.collectLogs(ctx, req, jobPod) + if err == nil { + if cancel { + cancellationMessage := "Build cancelled" + if cancellationDetails, ok := jobPod.GetAnnotations()["lagoon.sh/cancelReason"]; ok { + cancellationMessage = fmt.Sprintf("%v : %v", cancellationMessage, cancellationDetails) + } + allContainerLogs = append(allContainerLogs, []byte(fmt.Sprintf(` +======================================== +%v +========================================`, cancellationMessage))...) + } + } else { + allContainerLogs = []byte(fmt.Sprintf(` +======================================== +Build %s +========================================`, buildCondition)) + } + } + } else { + allContainerLogs = logs + } + + mergeMap := map[string]interface{}{ + "metadata": map[string]interface{}{ + "labels": map[string]interface{}{ + "lagoon.sh/buildStatus": buildCondition.String(), + "lagoon.sh/buildStarted": "true", + }, + }, + "statusMessages": map[string]interface{}{}, + } + + condition := lagooncrd.LagoonBuildConditions{ + Type: buildCondition, + Status: corev1.ConditionTrue, + LastTransitionTime: time.Now().UTC().Format(time.RFC3339), + } + if !lagooncrd.BuildContainsStatus(lagoonBuild.Status.Conditions, condition) { + lagoonBuild.Status.Conditions = append(lagoonBuild.Status.Conditions, condition) + mergeMap["status"] = map[string]interface{}{ + "conditions": lagoonBuild.Status.Conditions, + // don't save build logs in resource anymore + } + } + + // get the configmap for lagoon-env so we can use it for updating the deployment in lagoon + var lagoonEnv corev1.ConfigMap + if err := r.Get(ctx, types.NamespacedName{ + Namespace: jobPod.ObjectMeta.Namespace, + Name: "lagoon-env", + }, + &lagoonEnv, + ); err != nil { + // if there isn't a configmap, just info it and move on + // the updatedeployment function will see it as nil and not bother doing the bits that require the configmap + if r.EnableDebug { + opLog.Info(fmt.Sprintf("There is no configmap %s in namespace %s ", "lagoon-env", jobPod.ObjectMeta.Namespace)) + } + } + + // do any message publishing here, and update any pending messages if needed + pendingStatus, pendingStatusMessage := r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &jobPod, &lagoonEnv, namespace, buildCondition.ToLower()) + pendingEnvironment, pendingEnvironmentMessage := r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &jobPod, &lagoonEnv, namespace, buildCondition.ToLower()) + var pendingBuildLog bool + var pendingBuildLogMessage schema.LagoonLog + // if the container logs can't be retrieved, we don't want to send any build logs back, as this will nuke + // any previously received logs + if !strings.Contains(string(allContainerLogs), "unable to retrieve container logs for containerd") { + pendingBuildLog, pendingBuildLogMessage = r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &jobPod, namespace, buildCondition.ToLower(), allContainerLogs) + } + if pendingStatus || pendingEnvironment || pendingBuildLog { + mergeMap["metadata"].(map[string]interface{})["labels"].(map[string]interface{})["lagoon.sh/pendingMessages"] = "true" + if pendingStatus { + mergeMap["statusMessages"].(map[string]interface{})["statusMessage"] = pendingStatusMessage + } + if pendingEnvironment { + mergeMap["statusMessages"].(map[string]interface{})["environmentMessage"] = pendingEnvironmentMessage + } + // if the build log message is too long, don't save it + if pendingBuildLog && len(pendingBuildLogMessage.Message) > 1048576 { + mergeMap["statusMessages"].(map[string]interface{})["buildLogMessage"] = pendingBuildLogMessage + } + } + if !pendingStatus && !pendingEnvironment && !pendingBuildLog { + mergeMap["metadata"].(map[string]interface{})["labels"].(map[string]interface{})["lagoon.sh/pendingMessages"] = nil + mergeMap["statusMessages"] = nil + } + mergePatch, _ := json.Marshal(mergeMap) + // check if the build exists + if err := r.Get(ctx, req.NamespacedName, &lagoonBuild); err == nil { + // if it does, try to patch it + if err := r.Patch(ctx, &lagoonBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to update resource")) + } + } + // just delete the pod + // maybe if we move away from using BASH for the kubectl-build-deploy-dind scripts we could handle cancellations better + if cancel { + if err := r.Get(ctx, req.NamespacedName, &jobPod); err == nil { + if r.EnableDebug { + opLog.Info(fmt.Sprintf("Build pod exists %s", jobPod.ObjectMeta.Name)) + } + if err := r.Delete(ctx, &jobPod); err != nil { + return err + } + } + } + } + return nil +} diff --git a/controllers/v1beta2/podmonitor_controller.go b/controllers/v1beta2/podmonitor_controller.go new file mode 100644 index 00000000..9ae9f340 --- /dev/null +++ b/controllers/v1beta2/podmonitor_controller.go @@ -0,0 +1,225 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta2 + +import ( + "bytes" + "context" + "fmt" + "io" + "strconv" + + "github.com/go-logr/logr" + "github.com/hashicorp/golang-lru/v2/expirable" + lagooncrd "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" + "github.com/uselagoon/remote-controller/internal/helpers" + "github.com/uselagoon/remote-controller/internal/messenger" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/config" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +// LagoonMonitorReconciler reconciles a LagoonBuild object +type LagoonMonitorReconciler struct { + client.Client + Log logr.Logger + Scheme *runtime.Scheme + EnableMQ bool + Messaging *messenger.Messenger + ControllerNamespace string + NamespacePrefix string + RandomNamespacePrefix bool + EnableDebug bool + LagoonTargetName string + LFFQoSEnabled bool + BuildQoS BuildQoS + Cache *expirable.LRU[string, string] +} + +// slice of the different failure states of pods that we care about +// if we observe these on a pending pod, fail the build and get the logs +var failureStates = []string{ + "CrashLoopBackOff", + "ImagePullBackOff", +} + +// @TODO: all the things for now, review later +// +kubebuilder:rbac:groups="*",resources="*",verbs="*" + +// Reconcile runs when a request comes through +func (r *LagoonMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + opLog := r.Log.WithValues("lagoonmonitor", req.NamespacedName) + + var jobPod corev1.Pod + if err := r.Get(ctx, req.NamespacedName, &jobPod); err != nil { + return ctrl.Result{}, helpers.IgnoreNotFound(err) + } + + // if this is a lagoon task, then run the handle task monitoring process + if jobPod.ObjectMeta.Labels["lagoon.sh/jobType"] == "task" { + err := r.calculateTaskMetrics(ctx) + if err != nil { + opLog.Error(err, fmt.Sprintf("Unable to generate metrics.")) + } + if jobPod.ObjectMeta.DeletionTimestamp.IsZero() { + // pod is not being deleted + return ctrl.Result{}, r.handleTaskMonitor(ctx, opLog, req, jobPod) + } + // pod deletion request came through, check if this is an activestandby task, if it is, delete the activestandby role + if value, ok := jobPod.ObjectMeta.Labels["lagoon.sh/activeStandby"]; ok { + isActiveStandby, _ := strconv.ParseBool(value) + if isActiveStandby { + var destinationNamespace string + if value, ok := jobPod.ObjectMeta.Labels["lagoon.sh/activeStandbyDestinationNamespace"]; ok { + destinationNamespace = value + } + err := r.deleteActiveStandbyRole(ctx, destinationNamespace) + if err != nil { + return ctrl.Result{}, err + } + } + } + } + // if this is a lagoon build, then run the handle build monitoring process + if jobPod.ObjectMeta.Labels["lagoon.sh/jobType"] == "build" { + err := r.calculateBuildMetrics(ctx) + if err != nil { + opLog.Error(err, fmt.Sprintf("Unable to generate metrics.")) + } + if jobPod.ObjectMeta.DeletionTimestamp.IsZero() { + // pod is not being deleted + return ctrl.Result{}, r.handleBuildMonitor(ctx, opLog, req, jobPod) + } + + // a pod deletion request came through + // first try and clean up the pod and capture the logs and update + // the lagoonbuild that owns it with the status + var lagoonBuild lagooncrd.LagoonBuild + err = r.Get(ctx, types.NamespacedName{ + Namespace: jobPod.ObjectMeta.Namespace, + Name: jobPod.ObjectMeta.Labels["lagoon.sh/buildName"], + }, &lagoonBuild) + if err != nil { + opLog.Info(fmt.Sprintf("The build that started this pod may have been deleted or not started yet, continuing with cancellation if required.")) + err = r.updateDeploymentWithLogs(ctx, req, lagoonBuild, jobPod, nil, true) + if err != nil { + opLog.Error(err, fmt.Sprintf("Unable to update the LagoonBuild.")) + } + } else { + if helpers.ContainsString( + lagooncrd.BuildRunningPendingStatus, + lagoonBuild.Labels["lagoon.sh/buildStatus"], + ) { + opLog.Info(fmt.Sprintf("Attempting to update the LagoonBuild with cancellation if required.")) + // this will update the deployment back to lagoon if it can do so + // and should only update if the LagoonBuild is Pending or Running + err = r.updateDeploymentWithLogs(ctx, req, lagoonBuild, jobPod, nil, true) + if err != nil { + opLog.Error(err, fmt.Sprintf("Unable to update the LagoonBuild.")) + } + } + } + // if the update is successful or not, it will just continue on to check for pending builds + // in the event pending builds are not processed and the build pod itself has been deleted + // then manually patching the `LagoonBuild` with the label + // "lagoon.sh/buildStatus=Cancelled" + // should be enough to get things rolling again if no pending builds are being picked up + + // if we got any pending builds come through while one is running + // they will be processed here when any pods are cleaned up + // we check all `LagoonBuild` in the requested namespace + // if there are no running jobs, we check for any pending jobs + // sorted by their creation timestamp and set the first to running + if !r.LFFQoSEnabled { + // if qos is not enabled, then handle the check for pending builds here + opLog.Info(fmt.Sprintf("Checking for any pending builds.")) + runningBuilds := &lagooncrd.LagoonBuildList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(req.Namespace), + client.MatchingLabels(map[string]string{"lagoon.sh/buildStatus": lagooncrd.BuildStatusRunning.String()}), + }) + // list all builds in the namespace that have the running buildstatus + if err := r.List(ctx, runningBuilds, listOption); err != nil { + return ctrl.Result{}, fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err) + } + // if we have no running builds, then check for any pending builds + if len(runningBuilds.Items) == 0 { + return ctrl.Result{}, lagooncrd.CancelExtraBuilds(ctx, r.Client, opLog, req.Namespace, "Running") + } + } else { + // since qos handles pending build checks as part of its own operations, we can skip the running pod check step with no-op + if r.EnableDebug { + opLog.Info(fmt.Sprintf("No pending build check in namespaces when QoS is enabled")) + } + } + } + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the given manager +// and we set it to watch Pods with an event filter that contains our build label +func (r *LagoonMonitorReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&corev1.Pod{}). + WithEventFilter(PodPredicates{ + ControllerNamespace: r.ControllerNamespace, + }). + Complete(r) +} + +// getContainerLogs grabs the logs from a given container +func getContainerLogs(ctx context.Context, containerName string, request ctrl.Request) ([]byte, error) { + restCfg, err := config.GetConfig() + if err != nil { + return nil, fmt.Errorf("unable to get config: %v", err) + } + clientset, err := kubernetes.NewForConfig(restCfg) + if err != nil { + return nil, fmt.Errorf("unable to create client: %v", err) + } + req := clientset.CoreV1().Pods(request.Namespace).GetLogs(request.Name, &corev1.PodLogOptions{Container: containerName}) + podLogs, err := req.Stream(ctx) + if err != nil { + return nil, fmt.Errorf("error in opening stream: %v", err) + } + defer podLogs.Close() + buf := new(bytes.Buffer) + _, err = io.Copy(buf, podLogs) + if err != nil { + return nil, fmt.Errorf("error in copy information from podLogs to buffer: %v", err) + } + return buf.Bytes(), nil +} + +func (r *LagoonMonitorReconciler) collectLogs(ctx context.Context, req reconcile.Request, jobPod corev1.Pod) ([]byte, error) { + var allContainerLogs []byte + // grab all the logs from the containers in the task pod and just merge them all together + // we only have 1 container at the moment in a taskpod anyway so it doesn't matter + // if we do move to multi container tasks, then worry about it + for _, container := range jobPod.Spec.Containers { + cLogs, err := getContainerLogs(ctx, container.Name, req) + if err != nil { + return nil, fmt.Errorf("Unable to retrieve logs from pod: %v", err) + } + allContainerLogs = append(allContainerLogs, cLogs...) + } + return allContainerLogs, nil +} diff --git a/controllers/v1beta2/podmonitor_metrics.go b/controllers/v1beta2/podmonitor_metrics.go new file mode 100644 index 00000000..24520f51 --- /dev/null +++ b/controllers/v1beta2/podmonitor_metrics.go @@ -0,0 +1,51 @@ +package v1beta2 + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func (r *LagoonMonitorReconciler) calculateBuildMetrics(ctx context.Context) error { + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.MatchingLabels(map[string]string{ + "lagoon.sh/jobType": "build", + "lagoon.sh/controller": r.ControllerNamespace, + }), + }) + buildPods := &corev1.PodList{} + if err := r.List(ctx, buildPods, listOption); err != nil { + return fmt.Errorf("Unable to list builds in the cluster, there may be none or something went wrong: %v", err) + } + runningBuilds := float64(0) + for _, buildPod := range buildPods.Items { + if buildPod.Status.Phase == corev1.PodRunning { + runningBuilds = runningBuilds + 1 + } + } + buildsRunningGauge.Set(runningBuilds) + return nil +} + +func (r *LagoonMonitorReconciler) calculateTaskMetrics(ctx context.Context) error { + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.MatchingLabels(map[string]string{ + "lagoon.sh/jobType": "task", + "lagoon.sh/controller": r.ControllerNamespace, + }), + }) + taskPods := &corev1.PodList{} + if err := r.List(ctx, taskPods, listOption); err != nil { + return fmt.Errorf("Unable to list tasks in the cluster, there may be none or something went wrong: %v", err) + } + runningTasks := float64(0) + for _, taskPod := range taskPods.Items { + if taskPod.Status.Phase == corev1.PodRunning { + runningTasks = runningTasks + 1 + } + } + tasksRunningGauge.Set(runningTasks) + return nil +} diff --git a/controllers/v1beta2/podmonitor_taskhandlers.go b/controllers/v1beta2/podmonitor_taskhandlers.go new file mode 100644 index 00000000..5868e543 --- /dev/null +++ b/controllers/v1beta2/podmonitor_taskhandlers.go @@ -0,0 +1,401 @@ +package v1beta2 + +// this file is used by the `lagoonmonitor` controller + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "strings" + "time" + + "github.com/go-logr/logr" + "github.com/prometheus/client_golang/prometheus" + "github.com/uselagoon/machinery/api/schema" + lagooncrd "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" + "github.com/uselagoon/remote-controller/internal/helpers" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func (r *LagoonMonitorReconciler) handleTaskMonitor(ctx context.Context, opLog logr.Logger, req ctrl.Request, jobPod corev1.Pod) error { + // get the task associated to this pod, we wil need update it at some point + var lagoonTask lagooncrd.LagoonTask + err := r.Get(ctx, types.NamespacedName{ + Namespace: jobPod.ObjectMeta.Namespace, + Name: jobPod.ObjectMeta.Labels["lagoon.sh/taskName"], + }, &lagoonTask) + if err != nil { + return err + } + cancel := false + if cancelTask, ok := jobPod.ObjectMeta.Labels["lagoon.sh/cancelTask"]; ok { + cancel, _ = strconv.ParseBool(cancelTask) + } + _, ok := r.Cache.Get(lagoonTask.ObjectMeta.Name) + if ok { + opLog.Info(fmt.Sprintf("Cached cancellation exists for: %s", lagoonTask.ObjectMeta.Name)) + // this object exists in the cache meaning the task has been cancelled, set cancel to true and remove from cache + r.Cache.Remove(lagoonTask.ObjectMeta.Name) + cancel = true + } + if cancel { + opLog.Info(fmt.Sprintf("Attempting to cancel task %s", lagoonTask.ObjectMeta.Name)) + return r.updateTaskWithLogs(ctx, req, lagoonTask, jobPod, nil, cancel) + } + if jobPod.Status.Phase == corev1.PodPending { + for _, container := range jobPod.Status.ContainerStatuses { + if container.State.Waiting != nil && helpers.ContainsString(failureStates, container.State.Waiting.Reason) { + // if we have a failure state, then fail the task and get the logs from the container + opLog.Info(fmt.Sprintf("Task failed, container exit reason was: %v", container.State.Waiting.Reason)) + lagoonTask.Labels["lagoon.sh/taskStatus"] = lagooncrd.TaskStatusFailed.String() + if err := r.Update(ctx, &lagoonTask); err != nil { + return err + } + opLog.Info(fmt.Sprintf("Marked task %s as %s", lagoonTask.ObjectMeta.Name, lagooncrd.TaskStatusFailed.String())) + if err := r.Delete(ctx, &jobPod); err != nil { + return err + } + opLog.Info(fmt.Sprintf("Deleted failed task pod: %s", jobPod.ObjectMeta.Name)) + // update the status to failed on the deleted pod + // and set the terminate time to now, it is used when we update the deployment and environment + jobPod.Status.Phase = corev1.PodFailed + state := corev1.ContainerStatus{ + State: corev1.ContainerState{ + Terminated: &corev1.ContainerStateTerminated{ + FinishedAt: metav1.Time{Time: time.Now().UTC()}, + }, + }, + } + jobPod.Status.ContainerStatuses[0] = state + logMsg := fmt.Sprintf("%v: %v", container.State.Waiting.Reason, container.State.Waiting.Message) + return r.updateTaskWithLogs(ctx, req, lagoonTask, jobPod, []byte(logMsg), false) + } + } + return r.updateTaskWithLogs(ctx, req, lagoonTask, jobPod, nil, false) + } else if jobPod.Status.Phase == corev1.PodRunning { + // if the pod is running and detects a change to the pod (eg, detecting an updated lagoon.sh/taskStep label) + // then ship or store the logs + // get the task associated to this pod, the information in the resource is used for shipping the logs + var lagoonTask lagooncrd.LagoonTask + err := r.Get(ctx, + types.NamespacedName{ + Namespace: jobPod.ObjectMeta.Namespace, + Name: jobPod.ObjectMeta.Labels["lagoon.sh/taskName"], + }, &lagoonTask) + if err != nil { + return err + } + } + if jobPod.Status.Phase == corev1.PodFailed || jobPod.Status.Phase == corev1.PodSucceeded { + // get the task associated to this pod, we wil need update it at some point + var lagoonTask lagooncrd.LagoonTask + err := r.Get(ctx, types.NamespacedName{ + Namespace: jobPod.ObjectMeta.Namespace, + Name: jobPod.ObjectMeta.Labels["lagoon.sh/taskName"], + }, &lagoonTask) + if err != nil { + return err + } + } + // if it isn't pending, failed, or complete, it will be running, we should tell lagoon + return r.updateTaskWithLogs(ctx, req, lagoonTask, jobPod, nil, false) +} + +// taskLogsToLagoonLogs sends the task logs to the lagoon-logs message queue +// it contains the actual pod log output that is sent to elasticsearch, it is what eventually is displayed in the UI +func (r *LagoonMonitorReconciler) taskLogsToLagoonLogs(opLog logr.Logger, + lagoonTask *lagooncrd.LagoonTask, + jobPod *corev1.Pod, + condition string, + logs []byte, +) (bool, schema.LagoonLog) { + if r.EnableMQ && lagoonTask != nil { + msg := schema.LagoonLog{ + Severity: "info", + Project: lagoonTask.Spec.Project.Name, + Event: "task-logs:job-kubernetes:" + lagoonTask.ObjectMeta.Name, + Meta: &schema.LagoonLogMeta{ + Task: &lagoonTask.Spec.Task, + Environment: lagoonTask.Spec.Environment.Name, + JobName: lagoonTask.ObjectMeta.Name, + JobStatus: condition, + RemoteID: string(jobPod.ObjectMeta.UID), + Key: lagoonTask.Spec.Key, + Cluster: r.LagoonTargetName, + }, + } + if jobPod.Spec.NodeName != "" { + msg.Message = fmt.Sprintf(`================================================================================ +Logs on pod %s, assigned to node %s on cluster %s +================================================================================ +%s`, jobPod.ObjectMeta.Name, jobPod.Spec.NodeName, r.LagoonTargetName, logs) + } else { + msg.Message = fmt.Sprintf(`================================================================================ +Logs on pod %s, assigned to cluster %s +================================================================================ +%s`, jobPod.ObjectMeta.Name, r.LagoonTargetName, logs) + } + msgBytes, err := json.Marshal(msg) + if err != nil { + opLog.Error(err, "Unable to encode message as JSON") + } + if err := r.Messaging.Publish("lagoon-logs", msgBytes); err != nil { + // if we can't publish the message, set it as a pending message + // overwrite whatever is there as these are just current state messages so it doesn't + // really matter if we don't smootly transition in what we send back to lagoon + return true, msg + } + // if we are able to publish the message, then we need to remove any pending messages from the resource + // and make sure we don't try and publish again + } + return false, schema.LagoonLog{} +} + +// updateLagoonTask sends the status of the task and deployment to the controllerhandler message queue in lagoon, +// this is for the handler in lagoon to process. +func (r *LagoonMonitorReconciler) updateLagoonTask(ctx context.Context, opLog logr.Logger, + lagoonTask *lagooncrd.LagoonTask, + jobPod *corev1.Pod, + condition string, +) (bool, schema.LagoonMessage) { + if r.EnableMQ && lagoonTask != nil { + if condition == "failed" || condition == "complete" || condition == "cancelled" { + time.AfterFunc(31*time.Second, func() { + taskRunningStatus.Delete(prometheus.Labels{ + "task_namespace": lagoonTask.ObjectMeta.Namespace, + "task_name": lagoonTask.ObjectMeta.Name, + }) + }) + time.Sleep(2 * time.Second) // smol sleep to reduce race of final messages with previous messages + } + msg := schema.LagoonMessage{ + Type: "task", + Namespace: lagoonTask.ObjectMeta.Namespace, + Meta: &schema.LagoonLogMeta{ + Task: &lagoonTask.Spec.Task, + Environment: lagoonTask.Spec.Environment.Name, + Project: lagoonTask.Spec.Project.Name, + EnvironmentID: helpers.StringToUintPtr(lagoonTask.Spec.Environment.ID), + ProjectID: helpers.StringToUintPtr(lagoonTask.Spec.Project.ID), + JobName: lagoonTask.ObjectMeta.Name, + JobStatus: condition, + RemoteID: string(jobPod.ObjectMeta.UID), + Key: lagoonTask.Spec.Key, + Cluster: r.LagoonTargetName, + }, + } + if _, ok := jobPod.ObjectMeta.Annotations["lagoon.sh/taskData"]; ok { + // if the task contains `taskData` annotation, this is used to send data back to lagoon + // lagoon will use the data to perform an action against the api or something else + // the data in taskData should be base64 encoded + msg.Meta.AdvancedData = jobPod.ObjectMeta.Annotations["lagoon.sh/taskData"] + } + // we can add the task start time here + if jobPod.Status.StartTime != nil { + msg.Meta.StartTime = jobPod.Status.StartTime.Time.UTC().Format("2006-01-02 15:04:05") + } + // and then once the pod is terminated we can add the terminated time here + if jobPod.Status.ContainerStatuses != nil { + if jobPod.Status.ContainerStatuses[0].State.Terminated != nil { + msg.Meta.EndTime = jobPod.Status.ContainerStatuses[0].State.Terminated.FinishedAt.Time.UTC().Format("2006-01-02 15:04:05") + } + } + msgBytes, err := json.Marshal(msg) + if err != nil { + opLog.Error(err, "Unable to encode message as JSON") + } + if err := r.Messaging.Publish("lagoon-tasks:controller", msgBytes); err != nil { + // if we can't publish the message, set it as a pending message + // overwrite whatever is there as these are just current state messages so it doesn't + // really matter if we don't smootly transition in what we send back to lagoon + return true, msg + } + // if we are able to publish the message, then we need to remove any pending messages from the resource + // and make sure we don't try and publish again + } + return false, schema.LagoonMessage{} +} + +// taskStatusLogsToLagoonLogs sends the logs to lagoon-logs message queue, used for general messaging +func (r *LagoonMonitorReconciler) taskStatusLogsToLagoonLogs(opLog logr.Logger, + lagoonTask *lagooncrd.LagoonTask, + jobPod *corev1.Pod, + condition string, +) (bool, schema.LagoonLog) { + if r.EnableMQ && lagoonTask != nil { + msg := schema.LagoonLog{ + Severity: "info", + Project: lagoonTask.Spec.Project.Name, + Event: "task:job-kubernetes:" + condition, //@TODO: this probably needs to be changed to a new task event for the controller + Meta: &schema.LagoonLogMeta{ + Task: &lagoonTask.Spec.Task, + ProjectName: lagoonTask.Spec.Project.Name, + Environment: lagoonTask.Spec.Environment.Name, + EnvironmentID: helpers.StringToUintPtr(lagoonTask.Spec.Environment.ID), + ProjectID: helpers.StringToUintPtr(lagoonTask.Spec.Project.ID), + JobName: lagoonTask.ObjectMeta.Name, + JobStatus: condition, + RemoteID: string(jobPod.ObjectMeta.UID), + Key: lagoonTask.Spec.Key, + Cluster: r.LagoonTargetName, + }, + Message: fmt.Sprintf("*[%s]* Task `%s` *%s* %s", + lagoonTask.Spec.Project.Name, + lagoonTask.Spec.Task.ID, + lagoonTask.Spec.Task.Name, + condition, + ), + } + msgBytes, err := json.Marshal(msg) + if err != nil { + opLog.Error(err, "Unable to encode message as JSON") + } + if err := r.Messaging.Publish("lagoon-logs", msgBytes); err != nil { + // if we can't publish the message, set it as a pending message + // overwrite whatever is there as these are just current state messages so it doesn't + // really matter if we don't smootly transition in what we send back to lagoon + return true, msg + } + // if we are able to publish the message, then we need to remove any pending messages from the resource + // and make sure we don't try and publish again + } + return false, schema.LagoonLog{} +} + +// updateTaskWithLogs collects logs from the task containers and ships or stores them +func (r *LagoonMonitorReconciler) updateTaskWithLogs( + ctx context.Context, + req ctrl.Request, + lagoonTask lagooncrd.LagoonTask, + jobPod corev1.Pod, + logs []byte, + cancel bool, +) error { + opLog := r.Log.WithValues("lagoonmonitor", req.NamespacedName) + taskCondition := lagooncrd.GetTaskConditionFromPod(jobPod.Status.Phase) + collectLogs := true + if cancel { + taskCondition = lagooncrd.TaskStatusCancelled + } + // if the task status is Pending or Running + // then the taskCondition is Failed, Complete, or Cancelled + // then update the task to reflect the current pod status + // we do this so we don't update the status of the task again + if helpers.ContainsString( + lagooncrd.TaskRunningPendingStatus, + lagoonTask.Labels["lagoon.sh/taskStatus"], + ) || cancel { + opLog.Info( + fmt.Sprintf( + "Updating task status for %s to %v", + jobPod.ObjectMeta.Name, + taskCondition, + ), + ) + // grab all the logs from the containers in the task pod and just merge them all together + // we only have 1 container at the moment in a taskpod anyway so it doesn't matter + // if we do move to multi container tasks, then worry about it + var allContainerLogs []byte + var err error + if logs == nil { + if collectLogs { + allContainerLogs, err = r.collectLogs(ctx, req, jobPod) + if err == nil { + if cancel { + cancellationMessage := "Task cancelled" + if cancellationDetails, ok := jobPod.GetAnnotations()["lagoon.sh/cancelReason"]; ok { + cancellationMessage = fmt.Sprintf("%v : %v", cancellationMessage, cancellationDetails) + } + allContainerLogs = append(allContainerLogs, []byte(fmt.Sprintf(` +======================================== +%v +========================================`, cancellationMessage))...) + } + } else { + allContainerLogs = []byte(fmt.Sprintf(` +======================================== +Task %s +========================================`, taskCondition)) + } + } + } else { + allContainerLogs = logs + } + + mergeMap := map[string]interface{}{ + "metadata": map[string]interface{}{ + "labels": map[string]interface{}{ + "lagoon.sh/taskStatus": taskCondition.String(), + }, + }, + "statusMessages": map[string]interface{}{}, + } + + condition := lagooncrd.LagoonTaskConditions{ + Type: taskCondition, + Status: corev1.ConditionTrue, + LastTransitionTime: time.Now().UTC().Format(time.RFC3339), + } + if !lagooncrd.TaskContainsStatus(lagoonTask.Status.Conditions, condition) { + lagoonTask.Status.Conditions = append(lagoonTask.Status.Conditions, condition) + mergeMap["status"] = map[string]interface{}{ + "conditions": lagoonTask.Status.Conditions, + // don't save build logs in resource anymore + } + } + + // send any messages to lagoon message queues + // update the deployment with the status + pendingStatus, pendingStatusMessage := r.taskStatusLogsToLagoonLogs(opLog, &lagoonTask, &jobPod, taskCondition.ToLower()) + pendingEnvironment, pendingEnvironmentMessage := r.updateLagoonTask(ctx, opLog, &lagoonTask, &jobPod, taskCondition.ToLower()) + var pendingTaskLog bool + var pendingTaskLogMessage schema.LagoonLog + // if the container logs can't be retrieved, we don't want to send any task logs back, as this will nuke + // any previously received logs + if !strings.Contains(string(allContainerLogs), "unable to retrieve container logs for containerd") { + pendingTaskLog, pendingTaskLogMessage = r.taskLogsToLagoonLogs(opLog, &lagoonTask, &jobPod, taskCondition.ToLower(), allContainerLogs) + } + + if pendingStatus || pendingEnvironment || pendingTaskLog { + mergeMap["metadata"].(map[string]interface{})["labels"].(map[string]interface{})["lagoon.sh/pendingMessages"] = "true" + if pendingStatus { + mergeMap["statusMessages"].(map[string]interface{})["statusMessage"] = pendingStatusMessage + } + if pendingEnvironment { + mergeMap["statusMessages"].(map[string]interface{})["environmentMessage"] = pendingEnvironmentMessage + } + if pendingTaskLog { + mergeMap["statusMessages"].(map[string]interface{})["taskLogMessage"] = pendingTaskLogMessage + } + } + if !pendingStatus && !pendingEnvironment && !pendingTaskLog { + mergeMap["metadata"].(map[string]interface{})["labels"].(map[string]interface{})["lagoon.sh/pendingMessages"] = nil + mergeMap["statusMessages"] = nil + } + mergePatch, _ := json.Marshal(mergeMap) + // check if the task exists + if err := r.Get(ctx, req.NamespacedName, &lagoonTask); err == nil { + // if it does, try to patch it + if err := r.Patch(ctx, &lagoonTask, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to update resource")) + } + } + // just delete the pod + if cancel { + if err := r.Get(ctx, req.NamespacedName, &jobPod); err == nil { + if r.EnableDebug { + opLog.Info(fmt.Sprintf("Task pod exists %s", jobPod.ObjectMeta.Name)) + } + if err := r.Delete(ctx, &jobPod); err != nil { + return err + } + } + } + } + return nil +} diff --git a/controllers/v1beta2/predicates.go b/controllers/v1beta2/predicates.go new file mode 100644 index 00000000..bf8ac026 --- /dev/null +++ b/controllers/v1beta2/predicates.go @@ -0,0 +1,227 @@ +package v1beta2 + +// contains all the event watch conditions for secret and ingresses + +import ( + "regexp" + "time" + + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +// PodPredicates is used by the filter for the monitor controller to make sure the correct resources +// are acted on. +type PodPredicates struct { + predicate.Funcs + ControllerNamespace string +} + +// Create is used when a creation event is received by the controller. +func (p PodPredicates) Create(e event.CreateEvent) bool { + if controller, ok := e.Object.GetLabels()["lagoon.sh/controller"]; ok { + if controller == p.ControllerNamespace { + if value, ok := e.Object.GetLabels()["lagoon.sh/crdVersion"]; ok { + if value == crdVersion { + if value, ok := e.Object.GetLabels()["lagoon.sh/buildName"]; ok { + match, _ := regexp.MatchString("^lagoon-build", value) + return match + } + if value, ok := e.Object.GetLabels()["lagoon.sh/jobType"]; ok { + if value == "task" { + return true + } + } + } + } + } + } + return false +} + +// Delete is used when a deletion event is received by the controller. +func (p PodPredicates) Delete(e event.DeleteEvent) bool { + if controller, ok := e.Object.GetLabels()["lagoon.sh/controller"]; ok { + if controller == p.ControllerNamespace { + if value, ok := e.Object.GetLabels()["lagoon.sh/crdVersion"]; ok { + if value == crdVersion { + if value, ok := e.Object.GetLabels()["lagoon.sh/buildName"]; ok { + match, _ := regexp.MatchString("^lagoon-build", value) + return match + } + if value, ok := e.Object.GetLabels()["lagoon.sh/jobType"]; ok { + if value == "task" { + return true + } + } + } + } + } + } + return false +} + +// Update is used when an update event is received by the controller. +func (p PodPredicates) Update(e event.UpdateEvent) bool { + if controller, ok := e.ObjectOld.GetLabels()["lagoon.sh/controller"]; ok { + if controller == p.ControllerNamespace { + if value, ok := e.ObjectNew.GetLabels()["lagoon.sh/crdVersion"]; ok { + if value == crdVersion { + if _, okOld := e.ObjectOld.GetLabels()["lagoon.sh/buildName"]; okOld { + if value, ok := e.ObjectNew.GetLabels()["lagoon.sh/buildName"]; ok { + oldBuildStep := "running" + newBuildStep := "running" + if value, ok := e.ObjectNew.GetLabels()["lagoon.sh/buildStep"]; ok { + newBuildStep = value + } + if value, ok := e.ObjectOld.GetLabels()["lagoon.sh/buildStep"]; ok { + oldBuildStep = value + } + if newBuildStep != oldBuildStep { + buildStatus.With(prometheus.Labels{ + "build_namespace": e.ObjectOld.GetNamespace(), + "build_name": e.ObjectOld.GetName(), + "build_step": newBuildStep, + }).Set(1) + } + time.AfterFunc(31*time.Second, func() { + buildStatus.Delete(prometheus.Labels{ + "build_namespace": e.ObjectOld.GetNamespace(), + "build_name": e.ObjectOld.GetName(), + "build_step": oldBuildStep, + }) + }) + match, _ := regexp.MatchString("^lagoon-build", value) + return match + } + } + if _, ok := e.ObjectOld.GetLabels()["lagoon.sh/jobType"]; ok { + if value, ok := e.ObjectNew.GetLabels()["lagoon.sh/jobType"]; ok { + if value == "task" { + return true + } + } + } + } + } + } + } + return false +} + +// Generic is used when any other event is received by the controller. +func (p PodPredicates) Generic(e event.GenericEvent) bool { + if controller, ok := e.Object.GetLabels()["lagoon.sh/controller"]; ok { + if controller == p.ControllerNamespace { + if value, ok := e.Object.GetLabels()["lagoon.sh/crdVersion"]; ok { + if value == crdVersion { + if value, ok := e.Object.GetLabels()["lagoon.sh/buildName"]; ok { + match, _ := regexp.MatchString("^lagoon-build", value) + return match + } + if value, ok := e.Object.GetLabels()["lagoon.sh/jobType"]; ok { + if value == "task" { + return true + } + } + } + } + } + } + return false +} + +// BuildPredicates is used by the filter for the build controller to make sure the correct resources +// are acted on. +type BuildPredicates struct { + predicate.Funcs + ControllerNamespace string +} + +// Create is used when a creation event is received by the controller. +func (b BuildPredicates) Create(e event.CreateEvent) bool { + if controller, ok := e.Object.GetLabels()["lagoon.sh/controller"]; ok { + if controller == b.ControllerNamespace { + return true + } + } + return false +} + +// Delete is used when a deletion event is received by the controller. +func (b BuildPredicates) Delete(e event.DeleteEvent) bool { + if controller, ok := e.Object.GetLabels()["lagoon.sh/controller"]; ok { + if controller == b.ControllerNamespace { + return true + } + } + return false +} + +// Update is used when an update event is received by the controller. +func (b BuildPredicates) Update(e event.UpdateEvent) bool { + if controller, ok := e.ObjectOld.GetLabels()["lagoon.sh/controller"]; ok { + if controller == b.ControllerNamespace { + return true + } + } + return false +} + +// Generic is used when any other event is received by the controller. +func (b BuildPredicates) Generic(e event.GenericEvent) bool { + if controller, ok := e.Object.GetLabels()["lagoon.sh/controller"]; ok { + if controller == b.ControllerNamespace { + return true + } + } + return false +} + +// TaskPredicates is used by the filter for the task controller to make sure the correct resources +// are acted on. +type TaskPredicates struct { + predicate.Funcs + ControllerNamespace string +} + +// Create is used when a creation event is received by the controller. +func (t TaskPredicates) Create(e event.CreateEvent) bool { + if controller, ok := e.Object.GetLabels()["lagoon.sh/controller"]; ok { + if controller == t.ControllerNamespace { + return true + } + } + return false +} + +// Delete is used when a deletion event is received by the controller. +func (t TaskPredicates) Delete(e event.DeleteEvent) bool { + if controller, ok := e.Object.GetLabels()["lagoon.sh/controller"]; ok { + if controller == t.ControllerNamespace { + return true + } + } + return false +} + +// Update is used when an update event is received by the controller. +func (t TaskPredicates) Update(e event.UpdateEvent) bool { + if controller, ok := e.ObjectOld.GetLabels()["lagoon.sh/controller"]; ok { + if controller == t.ControllerNamespace { + return true + } + } + return false +} + +// Generic is used when any other event is received by the controller. +func (t TaskPredicates) Generic(e event.GenericEvent) bool { + if controller, ok := e.Object.GetLabels()["lagoon.sh/controller"]; ok { + if controller == t.ControllerNamespace { + return true + } + } + return false +} diff --git a/controllers/v1beta2/suite_test.go b/controllers/v1beta2/suite_test.go new file mode 100644 index 00000000..6d04f49a --- /dev/null +++ b/controllers/v1beta2/suite_test.go @@ -0,0 +1,80 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta2 + +import ( + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + lagooncrd "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + // +kubebuilder:scaffold:imports +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var cfg *rest.Config +var k8sClient client.Client +var testEnv *envtest.Environment + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Controller Suite") +} + +var _ = BeforeSuite(func(done Done) { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, + } + + var err error + cfg, err = testEnv.Start() + Expect(err).ToNot(HaveOccurred()) + Expect(cfg).ToNot(BeNil()) + + err = lagooncrd.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + err = lagooncrd.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + // +kubebuilder:scaffold:scheme + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).ToNot(HaveOccurred()) + Expect(k8sClient).ToNot(BeNil()) + + close(done) +}, 60) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + err := testEnv.Stop() + Expect(err).ToNot(HaveOccurred()) +}) diff --git a/controllers/v1beta2/task_controller.go b/controllers/v1beta2/task_controller.go new file mode 100644 index 00000000..c1d71a69 --- /dev/null +++ b/controllers/v1beta2/task_controller.go @@ -0,0 +1,678 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta2 + +import ( + "context" + "encoding/json" + "fmt" + "regexp" + "strconv" + + "github.com/go-logr/logr" + "github.com/prometheus/client_golang/prometheus" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + lagooncrd "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" + "github.com/uselagoon/remote-controller/internal/helpers" +) + +// LagoonTaskReconciler reconciles a LagoonTask object +type LagoonTaskReconciler struct { + client.Client + Log logr.Logger + Scheme *runtime.Scheme + ControllerNamespace string + NamespacePrefix string + RandomNamespacePrefix bool + LagoonAPIConfiguration helpers.LagoonAPIConfiguration + EnableDebug bool + LagoonTargetName string + ProxyConfig ProxyConfig +} + +var ( + taskFinalizer = "finalizer.lagoontask.crd.lagoon.sh/v1beta2" +) + +// +kubebuilder:rbac:groups=crd.lagoon.sh,resources=lagoontasks,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=crd.lagoon.sh,resources=lagoontasks/status,verbs=get;update;patch + +// Reconcile runs when a request comes through +func (r *LagoonTaskReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + opLog := r.Log.WithValues("lagoontask", req.NamespacedName) + + // your logic here + var lagoonTask lagooncrd.LagoonTask + if err := r.Get(ctx, req.NamespacedName, &lagoonTask); err != nil { + return ctrl.Result{}, helpers.IgnoreNotFound(err) + } + + // examine DeletionTimestamp to determine if object is under deletion + if lagoonTask.ObjectMeta.DeletionTimestamp.IsZero() { + // check if the task that has been recieved is a standard or advanced task + if lagoonTask.ObjectMeta.Labels["lagoon.sh/taskStatus"] == lagooncrd.TaskStatusPending.String() && + lagoonTask.ObjectMeta.Labels["lagoon.sh/taskType"] == lagooncrd.TaskTypeStandard.String() { + return ctrl.Result{}, r.createStandardTask(ctx, &lagoonTask, opLog) + } + if lagoonTask.ObjectMeta.Labels["lagoon.sh/taskStatus"] == lagooncrd.TaskStatusPending.String() && + lagoonTask.ObjectMeta.Labels["lagoon.sh/taskType"] == lagooncrd.TaskTypeAdvanced.String() { + return ctrl.Result{}, r.createAdvancedTask(ctx, &lagoonTask, opLog) + } + } else { + // The object is being deleted + if helpers.ContainsString(lagoonTask.ObjectMeta.Finalizers, taskFinalizer) { + // our finalizer is present, so lets handle any external dependency + if err := r.deleteExternalResources(ctx, &lagoonTask, req.NamespacedName.Namespace); err != nil { + // if fail to delete the external dependency here, return with error + // so that it can be retried + return ctrl.Result{}, err + } + // remove our finalizer from the list and update it. + lagoonTask.ObjectMeta.Finalizers = helpers.RemoveString(lagoonTask.ObjectMeta.Finalizers, taskFinalizer) + // use patches to avoid update errors + mergePatch, _ := json.Marshal(map[string]interface{}{ + "metadata": map[string]interface{}{ + "finalizers": lagoonTask.ObjectMeta.Finalizers, + }, + }) + if err := r.Patch(ctx, &lagoonTask, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { + return ctrl.Result{}, err + } + } + } + + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the given manager +func (r *LagoonTaskReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&lagooncrd.LagoonTask{}). + WithEventFilter(TaskPredicates{ + ControllerNamespace: r.ControllerNamespace, + }). + Complete(r) +} + +func (r *LagoonTaskReconciler) deleteExternalResources(ctx context.Context, lagoonTask *lagooncrd.LagoonTask, namespace string) error { + // delete any external resources if required + return nil +} + +// get the task pod information for kubernetes +func (r *LagoonTaskReconciler) getTaskPodDeployment(ctx context.Context, lagoonTask *lagooncrd.LagoonTask) (*corev1.Pod, error) { + deployments := &appsv1.DeploymentList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(lagoonTask.ObjectMeta.Namespace), + }) + err := r.List(ctx, deployments, listOption) + if err != nil { + return nil, fmt.Errorf( + "Unable to get deployments for project %s, environment %s: %v", + lagoonTask.Spec.Project.Name, + lagoonTask.Spec.Environment.Name, + err, + ) + } + if len(deployments.Items) > 0 { + hasService := false + for _, dep := range deployments.Items { + // grab the deployment that contains the task service we want to use + if dep.ObjectMeta.Name == lagoonTask.Spec.Task.Service { + hasService = true + // grab the container + for idx, depCon := range dep.Spec.Template.Spec.Containers { + // --- deprecate these at some point in favor of the `LAGOON_CONFIG_X` variants + dep.Spec.Template.Spec.Containers[idx].Env = append(dep.Spec.Template.Spec.Containers[idx].Env, corev1.EnvVar{ + Name: "TASK_API_HOST", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "TASK_API_HOST"), + }) + dep.Spec.Template.Spec.Containers[idx].Env = append(dep.Spec.Template.Spec.Containers[idx].Env, corev1.EnvVar{ + Name: "TASK_SSH_HOST", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "TASK_SSH_HOST"), + }) + dep.Spec.Template.Spec.Containers[idx].Env = append(dep.Spec.Template.Spec.Containers[idx].Env, corev1.EnvVar{ + Name: "TASK_SSH_PORT", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "TASK_SSH_PORT"), + }) + // ^^ --- + // add the API and SSH endpoint configuration to environments + dep.Spec.Template.Spec.Containers[idx].Env = append(dep.Spec.Template.Spec.Containers[idx].Env, corev1.EnvVar{ + Name: "LAGOON_CONFIG_API_HOST", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "LAGOON_CONFIG_API_HOST"), + }) + dep.Spec.Template.Spec.Containers[idx].Env = append(dep.Spec.Template.Spec.Containers[idx].Env, corev1.EnvVar{ + Name: "LAGOON_CONFIG_TOKEN_HOST", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "LAGOON_CONFIG_TOKEN_HOST"), + }) + dep.Spec.Template.Spec.Containers[idx].Env = append(dep.Spec.Template.Spec.Containers[idx].Env, corev1.EnvVar{ + Name: "LAGOON_CONFIG_TOKEN_PORT", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "LAGOON_CONFIG_TOKEN_PORT"), + }) + dep.Spec.Template.Spec.Containers[idx].Env = append(dep.Spec.Template.Spec.Containers[idx].Env, corev1.EnvVar{ + Name: "LAGOON_CONFIG_SSH_HOST", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "LAGOON_CONFIG_SSH_HOST"), + }) + dep.Spec.Template.Spec.Containers[idx].Env = append(dep.Spec.Template.Spec.Containers[idx].Env, corev1.EnvVar{ + Name: "LAGOON_CONFIG_SSH_PORT", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "LAGOON_CONFIG_SSH_PORT"), + }) + dep.Spec.Template.Spec.Containers[idx].Env = append(dep.Spec.Template.Spec.Containers[idx].Env, corev1.EnvVar{ + Name: "TASK_DATA_ID", + Value: lagoonTask.Spec.Task.ID, + }) + // add proxy variables to builds if they are defined + if r.ProxyConfig.HTTPProxy != "" { + dep.Spec.Template.Spec.Containers[idx].Env = append(dep.Spec.Template.Spec.Containers[idx].Env, corev1.EnvVar{ + Name: "HTTP_PROXY", + Value: r.ProxyConfig.HTTPProxy, + }) + dep.Spec.Template.Spec.Containers[idx].Env = append(dep.Spec.Template.Spec.Containers[idx].Env, corev1.EnvVar{ + Name: "http_proxy", + Value: r.ProxyConfig.HTTPProxy, + }) + } + if r.ProxyConfig.HTTPSProxy != "" { + dep.Spec.Template.Spec.Containers[idx].Env = append(dep.Spec.Template.Spec.Containers[idx].Env, corev1.EnvVar{ + Name: "HTTPS_PROXY", + Value: r.ProxyConfig.HTTPSProxy, + }) + dep.Spec.Template.Spec.Containers[idx].Env = append(dep.Spec.Template.Spec.Containers[idx].Env, corev1.EnvVar{ + Name: "https_proxy", + Value: r.ProxyConfig.HTTPSProxy, + }) + } + if r.ProxyConfig.NoProxy != "" { + dep.Spec.Template.Spec.Containers[idx].Env = append(dep.Spec.Template.Spec.Containers[idx].Env, corev1.EnvVar{ + Name: "NO_PROXY", + Value: r.ProxyConfig.NoProxy, + }) + dep.Spec.Template.Spec.Containers[idx].Env = append(dep.Spec.Template.Spec.Containers[idx].Env, corev1.EnvVar{ + Name: "no_proxy", + Value: r.ProxyConfig.NoProxy, + }) + } + if lagoonTask.Spec.Project.Variables.Project != nil { + // if this is 2 bytes long, then it means its just an empty json array + // we only want to add it if it is more than 2 bytes + if len(lagoonTask.Spec.Project.Variables.Project) > 2 { + dep.Spec.Template.Spec.Containers[idx].Env = append(dep.Spec.Template.Spec.Containers[idx].Env, corev1.EnvVar{ + Name: "LAGOON_PROJECT_VARIABLES", + Value: string(lagoonTask.Spec.Project.Variables.Project), + }) + } + } + if lagoonTask.Spec.Project.Variables.Environment != nil { + // if this is 2 bytes long, then it means its just an empty json array + // we only want to add it if it is more than 2 bytes + if len(lagoonTask.Spec.Project.Variables.Environment) > 2 { + dep.Spec.Template.Spec.Containers[idx].Env = append(dep.Spec.Template.Spec.Containers[idx].Env, corev1.EnvVar{ + Name: "LAGOON_ENVIRONMENT_VARIABLES", + Value: string(lagoonTask.Spec.Project.Variables.Environment), + }) + } + } + for idx2, env := range depCon.Env { + // remove any cronjobs from the envvars + if env.Name == "CRONJOBS" { + // Shift left one index. + copy(dep.Spec.Template.Spec.Containers[idx].Env[idx2:], dep.Spec.Template.Spec.Containers[idx].Env[idx2+1:]) + // Erase last element (write zero value). + dep.Spec.Template.Spec.Containers[idx].Env[len(dep.Spec.Template.Spec.Containers[idx].Env)-1] = corev1.EnvVar{} + dep.Spec.Template.Spec.Containers[idx].Env = dep.Spec.Template.Spec.Containers[idx].Env[:len(dep.Spec.Template.Spec.Containers[idx].Env)-1] + } + } + dep.Spec.Template.Spec.Containers[idx].Command = []string{"/sbin/tini", + "--", + "/lagoon/entrypoints.sh", + "/bin/sh", + "-c", + lagoonTask.Spec.Task.Command, + } + dep.Spec.Template.Spec.RestartPolicy = "Never" + taskPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: lagoonTask.ObjectMeta.Name, + Namespace: lagoonTask.ObjectMeta.Namespace, + Labels: map[string]string{ + "lagoon.sh/jobType": "task", + "lagoon.sh/taskName": lagoonTask.ObjectMeta.Name, + "lagoon.sh/crdVersion": crdVersion, + "lagoon.sh/controller": r.ControllerNamespace, + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: fmt.Sprintf("%v", lagooncrd.GroupVersion), + Kind: "LagoonTask", + Name: lagoonTask.ObjectMeta.Name, + UID: lagoonTask.UID, + }, + }, + }, + Spec: dep.Spec.Template.Spec, + } + // set the organization labels on task pods + if lagoonTask.Spec.Project.Organization != nil { + taskPod.ObjectMeta.Labels["organization.lagoon.sh/id"] = fmt.Sprintf("%d", *lagoonTask.Spec.Project.Organization.ID) + taskPod.ObjectMeta.Labels["organization.lagoon.sh/name"] = lagoonTask.Spec.Project.Organization.Name + } + return taskPod, nil + } + } + } + if !hasService { + return nil, fmt.Errorf( + "No matching service %s for project %s, environment %s: %v", + lagoonTask.Spec.Task.Service, + lagoonTask.Spec.Project.Name, + lagoonTask.Spec.Environment.Name, + err, + ) + } + } + // no deployments found return error + return nil, fmt.Errorf( + "No deployments %s for project %s, environment %s: %v", + lagoonTask.ObjectMeta.Namespace, + lagoonTask.Spec.Project.Name, + lagoonTask.Spec.Environment.Name, + err, + ) +} + +func (r *LagoonTaskReconciler) createStandardTask(ctx context.Context, lagoonTask *lagooncrd.LagoonTask, opLog logr.Logger) error { + newTaskPod := &corev1.Pod{} + var err error + + newTaskPod, err = r.getTaskPodDeployment(ctx, lagoonTask) + if err != nil { + opLog.Info(fmt.Sprintf("%v", err)) + //@TODO: send msg back and update task to failed? + return nil + } + opLog.Info(fmt.Sprintf("Checking task pod for: %s", lagoonTask.ObjectMeta.Name)) + // once the pod spec has been defined, check if it isn't already created + err = r.Get(ctx, types.NamespacedName{ + Namespace: lagoonTask.ObjectMeta.Namespace, + Name: newTaskPod.ObjectMeta.Name, + }, newTaskPod) + if err != nil { + // if it doesn't exist, then create the task pod + opLog.Info(fmt.Sprintf("Creating task pod for: %s", lagoonTask.ObjectMeta.Name)) + // create the task pod + if err := r.Create(ctx, newTaskPod); err != nil { + opLog.Info( + fmt.Sprintf( + "Unable to create task pod for project %s, environment %s: %v", + lagoonTask.Spec.Project.Name, + lagoonTask.Spec.Environment.Name, + err, + ), + ) + //@TODO: send msg back and update task to failed? + return nil + } + taskRunningStatus.With(prometheus.Labels{ + "task_namespace": lagoonTask.ObjectMeta.Namespace, + "task_name": lagoonTask.ObjectMeta.Name, + }).Set(1) + tasksStartedCounter.Inc() + } else { + opLog.Info(fmt.Sprintf("Task pod already running for: %s", lagoonTask.ObjectMeta.Name)) + } + // The object is not being deleted, so if it does not have our finalizer, + // then lets add the finalizer and update the object. This is equivalent + // registering our finalizer. + if !helpers.ContainsString(lagoonTask.ObjectMeta.Finalizers, taskFinalizer) { + lagoonTask.ObjectMeta.Finalizers = append(lagoonTask.ObjectMeta.Finalizers, taskFinalizer) + // use patches to avoid update errors + mergePatch, _ := json.Marshal(map[string]interface{}{ + "metadata": map[string]interface{}{ + "finalizers": lagoonTask.ObjectMeta.Finalizers, + }, + }) + if err := r.Patch(ctx, lagoonTask, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { + return err + } + } + return nil +} + +// createAdvancedTask allows running of more advanced tasks than the standard lagoon tasks +// see notes in the docs for infomration about advanced tasks +func (r *LagoonTaskReconciler) createAdvancedTask(ctx context.Context, lagoonTask *lagooncrd.LagoonTask, opLog logr.Logger) error { + additionalLabels := map[string]string{} + + // check if this is an activestandby task, if it is, create the activestandby role + if value, ok := lagoonTask.ObjectMeta.Labels["lagoon.sh/activeStandby"]; ok { + isActiveStandby, _ := strconv.ParseBool(value) + if isActiveStandby { + var sourceNamespace, destinationNamespace string + if value, ok := lagoonTask.ObjectMeta.Labels["lagoon.sh/activeStandbySourceNamespace"]; ok { + sourceNamespace = value + } + if value, ok := lagoonTask.ObjectMeta.Labels["lagoon.sh/activeStandbyDestinationNamespace"]; ok { + destinationNamespace = value + } + // create the role + binding to allow the service account to interact with both namespaces + err := r.createActiveStandbyRole(ctx, sourceNamespace, destinationNamespace) + if err != nil { + return err + } + additionalLabels["lagoon.sh/activeStandby"] = "true" + additionalLabels["lagoon.sh/activeStandbySourceNamespace"] = sourceNamespace + additionalLabels["lagoon.sh/activeStandbyDestinationNamespace"] = destinationNamespace + } + } + + // handle the volumes for sshkey + sshKeyVolume := corev1.Volume{ + Name: "lagoon-sshkey", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "lagoon-sshkey", + DefaultMode: helpers.Int32Ptr(420), + }, + }, + } + sshKeyVolumeMount := corev1.VolumeMount{ + Name: "lagoon-sshkey", + ReadOnly: true, + MountPath: "/var/run/secrets/lagoon/ssh", + } + volumes := []corev1.Volume{} + volumeMounts := []corev1.VolumeMount{} + if lagoonTask.Spec.AdvancedTask.SSHKey { + volumes = append(volumes, sshKeyVolume) + volumeMounts = append(volumeMounts, sshKeyVolumeMount) + } + if lagoonTask.Spec.AdvancedTask.DeployerToken { + // if this advanced task can access kubernetes, mount the token in + serviceAccount := &corev1.ServiceAccount{} + err := r.getServiceAccount(ctx, serviceAccount, lagoonTask.ObjectMeta.Namespace) + if err != nil { + return err + } + var serviceaccountTokenSecret string + for _, secret := range serviceAccount.Secrets { + match, _ := regexp.MatchString("^lagoon-deployer-token", secret.Name) + if match { + serviceaccountTokenSecret = secret.Name + break + } + } + // if the existing token exists, mount it + if serviceaccountTokenSecret != "" { + volumes = append(volumes, corev1.Volume{ + Name: serviceaccountTokenSecret, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: serviceaccountTokenSecret, + DefaultMode: helpers.Int32Ptr(420), + }, + }, + }) + // legacy tokens are mounted /var/run/secrets/lagoon/deployer + // new tokens using volume projection are mounted /var/run/secrets/kubernetes.io/serviceaccount/token + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: serviceaccountTokenSecret, + ReadOnly: true, + MountPath: "/var/run/secrets/lagoon/deployer", + }) + } + } + opLog.Info(fmt.Sprintf("Checking advanced task pod for: %s", lagoonTask.ObjectMeta.Name)) + // once the pod spec has been defined, check if it isn't already created + podEnvs := []corev1.EnvVar{ + { + Name: "JSON_PAYLOAD", + Value: string(lagoonTask.Spec.AdvancedTask.JSONPayload), + }, + { + Name: "NAMESPACE", + Value: lagoonTask.ObjectMeta.Namespace, + }, + { + Name: "PODNAME", + Value: lagoonTask.ObjectMeta.Name, + }, + { + Name: "LAGOON_PROJECT", + Value: lagoonTask.Spec.Project.Name, + }, + { + Name: "LAGOON_GIT_BRANCH", + Value: lagoonTask.Spec.Environment.Name, + }, + { + Name: "TASK_API_HOST", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "TASK_API_HOST"), + }, + { + Name: "TASK_SSH_HOST", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "TASK_SSH_HOST"), + }, + { + Name: "TASK_SSH_PORT", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "TASK_SSH_PORT"), + }, + { + Name: "LAGOON_CONFIG_API_HOST", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "LAGOON_CONFIG_API_HOST"), + }, + { + Name: "LAGOON_CONFIG_SSH_HOST", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "LAGOON_CONFIG_SSH_HOST"), + }, + { + Name: "LAGOON_CONFIG_SSH_PORT", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "LAGOON_CONFIG_SSH_PORT"), + }, + { + Name: "LAGOON_CONFIG_TOKEN_HOST", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "LAGOON_CONFIG_TOKEN_HOST"), + }, + { + Name: "LAGOON_CONFIG_TOKEN_PORT", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "LAGOON_CONFIG_TOKEN_PORT"), + }, + { + Name: "TASK_DATA_ID", + Value: lagoonTask.Spec.Task.ID, + }, + } + if lagoonTask.Spec.Project.Variables.Project != nil { + // if this is 2 bytes long, then it means its just an empty json array + // we only want to add it if it is more than 2 bytes + if len(lagoonTask.Spec.Project.Variables.Project) > 2 { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "LAGOON_PROJECT_VARIABLES", + Value: string(lagoonTask.Spec.Project.Variables.Project), + }) + } + } + if lagoonTask.Spec.Project.Variables.Environment != nil { + // if this is 2 bytes long, then it means its just an empty json array + // we only want to add it if it is more than 2 bytes + if len(lagoonTask.Spec.Project.Variables.Environment) > 2 { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "LAGOON_ENVIRONMENT_VARIABLES", + Value: string(lagoonTask.Spec.Project.Variables.Environment), + }) + } + } + newPod := &corev1.Pod{} + err := r.Get(ctx, types.NamespacedName{ + Namespace: lagoonTask.ObjectMeta.Namespace, + Name: lagoonTask.ObjectMeta.Name, + }, newPod) + if err != nil { + newPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: lagoonTask.ObjectMeta.Name, + Namespace: lagoonTask.ObjectMeta.Namespace, + Labels: map[string]string{ + "lagoon.sh/jobType": "task", + "lagoon.sh/taskName": lagoonTask.ObjectMeta.Name, + "lagoon.sh/crdVersion": crdVersion, + "lagoon.sh/controller": r.ControllerNamespace, + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: fmt.Sprintf("%v", lagooncrd.GroupVersion), + Kind: "LagoonTask", + Name: lagoonTask.ObjectMeta.Name, + UID: lagoonTask.UID, + }, + }, + }, + Spec: corev1.PodSpec{ + RestartPolicy: "Never", + Volumes: volumes, + Containers: []corev1.Container{ + { + Name: "lagoon-task", + Image: lagoonTask.Spec.AdvancedTask.RunnerImage, + ImagePullPolicy: "Always", + EnvFrom: []corev1.EnvFromSource{ + { + ConfigMapRef: &corev1.ConfigMapEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "lagoon-env", + }, + }, + }, + }, + Env: podEnvs, + VolumeMounts: volumeMounts, + }, + }, + }, + } + if lagoonTask.Spec.Project.Organization != nil { + newPod.ObjectMeta.Labels["organization.lagoon.sh/id"] = fmt.Sprintf("%d", *lagoonTask.Spec.Project.Organization.ID) + newPod.ObjectMeta.Labels["organization.lagoon.sh/name"] = lagoonTask.Spec.Project.Organization.Name + } + if lagoonTask.Spec.AdvancedTask.DeployerToken { + // start this with the serviceaccount so that it gets the token mounted into it + newPod.Spec.ServiceAccountName = "lagoon-deployer" + } + opLog.Info(fmt.Sprintf("Creating advanced task pod for: %s", lagoonTask.ObjectMeta.Name)) + + //Decorate the pod spec with additional details + + //dynamic secrets + secrets, err := getSecretsForNamespace(r.Client, lagoonTask.Namespace) + secrets = filterDynamicSecrets(secrets) + if err != nil { + return err + } + + const dynamicSecretVolumeNamePrefex = "dynamic-" + for _, secret := range secrets { + volumeMountName := dynamicSecretVolumeNamePrefex + secret.Name + v := corev1.Volume{ + Name: volumeMountName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: secret.Name, + DefaultMode: helpers.Int32Ptr(444), + }, + }, + } + newPod.Spec.Volumes = append(newPod.Spec.Volumes, v) + + //now add the volume mount + vm := corev1.VolumeMount{ + Name: volumeMountName, + ReadOnly: true, + MountPath: "/var/run/secrets/lagoon/dynamic/" + secret.Name, + } + + newPod.Spec.Containers[0].VolumeMounts = append(newPod.Spec.Containers[0].VolumeMounts, vm) + } + + if err := r.Create(ctx, newPod); err != nil { + opLog.Info( + fmt.Sprintf( + "Unable to create advanced task pod for project %s, environment %s: %v", + lagoonTask.Spec.Project.Name, + lagoonTask.Spec.Environment.Name, + err, + ), + ) + return err + } + taskRunningStatus.With(prometheus.Labels{ + "task_namespace": lagoonTask.ObjectMeta.Namespace, + "task_name": lagoonTask.ObjectMeta.Name, + }).Set(1) + tasksStartedCounter.Inc() + } else { + opLog.Info(fmt.Sprintf("Advanced task pod already running for: %s", lagoonTask.ObjectMeta.Name)) + } + return nil +} + +// getSecretsForNamespace is a convenience function to pull a list of secrets for a given namespace +func getSecretsForNamespace(k8sClient client.Client, namespace string) (map[string]corev1.Secret, error) { + secretList := &corev1.SecretList{} + err := k8sClient.List(context.Background(), secretList, &client.ListOptions{Namespace: namespace}) + if err != nil { + return nil, err + } + + secrets := map[string]corev1.Secret{} + for _, secret := range secretList.Items { + secrets[secret.Name] = secret + } + + return secrets, nil +} + +// filterDynamicSecrets will, given a map of secrets, filter those that match the dynamic secret label +func filterDynamicSecrets(secrets map[string]corev1.Secret) map[string]corev1.Secret { + filteredSecrets := map[string]corev1.Secret{} + for secretName, secret := range secrets { + if _, ok := secret.Labels["lagoon.sh/dynamic-secret"]; ok { + filteredSecrets[secretName] = secret + } + } + return filteredSecrets +} + +// getServiceAccount will get the service account if it exists +func (r *LagoonTaskReconciler) getServiceAccount(ctx context.Context, serviceAccount *corev1.ServiceAccount, ns string) error { + serviceAccount.ObjectMeta = metav1.ObjectMeta{ + Name: "lagoon-deployer", + Namespace: ns, + } + err := r.Get(ctx, types.NamespacedName{ + Namespace: ns, + Name: "lagoon-deployer", + }, serviceAccount) + if err != nil { + return err + } + return nil +} diff --git a/controllers/v1beta2/task_helpers.go b/controllers/v1beta2/task_helpers.go new file mode 100644 index 00000000..017857dc --- /dev/null +++ b/controllers/v1beta2/task_helpers.go @@ -0,0 +1,59 @@ +package v1beta2 + +import ( + "context" + "fmt" + + "github.com/uselagoon/remote-controller/internal/helpers" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +// createActiveStandbyRole will create the rolebinding for allowing lagoon-deployer to talk between namespaces for active/standby functionality +func (r *LagoonTaskReconciler) createActiveStandbyRole(ctx context.Context, sourceNamespace, destinationNamespace string) error { + activeStandbyRoleBinding := &rbacv1.RoleBinding{} + activeStandbyRoleBinding.ObjectMeta = metav1.ObjectMeta{ + Name: "lagoon-deployer-activestandby", + Namespace: destinationNamespace, + } + activeStandbyRoleBinding.RoleRef = rbacv1.RoleRef{ + Name: "admin", + Kind: "ClusterRole", + APIGroup: "rbac.authorization.k8s.io", + } + activeStandbyRoleBinding.Subjects = []rbacv1.Subject{ + { + Name: "lagoon-deployer", + Kind: "ServiceAccount", + Namespace: sourceNamespace, + }, + } + err := r.Get(ctx, types.NamespacedName{ + Namespace: destinationNamespace, + Name: "lagoon-deployer-activestandby", + }, activeStandbyRoleBinding) + if err != nil { + if err := r.Create(ctx, activeStandbyRoleBinding); err != nil { + return fmt.Errorf("There was an error creating the lagoon-deployer-activestandby role binding. Error was: %v", err) + } + } + return nil +} + +// deleteActiveStandbyRole +func (r *LagoonMonitorReconciler) deleteActiveStandbyRole(ctx context.Context, destinationNamespace string) error { + activeStandbyRoleBinding := &rbacv1.RoleBinding{} + err := r.Get(ctx, types.NamespacedName{ + Namespace: destinationNamespace, + Name: "lagoon-deployer-activestandby", + }, activeStandbyRoleBinding) + if err != nil { + helpers.IgnoreNotFound(err) + } + err = r.Delete(ctx, activeStandbyRoleBinding) + if err != nil { + return fmt.Errorf("Unable to delete lagoon-deployer-activestandby role binding") + } + return nil +} diff --git a/internal/harbor/harbor_credentialrotation.go b/internal/harbor/harbor_credentialrotation.go index 7a3bc4e7..43c97d90 100644 --- a/internal/harbor/harbor_credentialrotation.go +++ b/internal/harbor/harbor_credentialrotation.go @@ -7,6 +7,7 @@ import ( "time" lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" + lagoonv1beta2 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" "github.com/uselagoon/remote-controller/internal/helpers" corev1 "k8s.io/api/core/v1" @@ -45,7 +46,8 @@ func (h *Harbor) RotateRobotCredentials(ctx context.Context, cl client.Client) { opLog.Info(fmt.Sprintf("Checking if %s needs robot credentials rotated", ns.ObjectMeta.Name)) // check for running builds! runningBuildsv1beta1 := lagoonv1beta1.CheckRunningBuilds(ctx, h.ControllerNamespace, opLog, cl, ns) - if !runningBuildsv1beta1 { + runningBuildsv1beta2 := lagoonv1beta2.CheckRunningBuilds(ctx, h.ControllerNamespace, opLog, cl, ns) + if !runningBuildsv1beta1 && !runningBuildsv1beta2 { rotated, err := h.RotateRobotCredential(ctx, cl, ns, false) if err != nil { opLog.Error(err, "error") diff --git a/internal/messenger/consumer.go b/internal/messenger/consumer.go index 51173b37..b556300d 100644 --- a/internal/messenger/consumer.go +++ b/internal/messenger/consumer.go @@ -11,6 +11,7 @@ import ( "github.com/cheshir/go-mq/v2" "github.com/uselagoon/machinery/api/schema" lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" + lagoonv1beta2 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" "github.com/uselagoon/remote-controller/internal/helpers" "gopkg.in/matryer/try.v1" corev1 "k8s.io/api/core/v1" @@ -67,7 +68,7 @@ func (m *Messenger) Consumer(targetName string) { //error { err = messageQueue.SetConsumerHandler("builddeploy-queue", func(message mq.Message) { if err == nil { // unmarshal the body into a lagoonbuild - newBuild := &lagoonv1beta1.LagoonBuild{} + newBuild := &lagoonv1beta2.LagoonBuild{} json.Unmarshal(message.Body(), newBuild) // new builds that come in should initially get created in the controllers own // namespace before being handled and re-created in the correct namespace @@ -234,7 +235,7 @@ func (m *Messenger) Consumer(targetName string) { //error { err = messageQueue.SetConsumerHandler("jobs-queue", func(message mq.Message) { if err == nil { // unmarshall the message into a remove task to be processed - jobSpec := &lagoonv1beta1.LagoonTaskSpec{} + jobSpec := &lagoonv1beta2.LagoonTaskSpec{} json.Unmarshal(message.Body(), jobSpec) namespace := helpers.GenerateNamespaceName( jobSpec.Project.NamespacePattern, // the namespace pattern or `openshiftProjectPattern` from Lagoon is never received by the controller @@ -252,14 +253,14 @@ func (m *Messenger) Consumer(targetName string) { //error { namespace, ), ) - job := &lagoonv1beta1.LagoonTask{} + job := &lagoonv1beta2.LagoonTask{} job.Spec = *jobSpec // set the namespace to the `openshiftProjectName` from the environment job.ObjectMeta.Namespace = namespace job.SetLabels( map[string]string{ - "lagoon.sh/taskType": lagoonv1beta1.TaskTypeStandard.String(), - "lagoon.sh/taskStatus": lagoonv1beta1.TaskStatusPending.String(), + "lagoon.sh/taskType": lagoonv1beta2.TaskTypeStandard.String(), + "lagoon.sh/taskStatus": lagoonv1beta2.TaskStatusPending.String(), "lagoon.sh/controller": m.ControllerNamespace, }, ) @@ -292,7 +293,7 @@ func (m *Messenger) Consumer(targetName string) { //error { if err == nil { opLog := ctrl.Log.WithName("handlers").WithName("LagoonTasks") // unmarshall the message into a remove task to be processed - jobSpec := &lagoonv1beta1.LagoonTaskSpec{} + jobSpec := &lagoonv1beta2.LagoonTaskSpec{} json.Unmarshal(message.Body(), jobSpec) // check which key has been received namespace := helpers.GenerateNamespaceName( @@ -314,18 +315,43 @@ func (m *Messenger) Consumer(targetName string) { //error { ), ) m.Cache.Add(jobSpec.Misc.Name, jobSpec.Project.Name) - _, v1beta1Bytes, err := lagoonv1beta1.CancelBuild(ctx, m.Client, namespace, message.Body()) + // check if there is a v1beta2 task to cancel + hasv1beta2Build, v1beta2Bytes, err := lagoonv1beta2.CancelBuild(ctx, m.Client, namespace, message.Body()) + if err != nil { + //@TODO: send msg back to lagoon and update task to failed? + message.Ack(false) // ack to remove from queue + return + } + hasv1beta1Build, v1beta1Bytes, err := lagoonv1beta1.CancelBuild(ctx, m.Client, namespace, message.Body()) if err != nil { //@TODO: send msg back to lagoon and update build to failed? message.Ack(false) // ack to remove from queue return } - if v1beta1Bytes != nil { - // if v1beta1 has a build, send its response - if err := m.Publish("lagoon-tasks:controller", v1beta1Bytes); err != nil { - opLog.Error(err, "Unable to publish message.") - message.Ack(false) // ack to remove from queue - return + if v1beta1Bytes != nil && v1beta2Bytes != nil { + // check if either v1beta2 or v1beta1 have a matching build + if hasv1beta2Build && !hasv1beta1Build { + // if v1beta2 has a build, send its response + if err := m.Publish("lagoon-tasks:controller", v1beta2Bytes); err != nil { + opLog.Error(err, "Unable to publish message.") + message.Ack(false) // ack to remove from queue + return + } + } else if !hasv1beta2Build && hasv1beta1Build { + // if v1beta1 has a build, send its response + if err := m.Publish("lagoon-tasks:controller", v1beta1Bytes); err != nil { + opLog.Error(err, "Unable to publish message.") + message.Ack(false) // ack to remove from queue + return + } + } else { + // if both v1beta2 and v1beta1 say there is no associated build + // then respond with the v1beta2 response + if err := m.Publish("lagoon-tasks:controller", v1beta2Bytes); err != nil { + opLog.Error(err, "Unable to publish message.") + message.Ack(false) // ack to remove from queue + return + } } } case "deploytarget:task:cancel", "kubernetes:task:cancel": @@ -338,17 +364,43 @@ func (m *Messenger) Consumer(targetName string) { //error { ), ) m.Cache.Add(jobSpec.Task.TaskName, jobSpec.Project.Name) - _, v1beta1Bytes, err := lagoonv1beta1.CancelTask(ctx, m.Client, namespace, message.Body()) + // check if there is a v1beta2 task to cancel + hasv1beta2Task, v1beta2Bytes, err := lagoonv1beta2.CancelTask(ctx, m.Client, namespace, message.Body()) if err != nil { //@TODO: send msg back to lagoon and update task to failed? message.Ack(false) // ack to remove from queue return } - if v1beta1Bytes != nil { - if err := m.Publish("lagoon-tasks:controller", v1beta1Bytes); err != nil { - opLog.Error(err, "Unable to publish message.") - message.Ack(false) // ack to remove from queue - return + hasv1beta1Task, v1beta1Bytes, err := lagoonv1beta1.CancelTask(ctx, m.Client, namespace, message.Body()) + if err != nil { + //@TODO: send msg back to lagoon and update task to failed? + message.Ack(false) // ack to remove from queue + return + } + if v1beta1Bytes != nil && v1beta2Bytes != nil { + // check if either v1beta2 or v1beta1 have a matching task + if hasv1beta2Task && !hasv1beta1Task { + // if v1beta2 has a task, send its response + if err := m.Publish("lagoon-tasks:controller", v1beta2Bytes); err != nil { + opLog.Error(err, "Unable to publish message.") + message.Ack(false) // ack to remove from queue + return + } + } else if !hasv1beta2Task && hasv1beta1Task { + // if v1beta1 has a task, send its response + if err := m.Publish("lagoon-tasks:controller", v1beta1Bytes); err != nil { + opLog.Error(err, "Unable to publish message.") + message.Ack(false) // ack to remove from queue + return + } + } else { + // if both v1beta2 and v1beta1 say there is no associated task + // then respond with the v1beta2 response + if err := m.Publish("lagoon-tasks:controller", v1beta2Bytes); err != nil { + opLog.Error(err, "Unable to publish message.") + message.Ack(false) // ack to remove from queue + return + } } } case "deploytarget:restic:backup:restore", "kubernetes:restic:backup:restore": diff --git a/internal/messenger/tasks_handler.go b/internal/messenger/tasks_handler.go index 2b8b68a0..3a88b464 100644 --- a/internal/messenger/tasks_handler.go +++ b/internal/messenger/tasks_handler.go @@ -6,7 +6,7 @@ import ( "encoding/json" "fmt" - lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" + lagoonv1beta2 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" "github.com/uselagoon/remote-controller/internal/helpers" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" @@ -18,7 +18,7 @@ type ActiveStandbyPayload struct { } // IngressRouteMigration handles running the ingress migrations. -func (m *Messenger) IngressRouteMigration(namespace string, jobSpec *lagoonv1beta1.LagoonTaskSpec) error { +func (m *Messenger) IngressRouteMigration(namespace string, jobSpec *lagoonv1beta2.LagoonTaskSpec) error { // always set these to true for ingress migration tasks jobSpec.AdvancedTask.DeployerToken = true jobSpec.AdvancedTask.SSHKey = true @@ -26,7 +26,7 @@ func (m *Messenger) IngressRouteMigration(namespace string, jobSpec *lagoonv1bet } // ActiveStandbySwitch handles running the active standby switch setup advanced task. -func (m *Messenger) ActiveStandbySwitch(namespace string, jobSpec *lagoonv1beta1.LagoonTaskSpec) error { +func (m *Messenger) ActiveStandbySwitch(namespace string, jobSpec *lagoonv1beta2.LagoonTaskSpec) error { // always set these to true for ingress migration tasks jobSpec.AdvancedTask.DeployerToken = true jobSpec.AdvancedTask.SSHKey = true @@ -47,30 +47,30 @@ func (m *Messenger) ActiveStandbySwitch(namespace string, jobSpec *lagoonv1beta1 } // AdvancedTask handles running the ingress migrations. -func (m *Messenger) AdvancedTask(namespace string, jobSpec *lagoonv1beta1.LagoonTaskSpec) error { +func (m *Messenger) AdvancedTask(namespace string, jobSpec *lagoonv1beta2.LagoonTaskSpec) error { return m.createAdvancedTask(namespace, jobSpec, nil) } // CreateAdvancedTask takes care of creating actual advanced tasks -func (m *Messenger) createAdvancedTask(namespace string, jobSpec *lagoonv1beta1.LagoonTaskSpec, additionalLabels map[string]string) error { +func (m *Messenger) createAdvancedTask(namespace string, jobSpec *lagoonv1beta2.LagoonTaskSpec, additionalLabels map[string]string) error { return createAdvancedTask(namespace, jobSpec, m, additionalLabels) } // CreateAdvancedTask takes care of creating actual advanced tasks -func createAdvancedTask(namespace string, jobSpec *lagoonv1beta1.LagoonTaskSpec, m *Messenger, additionalLabels map[string]string) error { +func createAdvancedTask(namespace string, jobSpec *lagoonv1beta2.LagoonTaskSpec, m *Messenger, additionalLabels map[string]string) error { opLog := ctrl.Log.WithName("handlers").WithName("LagoonTasks") // create the advanced task taskName := fmt.Sprintf("lagoon-advanced-task-%s", helpers.RandString(6)) if jobSpec.Task.TaskName != "" { taskName = jobSpec.Task.TaskName } - task := lagoonv1beta1.LagoonTask{ + task := lagoonv1beta2.LagoonTask{ ObjectMeta: metav1.ObjectMeta{ Name: taskName, Namespace: namespace, Labels: map[string]string{ - "lagoon.sh/taskType": lagoonv1beta1.TaskTypeAdvanced.String(), - "lagoon.sh/taskStatus": lagoonv1beta1.TaskStatusPending.String(), + "lagoon.sh/taskType": lagoonv1beta2.TaskTypeAdvanced.String(), + "lagoon.sh/taskStatus": lagoonv1beta2.TaskStatusPending.String(), "lagoon.sh/controller": m.ControllerNamespace, }, }, diff --git a/internal/messenger/tasks_restore.go b/internal/messenger/tasks_restore.go index 315ddb4d..a380c4a1 100644 --- a/internal/messenger/tasks_restore.go +++ b/internal/messenger/tasks_restore.go @@ -6,7 +6,7 @@ import ( "fmt" "github.com/go-logr/logr" - lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" + lagoonv1beta2 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" "github.com/uselagoon/remote-controller/internal/helpers" ctrl "sigs.k8s.io/controller-runtime" @@ -17,7 +17,7 @@ import ( ) // ResticRestore handles creating the restic restore jobs. -func (m *Messenger) ResticRestore(namespace string, jobSpec *lagoonv1beta1.LagoonTaskSpec) error { +func (m *Messenger) ResticRestore(namespace string, jobSpec *lagoonv1beta2.LagoonTaskSpec) error { opLog := ctrl.Log.WithName("handlers").WithName("LagoonTasks") vers, err := checkRestoreVersionFromCore(jobSpec.Misc.MiscResource) if err != nil { @@ -97,7 +97,7 @@ func checkRestoreVersionFromCore(resource []byte) (string, error) { } // createv1alpha1Restore will create a restore task using the restores.backup.appuio.ch v1alpha1 api (k8up v1) -func (m *Messenger) createv1alpha1Restore(opLog logr.Logger, namespace string, jobSpec *lagoonv1beta1.LagoonTaskSpec) error { +func (m *Messenger) createv1alpha1Restore(opLog logr.Logger, namespace string, jobSpec *lagoonv1beta2.LagoonTaskSpec) error { restorev1alpha1 := &k8upv1alpha1.Restore{} if err := json.Unmarshal(jobSpec.Misc.MiscResource, restorev1alpha1); err != nil { opLog.Error(err, @@ -122,7 +122,7 @@ func (m *Messenger) createv1alpha1Restore(opLog logr.Logger, namespace string, j } // createv1Restore will create a restore task using the restores.k8up.io v1 api (k8up v2) -func (m *Messenger) createv1Restore(opLog logr.Logger, namespace string, jobSpec *lagoonv1beta1.LagoonTaskSpec) error { +func (m *Messenger) createv1Restore(opLog logr.Logger, namespace string, jobSpec *lagoonv1beta2.LagoonTaskSpec) error { restorev1 := &k8upv1.Restore{} if err := json.Unmarshal(jobSpec.Misc.MiscResource, restorev1); err != nil { opLog.Error(err, diff --git a/internal/utilities/deletions/process.go b/internal/utilities/deletions/process.go index f6909575..aa3354a5 100644 --- a/internal/utilities/deletions/process.go +++ b/internal/utilities/deletions/process.go @@ -10,6 +10,7 @@ import ( "k8s.io/apimachinery/pkg/types" lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" + lagoonv1beta2 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" "github.com/uselagoon/remote-controller/internal/harbor" ) @@ -77,9 +78,15 @@ func (d *Deletions) ProcessDeletion(ctx context.Context, opLog logr.Logger, name if del := lagoonv1beta1.DeleteLagoonTasks(ctx, opLog.WithName("DeleteLagoonTasks"), d.Client, namespace.ObjectMeta.Name, project, environment); del == false { return fmt.Errorf("error deleting tasks") } + if del := lagoonv1beta2.DeleteLagoonTasks(ctx, opLog.WithName("DeleteLagoonTasks"), d.Client, namespace.ObjectMeta.Name, project, environment); del == false { + return fmt.Errorf("error deleting tasks") + } if del := lagoonv1beta1.DeleteLagoonBuilds(ctx, opLog.WithName("DeleteLagoonBuilds"), d.Client, namespace.ObjectMeta.Name, project, environment); del == false { return fmt.Errorf("error deleting builds") } + if del := lagoonv1beta2.DeleteLagoonBuilds(ctx, opLog.WithName("DeleteLagoonBuilds"), d.Client, namespace.ObjectMeta.Name, project, environment); del == false { + return fmt.Errorf("error deleting builds") + } if del := d.DeleteDeployments(ctx, opLog.WithName("DeleteDeployments"), namespace.ObjectMeta.Name, project, environment); del == false { return fmt.Errorf("error deleting deployments") } diff --git a/main.go b/main.go index 9aef681d..1e66c0cf 100644 --- a/main.go +++ b/main.go @@ -45,8 +45,10 @@ import ( "github.com/hashicorp/golang-lru/v2/expirable" k8upv1 "github.com/k8up-io/k8up/v2/api/v1" lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" + lagoonv1beta2 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" harborctrl "github.com/uselagoon/remote-controller/controllers/harbor" lagoonv1beta1ctrl "github.com/uselagoon/remote-controller/controllers/v1beta1" + lagoonv1beta2ctrl "github.com/uselagoon/remote-controller/controllers/v1beta2" "github.com/uselagoon/remote-controller/internal/messenger" k8upv1alpha1 "github.com/vshn/k8up/api/v1alpha1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" @@ -75,6 +77,7 @@ func init() { _ = clientgoscheme.AddToScheme(scheme) _ = lagoonv1beta1.AddToScheme(scheme) + _ = lagoonv1beta2.AddToScheme(scheme) _ = k8upv1.AddToScheme(scheme) _ = k8upv1alpha1.AddToScheme(scheme) _ = apiextensionsv1.AddToScheme(scheme) @@ -663,6 +666,11 @@ func main() { DefaultValue: qosDefaultValue, } + buildQoSConfigv1beta2 := lagoonv1beta2ctrl.BuildQoS{ + MaxBuilds: qosMaxBuilds, + DefaultValue: qosDefaultValue, + } + resourceCleanup := pruner.New(mgr.GetClient(), buildsToKeep, buildPodsToKeep, @@ -680,6 +688,7 @@ func main() { // use cron to run a lagoonbuild cleanup task // this will check any Lagoon builds and attempt to delete them c.AddFunc(buildsCleanUpCron, func() { + lagoonv1beta2.LagoonBuildPruner(context.Background(), mgr.GetClient(), controllerNamespace, buildsToKeep) lagoonv1beta1.LagoonBuildPruner(context.Background(), mgr.GetClient(), controllerNamespace, buildsToKeep) }) } @@ -689,6 +698,7 @@ func main() { // use cron to run a build pod cleanup task // this will check any Lagoon build pods and attempt to delete them c.AddFunc(buildPodCleanUpCron, func() { + lagoonv1beta2.BuildPodPruner(context.Background(), mgr.GetClient(), controllerNamespace, buildPodsToKeep) lagoonv1beta1.BuildPodPruner(context.Background(), mgr.GetClient(), controllerNamespace, buildPodsToKeep) }) } @@ -698,6 +708,7 @@ func main() { // use cron to run a lagoontask cleanup task // this will check any Lagoon tasks and attempt to delete them c.AddFunc(taskCleanUpCron, func() { + lagoonv1beta2.LagoonTaskPruner(context.Background(), mgr.GetClient(), controllerNamespace, tasksToKeep) lagoonv1beta1.LagoonTaskPruner(context.Background(), mgr.GetClient(), controllerNamespace, tasksToKeep) }) } @@ -707,6 +718,7 @@ func main() { // use cron to run a task pod cleanup task // this will check any Lagoon task pods and attempt to delete them c.AddFunc(taskPodCleanUpCron, func() { + lagoonv1beta2.TaskPodPruner(context.Background(), mgr.GetClient(), controllerNamespace, taskPodsToKeep) lagoonv1beta1.TaskPodPruner(context.Background(), mgr.GetClient(), controllerNamespace, taskPodsToKeep) }) } @@ -846,6 +858,112 @@ func main() { os.Exit(1) } + // v1beta2 is the latest version + if err = (&lagoonv1beta2ctrl.LagoonBuildReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("v1beta2").WithName("LagoonBuild"), + Scheme: mgr.GetScheme(), + EnableMQ: enableMQ, + BuildImage: overrideBuildDeployImage, + Messaging: messaging, + NamespacePrefix: namespacePrefix, + RandomNamespacePrefix: randomPrefix, + ControllerNamespace: controllerNamespace, + EnableDebug: enableDebug, + FastlyServiceID: fastlyServiceID, + FastlyWatchStatus: fastlyWatchStatus, + BuildPodRunAsUser: int64(buildPodRunAsUser), + BuildPodRunAsGroup: int64(buildPodRunAsGroup), + BuildPodFSGroup: int64(buildPodFSGroup), + BackupConfig: lagoonv1beta2ctrl.BackupConfig{ + BackupDefaultSchedule: backupDefaultSchedule, + BackupDefaultDevelopmentSchedule: backupDefaultDevelopmentSchedule, + BackupDefaultPullrequestSchedule: backupDefaultPullrequestSchedule, + BackupDefaultDevelopmentRetention: backupDefaultDevelopmentRetention, + BackupDefaultPullrequestRetention: backupDefaultPullrequestRetention, + BackupDefaultMonthlyRetention: backupDefaultMonthlyRetention, + BackupDefaultWeeklyRetention: backupDefaultWeeklyRetention, + BackupDefaultDailyRetention: backupDefaultDailyRetention, + BackupDefaultHourlyRetention: backupDefaultHourlyRetention, + }, + // Lagoon feature flags + LFFForceRootlessWorkload: lffForceRootlessWorkload, + LFFDefaultRootlessWorkload: lffDefaultRootlessWorkload, + LFFForceIsolationNetworkPolicy: lffForceIsolationNetworkPolicy, + LFFDefaultIsolationNetworkPolicy: lffDefaultIsolationNetworkPolicy, + LFFForceInsights: lffForceInsights, + LFFDefaultInsights: lffDefaultInsights, + LFFForceRWX2RWO: lffForceRWX2RWO, + LFFDefaultRWX2RWO: lffDefaultRWX2RWO, + LFFBackupWeeklyRandom: lffBackupWeeklyRandom, + LFFRouterURL: lffRouterURL, + LFFHarborEnabled: lffHarborEnabled, + Harbor: harborConfig, + LFFQoSEnabled: lffQoSEnabled, + BuildQoS: buildQoSConfigv1beta2, + NativeCronPodMinFrequency: nativeCronPodMinFrequency, + LagoonTargetName: lagoonTargetName, + LagoonFeatureFlags: helpers.GetLagoonFeatureFlags(), + LagoonAPIConfiguration: helpers.LagoonAPIConfiguration{ + APIHost: lagoonAPIHost, + TokenHost: lagoonTokenHost, + TokenPort: lagoonTokenPort, + SSHHost: lagoonSSHHost, + SSHPort: lagoonSSHPort, + }, + ProxyConfig: lagoonv1beta2ctrl.ProxyConfig{ + HTTPProxy: httpProxy, + HTTPSProxy: httpsProxy, + NoProxy: noProxy, + }, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "LagoonBuild") + os.Exit(1) + } + if err = (&lagoonv1beta2ctrl.LagoonMonitorReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("v1beta2").WithName("LagoonMonitor"), + Scheme: mgr.GetScheme(), + EnableMQ: enableMQ, + Messaging: messaging, + ControllerNamespace: controllerNamespace, + NamespacePrefix: namespacePrefix, + RandomNamespacePrefix: randomPrefix, + EnableDebug: enableDebug, + LagoonTargetName: lagoonTargetName, + LFFQoSEnabled: lffQoSEnabled, + BuildQoS: buildQoSConfigv1beta2, + Cache: cache, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "LagoonMonitor") + os.Exit(1) + } + if err = (&lagoonv1beta2ctrl.LagoonTaskReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("v1beta2").WithName("LagoonTask"), + Scheme: mgr.GetScheme(), + ControllerNamespace: controllerNamespace, + NamespacePrefix: namespacePrefix, + RandomNamespacePrefix: randomPrefix, + LagoonAPIConfiguration: helpers.LagoonAPIConfiguration{ + APIHost: lagoonAPIHost, + TokenHost: lagoonTokenHost, + TokenPort: lagoonTokenPort, + SSHHost: lagoonSSHHost, + SSHPort: lagoonSSHPort, + }, + EnableDebug: enableDebug, + LagoonTargetName: lagoonTargetName, + ProxyConfig: lagoonv1beta2ctrl.ProxyConfig{ + HTTPProxy: httpProxy, + HTTPSProxy: httpsProxy, + NoProxy: noProxy, + }, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "LagoonTask") + os.Exit(1) + } + // for now the namespace reconciler only needs to run if harbor is enabled so that we can watch the namespace for rotation label events if lffHarborEnabled { if err = (&harborctrl.HarborCredentialReconciler{ diff --git a/test-resources/dynamic-secret-in-task-project1.yaml b/test-resources/dynamic-secret-in-task-project1.yaml index e242a0a6..85bdc21a 100644 --- a/test-resources/dynamic-secret-in-task-project1.yaml +++ b/test-resources/dynamic-secret-in-task-project1.yaml @@ -1,4 +1,4 @@ -apiVersion: crd.lagoon.sh/v1beta1 +apiVersion: crd.lagoon.sh/v1beta2 kind: LagoonTask metadata: name: "lagoon-advanced-task-example-task-project-1" diff --git a/test-resources/example-project1.yaml b/test-resources/example-project1.yaml index 355f8fb8..ef3f47e9 100644 --- a/test-resources/example-project1.yaml +++ b/test-resources/example-project1.yaml @@ -1,5 +1,5 @@ kind: LagoonBuild -apiVersion: crd.lagoon.sh/v1beta1 +apiVersion: crd.lagoon.sh/v1beta2 metadata: name: lagoon-build-7m5zypx spec: diff --git a/test-resources/example-project2.yaml b/test-resources/example-project2.yaml index 67451af2..1d30479d 100644 --- a/test-resources/example-project2.yaml +++ b/test-resources/example-project2.yaml @@ -1,5 +1,5 @@ kind: LagoonBuild -apiVersion: crd.lagoon.sh/v1beta1 +apiVersion: crd.lagoon.sh/v1beta2 metadata: name: lagoon-build-8m5zypx spec: diff --git a/test-resources/example-project3.yaml b/test-resources/example-project3.yaml new file mode 100644 index 00000000..57cba070 --- /dev/null +++ b/test-resources/example-project3.yaml @@ -0,0 +1,32 @@ +kind: LagoonBuild +apiVersion: crd.lagoon.sh/v1beta1 +metadata: + name: lagoon-build-1m5zypx +spec: + build: + ci: 'true' #to make sure that readwritemany is changed to readwriteonce + type: branch + gitReference: origin/main + project: + name: nginx-example + environment: main + organization: + id: 123 + name: test-org + uiLink: https://dashboard.amazeeio.cloud/projects/project/project-environment/deployments/lagoon-build-7m5zypx + routerPattern: 'main-nginx-example' + environmentType: production + productionEnvironment: main + standbyEnvironment: master + gitUrl: https://github.com/shreddedbacon/lagoon-nginx-example.git + deployTarget: kind + projectSecret: 4d6e7dd0f013a75d62a0680139fa82d350c2a1285f43f867535bad1143f228b1 + key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlDWFFJQkFBS0JnUUNjc1g2RG5KNXpNb0RqQ2R6a1JFOEg2TEh2TDQzaUhsekJLTWo4T1VNV05ZZG5YekdqCkR5Mkp1anQ3ZDNlMTVLeC8zOFo5UzJLdHNnVFVtWi9lUlRQSTdabE1idHRJK250UmtyblZLblBWNzhEeEFKNW8KTGZtQndmdWE2MnlVYnl0cnpYQ2pwVVJrQUlBMEZiR2VqS2Rvd3cxcnZGMzJoZFUzQ3ZIcG5rKzE2d0lEQVFBQgpBb0dCQUkrV0dyL1NDbVMzdCtIVkRPVGtMNk9vdVR6Y1QrRVFQNkVGbGIrRFhaV0JjZFhwSnB3c2NXZFBEK2poCkhnTEJUTTFWS3hkdnVEcEE4aW83cUlMTzJWYm1MeGpNWGk4TUdwY212dXJFNVJydTZTMXJzRDl2R0c5TGxoR3UKK0pUSmViMVdaZFduWFZ2am5LbExrWEV1eUthbXR2Z253Um5xNld5V05OazJ6SktoQWtFQThFenpxYnowcFVuTApLc241K2k0NUdoRGVpRTQvajRtamo1b1FHVzJxbUZWT2pHaHR1UGpaM2lwTis0RGlTRkFyMkl0b2VlK085d1pyCkRINHBkdU5YOFFKQkFLYnVOQ3dXK29sYXA4R2pUSk1TQjV1MW8wMVRHWFdFOGhVZG1leFBBdjl0cTBBT0gzUUQKUTIrM0RsaVY0ektoTlMra2xaSkVjNndzS0YyQmJIby81NXNDUVFETlBJd24vdERja3loSkJYVFJyc1RxZEZuOApCUWpZYVhBZTZEQ3o1eXg3S3ZFSmp1K1h1a01xTXV1ajBUSnpITFkySHVzK3FkSnJQVG9VMDNSS3JHV2hBa0JFCnB3aXI3Vk5pYy9jMFN2MnVLcWNZWWM1a2ViMnB1R0I3VUs1Q0lvaWdGakZzNmFJRDYyZXJwVVJ3S0V6RlFNbUgKNjQ5Y0ZXemhMVlA0aU1iZFREVHJBa0FFMTZXU1A3WXBWOHV1eFVGMGV0L3lFR3dURVpVU2R1OEppSTBHN0tqagpqcVR6RjQ3YkJZc0pIYTRYcWpVb2E3TXgwcS9FSUtRWkJ2NGFvQm42bGFOQwotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQ== + monitoring: + contact: '1234' + statuspageID: '1234' + variables: + project: W3sibmFtZSI6IkxBR09PTl9TWVNURU1fUk9VVEVSX1BBVFRFUk4iLCJ2YWx1ZSI6IiR7ZW52aXJvbm1lbnR9LiR7cHJvamVjdH0uZXhhbXBsZS5jb20iLCJzY29wZSI6ImludGVybmFsX3N5c3RlbSJ9XQ== + environment: W10= + branch: + name: main \ No newline at end of file