From 12e74a154c149ae4db454b34b0ee2006afd8b1a3 Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Fri, 5 Jan 2024 20:17:54 +1100 Subject: [PATCH 01/17] feat: support for services v2 --- apis/lagoon/v1beta1/lagoonmessaging_types.go | 6 +++ apis/lagoon/v1beta1/zz_generated.deepcopy.go | 20 ++++++++ .../crd/bases/crd.lagoon.sh_lagoonbuilds.yaml | 48 +++++++++++++++++++ .../crd/bases/crd.lagoon.sh_lagoontasks.yaml | 48 +++++++++++++++++++ controllers/v1beta1/build_deletionhandlers.go | 13 +++++ .../v1beta1/podmonitor_buildhandlers.go | 15 +++++- 6 files changed, 149 insertions(+), 1 deletion(-) diff --git a/apis/lagoon/v1beta1/lagoonmessaging_types.go b/apis/lagoon/v1beta1/lagoonmessaging_types.go index f9fe6ecd..7ee42727 100644 --- a/apis/lagoon/v1beta1/lagoonmessaging_types.go +++ b/apis/lagoon/v1beta1/lagoonmessaging_types.go @@ -35,12 +35,18 @@ type LagoonLogMeta struct { Routes []string `json:"routes,omitempty"` StartTime string `json:"startTime,omitempty"` Services []string `json:"services,omitempty"` + ServicesV2 []LagoonService `json:"servicesv2,omitempty"` Task *LagoonTaskInfo `json:"task,omitempty"` Key string `json:"key,omitempty"` AdvancedData string `json:"advancedData,omitempty"` Cluster string `json:"clusterName,omitempty"` } +type LagoonService struct { + Name string `json:"name"` + Type string `json:"type"` +} + // LagoonMessage is used for sending build info back to Lagoon // messaging queue to update the environment or deployment type LagoonMessage struct { diff --git a/apis/lagoon/v1beta1/zz_generated.deepcopy.go b/apis/lagoon/v1beta1/zz_generated.deepcopy.go index 842af614..56844afb 100644 --- a/apis/lagoon/v1beta1/zz_generated.deepcopy.go +++ b/apis/lagoon/v1beta1/zz_generated.deepcopy.go @@ -241,6 +241,11 @@ func (in *LagoonLogMeta) DeepCopyInto(out *LagoonLogMeta) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.ServicesV2 != nil { + in, out := &in.ServicesV2, &out.ServicesV2 + *out = make([]LagoonService, len(*in)) + copy(*out, *in) + } if in.Task != nil { in, out := &in.Task, &out.Task *out = new(LagoonTaskInfo) @@ -318,6 +323,21 @@ func (in *LagoonMiscInfo) DeepCopy() *LagoonMiscInfo { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LagoonService) DeepCopyInto(out *LagoonService) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonService. +func (in *LagoonService) DeepCopy() *LagoonService { + if in == nil { + return nil + } + out := new(LagoonService) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LagoonStatusMessages) DeepCopyInto(out *LagoonStatusMessages) { *out = *in diff --git a/config/crd/bases/crd.lagoon.sh_lagoonbuilds.yaml b/config/crd/bases/crd.lagoon.sh_lagoonbuilds.yaml index bbc2dc2d..7f97619c 100644 --- a/config/crd/bases/crd.lagoon.sh_lagoonbuilds.yaml +++ b/config/crd/bases/crd.lagoon.sh_lagoonbuilds.yaml @@ -265,6 +265,18 @@ spec: items: type: string type: array + servicesv2: + items: + properties: + name: + type: string + type: + type: string + required: + - name + - type + type: object + type: array startTime: type: string task: @@ -354,6 +366,18 @@ spec: items: type: string type: array + servicesv2: + items: + properties: + name: + type: string + type: + type: string + required: + - name + - type + type: object + type: array startTime: type: string task: @@ -445,6 +469,18 @@ spec: items: type: string type: array + servicesv2: + items: + properties: + name: + type: string + type: + type: string + required: + - name + - type + type: object + type: array startTime: type: string task: @@ -538,6 +574,18 @@ spec: items: type: string type: array + servicesv2: + items: + properties: + name: + type: string + type: + type: string + required: + - name + - type + type: object + type: array startTime: type: string task: diff --git a/config/crd/bases/crd.lagoon.sh_lagoontasks.yaml b/config/crd/bases/crd.lagoon.sh_lagoontasks.yaml index f1394f91..4bbe35b9 100644 --- a/config/crd/bases/crd.lagoon.sh_lagoontasks.yaml +++ b/config/crd/bases/crd.lagoon.sh_lagoontasks.yaml @@ -247,6 +247,18 @@ spec: items: type: string type: array + servicesv2: + items: + properties: + name: + type: string + type: + type: string + required: + - name + - type + type: object + type: array startTime: type: string task: @@ -336,6 +348,18 @@ spec: items: type: string type: array + servicesv2: + items: + properties: + name: + type: string + type: + type: string + required: + - name + - type + type: object + type: array startTime: type: string task: @@ -427,6 +451,18 @@ spec: items: type: string type: array + servicesv2: + items: + properties: + name: + type: string + type: + type: string + required: + - name + - type + type: object + type: array startTime: type: string task: @@ -520,6 +556,18 @@ spec: items: type: string type: array + servicesv2: + items: + properties: + name: + type: string + type: + type: string + required: + - name + - type + type: object + type: array startTime: type: string task: diff --git a/controllers/v1beta1/build_deletionhandlers.go b/controllers/v1beta1/build_deletionhandlers.go index fba4c0e2..d4ec0ff7 100644 --- a/controllers/v1beta1/build_deletionhandlers.go +++ b/controllers/v1beta1/build_deletionhandlers.go @@ -313,21 +313,34 @@ func (r *LagoonBuildReconciler) updateDeploymentAndEnvironmentTask(ctx context.C }) podList := &corev1.PodList{} serviceNames := []string{} + services := []lagoonv1beta1.LagoonService{} if err := r.List(context.TODO(), podList, listOption); err == nil { // generate the list of services to add to the environment for _, pod := range podList.Items { + var serviceName, serviceType string if _, ok := pod.ObjectMeta.Labels["lagoon.sh/service"]; ok { for _, container := range pod.Spec.Containers { serviceNames = append(serviceNames, container.Name) + serviceName = container.Name } } if _, ok := pod.ObjectMeta.Labels["service"]; ok { + // @TODO: remove this as this label shouldn't exist by now for _, container := range pod.Spec.Containers { serviceNames = append(serviceNames, container.Name) + serviceName = container.Name } } + if sType, ok := pod.ObjectMeta.Labels["lagoon.sh/service-type"]; ok { + serviceType = sType + } + services = append(services, lagoonv1beta1.LagoonService{ + Name: serviceName, + Type: serviceType, + }) } msg.Meta.Services = serviceNames + msg.Meta.ServicesV2 = services } // if we aren't being provided the lagoon config, we can skip adding the routes etc if lagoonEnv != nil { diff --git a/controllers/v1beta1/podmonitor_buildhandlers.go b/controllers/v1beta1/podmonitor_buildhandlers.go index 0159cb65..169cbebf 100644 --- a/controllers/v1beta1/podmonitor_buildhandlers.go +++ b/controllers/v1beta1/podmonitor_buildhandlers.go @@ -296,21 +296,34 @@ func (r *LagoonMonitorReconciler) updateDeploymentAndEnvironmentTask(ctx context }) depList := &appsv1.DeploymentList{} serviceNames := []string{} + services := []lagoonv1beta1.LagoonService{} if err := r.List(context.TODO(), depList, listOption); err == nil { - // generate the list of services to add to the environment + // generate the list of services to add or update to the environment for _, deployment := range depList.Items { + var serviceName, serviceType string if _, ok := deployment.ObjectMeta.Labels["lagoon.sh/service"]; ok { for _, container := range deployment.Spec.Template.Spec.Containers { serviceNames = append(serviceNames, container.Name) + serviceName = container.Name } } if _, ok := deployment.ObjectMeta.Labels["service"]; ok { + // @TODO: remove this as this label shouldn't exist by now for _, container := range deployment.Spec.Template.Spec.Containers { serviceNames = append(serviceNames, container.Name) + serviceName = container.Name } } + if sType, ok := deployment.ObjectMeta.Labels["lagoon.sh/service-type"]; ok { + serviceType = sType + } + services = append(services, lagoonv1beta1.LagoonService{ + Name: serviceName, + Type: serviceType, + }) } msg.Meta.Services = serviceNames + msg.Meta.ServicesV2 = services } // if we aren't being provided the lagoon config, we can skip adding the routes etc if lagoonEnv != nil { From f1317d037a8250820501f5179e3ff89163e70342 Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Fri, 5 Jan 2024 20:32:18 +1100 Subject: [PATCH 02/17] chore: remove pendingmessage functionality --- apis/lagoon/v1beta1/lagoonbuild_types.go | 5 +- apis/lagoon/v1beta1/lagoonmessaging_types.go | 9 - apis/lagoon/v1beta1/lagoontask_types.go | 5 +- apis/lagoon/v1beta1/zz_generated.deepcopy.go | 45 --- .../crd/bases/crd.lagoon.sh_lagoonbuilds.yaml | 371 ------------------ .../crd/bases/crd.lagoon.sh_lagoontasks.yaml | 371 ------------------ controllers/v1beta1/build_deletionhandlers.go | 114 +----- .../v1beta1/podmonitor_buildhandlers.go | 48 +-- .../v1beta1/podmonitor_taskhandlers.go | 48 +-- internal/messenger/pending_messages.go | 151 ------- main.go | 12 +- 11 files changed, 42 insertions(+), 1137 deletions(-) delete mode 100644 internal/messenger/pending_messages.go diff --git a/apis/lagoon/v1beta1/lagoonbuild_types.go b/apis/lagoon/v1beta1/lagoonbuild_types.go index c901ef22..16f0c706 100644 --- a/apis/lagoon/v1beta1/lagoonbuild_types.go +++ b/apis/lagoon/v1beta1/lagoonbuild_types.go @@ -90,9 +90,8 @@ type LagoonBuild struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` - Spec LagoonBuildSpec `json:"spec,omitempty"` - Status LagoonBuildStatus `json:"status,omitempty"` - StatusMessages *LagoonStatusMessages `json:"statusMessages,omitempty"` + Spec LagoonBuildSpec `json:"spec,omitempty"` + Status LagoonBuildStatus `json:"status,omitempty"` } // +kubebuilder:object:root=true diff --git a/apis/lagoon/v1beta1/lagoonmessaging_types.go b/apis/lagoon/v1beta1/lagoonmessaging_types.go index f9fe6ecd..fe1fb2e0 100644 --- a/apis/lagoon/v1beta1/lagoonmessaging_types.go +++ b/apis/lagoon/v1beta1/lagoonmessaging_types.go @@ -47,13 +47,4 @@ type LagoonMessage struct { Type string `json:"type,omitempty"` Namespace string `json:"namespace,omitempty"` Meta *LagoonLogMeta `json:"meta,omitempty"` - // BuildInfo *LagoonBuildInfo `json:"buildInfo,omitempty"` -} - -// LagoonStatusMessages is where unsent messages are stored for re-sending. -type LagoonStatusMessages struct { - StatusMessage *LagoonLog `json:"statusMessage,omitempty"` - BuildLogMessage *LagoonLog `json:"buildLogMessage,omitempty"` - TaskLogMessage *LagoonLog `json:"taskLogMessage,omitempty"` - EnvironmentMessage *LagoonMessage `json:"environmentMessage,omitempty"` } diff --git a/apis/lagoon/v1beta1/lagoontask_types.go b/apis/lagoon/v1beta1/lagoontask_types.go index 3231e530..aaffcff4 100644 --- a/apis/lagoon/v1beta1/lagoontask_types.go +++ b/apis/lagoon/v1beta1/lagoontask_types.go @@ -158,9 +158,8 @@ type LagoonTask struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` - Spec LagoonTaskSpec `json:"spec,omitempty"` - Status LagoonTaskStatus `json:"status,omitempty"` - StatusMessages *LagoonStatusMessages `json:"statusMessages,omitempty"` + Spec LagoonTaskSpec `json:"spec,omitempty"` + Status LagoonTaskStatus `json:"status,omitempty"` } // +kubebuilder:object:root=true diff --git a/apis/lagoon/v1beta1/zz_generated.deepcopy.go b/apis/lagoon/v1beta1/zz_generated.deepcopy.go index 842af614..45931b74 100644 --- a/apis/lagoon/v1beta1/zz_generated.deepcopy.go +++ b/apis/lagoon/v1beta1/zz_generated.deepcopy.go @@ -81,11 +81,6 @@ func (in *LagoonBuild) DeepCopyInto(out *LagoonBuild) { in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) - if in.StatusMessages != nil { - in, out := &in.StatusMessages, &out.StatusMessages - *out = new(LagoonStatusMessages) - (*in).DeepCopyInto(*out) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonBuild. @@ -318,41 +313,6 @@ func (in *LagoonMiscInfo) DeepCopy() *LagoonMiscInfo { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *LagoonStatusMessages) DeepCopyInto(out *LagoonStatusMessages) { - *out = *in - if in.StatusMessage != nil { - in, out := &in.StatusMessage, &out.StatusMessage - *out = new(LagoonLog) - (*in).DeepCopyInto(*out) - } - if in.BuildLogMessage != nil { - in, out := &in.BuildLogMessage, &out.BuildLogMessage - *out = new(LagoonLog) - (*in).DeepCopyInto(*out) - } - if in.TaskLogMessage != nil { - in, out := &in.TaskLogMessage, &out.TaskLogMessage - *out = new(LagoonLog) - (*in).DeepCopyInto(*out) - } - if in.EnvironmentMessage != nil { - in, out := &in.EnvironmentMessage, &out.EnvironmentMessage - *out = new(LagoonMessage) - (*in).DeepCopyInto(*out) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonStatusMessages. -func (in *LagoonStatusMessages) DeepCopy() *LagoonStatusMessages { - if in == nil { - return nil - } - out := new(LagoonStatusMessages) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LagoonTask) DeepCopyInto(out *LagoonTask) { *out = *in @@ -360,11 +320,6 @@ func (in *LagoonTask) DeepCopyInto(out *LagoonTask) { in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) - if in.StatusMessages != nil { - in, out := &in.StatusMessages, &out.StatusMessages - *out = new(LagoonStatusMessages) - (*in).DeepCopyInto(*out) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonTask. diff --git a/config/crd/bases/crd.lagoon.sh_lagoonbuilds.yaml b/config/crd/bases/crd.lagoon.sh_lagoonbuilds.yaml index bbc2dc2d..c8a2c299 100644 --- a/config/crd/bases/crd.lagoon.sh_lagoonbuilds.yaml +++ b/config/crd/bases/crd.lagoon.sh_lagoonbuilds.yaml @@ -201,377 +201,6 @@ spec: format: byte type: string type: object - statusMessages: - description: LagoonStatusMessages is where unsent messages are stored - for re-sending. - properties: - buildLogMessage: - description: LagoonLog is used to sendToLagoonLogs messaging queue - this is general logging information - properties: - event: - type: string - message: - type: string - meta: - description: LagoonLogMeta is the metadata that is used by logging - in Lagoon. - properties: - advancedData: - type: string - branchName: - type: string - buildName: - type: string - buildPhase: - type: string - buildStatus: - type: string - buildStep: - type: string - clusterName: - type: string - endTime: - type: string - environment: - type: string - environmentId: - type: integer - jobName: - type: string - jobStatus: - type: string - jobStep: - type: string - key: - type: string - logLink: - type: string - project: - type: string - projectId: - type: integer - projectName: - type: string - remoteId: - type: string - route: - type: string - routes: - items: - type: string - type: array - services: - items: - type: string - type: array - startTime: - type: string - task: - description: LagoonTaskInfo defines what a task can use to - communicate with Lagoon via SSH/API. - properties: - apiHost: - type: string - command: - type: string - id: - type: string - name: - type: string - service: - type: string - sshHost: - type: string - sshPort: - type: string - taskName: - type: string - required: - - id - type: object - type: object - project: - type: string - severity: - type: string - uuid: - type: string - type: object - environmentMessage: - description: LagoonMessage is used for sending build info back to - Lagoon messaging queue to update the environment or deployment - properties: - meta: - description: LagoonLogMeta is the metadata that is used by logging - in Lagoon. - properties: - advancedData: - type: string - branchName: - type: string - buildName: - type: string - buildPhase: - type: string - buildStatus: - type: string - buildStep: - type: string - clusterName: - type: string - endTime: - type: string - environment: - type: string - environmentId: - type: integer - jobName: - type: string - jobStatus: - type: string - jobStep: - type: string - key: - type: string - logLink: - type: string - project: - type: string - projectId: - type: integer - projectName: - type: string - remoteId: - type: string - route: - type: string - routes: - items: - type: string - type: array - services: - items: - type: string - type: array - startTime: - type: string - task: - description: LagoonTaskInfo defines what a task can use to - communicate with Lagoon via SSH/API. - properties: - apiHost: - type: string - command: - type: string - id: - type: string - name: - type: string - service: - type: string - sshHost: - type: string - sshPort: - type: string - taskName: - type: string - required: - - id - type: object - type: object - namespace: - type: string - type: - type: string - type: object - statusMessage: - description: LagoonLog is used to sendToLagoonLogs messaging queue - this is general logging information - properties: - event: - type: string - message: - type: string - meta: - description: LagoonLogMeta is the metadata that is used by logging - in Lagoon. - properties: - advancedData: - type: string - branchName: - type: string - buildName: - type: string - buildPhase: - type: string - buildStatus: - type: string - buildStep: - type: string - clusterName: - type: string - endTime: - type: string - environment: - type: string - environmentId: - type: integer - jobName: - type: string - jobStatus: - type: string - jobStep: - type: string - key: - type: string - logLink: - type: string - project: - type: string - projectId: - type: integer - projectName: - type: string - remoteId: - type: string - route: - type: string - routes: - items: - type: string - type: array - services: - items: - type: string - type: array - startTime: - type: string - task: - description: LagoonTaskInfo defines what a task can use to - communicate with Lagoon via SSH/API. - properties: - apiHost: - type: string - command: - type: string - id: - type: string - name: - type: string - service: - type: string - sshHost: - type: string - sshPort: - type: string - taskName: - type: string - required: - - id - type: object - type: object - project: - type: string - severity: - type: string - uuid: - type: string - type: object - taskLogMessage: - description: LagoonLog is used to sendToLagoonLogs messaging queue - this is general logging information - properties: - event: - type: string - message: - type: string - meta: - description: LagoonLogMeta is the metadata that is used by logging - in Lagoon. - properties: - advancedData: - type: string - branchName: - type: string - buildName: - type: string - buildPhase: - type: string - buildStatus: - type: string - buildStep: - type: string - clusterName: - type: string - endTime: - type: string - environment: - type: string - environmentId: - type: integer - jobName: - type: string - jobStatus: - type: string - jobStep: - type: string - key: - type: string - logLink: - type: string - project: - type: string - projectId: - type: integer - projectName: - type: string - remoteId: - type: string - route: - type: string - routes: - items: - type: string - type: array - services: - items: - type: string - type: array - startTime: - type: string - task: - description: LagoonTaskInfo defines what a task can use to - communicate with Lagoon via SSH/API. - properties: - apiHost: - type: string - command: - type: string - id: - type: string - name: - type: string - service: - type: string - sshHost: - type: string - sshPort: - type: string - taskName: - type: string - required: - - id - type: object - type: object - project: - type: string - severity: - type: string - uuid: - type: string - type: object - type: object type: object served: true storage: true diff --git a/config/crd/bases/crd.lagoon.sh_lagoontasks.yaml b/config/crd/bases/crd.lagoon.sh_lagoontasks.yaml index f1394f91..d17f6265 100644 --- a/config/crd/bases/crd.lagoon.sh_lagoontasks.yaml +++ b/config/crd/bases/crd.lagoon.sh_lagoontasks.yaml @@ -183,377 +183,6 @@ spec: format: byte type: string type: object - statusMessages: - description: LagoonStatusMessages is where unsent messages are stored - for re-sending. - properties: - buildLogMessage: - description: LagoonLog is used to sendToLagoonLogs messaging queue - this is general logging information - properties: - event: - type: string - message: - type: string - meta: - description: LagoonLogMeta is the metadata that is used by logging - in Lagoon. - properties: - advancedData: - type: string - branchName: - type: string - buildName: - type: string - buildPhase: - type: string - buildStatus: - type: string - buildStep: - type: string - clusterName: - type: string - endTime: - type: string - environment: - type: string - environmentId: - type: integer - jobName: - type: string - jobStatus: - type: string - jobStep: - type: string - key: - type: string - logLink: - type: string - project: - type: string - projectId: - type: integer - projectName: - type: string - remoteId: - type: string - route: - type: string - routes: - items: - type: string - type: array - services: - items: - type: string - type: array - startTime: - type: string - task: - description: LagoonTaskInfo defines what a task can use to - communicate with Lagoon via SSH/API. - properties: - apiHost: - type: string - command: - type: string - id: - type: string - name: - type: string - service: - type: string - sshHost: - type: string - sshPort: - type: string - taskName: - type: string - required: - - id - type: object - type: object - project: - type: string - severity: - type: string - uuid: - type: string - type: object - environmentMessage: - description: LagoonMessage is used for sending build info back to - Lagoon messaging queue to update the environment or deployment - properties: - meta: - description: LagoonLogMeta is the metadata that is used by logging - in Lagoon. - properties: - advancedData: - type: string - branchName: - type: string - buildName: - type: string - buildPhase: - type: string - buildStatus: - type: string - buildStep: - type: string - clusterName: - type: string - endTime: - type: string - environment: - type: string - environmentId: - type: integer - jobName: - type: string - jobStatus: - type: string - jobStep: - type: string - key: - type: string - logLink: - type: string - project: - type: string - projectId: - type: integer - projectName: - type: string - remoteId: - type: string - route: - type: string - routes: - items: - type: string - type: array - services: - items: - type: string - type: array - startTime: - type: string - task: - description: LagoonTaskInfo defines what a task can use to - communicate with Lagoon via SSH/API. - properties: - apiHost: - type: string - command: - type: string - id: - type: string - name: - type: string - service: - type: string - sshHost: - type: string - sshPort: - type: string - taskName: - type: string - required: - - id - type: object - type: object - namespace: - type: string - type: - type: string - type: object - statusMessage: - description: LagoonLog is used to sendToLagoonLogs messaging queue - this is general logging information - properties: - event: - type: string - message: - type: string - meta: - description: LagoonLogMeta is the metadata that is used by logging - in Lagoon. - properties: - advancedData: - type: string - branchName: - type: string - buildName: - type: string - buildPhase: - type: string - buildStatus: - type: string - buildStep: - type: string - clusterName: - type: string - endTime: - type: string - environment: - type: string - environmentId: - type: integer - jobName: - type: string - jobStatus: - type: string - jobStep: - type: string - key: - type: string - logLink: - type: string - project: - type: string - projectId: - type: integer - projectName: - type: string - remoteId: - type: string - route: - type: string - routes: - items: - type: string - type: array - services: - items: - type: string - type: array - startTime: - type: string - task: - description: LagoonTaskInfo defines what a task can use to - communicate with Lagoon via SSH/API. - properties: - apiHost: - type: string - command: - type: string - id: - type: string - name: - type: string - service: - type: string - sshHost: - type: string - sshPort: - type: string - taskName: - type: string - required: - - id - type: object - type: object - project: - type: string - severity: - type: string - uuid: - type: string - type: object - taskLogMessage: - description: LagoonLog is used to sendToLagoonLogs messaging queue - this is general logging information - properties: - event: - type: string - message: - type: string - meta: - description: LagoonLogMeta is the metadata that is used by logging - in Lagoon. - properties: - advancedData: - type: string - branchName: - type: string - buildName: - type: string - buildPhase: - type: string - buildStatus: - type: string - buildStep: - type: string - clusterName: - type: string - endTime: - type: string - environment: - type: string - environmentId: - type: integer - jobName: - type: string - jobStatus: - type: string - jobStep: - type: string - key: - type: string - logLink: - type: string - project: - type: string - projectId: - type: integer - projectName: - type: string - remoteId: - type: string - route: - type: string - routes: - items: - type: string - type: array - services: - items: - type: string - type: array - startTime: - type: string - task: - description: LagoonTaskInfo defines what a task can use to - communicate with Lagoon via SSH/API. - properties: - apiHost: - type: string - command: - type: string - id: - type: string - name: - type: string - service: - type: string - sshHost: - type: string - sshPort: - type: string - taskName: - type: string - required: - - id - type: object - type: object - project: - type: string - severity: - type: string - uuid: - type: string - type: object - type: object type: object served: true storage: true diff --git a/controllers/v1beta1/build_deletionhandlers.go b/controllers/v1beta1/build_deletionhandlers.go index fba4c0e2..03c773ad 100644 --- a/controllers/v1beta1/build_deletionhandlers.go +++ b/controllers/v1beta1/build_deletionhandlers.go @@ -260,15 +260,9 @@ func (r *LagoonBuildReconciler) buildLogsToLagoonLogs(ctx context.Context, // bother to patch the resource?? // leave it for now cause the resource will just be deleted anyway if err := r.Messaging.Publish("lagoon-logs", msgBytes); err != nil { - // if we can't publish the message, set it as a pending message - // overwrite whatever is there as these are just current state messages so it doesn't - // really matter if we don't smootly transition in what we send back to lagoon - r.updateBuildLogMessage(ctx, lagoonBuild, msg) + // if we can't publish the message, just return return } - // if we are able to publish the message, then we need to remove any pending messages from the resource - // and make sure we don't try and publish again - r.removeBuildPendingMessageStatus(ctx, lagoonBuild) } } @@ -351,15 +345,9 @@ func (r *LagoonBuildReconciler) updateDeploymentAndEnvironmentTask(ctx context.C // bother to patch the resource?? // leave it for now cause the resource will just be deleted anyway if err := r.Messaging.Publish("lagoon-tasks:controller", msgBytes); err != nil { - // if we can't publish the message, set it as a pending message - // overwrite whatever is there as these are just current state messages so it doesn't - // really matter if we don't smootly transition in what we send back to lagoon - r.updateEnvironmentMessage(ctx, lagoonBuild, msg) + // if we can't publish the message, just return return } - // if we are able to publish the message, then we need to remove any pending messages from the resource - // and make sure we don't try and publish again - r.removeBuildPendingMessageStatus(ctx, lagoonBuild) } } @@ -411,104 +399,8 @@ func (r *LagoonBuildReconciler) buildStatusLogsToLagoonLogs(ctx context.Context, // bother to patch the resource?? // leave it for now cause the resource will just be deleted anyway if err := r.Messaging.Publish("lagoon-logs", msgBytes); err != nil { - // if we can't publish the message, set it as a pending message - // overwrite whatever is there as these are just current state messages so it doesn't - // really matter if we don't smootly transition in what we send back to lagoon - r.updateBuildStatusMessage(ctx, lagoonBuild, msg) + // if we can't publish the message, just return return } - // if we are able to publish the message, then we need to remove any pending messages from the resource - // and make sure we don't try and publish again - r.removeBuildPendingMessageStatus(ctx, lagoonBuild) } } - -// updateEnvironmentMessage this is called if the message queue is unavailable, it stores the message that would be sent in the lagoon build -func (r *LagoonBuildReconciler) updateEnvironmentMessage(ctx context.Context, - lagoonBuild *lagoonv1beta1.LagoonBuild, - envMessage lagoonv1beta1.LagoonMessage, -) error { - // set the transition time - mergePatch, _ := json.Marshal(map[string]interface{}{ - "metadata": map[string]interface{}{ - "labels": map[string]interface{}{ - "lagoon.sh/pendingMessages": "true", - }, - }, - "statusMessages": map[string]interface{}{ - "environmentMessage": envMessage, - }, - }) - if err := r.Patch(ctx, lagoonBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { - return fmt.Errorf("Unable to update status condition: %v", err) - } - return nil -} - -// updateBuildStatusMessage this is called if the message queue is unavailable, it stores the message that would be sent in the lagoon build -func (r *LagoonBuildReconciler) updateBuildStatusMessage(ctx context.Context, - lagoonBuild *lagoonv1beta1.LagoonBuild, - statusMessage lagoonv1beta1.LagoonLog, -) error { - // set the transition time - mergePatch, _ := json.Marshal(map[string]interface{}{ - "metadata": map[string]interface{}{ - "labels": map[string]interface{}{ - "lagoon.sh/pendingMessages": "true", - }, - }, - "statusMessages": map[string]interface{}{ - "statusMessage": statusMessage, - }, - }) - if err := r.Patch(ctx, lagoonBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { - return fmt.Errorf("Unable to update status condition: %v", err) - } - return nil -} - -// removeBuildPendingMessageStatus purges the status messages from the resource once they are successfully re-sent -func (r *LagoonBuildReconciler) removeBuildPendingMessageStatus(ctx context.Context, - lagoonBuild *lagoonv1beta1.LagoonBuild, -) error { - // if we have the pending messages label as true, then we want to remove this label and any pending statusmessages - // so we can avoid double handling, or an old pending message from being sent after a new pending message - if val, ok := lagoonBuild.ObjectMeta.Labels["lagoon.sh/pendingMessages"]; !ok { - if val == "true" { - mergePatch, _ := json.Marshal(map[string]interface{}{ - "metadata": map[string]interface{}{ - "labels": map[string]interface{}{ - "lagoon.sh/pendingMessages": "false", - }, - }, - "statusMessages": nil, - }) - if err := r.Patch(ctx, lagoonBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { - return fmt.Errorf("Unable to update status condition: %v", err) - } - } - } - return nil -} - -// updateBuildLogMessage this is called if the message queue is unavailable, it stores the message that would be sent in the lagoon build -func (r *LagoonBuildReconciler) updateBuildLogMessage(ctx context.Context, - lagoonBuild *lagoonv1beta1.LagoonBuild, - buildMessage lagoonv1beta1.LagoonLog, -) error { - // set the transition time - mergePatch, _ := json.Marshal(map[string]interface{}{ - "metadata": map[string]interface{}{ - "labels": map[string]interface{}{ - "lagoon.sh/pendingMessages": "true", - }, - }, - "statusMessages": map[string]interface{}{ - "buildLogMessage": buildMessage, - }, - }) - if err := r.Patch(ctx, lagoonBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { - return fmt.Errorf("Unable to update status condition: %v", err) - } - return nil -} diff --git a/controllers/v1beta1/podmonitor_buildhandlers.go b/controllers/v1beta1/podmonitor_buildhandlers.go index 0159cb65..d41de695 100644 --- a/controllers/v1beta1/podmonitor_buildhandlers.go +++ b/controllers/v1beta1/podmonitor_buildhandlers.go @@ -143,7 +143,7 @@ func (r *LagoonMonitorReconciler) buildLogsToLagoonLogs(ctx context.Context, namespace *corev1.Namespace, condition string, logs []byte, -) (bool, lagoonv1beta1.LagoonLog) { +) error { if r.EnableMQ { buildStep := "running" if condition == "failed" || condition == "complete" || condition == "cancelled" { @@ -208,7 +208,7 @@ Logs on pod %s, assigned to cluster %s // overwrite whatever is there as these are just current state messages so it doesn't // really matter if we don't smootly transition in what we send back to lagoon // r.updateBuildLogMessage(ctx, lagoonBuild, msg) - return true, msg + return err } if r.EnableDebug { opLog.Info( @@ -222,7 +222,7 @@ Logs on pod %s, assigned to cluster %s // if we are able to publish the message, then we need to remove any pending messages from the resource // and make sure we don't try and publish again } - return false, lagoonv1beta1.LagoonLog{} + return nil } // updateDeploymentAndEnvironmentTask sends the status of the build and deployment to the controllerhandler message queue in lagoon, @@ -234,7 +234,7 @@ func (r *LagoonMonitorReconciler) updateDeploymentAndEnvironmentTask(ctx context lagoonEnv *corev1.ConfigMap, namespace *corev1.Namespace, condition string, -) (bool, lagoonv1beta1.LagoonMessage) { +) error { if r.EnableMQ { buildStep := "running" if condition == "failed" || condition == "complete" || condition == "cancelled" { @@ -346,7 +346,7 @@ func (r *LagoonMonitorReconciler) updateDeploymentAndEnvironmentTask(ctx context // if we can't publish the message, set it as a pending message // overwrite whatever is there as these are just current state messages so it doesn't // really matter if we don't smootly transition in what we send back to lagoon - return true, msg + return err } if r.EnableDebug { opLog.Info( @@ -359,7 +359,7 @@ func (r *LagoonMonitorReconciler) updateDeploymentAndEnvironmentTask(ctx context // if we are able to publish the message, then we need to remove any pending messages from the resource // and make sure we don't try and publish again } - return false, lagoonv1beta1.LagoonMessage{} + return nil } // buildStatusLogsToLagoonLogs sends the logs to lagoon-logs message queue, used for general messaging @@ -370,7 +370,7 @@ func (r *LagoonMonitorReconciler) buildStatusLogsToLagoonLogs(ctx context.Contex lagoonEnv *corev1.ConfigMap, namespace *corev1.Namespace, condition string, -) (bool, lagoonv1beta1.LagoonLog) { +) error { if r.EnableMQ { buildStep := "running" if condition == "failed" || condition == "complete" || condition == "cancelled" { @@ -441,7 +441,7 @@ func (r *LagoonMonitorReconciler) buildStatusLogsToLagoonLogs(ctx context.Contex // if we can't publish the message, set it as a pending message // overwrite whatever is there as these are just current state messages so it doesn't // really matter if we don't smootly transition in what we send back to lagoon - return true, msg + return err } if r.EnableDebug { opLog.Info( @@ -455,7 +455,7 @@ func (r *LagoonMonitorReconciler) buildStatusLogsToLagoonLogs(ctx context.Contex // if we are able to publish the message, then we need to remove any pending messages from the resource // and make sure we don't try and publish again } - return false, lagoonv1beta1.LagoonLog{} + return nil } // updateDeploymentWithLogs collects logs from the build containers and ships or stores them @@ -542,7 +542,6 @@ Build %s "lagoon.sh/buildStarted": "true", }, }, - "statusMessages": map[string]interface{}{}, } condition := lagoonv1beta1.LagoonBuildConditions{ @@ -574,31 +573,18 @@ Build %s } // do any message publishing here, and update any pending messages if needed - pendingStatus, pendingStatusMessage := r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &jobPod, &lagoonEnv, namespace, buildCondition.ToLower()) - pendingEnvironment, pendingEnvironmentMessage := r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &jobPod, &lagoonEnv, namespace, buildCondition.ToLower()) - var pendingBuildLog bool - var pendingBuildLogMessage lagoonv1beta1.LagoonLog + if err = r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &jobPod, &lagoonEnv, namespace, buildCondition.ToLower()); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to publish build status logs")) + } + if err = r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &jobPod, &lagoonEnv, namespace, buildCondition.ToLower()); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to publish build update")) + } // if the container logs can't be retrieved, we don't want to send any build logs back, as this will nuke // any previously received logs if !strings.Contains(string(allContainerLogs), "unable to retrieve container logs for containerd") { - pendingBuildLog, pendingBuildLogMessage = r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &jobPod, namespace, buildCondition.ToLower(), allContainerLogs) - } - if pendingStatus || pendingEnvironment || pendingBuildLog { - mergeMap["metadata"].(map[string]interface{})["labels"].(map[string]interface{})["lagoon.sh/pendingMessages"] = "true" - if pendingStatus { - mergeMap["statusMessages"].(map[string]interface{})["statusMessage"] = pendingStatusMessage + if err = r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &jobPod, namespace, buildCondition.ToLower(), allContainerLogs); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to publish build logs")) } - if pendingEnvironment { - mergeMap["statusMessages"].(map[string]interface{})["environmentMessage"] = pendingEnvironmentMessage - } - // if the build log message is too long, don't save it - if pendingBuildLog && len(pendingBuildLogMessage.Message) > 1048576 { - mergeMap["statusMessages"].(map[string]interface{})["buildLogMessage"] = pendingBuildLogMessage - } - } - if !pendingStatus && !pendingEnvironment && !pendingBuildLog { - mergeMap["metadata"].(map[string]interface{})["labels"].(map[string]interface{})["lagoon.sh/pendingMessages"] = nil - mergeMap["statusMessages"] = nil } mergePatch, _ := json.Marshal(mergeMap) // check if the build exists diff --git a/controllers/v1beta1/podmonitor_taskhandlers.go b/controllers/v1beta1/podmonitor_taskhandlers.go index 5ffe9305..01960323 100644 --- a/controllers/v1beta1/podmonitor_taskhandlers.go +++ b/controllers/v1beta1/podmonitor_taskhandlers.go @@ -112,7 +112,7 @@ func (r *LagoonMonitorReconciler) taskLogsToLagoonLogs(opLog logr.Logger, jobPod *corev1.Pod, condition string, logs []byte, -) (bool, lagoonv1beta1.LagoonLog) { +) error { if r.EnableMQ && lagoonTask != nil { msg := lagoonv1beta1.LagoonLog{ Severity: "info", @@ -147,12 +147,12 @@ Logs on pod %s, assigned to cluster %s // if we can't publish the message, set it as a pending message // overwrite whatever is there as these are just current state messages so it doesn't // really matter if we don't smootly transition in what we send back to lagoon - return true, msg + return err } // if we are able to publish the message, then we need to remove any pending messages from the resource // and make sure we don't try and publish again } - return false, lagoonv1beta1.LagoonLog{} + return nil } // updateLagoonTask sends the status of the task and deployment to the controllerhandler message queue in lagoon, @@ -161,7 +161,7 @@ func (r *LagoonMonitorReconciler) updateLagoonTask(ctx context.Context, opLog lo lagoonTask *lagoonv1beta1.LagoonTask, jobPod *corev1.Pod, condition string, -) (bool, lagoonv1beta1.LagoonMessage) { +) error { if r.EnableMQ && lagoonTask != nil { if condition == "failed" || condition == "complete" || condition == "cancelled" { time.AfterFunc(31*time.Second, func() { @@ -212,12 +212,12 @@ func (r *LagoonMonitorReconciler) updateLagoonTask(ctx context.Context, opLog lo // if we can't publish the message, set it as a pending message // overwrite whatever is there as these are just current state messages so it doesn't // really matter if we don't smootly transition in what we send back to lagoon - return true, msg + return err } // if we are able to publish the message, then we need to remove any pending messages from the resource // and make sure we don't try and publish again } - return false, lagoonv1beta1.LagoonMessage{} + return nil } // taskStatusLogsToLagoonLogs sends the logs to lagoon-logs message queue, used for general messaging @@ -225,7 +225,7 @@ func (r *LagoonMonitorReconciler) taskStatusLogsToLagoonLogs(opLog logr.Logger, lagoonTask *lagoonv1beta1.LagoonTask, jobPod *corev1.Pod, condition string, -) (bool, lagoonv1beta1.LagoonLog) { +) error { if r.EnableMQ && lagoonTask != nil { msg := lagoonv1beta1.LagoonLog{ Severity: "info", @@ -258,12 +258,12 @@ func (r *LagoonMonitorReconciler) taskStatusLogsToLagoonLogs(opLog logr.Logger, // if we can't publish the message, set it as a pending message // overwrite whatever is there as these are just current state messages so it doesn't // really matter if we don't smootly transition in what we send back to lagoon - return true, msg + return err } // if we are able to publish the message, then we need to remove any pending messages from the resource // and make sure we don't try and publish again } - return false, lagoonv1beta1.LagoonLog{} + return nil } // updateTaskWithLogs collects logs from the task containers and ships or stores them @@ -332,7 +332,6 @@ Task %s "lagoon.sh/taskStatus": taskCondition.String(), }, }, - "statusMessages": map[string]interface{}{}, } condition := lagoonv1beta1.LagoonTaskConditions{ @@ -350,31 +349,18 @@ Task %s // send any messages to lagoon message queues // update the deployment with the status - pendingStatus, pendingStatusMessage := r.taskStatusLogsToLagoonLogs(opLog, &lagoonTask, &jobPod, taskCondition.ToLower()) - pendingEnvironment, pendingEnvironmentMessage := r.updateLagoonTask(ctx, opLog, &lagoonTask, &jobPod, taskCondition.ToLower()) - var pendingTaskLog bool - var pendingTaskLogMessage lagoonv1beta1.LagoonLog + if err = r.taskStatusLogsToLagoonLogs(opLog, &lagoonTask, &jobPod, taskCondition.ToLower()); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to publish task status logs")) + } + if err = r.updateLagoonTask(ctx, opLog, &lagoonTask, &jobPod, taskCondition.ToLower()); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to publish task update")) + } // if the container logs can't be retrieved, we don't want to send any task logs back, as this will nuke // any previously received logs if !strings.Contains(string(allContainerLogs), "unable to retrieve container logs for containerd") { - pendingTaskLog, pendingTaskLogMessage = r.taskLogsToLagoonLogs(opLog, &lagoonTask, &jobPod, taskCondition.ToLower(), allContainerLogs) - } - - if pendingStatus || pendingEnvironment || pendingTaskLog { - mergeMap["metadata"].(map[string]interface{})["labels"].(map[string]interface{})["lagoon.sh/pendingMessages"] = "true" - if pendingStatus { - mergeMap["statusMessages"].(map[string]interface{})["statusMessage"] = pendingStatusMessage - } - if pendingEnvironment { - mergeMap["statusMessages"].(map[string]interface{})["environmentMessage"] = pendingEnvironmentMessage + if err = r.taskLogsToLagoonLogs(opLog, &lagoonTask, &jobPod, taskCondition.ToLower(), allContainerLogs); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to publish task logs")) } - if pendingTaskLog { - mergeMap["statusMessages"].(map[string]interface{})["taskLogMessage"] = pendingTaskLogMessage - } - } - if !pendingStatus && !pendingEnvironment && !pendingTaskLog { - mergeMap["metadata"].(map[string]interface{})["labels"].(map[string]interface{})["lagoon.sh/pendingMessages"] = nil - mergeMap["statusMessages"] = nil } mergePatch, _ := json.Marshal(mergeMap) // check if the task exists diff --git a/internal/messenger/pending_messages.go b/internal/messenger/pending_messages.go deleted file mode 100644 index 26658df6..00000000 --- a/internal/messenger/pending_messages.go +++ /dev/null @@ -1,151 +0,0 @@ -package messenger - -import ( - "context" - "encoding/json" - "fmt" - - "github.com/go-logr/logr" - lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" - "k8s.io/apimachinery/pkg/types" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -// GetPendingMessages will get any pending messages from the queue and attempt to publish them if possible -func (m *Messenger) GetPendingMessages() { - opLog := ctrl.Log.WithName("handlers").WithName("PendingMessages") - ctx := context.Background() - opLog.Info(fmt.Sprintf("Checking pending build messages across all namespaces")) - m.pendingBuildLogMessages(ctx, opLog) - opLog.Info(fmt.Sprintf("Checking pending task messages across all namespaces")) - m.pendingTaskLogMessages(ctx, opLog) -} - -func (m *Messenger) pendingBuildLogMessages(ctx context.Context, opLog logr.Logger) { - pendingMsgs := &lagoonv1beta1.LagoonBuildList{} - listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ - client.MatchingLabels(map[string]string{ - "lagoon.sh/pendingMessages": "true", - "lagoon.sh/controller": m.ControllerNamespace, - }), - }) - if err := m.Client.List(ctx, pendingMsgs, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list LagoonBuilds, there may be none or something went wrong")) - return - } - for _, build := range pendingMsgs.Items { - // get the latest resource in case it has been updated since the loop started - if err := m.Client.Get(ctx, types.NamespacedName{ - Name: build.ObjectMeta.Name, - Namespace: build.ObjectMeta.Namespace, - }, &build); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to get LagoonBuild, something went wrong")) - break - } - opLog.Info(fmt.Sprintf("LagoonBuild %s has pending messages, attempting to re-send", build.ObjectMeta.Name)) - - // try to re-publish message or break and try the next build with pending message - if build.StatusMessages.StatusMessage != nil { - statusBytes, _ := json.Marshal(build.StatusMessages.StatusMessage) - if err := m.Publish("lagoon-logs", statusBytes); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to publush message")) - break - } - } - if build.StatusMessages.BuildLogMessage != nil { - logBytes, _ := json.Marshal(build.StatusMessages.BuildLogMessage) - if err := m.Publish("lagoon-logs", logBytes); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to publush message")) - break - } - } - if build.StatusMessages.EnvironmentMessage != nil { - envBytes, _ := json.Marshal(build.StatusMessages.EnvironmentMessage) - if err := m.Publish("lagoon-tasks:controller", envBytes); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to publush message")) - break - } - } - // if we managed to send all the pending messages, then update the resource to remove the pending state - // so we don't send the same message multiple times - opLog.Info(fmt.Sprintf("Sent pending messages for LagoonBuild %s", build.ObjectMeta.Name)) - mergePatch, _ := json.Marshal(map[string]interface{}{ - "metadata": map[string]interface{}{ - "labels": map[string]interface{}{ - "lagoon.sh/pendingMessages": "false", - }, - }, - "statusMessages": nil, - }) - if err := m.Client.Patch(ctx, &build, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to update status condition")) - break - } - } - return -} - -func (m *Messenger) pendingTaskLogMessages(ctx context.Context, opLog logr.Logger) { - pendingMsgs := &lagoonv1beta1.LagoonTaskList{} - listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ - client.MatchingLabels(map[string]string{ - "lagoon.sh/pendingMessages": "true", - "lagoon.sh/controller": m.ControllerNamespace, - }), - }) - if err := m.Client.List(ctx, pendingMsgs, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list LagoonBuilds, there may be none or something went wrong")) - return - } - for _, task := range pendingMsgs.Items { - // get the latest resource in case it has been updated since the loop started - if err := m.Client.Get(ctx, types.NamespacedName{ - Name: task.ObjectMeta.Name, - Namespace: task.ObjectMeta.Namespace, - }, &task); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to get LagoonBuild, something went wrong")) - break - } - opLog.Info(fmt.Sprintf("LagoonTasl %s has pending messages, attempting to re-send", task.ObjectMeta.Name)) - - // try to re-publish message or break and try the next build with pending message - if task.StatusMessages.StatusMessage != nil { - statusBytes, _ := json.Marshal(task.StatusMessages.StatusMessage) - if err := m.Publish("lagoon-logs", statusBytes); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to publush message")) - break - } - } - if task.StatusMessages.TaskLogMessage != nil { - taskLogBytes, _ := json.Marshal(task.StatusMessages.TaskLogMessage) - if err := m.Publish("lagoon-logs", taskLogBytes); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to publush message")) - break - } - } - if task.StatusMessages.EnvironmentMessage != nil { - envBytes, _ := json.Marshal(task.StatusMessages.EnvironmentMessage) - if err := m.Publish("lagoon-tasks:controller", envBytes); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to publush message")) - break - } - } - // if we managed to send all the pending messages, then update the resource to remove the pending state - // so we don't send the same message multiple times - opLog.Info(fmt.Sprintf("Sent pending messages for LagoonTask %s", task.ObjectMeta.Name)) - mergePatch, _ := json.Marshal(map[string]interface{}{ - "metadata": map[string]interface{}{ - "labels": map[string]interface{}{ - "lagoon.sh/pendingMessages": "false", - }, - }, - "statusMessages": nil, - }) - if err := m.Client.Patch(ctx, &task, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to update status condition")) - break - } - } - return -} diff --git a/main.go b/main.go index 7238b1ab..c9034592 100644 --- a/main.go +++ b/main.go @@ -85,7 +85,6 @@ func main() { var enableLeaderElection bool var enableMQ bool var leaderElectionID string - var pendingMessageCron string var mqWorkers int var rabbitRetryInterval int var startupConnectionAttempts int @@ -191,8 +190,7 @@ func main() { "The retry interval for rabbitmq.") flag.StringVar(&leaderElectionID, "leader-election-id", "lagoon-builddeploy-leader-election-helper", "The ID to use for leader election.") - flag.StringVar(&pendingMessageCron, "pending-message-cron", "15,45 * * * *", - "The cron definition for pending messages.") + flag.String("pending-message-cron", "", "This does nothing and will be removed in a future version.") flag.IntVar(&startupConnectionAttempts, "startup-connection-attempts", 10, "The number of startup attempts before exiting.") flag.IntVar(&startupConnectionInterval, "startup-connection-interval-seconds", 30, @@ -375,7 +373,6 @@ func main() { mqPass = helpers.GetEnv("RABBITMQ_PASSWORD", mqPass) mqHost = helpers.GetEnv("RABBITMQ_HOSTNAME", mqHost) lagoonTargetName = helpers.GetEnv("LAGOON_TARGET_NAME", lagoonTargetName) - pendingMessageCron = helpers.GetEnv("PENDING_MESSAGE_CRON", pendingMessageCron) overrideBuildDeployImage = helpers.GetEnv("OVERRIDE_BUILD_DEPLOY_DIND_IMAGE", overrideBuildDeployImage) namespacePrefix = helpers.GetEnv("NAMESPACE_PREFIX", namespacePrefix) if len(namespacePrefix) > 8 { @@ -657,13 +654,6 @@ func main() { if enableMQ { setupLog.Info("starting messaging handler") go messaging.Consumer(lagoonTargetName) - - // use cron to run a pending message task - // this will check any `LagoonBuild` resources for the pendingMessages label - // and attempt to re-publish them - c.AddFunc(pendingMessageCron, func() { - messaging.GetPendingMessages() - }) } buildQoSConfig := lagoonv1beta1ctrl.BuildQoS{ From c5c260a16edbb154695f4202a3514721a8062377 Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Tue, 9 Jan 2024 15:52:36 +1100 Subject: [PATCH 03/17] feat: support for passing new env service info to api --- apis/lagoon/v1beta1/lagoonmessaging_types.go | 64 ++++++----- apis/lagoon/v1beta1/zz_generated.deepcopy.go | 4 +- .../crd/bases/crd.lagoon.sh_lagoonbuilds.yaml | 108 ++++++++++-------- .../crd/bases/crd.lagoon.sh_lagoontasks.yaml | 108 ++++++++++-------- controllers/v1beta1/build_deletionhandlers.go | 23 ++-- .../v1beta1/podmonitor_buildhandlers.go | 23 ++-- 6 files changed, 179 insertions(+), 151 deletions(-) diff --git a/apis/lagoon/v1beta1/lagoonmessaging_types.go b/apis/lagoon/v1beta1/lagoonmessaging_types.go index 7ee42727..789c01da 100644 --- a/apis/lagoon/v1beta1/lagoonmessaging_types.go +++ b/apis/lagoon/v1beta1/lagoonmessaging_types.go @@ -15,36 +15,46 @@ type LagoonLog struct { // LagoonLogMeta is the metadata that is used by logging in Lagoon. type LagoonLogMeta struct { - BranchName string `json:"branchName,omitempty"` - BuildName string `json:"buildName,omitempty"` - BuildPhase string `json:"buildPhase,omitempty"` // @TODO: deprecate once controller-handler is fixed - BuildStatus string `json:"buildStatus,omitempty"` - BuildStep string `json:"buildStep,omitempty"` - EndTime string `json:"endTime,omitempty"` - Environment string `json:"environment,omitempty"` - EnvironmentID *uint `json:"environmentId,omitempty"` - JobName string `json:"jobName,omitempty"` // used by tasks/jobs - JobStatus string `json:"jobStatus,omitempty"` // used by tasks/jobs - JobStep string `json:"jobStep,omitempty"` // used by tasks/jobs - LogLink string `json:"logLink,omitempty"` - Project string `json:"project,omitempty"` - ProjectID *uint `json:"projectId,omitempty"` - ProjectName string `json:"projectName,omitempty"` - RemoteID string `json:"remoteId,omitempty"` - Route string `json:"route,omitempty"` - Routes []string `json:"routes,omitempty"` - StartTime string `json:"startTime,omitempty"` - Services []string `json:"services,omitempty"` - ServicesV2 []LagoonService `json:"servicesv2,omitempty"` - Task *LagoonTaskInfo `json:"task,omitempty"` - Key string `json:"key,omitempty"` - AdvancedData string `json:"advancedData,omitempty"` - Cluster string `json:"clusterName,omitempty"` + BranchName string `json:"branchName,omitempty"` + BuildName string `json:"buildName,omitempty"` + BuildPhase string `json:"buildPhase,omitempty"` // @TODO: deprecate once controller-handler is fixed + BuildStatus string `json:"buildStatus,omitempty"` + BuildStep string `json:"buildStep,omitempty"` + EndTime string `json:"endTime,omitempty"` + Environment string `json:"environment,omitempty"` + EnvironmentID *uint `json:"environmentId,omitempty"` + JobName string `json:"jobName,omitempty"` // used by tasks/jobs + JobStatus string `json:"jobStatus,omitempty"` // used by tasks/jobs + JobStep string `json:"jobStep,omitempty"` // used by tasks/jobs + LogLink string `json:"logLink,omitempty"` + Project string `json:"project,omitempty"` + ProjectID *uint `json:"projectId,omitempty"` + ProjectName string `json:"projectName,omitempty"` + RemoteID string `json:"remoteId,omitempty"` + Route string `json:"route,omitempty"` + Routes []string `json:"routes,omitempty"` + StartTime string `json:"startTime,omitempty"` + Services []string `json:"services,omitempty"` + EnvironmentServices []LagoonService `json:"environmentServices,omitempty"` + Task *LagoonTaskInfo `json:"task,omitempty"` + Key string `json:"key,omitempty"` + AdvancedData string `json:"advancedData,omitempty"` + Cluster string `json:"clusterName,omitempty"` } +// LagoonService is the same as EnvironmentService type from the lagoon Schema type LagoonService struct { - Name string `json:"name"` - Type string `json:"type"` + ID int `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Type string `json:"type,omitempty"` + Updated string `json:"updated,omitempty"` + Containers []EnvironmentContainer `json:"containers,omitempty"` + Created string `json:"created,omitempty"` +} + +// EnvironmentService is based on the Lagoon API type. +type EnvironmentContainer struct { + Name string `json:"name,omitempty"` } // LagoonMessage is used for sending build info back to Lagoon diff --git a/apis/lagoon/v1beta1/zz_generated.deepcopy.go b/apis/lagoon/v1beta1/zz_generated.deepcopy.go index 56844afb..d4d52735 100644 --- a/apis/lagoon/v1beta1/zz_generated.deepcopy.go +++ b/apis/lagoon/v1beta1/zz_generated.deepcopy.go @@ -241,8 +241,8 @@ func (in *LagoonLogMeta) DeepCopyInto(out *LagoonLogMeta) { *out = make([]string, len(*in)) copy(*out, *in) } - if in.ServicesV2 != nil { - in, out := &in.ServicesV2, &out.ServicesV2 + if in.EnvironmentServices != nil { + in, out := &in.EnvironmentServices, &out.EnvironmentServices *out = make([]LagoonService, len(*in)) copy(*out, *in) } diff --git a/config/crd/bases/crd.lagoon.sh_lagoonbuilds.yaml b/config/crd/bases/crd.lagoon.sh_lagoonbuilds.yaml index 7f97619c..10599775 100644 --- a/config/crd/bases/crd.lagoon.sh_lagoonbuilds.yaml +++ b/config/crd/bases/crd.lagoon.sh_lagoonbuilds.yaml @@ -237,6 +237,21 @@ spec: type: string environmentId: type: integer + environmentServices: + items: + properties: + created: + type: string + id: + type: integer + name: + type: string + type: + type: string + updated: + type: string + type: object + type: array jobName: type: string jobStatus: @@ -265,18 +280,6 @@ spec: items: type: string type: array - servicesv2: - items: - properties: - name: - type: string - type: - type: string - required: - - name - - type - type: object - type: array startTime: type: string task: @@ -338,6 +341,21 @@ spec: type: string environmentId: type: integer + environmentServices: + items: + properties: + created: + type: string + id: + type: integer + name: + type: string + type: + type: string + updated: + type: string + type: object + type: array jobName: type: string jobStatus: @@ -366,18 +384,6 @@ spec: items: type: string type: array - servicesv2: - items: - properties: - name: - type: string - type: - type: string - required: - - name - - type - type: object - type: array startTime: type: string task: @@ -441,6 +447,21 @@ spec: type: string environmentId: type: integer + environmentServices: + items: + properties: + created: + type: string + id: + type: integer + name: + type: string + type: + type: string + updated: + type: string + type: object + type: array jobName: type: string jobStatus: @@ -469,18 +490,6 @@ spec: items: type: string type: array - servicesv2: - items: - properties: - name: - type: string - type: - type: string - required: - - name - - type - type: object - type: array startTime: type: string task: @@ -546,6 +555,21 @@ spec: type: string environmentId: type: integer + environmentServices: + items: + properties: + created: + type: string + id: + type: integer + name: + type: string + type: + type: string + updated: + type: string + type: object + type: array jobName: type: string jobStatus: @@ -574,18 +598,6 @@ spec: items: type: string type: array - servicesv2: - items: - properties: - name: - type: string - type: - type: string - required: - - name - - type - type: object - type: array startTime: type: string task: diff --git a/config/crd/bases/crd.lagoon.sh_lagoontasks.yaml b/config/crd/bases/crd.lagoon.sh_lagoontasks.yaml index 4bbe35b9..1ce640e0 100644 --- a/config/crd/bases/crd.lagoon.sh_lagoontasks.yaml +++ b/config/crd/bases/crd.lagoon.sh_lagoontasks.yaml @@ -219,6 +219,21 @@ spec: type: string environmentId: type: integer + environmentServices: + items: + properties: + created: + type: string + id: + type: integer + name: + type: string + type: + type: string + updated: + type: string + type: object + type: array jobName: type: string jobStatus: @@ -247,18 +262,6 @@ spec: items: type: string type: array - servicesv2: - items: - properties: - name: - type: string - type: - type: string - required: - - name - - type - type: object - type: array startTime: type: string task: @@ -320,6 +323,21 @@ spec: type: string environmentId: type: integer + environmentServices: + items: + properties: + created: + type: string + id: + type: integer + name: + type: string + type: + type: string + updated: + type: string + type: object + type: array jobName: type: string jobStatus: @@ -348,18 +366,6 @@ spec: items: type: string type: array - servicesv2: - items: - properties: - name: - type: string - type: - type: string - required: - - name - - type - type: object - type: array startTime: type: string task: @@ -423,6 +429,21 @@ spec: type: string environmentId: type: integer + environmentServices: + items: + properties: + created: + type: string + id: + type: integer + name: + type: string + type: + type: string + updated: + type: string + type: object + type: array jobName: type: string jobStatus: @@ -451,18 +472,6 @@ spec: items: type: string type: array - servicesv2: - items: - properties: - name: - type: string - type: - type: string - required: - - name - - type - type: object - type: array startTime: type: string task: @@ -528,6 +537,21 @@ spec: type: string environmentId: type: integer + environmentServices: + items: + properties: + created: + type: string + id: + type: integer + name: + type: string + type: + type: string + updated: + type: string + type: object + type: array jobName: type: string jobStatus: @@ -556,18 +580,6 @@ spec: items: type: string type: array - servicesv2: - items: - properties: - name: - type: string - type: - type: string - required: - - name - - type - type: object - type: array startTime: type: string task: diff --git a/controllers/v1beta1/build_deletionhandlers.go b/controllers/v1beta1/build_deletionhandlers.go index d4ec0ff7..0ab6450b 100644 --- a/controllers/v1beta1/build_deletionhandlers.go +++ b/controllers/v1beta1/build_deletionhandlers.go @@ -318,29 +318,26 @@ func (r *LagoonBuildReconciler) updateDeploymentAndEnvironmentTask(ctx context.C // generate the list of services to add to the environment for _, pod := range podList.Items { var serviceName, serviceType string - if _, ok := pod.ObjectMeta.Labels["lagoon.sh/service"]; ok { + containers := []lagoonv1beta1.EnvironmentContainer{} + if name, ok := pod.ObjectMeta.Labels["lagoon.sh/service"]; ok { + serviceName = name + serviceNames = append(serviceNames, serviceName) for _, container := range pod.Spec.Containers { - serviceNames = append(serviceNames, container.Name) - serviceName = container.Name - } - } - if _, ok := pod.ObjectMeta.Labels["service"]; ok { - // @TODO: remove this as this label shouldn't exist by now - for _, container := range pod.Spec.Containers { - serviceNames = append(serviceNames, container.Name) - serviceName = container.Name + containers = append(containers, lagoonv1beta1.EnvironmentContainer{Name: container.Name}) } } if sType, ok := pod.ObjectMeta.Labels["lagoon.sh/service-type"]; ok { serviceType = sType } + // probably need to collect dbaas consumers too at some stage services = append(services, lagoonv1beta1.LagoonService{ - Name: serviceName, - Type: serviceType, + Name: serviceName, + Type: serviceType, + Containers: containers, }) } msg.Meta.Services = serviceNames - msg.Meta.ServicesV2 = services + msg.Meta.EnvironmentServices = services } // if we aren't being provided the lagoon config, we can skip adding the routes etc if lagoonEnv != nil { diff --git a/controllers/v1beta1/podmonitor_buildhandlers.go b/controllers/v1beta1/podmonitor_buildhandlers.go index 169cbebf..df3396d3 100644 --- a/controllers/v1beta1/podmonitor_buildhandlers.go +++ b/controllers/v1beta1/podmonitor_buildhandlers.go @@ -301,29 +301,26 @@ func (r *LagoonMonitorReconciler) updateDeploymentAndEnvironmentTask(ctx context // generate the list of services to add or update to the environment for _, deployment := range depList.Items { var serviceName, serviceType string - if _, ok := deployment.ObjectMeta.Labels["lagoon.sh/service"]; ok { + containers := []lagoonv1beta1.EnvironmentContainer{} + if name, ok := deployment.ObjectMeta.Labels["lagoon.sh/service"]; ok { + serviceName = name + serviceNames = append(serviceNames, serviceName) for _, container := range deployment.Spec.Template.Spec.Containers { - serviceNames = append(serviceNames, container.Name) - serviceName = container.Name - } - } - if _, ok := deployment.ObjectMeta.Labels["service"]; ok { - // @TODO: remove this as this label shouldn't exist by now - for _, container := range deployment.Spec.Template.Spec.Containers { - serviceNames = append(serviceNames, container.Name) - serviceName = container.Name + containers = append(containers, lagoonv1beta1.EnvironmentContainer{Name: container.Name}) } } if sType, ok := deployment.ObjectMeta.Labels["lagoon.sh/service-type"]; ok { serviceType = sType } + // probably need to collect dbaas consumers too at some stage services = append(services, lagoonv1beta1.LagoonService{ - Name: serviceName, - Type: serviceType, + Name: serviceName, + Type: serviceType, + Containers: containers, }) } msg.Meta.Services = serviceNames - msg.Meta.ServicesV2 = services + msg.Meta.EnvironmentServices = services } // if we aren't being provided the lagoon config, we can skip adding the routes etc if lagoonEnv != nil { From b32c8ebd97c6fe3f2038fde398507b87711a9002 Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Tue, 9 Jan 2024 19:21:55 +1100 Subject: [PATCH 04/17] refactor: move v1beta1 related functions into their own api or controllers for easier isolation --- apis/lagoon/v1beta1/helpers_test.go | 105 +++++ apis/lagoon/v1beta1/lagoonbuild_helpers.go | 444 ++++++++++++++++++ apis/lagoon/v1beta1/lagoonmessaging_types.go | 60 +-- apis/lagoon/v1beta1/lagoontask_helpers.go | 292 ++++++++++++ apis/lagoon/v1beta1/lagoontask_types.go | 3 +- apis/lagoon/v1beta1/zz_generated.deepcopy.go | 92 +--- .../harborcredential_controller.go | 0 controllers/harbor/predicates.go | 61 +++ controllers/v1beta1/build_controller.go | 4 +- controllers/v1beta1/build_deletionhandlers.go | 25 +- controllers/v1beta1/build_helpers.go | 2 +- controllers/v1beta1/build_qoshandler.go | 2 +- controllers/v1beta1/build_standardhandler.go | 2 +- .../v1beta1/podmonitor_buildhandlers.go | 35 +- controllers/v1beta1/podmonitor_controller.go | 4 +- .../v1beta1/podmonitor_taskhandlers.go | 33 +- controllers/v1beta1/predicates.go | 53 --- go.mod | 6 +- go.sum | 14 +- internal/harbor/harbor_credentialrotation.go | 29 +- internal/helpers/helpers.go | 151 ------ internal/helpers/helpers_test.go | 102 ---- internal/messenger/consumer.go | 30 +- internal/messenger/pending_messages.go | 151 ------ internal/messenger/tasks_handler.go | 221 --------- internal/utilities/deletions/lagoon.go | 98 ---- internal/utilities/deletions/process.go | 5 +- internal/utilities/pruner/build_pruner.go | 127 ----- .../utilities/pruner/old_process_pruner.go | 7 +- internal/utilities/pruner/task_pruner.go | 127 ----- main.go | 33 +- 31 files changed, 1028 insertions(+), 1290 deletions(-) create mode 100644 apis/lagoon/v1beta1/helpers_test.go create mode 100644 apis/lagoon/v1beta1/lagoonbuild_helpers.go create mode 100644 apis/lagoon/v1beta1/lagoontask_helpers.go rename controllers/{v1beta1 => harbor}/harborcredential_controller.go (100%) create mode 100644 controllers/harbor/predicates.go delete mode 100644 internal/messenger/pending_messages.go delete mode 100644 internal/utilities/deletions/lagoon.go delete mode 100644 internal/utilities/pruner/build_pruner.go delete mode 100644 internal/utilities/pruner/task_pruner.go diff --git a/apis/lagoon/v1beta1/helpers_test.go b/apis/lagoon/v1beta1/helpers_test.go new file mode 100644 index 00000000..69c2a02b --- /dev/null +++ b/apis/lagoon/v1beta1/helpers_test.go @@ -0,0 +1,105 @@ +package v1beta1 + +import ( + "testing" +) + +func TestCheckLagoonVersion(t *testing.T) { + type args struct { + build *LagoonBuild + checkVersion string + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "test1", + args: args{ + build: &LagoonBuild{ + Spec: LagoonBuildSpec{ + Project: Project{ + Variables: LagoonVariables{ + Project: []byte(`[{"name":"LAGOON_SYSTEM_CORE_VERSION","value":"v2.12.0","scope":"internal_system"}]`), + }, + }, + }, + }, + checkVersion: "2.12.0", + }, + want: true, + }, + { + name: "test2", + args: args{ + build: &LagoonBuild{ + Spec: LagoonBuildSpec{ + Project: Project{ + Variables: LagoonVariables{ + Project: []byte(`[{"name":"LAGOON_SYSTEM_CORE_VERSION","value":"v2.11.0","scope":"internal_system"}]`), + }, + }, + }, + }, + checkVersion: "2.12.0", + }, + want: false, + }, + { + name: "test3", + args: args{ + build: &LagoonBuild{ + Spec: LagoonBuildSpec{ + Project: Project{ + Variables: LagoonVariables{ + Project: []byte(`[]`), + }, + }, + }, + }, + checkVersion: "2.12.0", + }, + want: false, + }, + { + name: "test4", + args: args{ + build: &LagoonBuild{ + Spec: LagoonBuildSpec{ + Project: Project{ + Variables: LagoonVariables{ + Project: []byte(`[{"name":"LAGOON_SYSTEM_CORE_VERSION","value":"v2.12.0","scope":"internal_system"}]`), + }, + }, + }, + }, + checkVersion: "v2.12.0", + }, + want: true, + }, + { + name: "test5", + args: args{ + build: &LagoonBuild{ + Spec: LagoonBuildSpec{ + Project: Project{ + Variables: LagoonVariables{ + Project: []byte(`[{"name":"LAGOON_SYSTEM_CORE_VERSION","value":"v2.11.0","scope":"internal_system"}]`), + }, + }, + }, + }, + checkVersion: "v2.12.0", + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := CheckLagoonVersion(tt.args.build, tt.args.checkVersion); got != tt.want { + t.Errorf("CheckLagoonVersion() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/apis/lagoon/v1beta1/lagoonbuild_helpers.go b/apis/lagoon/v1beta1/lagoonbuild_helpers.go new file mode 100644 index 00000000..f32a45ad --- /dev/null +++ b/apis/lagoon/v1beta1/lagoonbuild_helpers.go @@ -0,0 +1,444 @@ +package v1beta1 + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + "github.com/go-logr/logr" + "github.com/hashicorp/go-version" + "github.com/uselagoon/machinery/api/schema" + "github.com/uselagoon/remote-controller/internal/helpers" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var ( + // BuildRunningPendingStatus . + BuildRunningPendingStatus = []string{ + BuildStatusPending.String(), + BuildStatusQueued.String(), + BuildStatusRunning.String(), + } + // BuildCompletedCancelledFailedStatus . + BuildCompletedCancelledFailedStatus = []string{ + BuildStatusFailed.String(), + BuildStatusComplete.String(), + BuildStatusCancelled.String(), + } +) + +// BuildContainsStatus . +func BuildContainsStatus(slice []LagoonBuildConditions, s LagoonBuildConditions) bool { + for _, item := range slice { + if item == s { + return true + } + } + return false +} + +// RemoveBuild remove a LagoonBuild from a slice of LagoonBuilds +func RemoveBuild(slice []LagoonBuild, s LagoonBuild) []LagoonBuild { + result := []LagoonBuild{} + for _, item := range slice { + if item.ObjectMeta.Name == s.ObjectMeta.Name { + continue + } + result = append(result, item) + } + return result +} + +// Check if the version of lagoon provided in the internal_system scope variable is greater than or equal to the checked version +func CheckLagoonVersion(build *LagoonBuild, checkVersion string) bool { + lagoonProjectVariables := &[]helpers.LagoonEnvironmentVariable{} + json.Unmarshal(build.Spec.Project.Variables.Project, lagoonProjectVariables) + lagoonVersion, err := helpers.GetLagoonVariable("LAGOON_SYSTEM_CORE_VERSION", []string{"internal_system"}, *lagoonProjectVariables) + if err != nil { + return false + } + aVer, err := version.NewSemver(lagoonVersion.Value) + if err != nil { + return false + } + bVer, err := version.NewSemver(checkVersion) + if err != nil { + return false + } + return aVer.GreaterThanOrEqual(bVer) +} + +// CancelExtraBuilds cancels extra builds. +func CancelExtraBuilds(ctx context.Context, r client.Client, opLog logr.Logger, ns string, status string) error { + pendingBuilds := &LagoonBuildList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(ns), + client.MatchingLabels(map[string]string{"lagoon.sh/buildStatus": BuildStatusPending.String()}), + }) + if err := r.List(ctx, pendingBuilds, listOption); err != nil { + return fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err) + } + if len(pendingBuilds.Items) > 0 { + // opLog.Info(fmt.Sprintf("There are %v pending builds", len(pendingBuilds.Items))) + // if we have any pending builds, then grab the latest one and make it running + // if there are any other pending builds, cancel them so only the latest one runs + sort.Slice(pendingBuilds.Items, func(i, j int) bool { + return pendingBuilds.Items[i].ObjectMeta.CreationTimestamp.After(pendingBuilds.Items[j].ObjectMeta.CreationTimestamp.Time) + }) + for idx, pBuild := range pendingBuilds.Items { + pendingBuild := pBuild.DeepCopy() + if idx == 0 { + pendingBuild.Labels["lagoon.sh/buildStatus"] = status + } else { + // cancel any other pending builds + opLog.Info(fmt.Sprintf("Setting build %s as cancelled", pendingBuild.ObjectMeta.Name)) + pendingBuild.Labels["lagoon.sh/buildStatus"] = BuildStatusCancelled.String() + pendingBuild.Labels["lagoon.sh/cancelledByNewBuild"] = "true" + } + if err := r.Update(ctx, pendingBuild); err != nil { + return err + } + } + } + return nil +} + +func GetBuildConditionFromPod(phase corev1.PodPhase) BuildStatusType { + var buildCondition BuildStatusType + switch phase { + case corev1.PodFailed: + buildCondition = BuildStatusFailed + case corev1.PodSucceeded: + buildCondition = BuildStatusComplete + case corev1.PodPending: + buildCondition = BuildStatusPending + case corev1.PodRunning: + buildCondition = BuildStatusRunning + } + return buildCondition +} + +func GetTaskConditionFromPod(phase corev1.PodPhase) TaskStatusType { + var taskCondition TaskStatusType + switch phase { + case corev1.PodFailed: + taskCondition = TaskStatusFailed + case corev1.PodSucceeded: + taskCondition = TaskStatusComplete + case corev1.PodPending: + taskCondition = TaskStatusPending + case corev1.PodRunning: + taskCondition = TaskStatusRunning + } + return taskCondition +} + +func CheckRunningBuilds(ctx context.Context, cns string, opLog logr.Logger, cl client.Client, ns corev1.Namespace) bool { + lagoonBuilds := &LagoonBuildList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(ns.ObjectMeta.Name), + client.MatchingLabels(map[string]string{ + "lagoon.sh/controller": cns, // created by this controller + }), + }) + if err := cl.List(context.Background(), lagoonBuilds, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list Lagoon build pods, there may be none or something went wrong")) + return false + } + runningBuilds := false + sort.Slice(lagoonBuilds.Items, func(i, j int) bool { + return lagoonBuilds.Items[i].ObjectMeta.CreationTimestamp.After(lagoonBuilds.Items[j].ObjectMeta.CreationTimestamp.Time) + }) + // if there are any builds pending or running, don't try and refresh the credentials as this + // could break the build + if len(lagoonBuilds.Items) > 0 { + if helpers.ContainsString( + BuildRunningPendingStatus, + lagoonBuilds.Items[0].Labels["lagoon.sh/buildStatus"], + ) { + runningBuilds = true + } + } + return runningBuilds +} + +// DeleteLagoonBuilds will delete any lagoon builds from the namespace. +func DeleteLagoonBuilds(ctx context.Context, opLog logr.Logger, cl client.Client, ns, project, environment string) bool { + lagoonBuilds := &LagoonBuildList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(ns), + }) + if err := cl.List(ctx, lagoonBuilds, listOption); err != nil { + opLog.Error(err, + fmt.Sprintf( + "Unable to list lagoon build in namespace %s for project %s, environment %s", + ns, + project, + environment, + ), + ) + return false + } + for _, lagoonBuild := range lagoonBuilds.Items { + if err := cl.Delete(ctx, &lagoonBuild); helpers.IgnoreNotFound(err) != nil { + opLog.Error(err, + fmt.Sprintf( + "Unable to delete lagoon build %s in %s for project %s, environment %s", + lagoonBuild.ObjectMeta.Name, + ns, + project, + environment, + ), + ) + return false + } + opLog.Info( + fmt.Sprintf( + "Deleted lagoon build %s in %s for project %s, environment %s", + lagoonBuild.ObjectMeta.Name, + ns, + project, + environment, + ), + ) + } + return true +} + +func LagoonBuildPruner(ctx context.Context, cl client.Client, cns string, buildsToKeep int) { + opLog := ctrl.Log.WithName("utilities").WithName("LagoonBuildPruner") + namespaces := &corev1.NamespaceList{} + labelRequirements, _ := labels.NewRequirement("lagoon.sh/environmentType", selection.Exists, nil) + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.MatchingLabelsSelector{ + Selector: labels.NewSelector().Add(*labelRequirements), + }, + }) + if err := cl.List(ctx, namespaces, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list namespaces created by Lagoon, there may be none or something went wrong")) + return + } + for _, ns := range namespaces.Items { + if ns.Status.Phase == corev1.NamespaceTerminating { + // if the namespace is terminating, don't try to renew the robot credentials + opLog.Info(fmt.Sprintf("Namespace %s is being terminated, aborting build pruner", ns.ObjectMeta.Name)) + continue + } + opLog.Info(fmt.Sprintf("Checking LagoonBuilds in namespace %s", ns.ObjectMeta.Name)) + lagoonBuilds := &LagoonBuildList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(ns.ObjectMeta.Name), + client.MatchingLabels(map[string]string{ + "lagoon.sh/controller": cns, // created by this controller + }), + }) + if err := cl.List(ctx, lagoonBuilds, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list LagoonBuild resources, there may be none or something went wrong")) + continue + } + // sort the build pods by creation timestamp + sort.Slice(lagoonBuilds.Items, func(i, j int) bool { + return lagoonBuilds.Items[i].ObjectMeta.CreationTimestamp.After(lagoonBuilds.Items[j].ObjectMeta.CreationTimestamp.Time) + }) + if len(lagoonBuilds.Items) > buildsToKeep { + for idx, lagoonBuild := range lagoonBuilds.Items { + if idx >= buildsToKeep { + if helpers.ContainsString( + BuildCompletedCancelledFailedStatus, + lagoonBuild.ObjectMeta.Labels["lagoon.sh/buildStatus"], + ) { + opLog.Info(fmt.Sprintf("Cleaning up LagoonBuild %s", lagoonBuild.ObjectMeta.Name)) + if err := cl.Delete(ctx, &lagoonBuild); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to update status condition")) + break + } + } + } + } + } + } + return +} + +// BuildPodPruner will prune any build pods that are hanging around. +func BuildPodPruner(ctx context.Context, cl client.Client, cns string, buildPodsToKeep int) { + opLog := ctrl.Log.WithName("utilities").WithName("BuildPodPruner") + namespaces := &corev1.NamespaceList{} + labelRequirements, _ := labels.NewRequirement("lagoon.sh/environmentType", selection.Exists, nil) + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.MatchingLabelsSelector{ + Selector: labels.NewSelector().Add(*labelRequirements), + }, + }) + if err := cl.List(ctx, namespaces, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list namespaces created by Lagoon, there may be none or something went wrong")) + return + } + for _, ns := range namespaces.Items { + if ns.Status.Phase == corev1.NamespaceTerminating { + // if the namespace is terminating, don't try to renew the robot credentials + opLog.Info(fmt.Sprintf("Namespace %s is being terminated, aborting build pod pruner", ns.ObjectMeta.Name)) + return + } + opLog.Info(fmt.Sprintf("Checking Lagoon build pods in namespace %s", ns.ObjectMeta.Name)) + buildPods := &corev1.PodList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(ns.ObjectMeta.Name), + client.MatchingLabels(map[string]string{ + "lagoon.sh/jobType": "build", + "lagoon.sh/controller": cns, // created by this controller + }), + }) + if err := cl.List(ctx, buildPods, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list Lagoon build pods, there may be none or something went wrong")) + return + } + // sort the build pods by creation timestamp + sort.Slice(buildPods.Items, func(i, j int) bool { + return buildPods.Items[i].ObjectMeta.CreationTimestamp.After(buildPods.Items[j].ObjectMeta.CreationTimestamp.Time) + }) + if len(buildPods.Items) > buildPodsToKeep { + for idx, pod := range buildPods.Items { + if idx >= buildPodsToKeep { + if pod.Status.Phase == corev1.PodFailed || + pod.Status.Phase == corev1.PodSucceeded { + opLog.Info(fmt.Sprintf("Cleaning up pod %s", pod.ObjectMeta.Name)) + if err := cl.Delete(ctx, &pod); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to update status condition")) + break + } + } + } + } + } + } + return +} + +func updateLagoonBuild(opLog logr.Logger, namespace string, jobSpec LagoonTaskSpec, lagoonBuild *LagoonBuild) ([]byte, error) { + // if the build isn't found by the controller + // then publish a response back to controllerhandler to tell it to update the build to cancelled + // this allows us to update builds in the API that may have gone stale or not updated from `New`, `Pending`, or `Running` status + buildCondition := "cancelled" + if lagoonBuild != nil { + if val, ok := lagoonBuild.ObjectMeta.Labels["lagoon.sh/buildStatus"]; ok { + // if the build isnt running,pending,queued, then set the buildcondition to the value failed/complete/cancelled + if !helpers.ContainsString(BuildRunningPendingStatus, val) { + buildCondition = strings.ToLower(val) + } + } + } + msg := schema.LagoonMessage{ + Type: "build", + Namespace: namespace, + Meta: &schema.LagoonLogMeta{ + Environment: jobSpec.Environment.Name, + Project: jobSpec.Project.Name, + BuildStatus: buildCondition, + BuildName: jobSpec.Misc.Name, + }, + } + // set the start/end time to be now as the default + // to stop the duration counter in the ui + msg.Meta.StartTime = time.Now().UTC().Format("2006-01-02 15:04:05") + msg.Meta.EndTime = time.Now().UTC().Format("2006-01-02 15:04:05") + + // if possible, get the start and end times from the build resource, these will be sent back to lagoon to update the api + if lagoonBuild != nil && lagoonBuild.Status.Conditions != nil { + conditions := lagoonBuild.Status.Conditions + // sort the build conditions by time so the first and last can be extracted + sort.Slice(conditions, func(i, j int) bool { + iTime, _ := time.Parse("2006-01-02T15:04:05Z", conditions[i].LastTransitionTime) + jTime, _ := time.Parse("2006-01-02T15:04:05Z", conditions[j].LastTransitionTime) + return iTime.Before(jTime) + }) + // get the starting time, or fallback to default + sTime, err := time.Parse("2006-01-02T15:04:05Z", conditions[0].LastTransitionTime) + if err == nil { + msg.Meta.StartTime = sTime.Format("2006-01-02 15:04:05") + } + // get the ending time, or fallback to default + eTime, err := time.Parse("2006-01-02T15:04:05Z", conditions[len(conditions)-1].LastTransitionTime) + if err == nil { + msg.Meta.EndTime = eTime.Format("2006-01-02 15:04:05") + } + } + msgBytes, err := json.Marshal(msg) + if err != nil { + return nil, fmt.Errorf("Unable to encode message as JSON: %v", err) + } + return msgBytes, nil +} + +// CancelBuild handles cancelling builds or handling if a build no longer exists. +func CancelBuild(ctx context.Context, cl client.Client, namespace string, body []byte) (bool, []byte, error) { + opLog := ctrl.Log.WithName("handlers").WithName("LagoonTasks") + jobSpec := &LagoonTaskSpec{} + json.Unmarshal(body, jobSpec) + var jobPod corev1.Pod + if err := cl.Get(ctx, types.NamespacedName{ + Name: jobSpec.Misc.Name, + Namespace: namespace, + }, &jobPod); err != nil { + opLog.Info(fmt.Sprintf( + "Unable to find build pod %s to cancel it. Checking to see if LagoonBuild exists.", + jobSpec.Misc.Name, + )) + // since there was no build pod, check for the lagoon build resource + var lagoonBuild LagoonBuild + if err := cl.Get(ctx, types.NamespacedName{ + Name: jobSpec.Misc.Name, + Namespace: namespace, + }, &lagoonBuild); err != nil { + opLog.Info(fmt.Sprintf( + "Unable to find build %s to cancel it. Sending response to Lagoon to update the build to cancelled.", + jobSpec.Misc.Name, + )) + // if there is no pod or build, update the build in Lagoon to cancelled, assume completely cancelled with no other information + // and then send the response back to lagoon to say it was cancelled. + b, err := updateLagoonBuild(opLog, namespace, *jobSpec, nil) + return false, b, err + } + // as there is no build pod, but there is a lagoon build resource + // update it to cancelled so that the controller doesn't try to run it + // check if the build has existing status or not though to consume it + if helpers.ContainsString( + BuildRunningPendingStatus, + lagoonBuild.ObjectMeta.Labels["lagoon.sh/buildStatus"], + ) { + lagoonBuild.ObjectMeta.Labels["lagoon.sh/buildStatus"] = BuildStatusCancelled.String() + } + lagoonBuild.ObjectMeta.Labels["lagoon.sh/cancelBuildNoPod"] = "true" + if err := cl.Update(ctx, &lagoonBuild); err != nil { + opLog.Error(err, + fmt.Sprintf( + "Unable to update build %s to cancel it.", + jobSpec.Misc.Name, + ), + ) + return false, nil, err + } + // and then send the response back to lagoon to say it was cancelled. + b, err := updateLagoonBuild(opLog, namespace, *jobSpec, &lagoonBuild) + return true, b, err + } + jobPod.ObjectMeta.Labels["lagoon.sh/cancelBuild"] = "true" + if err := cl.Update(ctx, &jobPod); err != nil { + opLog.Error(err, + fmt.Sprintf( + "Unable to update build %s to cancel it.", + jobSpec.Misc.Name, + ), + ) + return false, nil, err + } + return false, nil, nil +} diff --git a/apis/lagoon/v1beta1/lagoonmessaging_types.go b/apis/lagoon/v1beta1/lagoonmessaging_types.go index f9fe6ecd..5480b387 100644 --- a/apis/lagoon/v1beta1/lagoonmessaging_types.go +++ b/apis/lagoon/v1beta1/lagoonmessaging_types.go @@ -1,59 +1,13 @@ package v1beta1 -// LagoonMessaging - -// LagoonLog is used to sendToLagoonLogs messaging queue -// this is general logging information -type LagoonLog struct { - Severity string `json:"severity,omitempty"` - Project string `json:"project,omitempty"` - UUID string `json:"uuid,omitempty"` - Event string `json:"event,omitempty"` - Meta *LagoonLogMeta `json:"meta,omitempty"` - Message string `json:"message,omitempty"` -} - -// LagoonLogMeta is the metadata that is used by logging in Lagoon. -type LagoonLogMeta struct { - BranchName string `json:"branchName,omitempty"` - BuildName string `json:"buildName,omitempty"` - BuildPhase string `json:"buildPhase,omitempty"` // @TODO: deprecate once controller-handler is fixed - BuildStatus string `json:"buildStatus,omitempty"` - BuildStep string `json:"buildStep,omitempty"` - EndTime string `json:"endTime,omitempty"` - Environment string `json:"environment,omitempty"` - EnvironmentID *uint `json:"environmentId,omitempty"` - JobName string `json:"jobName,omitempty"` // used by tasks/jobs - JobStatus string `json:"jobStatus,omitempty"` // used by tasks/jobs - JobStep string `json:"jobStep,omitempty"` // used by tasks/jobs - LogLink string `json:"logLink,omitempty"` - Project string `json:"project,omitempty"` - ProjectID *uint `json:"projectId,omitempty"` - ProjectName string `json:"projectName,omitempty"` - RemoteID string `json:"remoteId,omitempty"` - Route string `json:"route,omitempty"` - Routes []string `json:"routes,omitempty"` - StartTime string `json:"startTime,omitempty"` - Services []string `json:"services,omitempty"` - Task *LagoonTaskInfo `json:"task,omitempty"` - Key string `json:"key,omitempty"` - AdvancedData string `json:"advancedData,omitempty"` - Cluster string `json:"clusterName,omitempty"` -} - -// LagoonMessage is used for sending build info back to Lagoon -// messaging queue to update the environment or deployment -type LagoonMessage struct { - Type string `json:"type,omitempty"` - Namespace string `json:"namespace,omitempty"` - Meta *LagoonLogMeta `json:"meta,omitempty"` - // BuildInfo *LagoonBuildInfo `json:"buildInfo,omitempty"` -} +import "github.com/uselagoon/machinery/api/schema" // LagoonStatusMessages is where unsent messages are stored for re-sending. type LagoonStatusMessages struct { - StatusMessage *LagoonLog `json:"statusMessage,omitempty"` - BuildLogMessage *LagoonLog `json:"buildLogMessage,omitempty"` - TaskLogMessage *LagoonLog `json:"taskLogMessage,omitempty"` - EnvironmentMessage *LagoonMessage `json:"environmentMessage,omitempty"` + StatusMessage *schema.LagoonLog `json:"statusMessage,omitempty"` + BuildLogMessage *schema.LagoonLog `json:"buildLogMessage,omitempty"` + TaskLogMessage *schema.LagoonLog `json:"taskLogMessage,omitempty"` + // LagoonMessage is used for sending build info back to Lagoon + // messaging queue to update the environment or deployment + EnvironmentMessage *schema.LagoonMessage `json:"environmentMessage,omitempty"` } diff --git a/apis/lagoon/v1beta1/lagoontask_helpers.go b/apis/lagoon/v1beta1/lagoontask_helpers.go new file mode 100644 index 00000000..ba30f3c6 --- /dev/null +++ b/apis/lagoon/v1beta1/lagoontask_helpers.go @@ -0,0 +1,292 @@ +package v1beta1 + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "time" + + "github.com/go-logr/logr" + "github.com/uselagoon/machinery/api/schema" + "github.com/uselagoon/remote-controller/internal/helpers" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var ( + // TaskRunningPendingStatus . + TaskRunningPendingStatus = []string{ + TaskStatusPending.String(), + TaskStatusQueued.String(), + TaskStatusRunning.String(), + } + // TaskCompletedCancelledFailedStatus . + TaskCompletedCancelledFailedStatus = []string{ + TaskStatusFailed.String(), + TaskStatusComplete.String(), + TaskStatusCancelled.String(), + } +) + +// TaskContainsStatus . +func TaskContainsStatus(slice []LagoonTaskConditions, s LagoonTaskConditions) bool { + for _, item := range slice { + if item == s { + return true + } + } + return false +} + +// DeleteLagoonTasks will delete any lagoon tasks from the namespace. +func DeleteLagoonTasks(ctx context.Context, opLog logr.Logger, cl client.Client, ns, project, environment string) bool { + lagoonTasks := &LagoonTaskList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(ns), + }) + if err := cl.List(ctx, lagoonTasks, listOption); err != nil { + opLog.Error(err, + fmt.Sprintf( + "Unable to list lagoon task in namespace %s for project %s, environment %s", + ns, + project, + environment, + ), + ) + return false + } + for _, lagoonTask := range lagoonTasks.Items { + if err := cl.Delete(ctx, &lagoonTask); helpers.IgnoreNotFound(err) != nil { + opLog.Error(err, + fmt.Sprintf( + "Unable to delete lagoon task %s in %s for project %s, environment %s", + lagoonTask.ObjectMeta.Name, + ns, + project, + environment, + ), + ) + return false + } + opLog.Info( + fmt.Sprintf( + "Deleted lagoon task %s in %s for project %s, environment %s", + lagoonTask.ObjectMeta.Name, + ns, + project, + environment, + ), + ) + } + return true +} + +// LagoonTaskPruner will prune any build crds that are hanging around. +func LagoonTaskPruner(ctx context.Context, cl client.Client, cns string, tasksToKeep int) { + opLog := ctrl.Log.WithName("utilities").WithName("LagoonTaskPruner") + namespaces := &corev1.NamespaceList{} + labelRequirements, _ := labels.NewRequirement("lagoon.sh/environmentType", selection.Exists, nil) + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.MatchingLabelsSelector{ + Selector: labels.NewSelector().Add(*labelRequirements), + }, + }) + if err := cl.List(ctx, namespaces, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list namespaces created by Lagoon, there may be none or something went wrong")) + return + } + for _, ns := range namespaces.Items { + if ns.Status.Phase == corev1.NamespaceTerminating { + // if the namespace is terminating, don't try to renew the robot credentials + opLog.Info(fmt.Sprintf("Namespace %s is being terminated, aborting task pruner", ns.ObjectMeta.Name)) + continue + } + opLog.Info(fmt.Sprintf("Checking LagoonTasks in namespace %s", ns.ObjectMeta.Name)) + lagoonTasks := &LagoonTaskList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(ns.ObjectMeta.Name), + client.MatchingLabels(map[string]string{ + "lagoon.sh/controller": cns, // created by this controller + }), + }) + if err := cl.List(ctx, lagoonTasks, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list LagoonTask resources, there may be none or something went wrong")) + continue + } + // sort the build pods by creation timestamp + sort.Slice(lagoonTasks.Items, func(i, j int) bool { + return lagoonTasks.Items[i].ObjectMeta.CreationTimestamp.After(lagoonTasks.Items[j].ObjectMeta.CreationTimestamp.Time) + }) + if len(lagoonTasks.Items) > tasksToKeep { + for idx, lagoonTask := range lagoonTasks.Items { + if idx >= tasksToKeep { + if helpers.ContainsString( + TaskCompletedCancelledFailedStatus, + lagoonTask.ObjectMeta.Labels["lagoon.sh/taskStatus"], + ) { + opLog.Info(fmt.Sprintf("Cleaning up LagoonTask %s", lagoonTask.ObjectMeta.Name)) + if err := cl.Delete(ctx, &lagoonTask); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to update status condition")) + break + } + } + } + } + } + } + return +} + +// TaskPodPruner will prune any task pods that are hanging around. +func TaskPodPruner(ctx context.Context, cl client.Client, cns string, taskPodsToKeep int) { + opLog := ctrl.Log.WithName("utilities").WithName("TaskPodPruner") + namespaces := &corev1.NamespaceList{} + labelRequirements, _ := labels.NewRequirement("lagoon.sh/environmentType", selection.Exists, nil) + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.MatchingLabelsSelector{ + Selector: labels.NewSelector().Add(*labelRequirements), + }, + }) + if err := cl.List(ctx, namespaces, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list namespaces created by Lagoon, there may be none or something went wrong")) + return + } + for _, ns := range namespaces.Items { + if ns.Status.Phase == corev1.NamespaceTerminating { + // if the namespace is terminating, don't try to renew the robot credentials + opLog.Info(fmt.Sprintf("Namespace %s is being terminated, aborting task pod pruner", ns.ObjectMeta.Name)) + return + } + opLog.Info(fmt.Sprintf("Checking Lagoon task pods in namespace %s", ns.ObjectMeta.Name)) + taskPods := &corev1.PodList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(ns.ObjectMeta.Name), + client.MatchingLabels(map[string]string{ + "lagoon.sh/jobType": "task", + "lagoon.sh/controller": cns, // created by this controller + }), + }) + if err := cl.List(ctx, taskPods, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list Lagoon task pods, there may be none or something went wrong")) + return + } + // sort the build pods by creation timestamp + sort.Slice(taskPods.Items, func(i, j int) bool { + return taskPods.Items[i].ObjectMeta.CreationTimestamp.After(taskPods.Items[j].ObjectMeta.CreationTimestamp.Time) + }) + if len(taskPods.Items) > taskPodsToKeep { + for idx, pod := range taskPods.Items { + if idx >= taskPodsToKeep { + if pod.Status.Phase == corev1.PodFailed || + pod.Status.Phase == corev1.PodSucceeded { + opLog.Info(fmt.Sprintf("Cleaning up pod %s", pod.ObjectMeta.Name)) + if err := cl.Delete(ctx, &pod); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to delete pod")) + break + } + } + } + } + } + } + return +} + +func updateLagoonTask(opLog logr.Logger, namespace string, taskSpec LagoonTaskSpec) ([]byte, error) { + //@TODO: use `taskName` in the future only + taskName := fmt.Sprintf("lagoon-task-%s-%s", taskSpec.Task.ID, helpers.HashString(taskSpec.Task.ID)[0:6]) + if taskSpec.Task.TaskName != "" { + taskName = taskSpec.Task.TaskName + } + // if the task isn't found by the controller + // then publish a response back to controllerhandler to tell it to update the task to cancelled + // this allows us to update tasks in the API that may have gone stale or not updated from `New`, `Pending`, or `Running` status + msg := schema.LagoonMessage{ + Type: "task", + Namespace: namespace, + Meta: &schema.LagoonLogMeta{ + Environment: taskSpec.Environment.Name, + Project: taskSpec.Project.Name, + JobName: taskName, + JobStatus: "cancelled", + Task: &schema.LagoonTaskInfo{ + TaskName: taskSpec.Task.TaskName, + ID: taskSpec.Task.ID, + Name: taskSpec.Task.Name, + Service: taskSpec.Task.Service, + }, + }, + } + // if the task isn't found at all, then set the start/end time to be now + // to stop the duration counter in the ui + msg.Meta.StartTime = time.Now().UTC().Format("2006-01-02 15:04:05") + msg.Meta.EndTime = time.Now().UTC().Format("2006-01-02 15:04:05") + msgBytes, err := json.Marshal(msg) + if err != nil { + return nil, fmt.Errorf("Unable to encode message as JSON: %v", err) + } + return msgBytes, nil +} + +// CancelTask handles cancelling tasks or handling if a tasks no longer exists. +func CancelTask(ctx context.Context, cl client.Client, namespace string, body []byte) (bool, []byte, error) { + opLog := ctrl.Log.WithName("handlers").WithName("LagoonTasks") + jobSpec := &LagoonTaskSpec{} + json.Unmarshal(body, jobSpec) + var jobPod corev1.Pod + //@TODO: use `taskName` in the future only + taskName := fmt.Sprintf("lagoon-task-%s-%s", jobSpec.Task.ID, helpers.HashString(jobSpec.Task.ID)[0:6]) + if jobSpec.Task.TaskName != "" { + taskName = jobSpec.Task.TaskName + } + if err := cl.Get(ctx, types.NamespacedName{ + Name: taskName, + Namespace: namespace, + }, &jobPod); err != nil { + // since there was no task pod, check for the lagoon task resource + var lagoonTask LagoonTask + if err := cl.Get(ctx, types.NamespacedName{ + Name: taskName, + Namespace: namespace, + }, &lagoonTask); err != nil { + opLog.Info(fmt.Sprintf( + "Unable to find task %s to cancel it. Sending response to Lagoon to update the task to cancelled.", + taskName, + )) + // if there is no pod or task, update the task in Lagoon to cancelled + b, err := updateLagoonTask(opLog, namespace, *jobSpec) + return false, b, err + } + // as there is no task pod, but there is a lagoon task resource + // update it to cancelled so that the controller doesn't try to run it + lagoonTask.ObjectMeta.Labels["lagoon.sh/taskStatus"] = TaskStatusCancelled.String() + if err := cl.Update(ctx, &lagoonTask); err != nil { + opLog.Error(err, + fmt.Sprintf( + "Unable to update task %s to cancel it.", + taskName, + ), + ) + return false, nil, err + } + // and then send the response back to lagoon to say it was cancelled. + b, err := updateLagoonTask(opLog, namespace, *jobSpec) + return true, b, err + } + jobPod.ObjectMeta.Labels["lagoon.sh/cancelTask"] = "true" + if err := cl.Update(ctx, &jobPod); err != nil { + opLog.Error(err, + fmt.Sprintf( + "Unable to update task %s to cancel it.", + jobSpec.Misc.Name, + ), + ) + return false, nil, err + } + return false, nil, nil +} diff --git a/apis/lagoon/v1beta1/lagoontask_types.go b/apis/lagoon/v1beta1/lagoontask_types.go index 3231e530..bb48276d 100644 --- a/apis/lagoon/v1beta1/lagoontask_types.go +++ b/apis/lagoon/v1beta1/lagoontask_types.go @@ -22,6 +22,7 @@ import ( "strconv" "strings" + "github.com/uselagoon/machinery/api/schema" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -76,7 +77,7 @@ type LagoonTaskSpec struct { // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster // Important: Run "make" to regenerate code after modifying this file Key string `json:"key,omitempty"` - Task LagoonTaskInfo `json:"task,omitempty"` + Task schema.LagoonTaskInfo `json:"task,omitempty"` Project LagoonTaskProject `json:"project,omitempty"` Environment LagoonTaskEnvironment `json:"environment,omitempty"` Misc *LagoonMiscInfo `json:"misc,omitempty"` diff --git a/apis/lagoon/v1beta1/zz_generated.deepcopy.go b/apis/lagoon/v1beta1/zz_generated.deepcopy.go index 842af614..c11c785c 100644 --- a/apis/lagoon/v1beta1/zz_generated.deepcopy.go +++ b/apis/lagoon/v1beta1/zz_generated.deepcopy.go @@ -198,86 +198,6 @@ func (in *LagoonBuildStatus) DeepCopy() *LagoonBuildStatus { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *LagoonLog) DeepCopyInto(out *LagoonLog) { - *out = *in - if in.Meta != nil { - in, out := &in.Meta, &out.Meta - *out = new(LagoonLogMeta) - (*in).DeepCopyInto(*out) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonLog. -func (in *LagoonLog) DeepCopy() *LagoonLog { - if in == nil { - return nil - } - out := new(LagoonLog) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *LagoonLogMeta) DeepCopyInto(out *LagoonLogMeta) { - *out = *in - if in.EnvironmentID != nil { - in, out := &in.EnvironmentID, &out.EnvironmentID - *out = new(uint) - **out = **in - } - if in.ProjectID != nil { - in, out := &in.ProjectID, &out.ProjectID - *out = new(uint) - **out = **in - } - if in.Routes != nil { - in, out := &in.Routes, &out.Routes - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.Services != nil { - in, out := &in.Services, &out.Services - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.Task != nil { - in, out := &in.Task, &out.Task - *out = new(LagoonTaskInfo) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonLogMeta. -func (in *LagoonLogMeta) DeepCopy() *LagoonLogMeta { - if in == nil { - return nil - } - out := new(LagoonLogMeta) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *LagoonMessage) DeepCopyInto(out *LagoonMessage) { - *out = *in - if in.Meta != nil { - in, out := &in.Meta, &out.Meta - *out = new(LagoonLogMeta) - (*in).DeepCopyInto(*out) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonMessage. -func (in *LagoonMessage) DeepCopy() *LagoonMessage { - if in == nil { - return nil - } - out := new(LagoonMessage) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LagoonMiscBackupInfo) DeepCopyInto(out *LagoonMiscBackupInfo) { *out = *in @@ -323,23 +243,19 @@ func (in *LagoonStatusMessages) DeepCopyInto(out *LagoonStatusMessages) { *out = *in if in.StatusMessage != nil { in, out := &in.StatusMessage, &out.StatusMessage - *out = new(LagoonLog) - (*in).DeepCopyInto(*out) + *out = (*in).DeepCopy() } if in.BuildLogMessage != nil { in, out := &in.BuildLogMessage, &out.BuildLogMessage - *out = new(LagoonLog) - (*in).DeepCopyInto(*out) + *out = (*in).DeepCopy() } if in.TaskLogMessage != nil { in, out := &in.TaskLogMessage, &out.TaskLogMessage - *out = new(LagoonLog) - (*in).DeepCopyInto(*out) + *out = (*in).DeepCopy() } if in.EnvironmentMessage != nil { in, out := &in.EnvironmentMessage, &out.EnvironmentMessage - *out = new(LagoonMessage) - (*in).DeepCopyInto(*out) + *out = (*in).DeepCopy() } } diff --git a/controllers/v1beta1/harborcredential_controller.go b/controllers/harbor/harborcredential_controller.go similarity index 100% rename from controllers/v1beta1/harborcredential_controller.go rename to controllers/harbor/harborcredential_controller.go diff --git a/controllers/harbor/predicates.go b/controllers/harbor/predicates.go new file mode 100644 index 00000000..dab932dd --- /dev/null +++ b/controllers/harbor/predicates.go @@ -0,0 +1,61 @@ +package v1beta1 + +// contains all the event watch conditions for secret and ingresses + +import ( + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +// SecretPredicates defines the funcs for predicates +type SecretPredicates struct { + predicate.Funcs + ControllerNamespace string +} + +// Create is used when a creation event is received by the controller. +func (n SecretPredicates) Create(e event.CreateEvent) bool { + if controller, ok := e.Object.GetLabels()["lagoon.sh/controller"]; ok { + if controller == n.ControllerNamespace { + if val, ok := e.Object.GetLabels()["lagoon.sh/harbor-credential"]; ok { + if val == "true" { + return true + } + } + } + } + return false +} + +// Delete is used when a deletion event is received by the controller. +func (n SecretPredicates) Delete(e event.DeleteEvent) bool { + return false +} + +// Update is used when an update event is received by the controller. +func (n SecretPredicates) Update(e event.UpdateEvent) bool { + if controller, ok := e.ObjectOld.GetLabels()["lagoon.sh/controller"]; ok { + if controller == n.ControllerNamespace { + if val, ok := e.ObjectOld.GetLabels()["lagoon.sh/harbor-credential"]; ok { + if val == "true" { + return true + } + } + } + } + return false +} + +// Generic is used when any other event is received by the controller. +func (n SecretPredicates) Generic(e event.GenericEvent) bool { + if controller, ok := e.Object.GetLabels()["lagoon.sh/controller"]; ok { + if controller == n.ControllerNamespace { + if val, ok := e.Object.GetLabels()["lagoon.sh/harbor-credential"]; ok { + if val == "true" { + return true + } + } + } + } + return false +} diff --git a/controllers/v1beta1/build_controller.go b/controllers/v1beta1/build_controller.go index 421cdac3..db1f608c 100644 --- a/controllers/v1beta1/build_controller.go +++ b/controllers/v1beta1/build_controller.go @@ -259,7 +259,7 @@ func (r *LagoonBuildReconciler) createNamespaceBuild(ctx context.Context, // if everything is all good controller will handle the new build resource that gets created as it will have // the `lagoon.sh/buildStatus = Pending` now - err = helpers.CancelExtraBuilds(ctx, r.Client, opLog, namespace.ObjectMeta.Name, lagoonv1beta1.BuildStatusPending.String()) + err = lagoonv1beta1.CancelExtraBuilds(ctx, r.Client, opLog, namespace.ObjectMeta.Name, lagoonv1beta1.BuildStatusPending.String()) if err != nil { return ctrl.Result{}, err } @@ -292,7 +292,7 @@ func (r *LagoonBuildReconciler) createNamespaceBuild(ctx context.Context, } else { // get the status from the pod and update the build if lagoonBuildPod.Status.Phase == corev1.PodFailed || lagoonBuildPod.Status.Phase == corev1.PodSucceeded { - buildCondition = helpers.GetBuildConditionFromPod(lagoonBuildPod.Status.Phase) + buildCondition = lagoonv1beta1.GetBuildConditionFromPod(lagoonBuildPod.Status.Phase) opLog.Info(fmt.Sprintf("Setting build %s as %s", runningBuild.ObjectMeta.Name, buildCondition.String())) runningBuild.Labels["lagoon.sh/buildStatus"] = buildCondition.String() } else { diff --git a/controllers/v1beta1/build_deletionhandlers.go b/controllers/v1beta1/build_deletionhandlers.go index fba4c0e2..9af24062 100644 --- a/controllers/v1beta1/build_deletionhandlers.go +++ b/controllers/v1beta1/build_deletionhandlers.go @@ -11,6 +11,7 @@ import ( "time" "github.com/go-logr/logr" + "github.com/uselagoon/machinery/api/schema" lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" "github.com/uselagoon/remote-controller/internal/helpers" "gopkg.in/matryer/try.v1" @@ -102,7 +103,7 @@ func (r *LagoonBuildReconciler) deleteExternalResources( // if there are any running builds, check if it is the one currently being deleted if lagoonBuild.ObjectMeta.Name == runningBuild.ObjectMeta.Name { // if the one being deleted is a running one, remove it from the list of running builds - newRunningBuilds = helpers.RemoveBuild(newRunningBuilds, runningBuild) + newRunningBuilds = lagoonv1beta1.RemoveBuild(newRunningBuilds, runningBuild) } } // if the number of runningBuilds is 0 (excluding the one being deleted) @@ -125,7 +126,7 @@ func (r *LagoonBuildReconciler) deleteExternalResources( // if there are any pending builds, check if it is the one currently being deleted if lagoonBuild.ObjectMeta.Name == pendingBuild.ObjectMeta.Name { // if the one being deleted a the pending one, remove it from the list of pending builds - newPendingBuilds = helpers.RemoveBuild(newPendingBuilds, pendingBuild) + newPendingBuilds = lagoonv1beta1.RemoveBuild(newPendingBuilds, pendingBuild) } } // sort the pending builds by creation timestamp @@ -165,7 +166,7 @@ func (r *LagoonBuildReconciler) updateCancelledDeploymentWithLogs( // if it was already Failed or Completed, lagoon probably already knows // so we don't have to do anything else. if helpers.ContainsString( - helpers.BuildRunningPendingStatus, + lagoonv1beta1.BuildRunningPendingStatus, lagoonBuild.Labels["lagoon.sh/buildStatus"], ) { opLog.Info( @@ -234,11 +235,11 @@ func (r *LagoonBuildReconciler) buildLogsToLagoonLogs(ctx context.Context, if condition == lagoonv1beta1.BuildStatusCancelled { buildStep = "cancelled" } - msg := lagoonv1beta1.LagoonLog{ + msg := schema.LagoonLog{ Severity: "info", Project: lagoonBuild.Spec.Project.Name, Event: "build-logs:builddeploy-kubernetes:" + lagoonBuild.ObjectMeta.Name, - Meta: &lagoonv1beta1.LagoonLogMeta{ + Meta: &schema.LagoonLogMeta{ JobName: lagoonBuild.ObjectMeta.Name, // @TODO: remove once lagoon is corrected in controller-handler BuildName: lagoonBuild.ObjectMeta.Name, BuildPhase: buildCondition.ToLower(), // @TODO: same as buildstatus label, remove once lagoon is corrected in controller-handler @@ -290,10 +291,10 @@ func (r *LagoonBuildReconciler) updateDeploymentAndEnvironmentTask(ctx context.C r.RandomNamespacePrefix, ) if r.EnableMQ { - msg := lagoonv1beta1.LagoonMessage{ + msg := schema.LagoonMessage{ Type: "build", Namespace: namespace, - Meta: &lagoonv1beta1.LagoonLogMeta{ + Meta: &schema.LagoonLogMeta{ Environment: lagoonBuild.Spec.Project.Environment, Project: lagoonBuild.Spec.Project.Name, BuildPhase: buildCondition.ToLower(), @@ -372,11 +373,11 @@ func (r *LagoonBuildReconciler) buildStatusLogsToLagoonLogs(ctx context.Context, buildStep string, ) { if r.EnableMQ { - msg := lagoonv1beta1.LagoonLog{ + msg := schema.LagoonLog{ Severity: "info", Project: lagoonBuild.Spec.Project.Name, Event: "task:builddeploy-kubernetes:" + buildCondition.ToLower(), //@TODO: this probably needs to be changed to a new task event for the controller - Meta: &lagoonv1beta1.LagoonLogMeta{ + Meta: &schema.LagoonLogMeta{ ProjectName: lagoonBuild.Spec.Project.Name, BranchName: lagoonBuild.Spec.Project.Environment, BuildPhase: buildCondition.ToLower(), @@ -426,7 +427,7 @@ func (r *LagoonBuildReconciler) buildStatusLogsToLagoonLogs(ctx context.Context, // updateEnvironmentMessage this is called if the message queue is unavailable, it stores the message that would be sent in the lagoon build func (r *LagoonBuildReconciler) updateEnvironmentMessage(ctx context.Context, lagoonBuild *lagoonv1beta1.LagoonBuild, - envMessage lagoonv1beta1.LagoonMessage, + envMessage schema.LagoonMessage, ) error { // set the transition time mergePatch, _ := json.Marshal(map[string]interface{}{ @@ -448,7 +449,7 @@ func (r *LagoonBuildReconciler) updateEnvironmentMessage(ctx context.Context, // updateBuildStatusMessage this is called if the message queue is unavailable, it stores the message that would be sent in the lagoon build func (r *LagoonBuildReconciler) updateBuildStatusMessage(ctx context.Context, lagoonBuild *lagoonv1beta1.LagoonBuild, - statusMessage lagoonv1beta1.LagoonLog, + statusMessage schema.LagoonLog, ) error { // set the transition time mergePatch, _ := json.Marshal(map[string]interface{}{ @@ -494,7 +495,7 @@ func (r *LagoonBuildReconciler) removeBuildPendingMessageStatus(ctx context.Cont // updateBuildLogMessage this is called if the message queue is unavailable, it stores the message that would be sent in the lagoon build func (r *LagoonBuildReconciler) updateBuildLogMessage(ctx context.Context, lagoonBuild *lagoonv1beta1.LagoonBuild, - buildMessage lagoonv1beta1.LagoonLog, + buildMessage schema.LagoonLog, ) error { // set the transition time mergePatch, _ := json.Marshal(map[string]interface{}{ diff --git a/controllers/v1beta1/build_helpers.go b/controllers/v1beta1/build_helpers.go index 81831ab1..0ff88268 100644 --- a/controllers/v1beta1/build_helpers.go +++ b/controllers/v1beta1/build_helpers.go @@ -922,7 +922,7 @@ func (r *LagoonBuildReconciler) updateQueuedBuild( } // send any messages to lagoon message queues // update the deployment with the status, lagoon v2.12.0 supports queued status, otherwise use pending - if helpers.CheckLagoonVersion(&lagoonBuild, "2.12.0") { + if lagoonv1beta1.CheckLagoonVersion(&lagoonBuild, "2.12.0") { r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusQueued, fmt.Sprintf("queued %v/%v", queuePosition, queueLength)) r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusQueued, fmt.Sprintf("queued %v/%v", queuePosition, queueLength)) r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, allContainerLogs, lagoonv1beta1.BuildStatusQueued) diff --git a/controllers/v1beta1/build_qoshandler.go b/controllers/v1beta1/build_qoshandler.go index 51467ae2..f28cda4a 100644 --- a/controllers/v1beta1/build_qoshandler.go +++ b/controllers/v1beta1/build_qoshandler.go @@ -128,7 +128,7 @@ func (r *LagoonBuildReconciler) processQueue(ctx context.Context, opLog logr.Log } // if there are no running builds, check if there are any pending builds that can be started if len(runningNSBuilds.Items) == 0 { - if err := helpers.CancelExtraBuilds(ctx, r.Client, opLog, pBuild.ObjectMeta.Namespace, "Running"); err != nil { + if err := lagoonv1beta1.CancelExtraBuilds(ctx, r.Client, opLog, pBuild.ObjectMeta.Namespace, "Running"); err != nil { // only return if there is an error doing this operation // continue on otherwise to allow the queued status updater to run runningProcessQueue = false diff --git a/controllers/v1beta1/build_standardhandler.go b/controllers/v1beta1/build_standardhandler.go index 95bd74bf..6ce5fb41 100644 --- a/controllers/v1beta1/build_standardhandler.go +++ b/controllers/v1beta1/build_standardhandler.go @@ -52,7 +52,7 @@ func (r *LagoonBuildReconciler) standardBuildProcessor(ctx context.Context, // if there are no running builds, check if there are any pending builds that can be started if len(runningBuilds.Items) == 0 { - return ctrl.Result{}, helpers.CancelExtraBuilds(ctx, r.Client, opLog, req.Namespace, "Running") + return ctrl.Result{}, lagoonv1beta1.CancelExtraBuilds(ctx, r.Client, opLog, req.Namespace, "Running") } // The object is not being deleted, so if it does not have our finalizer, // then lets add the finalizer and update the object. This is equivalent diff --git a/controllers/v1beta1/podmonitor_buildhandlers.go b/controllers/v1beta1/podmonitor_buildhandlers.go index 0159cb65..8ddac801 100644 --- a/controllers/v1beta1/podmonitor_buildhandlers.go +++ b/controllers/v1beta1/podmonitor_buildhandlers.go @@ -12,6 +12,7 @@ import ( "github.com/go-logr/logr" "github.com/prometheus/client_golang/prometheus" + "github.com/uselagoon/machinery/api/schema" lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" "github.com/uselagoon/remote-controller/internal/helpers" appsv1 "k8s.io/api/apps/v1" @@ -143,7 +144,7 @@ func (r *LagoonMonitorReconciler) buildLogsToLagoonLogs(ctx context.Context, namespace *corev1.Namespace, condition string, logs []byte, -) (bool, lagoonv1beta1.LagoonLog) { +) (bool, schema.LagoonLog) { if r.EnableMQ { buildStep := "running" if condition == "failed" || condition == "complete" || condition == "cancelled" { @@ -170,11 +171,11 @@ func (r *LagoonMonitorReconciler) buildLogsToLagoonLogs(ctx context.Context, if value, ok := jobPod.Labels["lagoon.sh/buildRemoteID"]; ok { remoteId = value } - msg := lagoonv1beta1.LagoonLog{ + msg := schema.LagoonLog{ Severity: "info", Project: projectName, Event: "build-logs:builddeploy-kubernetes:" + jobPod.ObjectMeta.Name, - Meta: &lagoonv1beta1.LagoonLogMeta{ + Meta: &schema.LagoonLogMeta{ EnvironmentID: envID, ProjectID: projectID, BuildName: jobPod.ObjectMeta.Name, @@ -222,7 +223,7 @@ Logs on pod %s, assigned to cluster %s // if we are able to publish the message, then we need to remove any pending messages from the resource // and make sure we don't try and publish again } - return false, lagoonv1beta1.LagoonLog{} + return false, schema.LagoonLog{} } // updateDeploymentAndEnvironmentTask sends the status of the build and deployment to the controllerhandler message queue in lagoon, @@ -234,7 +235,7 @@ func (r *LagoonMonitorReconciler) updateDeploymentAndEnvironmentTask(ctx context lagoonEnv *corev1.ConfigMap, namespace *corev1.Namespace, condition string, -) (bool, lagoonv1beta1.LagoonMessage) { +) (bool, schema.LagoonMessage) { if r.EnableMQ { buildStep := "running" if condition == "failed" || condition == "complete" || condition == "cancelled" { @@ -270,10 +271,10 @@ func (r *LagoonMonitorReconciler) updateDeploymentAndEnvironmentTask(ctx context if value, ok := jobPod.Labels["lagoon.sh/buildRemoteID"]; ok { remoteId = value } - msg := lagoonv1beta1.LagoonMessage{ + msg := schema.LagoonMessage{ Type: "build", Namespace: namespace.ObjectMeta.Name, - Meta: &lagoonv1beta1.LagoonLogMeta{ + Meta: &schema.LagoonLogMeta{ Environment: envName, EnvironmentID: envID, Project: projectName, @@ -359,7 +360,7 @@ func (r *LagoonMonitorReconciler) updateDeploymentAndEnvironmentTask(ctx context // if we are able to publish the message, then we need to remove any pending messages from the resource // and make sure we don't try and publish again } - return false, lagoonv1beta1.LagoonMessage{} + return false, schema.LagoonMessage{} } // buildStatusLogsToLagoonLogs sends the logs to lagoon-logs message queue, used for general messaging @@ -370,7 +371,7 @@ func (r *LagoonMonitorReconciler) buildStatusLogsToLagoonLogs(ctx context.Contex lagoonEnv *corev1.ConfigMap, namespace *corev1.Namespace, condition string, -) (bool, lagoonv1beta1.LagoonLog) { +) (bool, schema.LagoonLog) { if r.EnableMQ { buildStep := "running" if condition == "failed" || condition == "complete" || condition == "cancelled" { @@ -393,11 +394,11 @@ func (r *LagoonMonitorReconciler) buildStatusLogsToLagoonLogs(ctx context.Contex pID, _ := strconv.Atoi(namespace.ObjectMeta.Labels["lagoon.sh/environment"]) projectID = helpers.UintPtr(uint(pID)) } - msg := lagoonv1beta1.LagoonLog{ + msg := schema.LagoonLog{ Severity: "info", Project: projectName, Event: "task:builddeploy-kubernetes:" + condition, //@TODO: this probably needs to be changed to a new task event for the controller - Meta: &lagoonv1beta1.LagoonLogMeta{ + Meta: &schema.LagoonLogMeta{ EnvironmentID: envID, ProjectID: projectID, ProjectName: projectName, @@ -455,7 +456,7 @@ func (r *LagoonMonitorReconciler) buildStatusLogsToLagoonLogs(ctx context.Contex // if we are able to publish the message, then we need to remove any pending messages from the resource // and make sure we don't try and publish again } - return false, lagoonv1beta1.LagoonLog{} + return false, schema.LagoonLog{} } // updateDeploymentWithLogs collects logs from the build containers and ships or stores them @@ -468,13 +469,13 @@ func (r *LagoonMonitorReconciler) updateDeploymentWithLogs( cancel bool, ) error { opLog := r.Log.WithValues("lagoonmonitor", req.NamespacedName) - buildCondition := helpers.GetBuildConditionFromPod(jobPod.Status.Phase) + buildCondition := lagoonv1beta1.GetBuildConditionFromPod(jobPod.Status.Phase) collectLogs := true if cancel { // only set the status to cancelled if the pod is running/pending/queued // otherwise send the existing status of complete/failed/cancelled if helpers.ContainsString( - helpers.BuildRunningPendingStatus, + lagoonv1beta1.BuildRunningPendingStatus, lagoonBuild.Labels["lagoon.sh/buildStatus"], ) { buildCondition = lagoonv1beta1.BuildStatusCancelled @@ -497,7 +498,7 @@ func (r *LagoonMonitorReconciler) updateDeploymentWithLogs( // if the buildstatus is pending or running, or the cancel flag is provided // send the update status to lagoon if helpers.ContainsString( - helpers.BuildRunningPendingStatus, + lagoonv1beta1.BuildRunningPendingStatus, lagoonBuild.Labels["lagoon.sh/buildStatus"], ) || cancel { opLog.Info( @@ -550,7 +551,7 @@ Build %s Status: corev1.ConditionTrue, LastTransitionTime: time.Now().UTC().Format(time.RFC3339), } - if !helpers.BuildContainsStatus(lagoonBuild.Status.Conditions, condition) { + if !lagoonv1beta1.BuildContainsStatus(lagoonBuild.Status.Conditions, condition) { lagoonBuild.Status.Conditions = append(lagoonBuild.Status.Conditions, condition) mergeMap["status"] = map[string]interface{}{ "conditions": lagoonBuild.Status.Conditions, @@ -577,7 +578,7 @@ Build %s pendingStatus, pendingStatusMessage := r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &jobPod, &lagoonEnv, namespace, buildCondition.ToLower()) pendingEnvironment, pendingEnvironmentMessage := r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &jobPod, &lagoonEnv, namespace, buildCondition.ToLower()) var pendingBuildLog bool - var pendingBuildLogMessage lagoonv1beta1.LagoonLog + var pendingBuildLogMessage schema.LagoonLog // if the container logs can't be retrieved, we don't want to send any build logs back, as this will nuke // any previously received logs if !strings.Contains(string(allContainerLogs), "unable to retrieve container logs for containerd") { diff --git a/controllers/v1beta1/podmonitor_controller.go b/controllers/v1beta1/podmonitor_controller.go index 185be749..a1ea2406 100644 --- a/controllers/v1beta1/podmonitor_controller.go +++ b/controllers/v1beta1/podmonitor_controller.go @@ -125,7 +125,7 @@ func (r *LagoonMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Reques } } else { if helpers.ContainsString( - helpers.BuildRunningPendingStatus, + lagoonv1beta1.BuildRunningPendingStatus, lagoonBuild.Labels["lagoon.sh/buildStatus"], ) { opLog.Info(fmt.Sprintf("Attempting to update the LagoonBuild with cancellation if required.")) @@ -162,7 +162,7 @@ func (r *LagoonMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Reques } // if we have no running builds, then check for any pending builds if len(runningBuilds.Items) == 0 { - return ctrl.Result{}, helpers.CancelExtraBuilds(ctx, r.Client, opLog, req.Namespace, "Running") + return ctrl.Result{}, lagoonv1beta1.CancelExtraBuilds(ctx, r.Client, opLog, req.Namespace, "Running") } } else { // since qos handles pending build checks as part of its own operations, we can skip the running pod check step with no-op diff --git a/controllers/v1beta1/podmonitor_taskhandlers.go b/controllers/v1beta1/podmonitor_taskhandlers.go index 5ffe9305..c5e34c1f 100644 --- a/controllers/v1beta1/podmonitor_taskhandlers.go +++ b/controllers/v1beta1/podmonitor_taskhandlers.go @@ -12,6 +12,7 @@ import ( "github.com/go-logr/logr" "github.com/prometheus/client_golang/prometheus" + "github.com/uselagoon/machinery/api/schema" lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" "github.com/uselagoon/remote-controller/internal/helpers" corev1 "k8s.io/api/core/v1" @@ -112,13 +113,13 @@ func (r *LagoonMonitorReconciler) taskLogsToLagoonLogs(opLog logr.Logger, jobPod *corev1.Pod, condition string, logs []byte, -) (bool, lagoonv1beta1.LagoonLog) { +) (bool, schema.LagoonLog) { if r.EnableMQ && lagoonTask != nil { - msg := lagoonv1beta1.LagoonLog{ + msg := schema.LagoonLog{ Severity: "info", Project: lagoonTask.Spec.Project.Name, Event: "task-logs:job-kubernetes:" + lagoonTask.ObjectMeta.Name, - Meta: &lagoonv1beta1.LagoonLogMeta{ + Meta: &schema.LagoonLogMeta{ Task: &lagoonTask.Spec.Task, Environment: lagoonTask.Spec.Environment.Name, JobName: lagoonTask.ObjectMeta.Name, @@ -152,7 +153,7 @@ Logs on pod %s, assigned to cluster %s // if we are able to publish the message, then we need to remove any pending messages from the resource // and make sure we don't try and publish again } - return false, lagoonv1beta1.LagoonLog{} + return false, schema.LagoonLog{} } // updateLagoonTask sends the status of the task and deployment to the controllerhandler message queue in lagoon, @@ -161,7 +162,7 @@ func (r *LagoonMonitorReconciler) updateLagoonTask(ctx context.Context, opLog lo lagoonTask *lagoonv1beta1.LagoonTask, jobPod *corev1.Pod, condition string, -) (bool, lagoonv1beta1.LagoonMessage) { +) (bool, schema.LagoonMessage) { if r.EnableMQ && lagoonTask != nil { if condition == "failed" || condition == "complete" || condition == "cancelled" { time.AfterFunc(31*time.Second, func() { @@ -172,10 +173,10 @@ func (r *LagoonMonitorReconciler) updateLagoonTask(ctx context.Context, opLog lo }) time.Sleep(2 * time.Second) // smol sleep to reduce race of final messages with previous messages } - msg := lagoonv1beta1.LagoonMessage{ + msg := schema.LagoonMessage{ Type: "task", Namespace: lagoonTask.ObjectMeta.Namespace, - Meta: &lagoonv1beta1.LagoonLogMeta{ + Meta: &schema.LagoonLogMeta{ Task: &lagoonTask.Spec.Task, Environment: lagoonTask.Spec.Environment.Name, Project: lagoonTask.Spec.Project.Name, @@ -217,7 +218,7 @@ func (r *LagoonMonitorReconciler) updateLagoonTask(ctx context.Context, opLog lo // if we are able to publish the message, then we need to remove any pending messages from the resource // and make sure we don't try and publish again } - return false, lagoonv1beta1.LagoonMessage{} + return false, schema.LagoonMessage{} } // taskStatusLogsToLagoonLogs sends the logs to lagoon-logs message queue, used for general messaging @@ -225,13 +226,13 @@ func (r *LagoonMonitorReconciler) taskStatusLogsToLagoonLogs(opLog logr.Logger, lagoonTask *lagoonv1beta1.LagoonTask, jobPod *corev1.Pod, condition string, -) (bool, lagoonv1beta1.LagoonLog) { +) (bool, schema.LagoonLog) { if r.EnableMQ && lagoonTask != nil { - msg := lagoonv1beta1.LagoonLog{ + msg := schema.LagoonLog{ Severity: "info", Project: lagoonTask.Spec.Project.Name, Event: "task:job-kubernetes:" + condition, //@TODO: this probably needs to be changed to a new task event for the controller - Meta: &lagoonv1beta1.LagoonLogMeta{ + Meta: &schema.LagoonLogMeta{ Task: &lagoonTask.Spec.Task, ProjectName: lagoonTask.Spec.Project.Name, Environment: lagoonTask.Spec.Environment.Name, @@ -263,7 +264,7 @@ func (r *LagoonMonitorReconciler) taskStatusLogsToLagoonLogs(opLog logr.Logger, // if we are able to publish the message, then we need to remove any pending messages from the resource // and make sure we don't try and publish again } - return false, lagoonv1beta1.LagoonLog{} + return false, schema.LagoonLog{} } // updateTaskWithLogs collects logs from the task containers and ships or stores them @@ -276,7 +277,7 @@ func (r *LagoonMonitorReconciler) updateTaskWithLogs( cancel bool, ) error { opLog := r.Log.WithValues("lagoonmonitor", req.NamespacedName) - taskCondition := helpers.GetTaskConditionFromPod(jobPod.Status.Phase) + taskCondition := lagoonv1beta1.GetTaskConditionFromPod(jobPod.Status.Phase) collectLogs := true if cancel { taskCondition = lagoonv1beta1.TaskStatusCancelled @@ -286,7 +287,7 @@ func (r *LagoonMonitorReconciler) updateTaskWithLogs( // then update the task to reflect the current pod status // we do this so we don't update the status of the task again if helpers.ContainsString( - helpers.TaskRunningPendingStatus, + lagoonv1beta1.TaskRunningPendingStatus, lagoonTask.Labels["lagoon.sh/taskStatus"], ) || cancel { opLog.Info( @@ -340,7 +341,7 @@ Task %s Status: corev1.ConditionTrue, LastTransitionTime: time.Now().UTC().Format(time.RFC3339), } - if !helpers.TaskContainsStatus(lagoonTask.Status.Conditions, condition) { + if !lagoonv1beta1.TaskContainsStatus(lagoonTask.Status.Conditions, condition) { lagoonTask.Status.Conditions = append(lagoonTask.Status.Conditions, condition) mergeMap["status"] = map[string]interface{}{ "conditions": lagoonTask.Status.Conditions, @@ -353,7 +354,7 @@ Task %s pendingStatus, pendingStatusMessage := r.taskStatusLogsToLagoonLogs(opLog, &lagoonTask, &jobPod, taskCondition.ToLower()) pendingEnvironment, pendingEnvironmentMessage := r.updateLagoonTask(ctx, opLog, &lagoonTask, &jobPod, taskCondition.ToLower()) var pendingTaskLog bool - var pendingTaskLogMessage lagoonv1beta1.LagoonLog + var pendingTaskLogMessage schema.LagoonLog // if the container logs can't be retrieved, we don't want to send any task logs back, as this will nuke // any previously received logs if !strings.Contains(string(allContainerLogs), "unable to retrieve container logs for containerd") { diff --git a/controllers/v1beta1/predicates.go b/controllers/v1beta1/predicates.go index 77aa324a..afb99a03 100644 --- a/controllers/v1beta1/predicates.go +++ b/controllers/v1beta1/predicates.go @@ -225,56 +225,3 @@ func (t TaskPredicates) Generic(e event.GenericEvent) bool { } return false } - -// SecretPredicates defines the funcs for predicates -type SecretPredicates struct { - predicate.Funcs - ControllerNamespace string -} - -// Create is used when a creation event is received by the controller. -func (n SecretPredicates) Create(e event.CreateEvent) bool { - if controller, ok := e.Object.GetLabels()["lagoon.sh/controller"]; ok { - if controller == n.ControllerNamespace { - if val, ok := e.Object.GetLabels()["lagoon.sh/harbor-credential"]; ok { - if val == "true" { - return true - } - } - } - } - return false -} - -// Delete is used when a deletion event is received by the controller. -func (n SecretPredicates) Delete(e event.DeleteEvent) bool { - return false -} - -// Update is used when an update event is received by the controller. -func (n SecretPredicates) Update(e event.UpdateEvent) bool { - if controller, ok := e.ObjectOld.GetLabels()["lagoon.sh/controller"]; ok { - if controller == n.ControllerNamespace { - if val, ok := e.ObjectOld.GetLabels()["lagoon.sh/harbor-credential"]; ok { - if val == "true" { - return true - } - } - } - } - return false -} - -// Generic is used when any other event is received by the controller. -func (n SecretPredicates) Generic(e event.GenericEvent) bool { - if controller, ok := e.Object.GetLabels()["lagoon.sh/controller"]; ok { - if controller == n.ControllerNamespace { - if val, ok := e.Object.GetLabels()["lagoon.sh/harbor-credential"]; ok { - if val == "true" { - return true - } - } - } - } - return false -} diff --git a/go.mod b/go.mod index 88770eda..c38f7648 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.27.7 github.com/prometheus/client_golang v1.15.1 + github.com/uselagoon/machinery v0.0.17-0.20240109062854-3b567fb41003 github.com/vshn/k8up v1.99.99 github.com/xhit/go-str2duration/v2 v2.0.0 gopkg.in/matryer/try.v1 v1.0.0-20150601225556-312d2599e12e @@ -55,6 +56,7 @@ require ( github.com/google/gnostic v0.5.7-v3refs // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/uuid v1.3.0 // indirect + github.com/guregu/null v4.0.0+incompatible // indirect github.com/imdario/mergo v0.3.13 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -83,8 +85,8 @@ require ( go.uber.org/zap v1.24.0 // indirect golang.org/x/net v0.10.0 // indirect golang.org/x/oauth2 v0.5.0 // indirect - golang.org/x/sys v0.8.0 // indirect - golang.org/x/term v0.8.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/term v0.15.0 // indirect golang.org/x/text v0.9.0 // indirect golang.org/x/time v0.3.0 // indirect gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect diff --git a/go.sum b/go.sum index 53473ad0..d7d22f0b 100644 --- a/go.sum +++ b/go.sum @@ -747,6 +747,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.3.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpg github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/guregu/null v4.0.0+incompatible h1:4zw0ckM7ECd6FNNddc3Fu4aty9nTlpkkzH7dPn4/4Gw= +github.com/guregu/null v4.0.0+incompatible/go.mod h1:ePGpQaN9cw0tj45IR5E5ehMvsFlLlQZAkkOXZurJ3NM= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -1240,6 +1242,10 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +github.com/uselagoon/machinery v0.0.17-0.20240109062106-ccc88289f3ca h1:Opwha2XIEUFIUbAmYy6xdxtbnk0DOjjYY4MkUSArzGI= +github.com/uselagoon/machinery v0.0.17-0.20240109062106-ccc88289f3ca/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU= +github.com/uselagoon/machinery v0.0.17-0.20240109062854-3b567fb41003 h1:fLhncKjshd8f0DBFB45u7OWsF1H6OgNmpkswJYqwCA8= +github.com/uselagoon/machinery v0.0.17-0.20240109062854-3b567fb41003/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU= github.com/uudashr/gocognit v0.0.0-20190926065955-1655d0de0517/go.mod h1:j44Ayx2KW4+oB6SWMv8KsmHzZrOInQav7D3cQMJ5JUM= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s= @@ -1654,16 +1660,16 @@ golang.org/x/sys v0.0.0-20220405210540-1e041c57c461/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/internal/harbor/harbor_credentialrotation.go b/internal/harbor/harbor_credentialrotation.go index 5256e316..7a3bc4e7 100644 --- a/internal/harbor/harbor_credentialrotation.go +++ b/internal/harbor/harbor_credentialrotation.go @@ -2,7 +2,6 @@ package harbor import ( "fmt" - "sort" "context" "time" @@ -45,32 +44,8 @@ func (h *Harbor) RotateRobotCredentials(ctx context.Context, cl client.Client) { } opLog.Info(fmt.Sprintf("Checking if %s needs robot credentials rotated", ns.ObjectMeta.Name)) // check for running builds! - lagoonBuilds := &lagoonv1beta1.LagoonBuildList{} - listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ - client.InNamespace(ns.ObjectMeta.Name), - client.MatchingLabels(map[string]string{ - "lagoon.sh/controller": h.ControllerNamespace, // created by this controller - }), - }) - if err := cl.List(context.Background(), lagoonBuilds, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list Lagoon build pods, there may be none or something went wrong")) - continue - } - runningBuilds := false - sort.Slice(lagoonBuilds.Items, func(i, j int) bool { - return lagoonBuilds.Items[i].ObjectMeta.CreationTimestamp.After(lagoonBuilds.Items[j].ObjectMeta.CreationTimestamp.Time) - }) - // if there are any builds pending or running, don't try and refresh the credentials as this - // could break the build - if len(lagoonBuilds.Items) > 0 { - if helpers.ContainsString( - helpers.BuildRunningPendingStatus, - lagoonBuilds.Items[0].Labels["lagoon.sh/buildStatus"], - ) { - runningBuilds = true - } - } - if !runningBuilds { + runningBuildsv1beta1 := lagoonv1beta1.CheckRunningBuilds(ctx, h.ControllerNamespace, opLog, cl, ns) + if !runningBuildsv1beta1 { rotated, err := h.RotateRobotCredential(ctx, cl, ns, false) if err != nil { opLog.Error(err, "error") diff --git a/internal/helpers/helpers.go b/internal/helpers/helpers.go index 809eef78..680141d4 100644 --- a/internal/helpers/helpers.go +++ b/internal/helpers/helpers.go @@ -1,53 +1,18 @@ package helpers import ( - "context" "crypto/sha1" "crypto/sha256" "encoding/base32" - "encoding/json" "fmt" "math/rand" "os" "regexp" - "sort" "strconv" "strings" "time" - "github.com/go-logr/logr" - "github.com/hashicorp/go-version" - lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" - corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -var ( - // BuildRunningPendingStatus . - BuildRunningPendingStatus = []string{ - lagoonv1beta1.BuildStatusPending.String(), - lagoonv1beta1.BuildStatusQueued.String(), - lagoonv1beta1.BuildStatusRunning.String(), - } - // BuildCompletedCancelledFailedStatus . - BuildCompletedCancelledFailedStatus = []string{ - lagoonv1beta1.BuildStatusFailed.String(), - lagoonv1beta1.BuildStatusComplete.String(), - lagoonv1beta1.BuildStatusCancelled.String(), - } - // TaskRunningPendingStatus . - TaskRunningPendingStatus = []string{ - lagoonv1beta1.TaskStatusPending.String(), - lagoonv1beta1.TaskStatusQueued.String(), - lagoonv1beta1.TaskStatusRunning.String(), - } - // TaskCompletedCancelledFailedStatus . - TaskCompletedCancelledFailedStatus = []string{ - lagoonv1beta1.TaskStatusFailed.String(), - lagoonv1beta1.TaskStatusComplete.String(), - lagoonv1beta1.TaskStatusCancelled.String(), - } ) const ( @@ -84,26 +49,6 @@ func RemoveString(slice []string, s string) (result []string) { return } -// BuildContainsStatus . -func BuildContainsStatus(slice []lagoonv1beta1.LagoonBuildConditions, s lagoonv1beta1.LagoonBuildConditions) bool { - for _, item := range slice { - if item == s { - return true - } - } - return false -} - -// TaskContainsStatus . -func TaskContainsStatus(slice []lagoonv1beta1.LagoonTaskConditions, s lagoonv1beta1.LagoonTaskConditions) bool { - for _, item := range slice { - if item == s { - return true - } - } - return false -} - // IntPtr . func IntPtr(i int) *int { var iPtr *int @@ -166,18 +111,6 @@ func HashString(s string) string { return fmt.Sprintf("%x", bs) } -// RemoveBuild remove a LagoonBuild from a slice of LagoonBuilds -func RemoveBuild(slice []lagoonv1beta1.LagoonBuild, s lagoonv1beta1.LagoonBuild) []lagoonv1beta1.LagoonBuild { - result := []lagoonv1beta1.LagoonBuild{} - for _, item := range slice { - if item.ObjectMeta.Name == s.ObjectMeta.Name { - continue - } - result = append(result, item) - } - return result -} - var lowerAlNum = regexp.MustCompile("[^a-z0-9]+") // ShortName returns a deterministic random short name of 8 lowercase @@ -254,90 +187,6 @@ func containsStr(s []string, str string) bool { return false } -// Check if the version of lagoon provided in the internal_system scope variable is greater than or equal to the checked version -func CheckLagoonVersion(build *lagoonv1beta1.LagoonBuild, checkVersion string) bool { - lagoonProjectVariables := &[]LagoonEnvironmentVariable{} - json.Unmarshal(build.Spec.Project.Variables.Project, lagoonProjectVariables) - lagoonVersion, err := GetLagoonVariable("LAGOON_SYSTEM_CORE_VERSION", []string{"internal_system"}, *lagoonProjectVariables) - if err != nil { - return false - } - aVer, err := version.NewSemver(lagoonVersion.Value) - if err != nil { - return false - } - bVer, err := version.NewSemver(checkVersion) - if err != nil { - return false - } - return aVer.GreaterThanOrEqual(bVer) -} - -// CancelExtraBuilds cancels extra builds. -func CancelExtraBuilds(ctx context.Context, r client.Client, opLog logr.Logger, ns string, status string) error { - pendingBuilds := &lagoonv1beta1.LagoonBuildList{} - listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ - client.InNamespace(ns), - client.MatchingLabels(map[string]string{"lagoon.sh/buildStatus": lagoonv1beta1.BuildStatusPending.String()}), - }) - if err := r.List(ctx, pendingBuilds, listOption); err != nil { - return fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err) - } - if len(pendingBuilds.Items) > 0 { - // opLog.Info(fmt.Sprintf("There are %v pending builds", len(pendingBuilds.Items))) - // if we have any pending builds, then grab the latest one and make it running - // if there are any other pending builds, cancel them so only the latest one runs - sort.Slice(pendingBuilds.Items, func(i, j int) bool { - return pendingBuilds.Items[i].ObjectMeta.CreationTimestamp.After(pendingBuilds.Items[j].ObjectMeta.CreationTimestamp.Time) - }) - for idx, pBuild := range pendingBuilds.Items { - pendingBuild := pBuild.DeepCopy() - if idx == 0 { - pendingBuild.Labels["lagoon.sh/buildStatus"] = status - } else { - // cancel any other pending builds - opLog.Info(fmt.Sprintf("Setting build %s as cancelled", pendingBuild.ObjectMeta.Name)) - pendingBuild.Labels["lagoon.sh/buildStatus"] = lagoonv1beta1.BuildStatusCancelled.String() - pendingBuild.Labels["lagoon.sh/cancelledByNewBuild"] = "true" - } - if err := r.Update(ctx, pendingBuild); err != nil { - return err - } - } - } - return nil -} - -func GetBuildConditionFromPod(phase corev1.PodPhase) lagoonv1beta1.BuildStatusType { - var buildCondition lagoonv1beta1.BuildStatusType - switch phase { - case corev1.PodFailed: - buildCondition = lagoonv1beta1.BuildStatusFailed - case corev1.PodSucceeded: - buildCondition = lagoonv1beta1.BuildStatusComplete - case corev1.PodPending: - buildCondition = lagoonv1beta1.BuildStatusPending - case corev1.PodRunning: - buildCondition = lagoonv1beta1.BuildStatusRunning - } - return buildCondition -} - -func GetTaskConditionFromPod(phase corev1.PodPhase) lagoonv1beta1.TaskStatusType { - var taskCondition lagoonv1beta1.TaskStatusType - switch phase { - case corev1.PodFailed: - taskCondition = lagoonv1beta1.TaskStatusFailed - case corev1.PodSucceeded: - taskCondition = lagoonv1beta1.TaskStatusComplete - case corev1.PodPending: - taskCondition = lagoonv1beta1.TaskStatusPending - case corev1.PodRunning: - taskCondition = lagoonv1beta1.TaskStatusRunning - } - return taskCondition -} - // GenerateNamespaceName handles the generation of the namespace name from environment and project name with prefixes and patterns func GenerateNamespaceName(pattern, environmentName, projectname, prefix, controllerNamespace string, randomPrefix bool) string { nsPattern := pattern diff --git a/internal/helpers/helpers_test.go b/internal/helpers/helpers_test.go index b0004c9b..e1181b7d 100644 --- a/internal/helpers/helpers_test.go +++ b/internal/helpers/helpers_test.go @@ -4,8 +4,6 @@ import ( "os" "reflect" "testing" - - lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" ) func TestShortName(t *testing.T) { @@ -258,103 +256,3 @@ func TestGetLagoonFeatureFlags(t *testing.T) { }) } } - -func TestCheckLagoonVersion(t *testing.T) { - type args struct { - build *lagoonv1beta1.LagoonBuild - checkVersion string - } - tests := []struct { - name string - args args - want bool - }{ - { - name: "test1", - args: args{ - build: &lagoonv1beta1.LagoonBuild{ - Spec: lagoonv1beta1.LagoonBuildSpec{ - Project: lagoonv1beta1.Project{ - Variables: lagoonv1beta1.LagoonVariables{ - Project: []byte(`[{"name":"LAGOON_SYSTEM_CORE_VERSION","value":"v2.12.0","scope":"internal_system"}]`), - }, - }, - }, - }, - checkVersion: "2.12.0", - }, - want: true, - }, - { - name: "test2", - args: args{ - build: &lagoonv1beta1.LagoonBuild{ - Spec: lagoonv1beta1.LagoonBuildSpec{ - Project: lagoonv1beta1.Project{ - Variables: lagoonv1beta1.LagoonVariables{ - Project: []byte(`[{"name":"LAGOON_SYSTEM_CORE_VERSION","value":"v2.11.0","scope":"internal_system"}]`), - }, - }, - }, - }, - checkVersion: "2.12.0", - }, - want: false, - }, - { - name: "test3", - args: args{ - build: &lagoonv1beta1.LagoonBuild{ - Spec: lagoonv1beta1.LagoonBuildSpec{ - Project: lagoonv1beta1.Project{ - Variables: lagoonv1beta1.LagoonVariables{ - Project: []byte(`[]`), - }, - }, - }, - }, - checkVersion: "2.12.0", - }, - want: false, - }, - { - name: "test4", - args: args{ - build: &lagoonv1beta1.LagoonBuild{ - Spec: lagoonv1beta1.LagoonBuildSpec{ - Project: lagoonv1beta1.Project{ - Variables: lagoonv1beta1.LagoonVariables{ - Project: []byte(`[{"name":"LAGOON_SYSTEM_CORE_VERSION","value":"v2.12.0","scope":"internal_system"}]`), - }, - }, - }, - }, - checkVersion: "v2.12.0", - }, - want: true, - }, - { - name: "test5", - args: args{ - build: &lagoonv1beta1.LagoonBuild{ - Spec: lagoonv1beta1.LagoonBuildSpec{ - Project: lagoonv1beta1.Project{ - Variables: lagoonv1beta1.LagoonVariables{ - Project: []byte(`[{"name":"LAGOON_SYSTEM_CORE_VERSION","value":"v2.11.0","scope":"internal_system"}]`), - }, - }, - }, - }, - checkVersion: "v2.12.0", - }, - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := CheckLagoonVersion(tt.args.build, tt.args.checkVersion); got != tt.want { - t.Errorf("CheckLagoonVersion() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/internal/messenger/consumer.go b/internal/messenger/consumer.go index a0b90853..51173b37 100644 --- a/internal/messenger/consumer.go +++ b/internal/messenger/consumer.go @@ -9,6 +9,7 @@ import ( "time" "github.com/cheshir/go-mq/v2" + "github.com/uselagoon/machinery/api/schema" lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" "github.com/uselagoon/remote-controller/internal/helpers" "gopkg.in/matryer/try.v1" @@ -150,10 +151,10 @@ func (m *Messenger) Consumer(targetName string) { //error { branch, ), ) - msg := lagoonv1beta1.LagoonMessage{ + msg := schema.LagoonMessage{ Type: "remove", Namespace: ns, - Meta: &lagoonv1beta1.LagoonLogMeta{ + Meta: &schema.LagoonLogMeta{ Project: project, Environment: branch, }, @@ -183,10 +184,10 @@ func (m *Messenger) Consumer(targetName string) { //error { go func() { err := m.DeletionHandler.ProcessDeletion(ctx, opLog, namespace) if err == nil { - msg := lagoonv1beta1.LagoonMessage{ + msg := schema.LagoonMessage{ Type: "remove", Namespace: namespace.ObjectMeta.Name, - Meta: &lagoonv1beta1.LagoonLogMeta{ + Meta: &schema.LagoonLogMeta{ Project: project, Environment: branch, }, @@ -313,12 +314,20 @@ func (m *Messenger) Consumer(targetName string) { //error { ), ) m.Cache.Add(jobSpec.Misc.Name, jobSpec.Project.Name) - err := m.CancelBuild(namespace, jobSpec) + _, v1beta1Bytes, err := lagoonv1beta1.CancelBuild(ctx, m.Client, namespace, message.Body()) if err != nil { - //@TODO: send msg back to lagoon and update task to failed? + //@TODO: send msg back to lagoon and update build to failed? message.Ack(false) // ack to remove from queue return } + if v1beta1Bytes != nil { + // if v1beta1 has a build, send its response + if err := m.Publish("lagoon-tasks:controller", v1beta1Bytes); err != nil { + opLog.Error(err, "Unable to publish message.") + message.Ack(false) // ack to remove from queue + return + } + } case "deploytarget:task:cancel", "kubernetes:task:cancel": opLog.Info( fmt.Sprintf( @@ -329,12 +338,19 @@ func (m *Messenger) Consumer(targetName string) { //error { ), ) m.Cache.Add(jobSpec.Task.TaskName, jobSpec.Project.Name) - err := m.CancelTask(namespace, jobSpec) + _, v1beta1Bytes, err := lagoonv1beta1.CancelTask(ctx, m.Client, namespace, message.Body()) if err != nil { //@TODO: send msg back to lagoon and update task to failed? message.Ack(false) // ack to remove from queue return } + if v1beta1Bytes != nil { + if err := m.Publish("lagoon-tasks:controller", v1beta1Bytes); err != nil { + opLog.Error(err, "Unable to publish message.") + message.Ack(false) // ack to remove from queue + return + } + } case "deploytarget:restic:backup:restore", "kubernetes:restic:backup:restore": opLog.Info( fmt.Sprintf( diff --git a/internal/messenger/pending_messages.go b/internal/messenger/pending_messages.go deleted file mode 100644 index 26658df6..00000000 --- a/internal/messenger/pending_messages.go +++ /dev/null @@ -1,151 +0,0 @@ -package messenger - -import ( - "context" - "encoding/json" - "fmt" - - "github.com/go-logr/logr" - lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" - "k8s.io/apimachinery/pkg/types" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -// GetPendingMessages will get any pending messages from the queue and attempt to publish them if possible -func (m *Messenger) GetPendingMessages() { - opLog := ctrl.Log.WithName("handlers").WithName("PendingMessages") - ctx := context.Background() - opLog.Info(fmt.Sprintf("Checking pending build messages across all namespaces")) - m.pendingBuildLogMessages(ctx, opLog) - opLog.Info(fmt.Sprintf("Checking pending task messages across all namespaces")) - m.pendingTaskLogMessages(ctx, opLog) -} - -func (m *Messenger) pendingBuildLogMessages(ctx context.Context, opLog logr.Logger) { - pendingMsgs := &lagoonv1beta1.LagoonBuildList{} - listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ - client.MatchingLabels(map[string]string{ - "lagoon.sh/pendingMessages": "true", - "lagoon.sh/controller": m.ControllerNamespace, - }), - }) - if err := m.Client.List(ctx, pendingMsgs, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list LagoonBuilds, there may be none or something went wrong")) - return - } - for _, build := range pendingMsgs.Items { - // get the latest resource in case it has been updated since the loop started - if err := m.Client.Get(ctx, types.NamespacedName{ - Name: build.ObjectMeta.Name, - Namespace: build.ObjectMeta.Namespace, - }, &build); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to get LagoonBuild, something went wrong")) - break - } - opLog.Info(fmt.Sprintf("LagoonBuild %s has pending messages, attempting to re-send", build.ObjectMeta.Name)) - - // try to re-publish message or break and try the next build with pending message - if build.StatusMessages.StatusMessage != nil { - statusBytes, _ := json.Marshal(build.StatusMessages.StatusMessage) - if err := m.Publish("lagoon-logs", statusBytes); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to publush message")) - break - } - } - if build.StatusMessages.BuildLogMessage != nil { - logBytes, _ := json.Marshal(build.StatusMessages.BuildLogMessage) - if err := m.Publish("lagoon-logs", logBytes); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to publush message")) - break - } - } - if build.StatusMessages.EnvironmentMessage != nil { - envBytes, _ := json.Marshal(build.StatusMessages.EnvironmentMessage) - if err := m.Publish("lagoon-tasks:controller", envBytes); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to publush message")) - break - } - } - // if we managed to send all the pending messages, then update the resource to remove the pending state - // so we don't send the same message multiple times - opLog.Info(fmt.Sprintf("Sent pending messages for LagoonBuild %s", build.ObjectMeta.Name)) - mergePatch, _ := json.Marshal(map[string]interface{}{ - "metadata": map[string]interface{}{ - "labels": map[string]interface{}{ - "lagoon.sh/pendingMessages": "false", - }, - }, - "statusMessages": nil, - }) - if err := m.Client.Patch(ctx, &build, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to update status condition")) - break - } - } - return -} - -func (m *Messenger) pendingTaskLogMessages(ctx context.Context, opLog logr.Logger) { - pendingMsgs := &lagoonv1beta1.LagoonTaskList{} - listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ - client.MatchingLabels(map[string]string{ - "lagoon.sh/pendingMessages": "true", - "lagoon.sh/controller": m.ControllerNamespace, - }), - }) - if err := m.Client.List(ctx, pendingMsgs, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list LagoonBuilds, there may be none or something went wrong")) - return - } - for _, task := range pendingMsgs.Items { - // get the latest resource in case it has been updated since the loop started - if err := m.Client.Get(ctx, types.NamespacedName{ - Name: task.ObjectMeta.Name, - Namespace: task.ObjectMeta.Namespace, - }, &task); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to get LagoonBuild, something went wrong")) - break - } - opLog.Info(fmt.Sprintf("LagoonTasl %s has pending messages, attempting to re-send", task.ObjectMeta.Name)) - - // try to re-publish message or break and try the next build with pending message - if task.StatusMessages.StatusMessage != nil { - statusBytes, _ := json.Marshal(task.StatusMessages.StatusMessage) - if err := m.Publish("lagoon-logs", statusBytes); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to publush message")) - break - } - } - if task.StatusMessages.TaskLogMessage != nil { - taskLogBytes, _ := json.Marshal(task.StatusMessages.TaskLogMessage) - if err := m.Publish("lagoon-logs", taskLogBytes); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to publush message")) - break - } - } - if task.StatusMessages.EnvironmentMessage != nil { - envBytes, _ := json.Marshal(task.StatusMessages.EnvironmentMessage) - if err := m.Publish("lagoon-tasks:controller", envBytes); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to publush message")) - break - } - } - // if we managed to send all the pending messages, then update the resource to remove the pending state - // so we don't send the same message multiple times - opLog.Info(fmt.Sprintf("Sent pending messages for LagoonTask %s", task.ObjectMeta.Name)) - mergePatch, _ := json.Marshal(map[string]interface{}{ - "metadata": map[string]interface{}{ - "labels": map[string]interface{}{ - "lagoon.sh/pendingMessages": "false", - }, - }, - "statusMessages": nil, - }) - if err := m.Client.Patch(ctx, &task, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to update status condition")) - break - } - } - return -} diff --git a/internal/messenger/tasks_handler.go b/internal/messenger/tasks_handler.go index a5ff074a..2b8b68a0 100644 --- a/internal/messenger/tasks_handler.go +++ b/internal/messenger/tasks_handler.go @@ -5,16 +5,10 @@ import ( "encoding/base64" "encoding/json" "fmt" - "sort" - "strings" - "time" - "github.com/go-logr/logr" lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" "github.com/uselagoon/remote-controller/internal/helpers" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" ) @@ -23,221 +17,6 @@ type ActiveStandbyPayload struct { DestinationNamespace string `json:"destinationNamespace"` } -// CancelBuild handles cancelling builds or handling if a build no longer exists. -func (m *Messenger) CancelBuild(namespace string, jobSpec *lagoonv1beta1.LagoonTaskSpec) error { - opLog := ctrl.Log.WithName("handlers").WithName("LagoonTasks") - var jobPod corev1.Pod - if err := m.Client.Get(context.Background(), types.NamespacedName{ - Name: jobSpec.Misc.Name, - Namespace: namespace, - }, &jobPod); err != nil { - opLog.Info(fmt.Sprintf( - "Unable to find build pod %s to cancel it. Checking to see if LagoonBuild exists.", - jobSpec.Misc.Name, - )) - // since there was no build pod, check for the lagoon build resource - var lagoonBuild lagoonv1beta1.LagoonBuild - if err := m.Client.Get(context.Background(), types.NamespacedName{ - Name: jobSpec.Misc.Name, - Namespace: namespace, - }, &lagoonBuild); err != nil { - opLog.Info(fmt.Sprintf( - "Unable to find build %s to cancel it. Sending response to Lagoon to update the build to cancelled.", - jobSpec.Misc.Name, - )) - // if there is no pod or build, update the build in Lagoon to cancelled, assume completely cancelled with no other information - m.updateLagoonBuild(opLog, namespace, *jobSpec, nil) - return nil - } - // as there is no build pod, but there is a lagoon build resource - // update it to cancelled so that the controller doesn't try to run it - // check if the build has existing status or not though to consume it - if helpers.ContainsString( - helpers.BuildRunningPendingStatus, - lagoonBuild.ObjectMeta.Labels["lagoon.sh/buildStatus"], - ) { - lagoonBuild.ObjectMeta.Labels["lagoon.sh/buildStatus"] = lagoonv1beta1.BuildStatusCancelled.String() - } - lagoonBuild.ObjectMeta.Labels["lagoon.sh/cancelBuildNoPod"] = "true" - if err := m.Client.Update(context.Background(), &lagoonBuild); err != nil { - opLog.Error(err, - fmt.Sprintf( - "Unable to update build %s to cancel it.", - jobSpec.Misc.Name, - ), - ) - return err - } - // and then send the response back to lagoon to say it was cancelled. - m.updateLagoonBuild(opLog, namespace, *jobSpec, &lagoonBuild) - return nil - } - jobPod.ObjectMeta.Labels["lagoon.sh/cancelBuild"] = "true" - if err := m.Client.Update(context.Background(), &jobPod); err != nil { - opLog.Error(err, - fmt.Sprintf( - "Unable to update build %s to cancel it.", - jobSpec.Misc.Name, - ), - ) - return err - } - return nil -} - -// CancelTask handles cancelling tasks or handling if a tasks no longer exists. -func (m *Messenger) CancelTask(namespace string, jobSpec *lagoonv1beta1.LagoonTaskSpec) error { - opLog := ctrl.Log.WithName("handlers").WithName("LagoonTasks") - var jobPod corev1.Pod - //@TODO: use `taskName` in the future only - taskName := fmt.Sprintf("lagoon-task-%s-%s", jobSpec.Task.ID, helpers.HashString(jobSpec.Task.ID)[0:6]) - if jobSpec.Task.TaskName != "" { - taskName = jobSpec.Task.TaskName - } - if err := m.Client.Get(context.Background(), types.NamespacedName{ - Name: taskName, - Namespace: namespace, - }, &jobPod); err != nil { - // since there was no task pod, check for the lagoon task resource - var lagoonTask lagoonv1beta1.LagoonTask - if err := m.Client.Get(context.Background(), types.NamespacedName{ - Name: taskName, - Namespace: namespace, - }, &lagoonTask); err != nil { - opLog.Info(fmt.Sprintf( - "Unable to find task %s to cancel it. Sending response to Lagoon to update the task to cancelled.", - taskName, - )) - // if there is no pod or task, update the task in Lagoon to cancelled - m.updateLagoonTask(opLog, namespace, *jobSpec) - return nil - } - // as there is no task pod, but there is a lagoon task resource - // update it to cancelled so that the controller doesn't try to run it - lagoonTask.ObjectMeta.Labels["lagoon.sh/taskStatus"] = lagoonv1beta1.TaskStatusCancelled.String() - if err := m.Client.Update(context.Background(), &lagoonTask); err != nil { - opLog.Error(err, - fmt.Sprintf( - "Unable to update task %s to cancel it.", - taskName, - ), - ) - return err - } - // and then send the response back to lagoon to say it was cancelled. - m.updateLagoonTask(opLog, namespace, *jobSpec) - return nil - } - jobPod.ObjectMeta.Labels["lagoon.sh/cancelTask"] = "true" - if err := m.Client.Update(context.Background(), &jobPod); err != nil { - opLog.Error(err, - fmt.Sprintf( - "Unable to update task %s to cancel it.", - jobSpec.Misc.Name, - ), - ) - return err - } - return nil -} - -func (m *Messenger) updateLagoonBuild(opLog logr.Logger, namespace string, jobSpec lagoonv1beta1.LagoonTaskSpec, lagoonBuild *lagoonv1beta1.LagoonBuild) { - // if the build isn't found by the controller - // then publish a response back to controllerhandler to tell it to update the build to cancelled - // this allows us to update builds in the API that may have gone stale or not updated from `New`, `Pending`, or `Running` status - buildCondition := "cancelled" - if lagoonBuild != nil { - if val, ok := lagoonBuild.ObjectMeta.Labels["lagoon.sh/buildStatus"]; ok { - // if the build isnt running,pending,queued, then set the buildcondition to the value failed/complete/cancelled - if !helpers.ContainsString(helpers.BuildRunningPendingStatus, val) { - buildCondition = strings.ToLower(val) - } - } - } - msg := lagoonv1beta1.LagoonMessage{ - Type: "build", - Namespace: namespace, - Meta: &lagoonv1beta1.LagoonLogMeta{ - Environment: jobSpec.Environment.Name, - Project: jobSpec.Project.Name, - BuildPhase: buildCondition, - BuildName: jobSpec.Misc.Name, - }, - } - // set the start/end time to be now as the default - // to stop the duration counter in the ui - msg.Meta.StartTime = time.Now().UTC().Format("2006-01-02 15:04:05") - msg.Meta.EndTime = time.Now().UTC().Format("2006-01-02 15:04:05") - - // if possible, get the start and end times from the build resource, these will be sent back to lagoon to update the api - if lagoonBuild != nil && lagoonBuild.Status.Conditions != nil { - conditions := lagoonBuild.Status.Conditions - // sort the build conditions by time so the first and last can be extracted - sort.Slice(conditions, func(i, j int) bool { - iTime, _ := time.Parse("2006-01-02T15:04:05Z", conditions[i].LastTransitionTime) - jTime, _ := time.Parse("2006-01-02T15:04:05Z", conditions[j].LastTransitionTime) - return iTime.Before(jTime) - }) - // get the starting time, or fallback to default - sTime, err := time.Parse("2006-01-02T15:04:05Z", conditions[0].LastTransitionTime) - if err == nil { - msg.Meta.StartTime = sTime.Format("2006-01-02 15:04:05") - } - // get the ending time, or fallback to default - eTime, err := time.Parse("2006-01-02T15:04:05Z", conditions[len(conditions)-1].LastTransitionTime) - if err == nil { - msg.Meta.EndTime = eTime.Format("2006-01-02 15:04:05") - } - } - msgBytes, err := json.Marshal(msg) - if err != nil { - opLog.Error(err, "Unable to encode message as JSON") - } - // publish the cancellation result back to lagoon - if err := m.Publish("lagoon-tasks:controller", msgBytes); err != nil { - opLog.Error(err, "Unable to publish message.") - } -} - -func (m *Messenger) updateLagoonTask(opLog logr.Logger, namespace string, jobSpec lagoonv1beta1.LagoonTaskSpec) { - //@TODO: use `taskName` in the future only - taskName := fmt.Sprintf("lagoon-task-%s-%s", jobSpec.Task.ID, helpers.HashString(jobSpec.Task.ID)[0:6]) - if jobSpec.Task.TaskName != "" { - taskName = jobSpec.Task.TaskName - } - // if the task isn't found by the controller - // then publish a response back to controllerhandler to tell it to update the task to cancelled - // this allows us to update tasks in the API that may have gone stale or not updated from `New`, `Pending`, or `Running` status - msg := lagoonv1beta1.LagoonMessage{ - Type: "task", - Namespace: namespace, - Meta: &lagoonv1beta1.LagoonLogMeta{ - Environment: jobSpec.Environment.Name, - Project: jobSpec.Project.Name, - JobName: taskName, - JobStatus: "cancelled", - Task: &lagoonv1beta1.LagoonTaskInfo{ - TaskName: jobSpec.Task.TaskName, - ID: jobSpec.Task.ID, - Name: jobSpec.Task.Name, - Service: jobSpec.Task.Service, - }, - }, - } - // if the task isn't found at all, then set the start/end time to be now - // to stop the duration counter in the ui - msg.Meta.StartTime = time.Now().UTC().Format("2006-01-02 15:04:05") - msg.Meta.EndTime = time.Now().UTC().Format("2006-01-02 15:04:05") - msgBytes, err := json.Marshal(msg) - if err != nil { - opLog.Error(err, "Unable to encode message as JSON") - } - // publish the cancellation result back to lagoon - if err := m.Publish("lagoon-tasks:controller", msgBytes); err != nil { - opLog.Error(err, "Unable to publish message.") - } -} - // IngressRouteMigration handles running the ingress migrations. func (m *Messenger) IngressRouteMigration(namespace string, jobSpec *lagoonv1beta1.LagoonTaskSpec) error { // always set these to true for ingress migration tasks diff --git a/internal/utilities/deletions/lagoon.go b/internal/utilities/deletions/lagoon.go deleted file mode 100644 index d894e877..00000000 --- a/internal/utilities/deletions/lagoon.go +++ /dev/null @@ -1,98 +0,0 @@ -package deletions - -import ( - "context" - "fmt" - - "github.com/go-logr/logr" - lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/uselagoon/remote-controller/internal/helpers" -) - -// DeleteLagoonBuilds will delete any lagoon builds from the namespace. -func (d *Deletions) DeleteLagoonBuilds(ctx context.Context, opLog logr.Logger, ns, project, environment string) bool { - lagoonBuilds := &lagoonv1beta1.LagoonBuildList{} - listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ - client.InNamespace(ns), - }) - if err := d.Client.List(ctx, lagoonBuilds, listOption); err != nil { - opLog.Error(err, - fmt.Sprintf( - "Unable to list lagoon build in namespace %s for project %s, environment %s", - ns, - project, - environment, - ), - ) - return false - } - for _, lagoonBuild := range lagoonBuilds.Items { - if err := d.Client.Delete(ctx, &lagoonBuild); helpers.IgnoreNotFound(err) != nil { - opLog.Error(err, - fmt.Sprintf( - "Unable to delete lagoon build %s in %s for project %s, environment %s", - lagoonBuild.ObjectMeta.Name, - ns, - project, - environment, - ), - ) - return false - } - opLog.Info( - fmt.Sprintf( - "Deleted lagoon build %s in %s for project %s, environment %s", - lagoonBuild.ObjectMeta.Name, - ns, - project, - environment, - ), - ) - } - return true -} - -// DeleteLagoonTasks will delete any lagoon tasks from the namespace. -func (d *Deletions) DeleteLagoonTasks(ctx context.Context, opLog logr.Logger, ns, project, environment string) bool { - lagoonTasks := &lagoonv1beta1.LagoonTaskList{} - listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ - client.InNamespace(ns), - }) - if err := d.Client.List(ctx, lagoonTasks, listOption); err != nil { - opLog.Error(err, - fmt.Sprintf( - "Unable to list lagoon task in namespace %s for project %s, environment %s", - ns, - project, - environment, - ), - ) - return false - } - for _, lagoonTask := range lagoonTasks.Items { - if err := d.Client.Delete(ctx, &lagoonTask); helpers.IgnoreNotFound(err) != nil { - opLog.Error(err, - fmt.Sprintf( - "Unable to delete lagoon task %s in %s for project %s, environment %s", - lagoonTask.ObjectMeta.Name, - ns, - project, - environment, - ), - ) - return false - } - opLog.Info( - fmt.Sprintf( - "Deleted lagoon task %s in %s for project %s, environment %s", - lagoonTask.ObjectMeta.Name, - ns, - project, - environment, - ), - ) - } - return true -} diff --git a/internal/utilities/deletions/process.go b/internal/utilities/deletions/process.go index 53acc365..c8521a3a 100644 --- a/internal/utilities/deletions/process.go +++ b/internal/utilities/deletions/process.go @@ -9,6 +9,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" "github.com/uselagoon/remote-controller/internal/harbor" ) @@ -97,10 +98,10 @@ func (d *Deletions) ProcessDeletion(ctx context.Context, opLog logr.Logger, name get any deployments/statefulsets/daemonsets then delete them */ - if del := d.DeleteLagoonTasks(ctx, opLog.WithName("DeleteLagoonTasks"), namespace.ObjectMeta.Name, project, environment); del == false { + if del := lagoonv1beta1.DeleteLagoonTasks(ctx, opLog.WithName("DeleteLagoonTasks"), d.Client, namespace.ObjectMeta.Name, project, environment); del == false { return fmt.Errorf("error deleting tasks") } - if del := d.DeleteLagoonBuilds(ctx, opLog.WithName("DeleteLagoonBuilds"), namespace.ObjectMeta.Name, project, environment); del == false { + if del := lagoonv1beta1.DeleteLagoonBuilds(ctx, opLog.WithName("DeleteLagoonBuilds"), d.Client, namespace.ObjectMeta.Name, project, environment); del == false { return fmt.Errorf("error deleting builds") } if del := d.DeleteDeployments(ctx, opLog.WithName("DeleteDeployments"), namespace.ObjectMeta.Name, project, environment); del == false { diff --git a/internal/utilities/pruner/build_pruner.go b/internal/utilities/pruner/build_pruner.go deleted file mode 100644 index 5568077c..00000000 --- a/internal/utilities/pruner/build_pruner.go +++ /dev/null @@ -1,127 +0,0 @@ -package pruner - -import ( - "context" - "fmt" - "sort" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/selection" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - - lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" - "github.com/uselagoon/remote-controller/internal/helpers" -) - -// LagoonBuildPruner will prune any build crds that are hanging around. -func (p *Pruner) LagoonBuildPruner() { - opLog := ctrl.Log.WithName("utilities").WithName("LagoonBuildPruner") - namespaces := &corev1.NamespaceList{} - labelRequirements, _ := labels.NewRequirement("lagoon.sh/environmentType", selection.Exists, nil) - listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ - client.MatchingLabelsSelector{ - Selector: labels.NewSelector().Add(*labelRequirements), - }, - }) - if err := p.Client.List(context.Background(), namespaces, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list namespaces created by Lagoon, there may be none or something went wrong")) - return - } - for _, ns := range namespaces.Items { - if ns.Status.Phase == corev1.NamespaceTerminating { - // if the namespace is terminating, don't try to renew the robot credentials - opLog.Info(fmt.Sprintf("Namespace %s is being terminated, aborting build pruner", ns.ObjectMeta.Name)) - continue - } - opLog.Info(fmt.Sprintf("Checking LagoonBuilds in namespace %s", ns.ObjectMeta.Name)) - lagoonBuilds := &lagoonv1beta1.LagoonBuildList{} - listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ - client.InNamespace(ns.ObjectMeta.Name), - client.MatchingLabels(map[string]string{ - "lagoon.sh/controller": p.ControllerNamespace, // created by this controller - }), - }) - if err := p.Client.List(context.Background(), lagoonBuilds, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list LagoonBuild resources, there may be none or something went wrong")) - continue - } - // sort the build pods by creation timestamp - sort.Slice(lagoonBuilds.Items, func(i, j int) bool { - return lagoonBuilds.Items[i].ObjectMeta.CreationTimestamp.After(lagoonBuilds.Items[j].ObjectMeta.CreationTimestamp.Time) - }) - if len(lagoonBuilds.Items) > p.BuildsToKeep { - for idx, lagoonBuild := range lagoonBuilds.Items { - if idx >= p.BuildsToKeep { - if helpers.ContainsString( - helpers.BuildCompletedCancelledFailedStatus, - lagoonBuild.ObjectMeta.Labels["lagoon.sh/buildStatus"], - ) { - opLog.Info(fmt.Sprintf("Cleaning up LagoonBuild %s", lagoonBuild.ObjectMeta.Name)) - if err := p.Client.Delete(context.Background(), &lagoonBuild); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to update status condition")) - break - } - } - } - } - } - } - return -} - -// BuildPodPruner will prune any build pods that are hanging around. -func (p *Pruner) BuildPodPruner() { - opLog := ctrl.Log.WithName("utilities").WithName("BuildPodPruner") - namespaces := &corev1.NamespaceList{} - labelRequirements, _ := labels.NewRequirement("lagoon.sh/environmentType", selection.Exists, nil) - listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ - client.MatchingLabelsSelector{ - Selector: labels.NewSelector().Add(*labelRequirements), - }, - }) - if err := p.Client.List(context.Background(), namespaces, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list namespaces created by Lagoon, there may be none or something went wrong")) - return - } - for _, ns := range namespaces.Items { - if ns.Status.Phase == corev1.NamespaceTerminating { - // if the namespace is terminating, don't try to renew the robot credentials - opLog.Info(fmt.Sprintf("Namespace %s is being terminated, aborting build pod pruner", ns.ObjectMeta.Name)) - return - } - opLog.Info(fmt.Sprintf("Checking Lagoon build pods in namespace %s", ns.ObjectMeta.Name)) - buildPods := &corev1.PodList{} - listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ - client.InNamespace(ns.ObjectMeta.Name), - client.MatchingLabels(map[string]string{ - "lagoon.sh/jobType": "build", - "lagoon.sh/controller": p.ControllerNamespace, // created by this controller - }), - }) - if err := p.Client.List(context.Background(), buildPods, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list Lagoon build pods, there may be none or something went wrong")) - return - } - // sort the build pods by creation timestamp - sort.Slice(buildPods.Items, func(i, j int) bool { - return buildPods.Items[i].ObjectMeta.CreationTimestamp.After(buildPods.Items[j].ObjectMeta.CreationTimestamp.Time) - }) - if len(buildPods.Items) > p.BuildPodsToKeep { - for idx, pod := range buildPods.Items { - if idx >= p.BuildPodsToKeep { - if pod.Status.Phase == corev1.PodFailed || - pod.Status.Phase == corev1.PodSucceeded { - opLog.Info(fmt.Sprintf("Cleaning up pod %s", pod.ObjectMeta.Name)) - if err := p.Client.Delete(context.Background(), &pod); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to update status condition")) - break - } - } - } - } - } - } - return -} diff --git a/internal/utilities/pruner/old_process_pruner.go b/internal/utilities/pruner/old_process_pruner.go index 4b74506d..8e43db04 100644 --- a/internal/utilities/pruner/old_process_pruner.go +++ b/internal/utilities/pruner/old_process_pruner.go @@ -4,16 +4,15 @@ import ( "context" "errors" "fmt" + "strconv" + "time" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/selection" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "strconv" - "time" - //lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" - //"github.com/uselagoon/remote-controller/internal/helpers" ) // LagoonOldProcPruner will identify and remove any long running builds or tasks. diff --git a/internal/utilities/pruner/task_pruner.go b/internal/utilities/pruner/task_pruner.go deleted file mode 100644 index 0dea5280..00000000 --- a/internal/utilities/pruner/task_pruner.go +++ /dev/null @@ -1,127 +0,0 @@ -package pruner - -import ( - "context" - "fmt" - "sort" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/selection" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - - lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" - "github.com/uselagoon/remote-controller/internal/helpers" -) - -// LagoonTaskPruner will prune any build crds that are hanging around. -func (p *Pruner) LagoonTaskPruner() { - opLog := ctrl.Log.WithName("utilities").WithName("LagoonTaskPruner") - namespaces := &corev1.NamespaceList{} - labelRequirements, _ := labels.NewRequirement("lagoon.sh/environmentType", selection.Exists, nil) - listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ - client.MatchingLabelsSelector{ - Selector: labels.NewSelector().Add(*labelRequirements), - }, - }) - if err := p.Client.List(context.Background(), namespaces, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list namespaces created by Lagoon, there may be none or something went wrong")) - return - } - for _, ns := range namespaces.Items { - if ns.Status.Phase == corev1.NamespaceTerminating { - // if the namespace is terminating, don't try to renew the robot credentials - opLog.Info(fmt.Sprintf("Namespace %s is being terminated, aborting task pruner", ns.ObjectMeta.Name)) - continue - } - opLog.Info(fmt.Sprintf("Checking LagoonTasks in namespace %s", ns.ObjectMeta.Name)) - lagoonTasks := &lagoonv1beta1.LagoonTaskList{} - listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ - client.InNamespace(ns.ObjectMeta.Name), - client.MatchingLabels(map[string]string{ - "lagoon.sh/controller": p.ControllerNamespace, // created by this controller - }), - }) - if err := p.Client.List(context.Background(), lagoonTasks, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list LagoonTask resources, there may be none or something went wrong")) - continue - } - // sort the build pods by creation timestamp - sort.Slice(lagoonTasks.Items, func(i, j int) bool { - return lagoonTasks.Items[i].ObjectMeta.CreationTimestamp.After(lagoonTasks.Items[j].ObjectMeta.CreationTimestamp.Time) - }) - if len(lagoonTasks.Items) > p.TasksToKeep { - for idx, lagoonTask := range lagoonTasks.Items { - if idx >= p.TasksToKeep { - if helpers.ContainsString( - helpers.TaskCompletedCancelledFailedStatus, - lagoonTask.ObjectMeta.Labels["lagoon.sh/taskStatus"], - ) { - opLog.Info(fmt.Sprintf("Cleaning up LagoonTask %s", lagoonTask.ObjectMeta.Name)) - if err := p.Client.Delete(context.Background(), &lagoonTask); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to update status condition")) - break - } - } - } - } - } - } - return -} - -// TaskPodPruner will prune any task pods that are hanging around. -func (p *Pruner) TaskPodPruner() { - opLog := ctrl.Log.WithName("utilities").WithName("TaskPodPruner") - namespaces := &corev1.NamespaceList{} - labelRequirements, _ := labels.NewRequirement("lagoon.sh/environmentType", selection.Exists, nil) - listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ - client.MatchingLabelsSelector{ - Selector: labels.NewSelector().Add(*labelRequirements), - }, - }) - if err := p.Client.List(context.Background(), namespaces, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list namespaces created by Lagoon, there may be none or something went wrong")) - return - } - for _, ns := range namespaces.Items { - if ns.Status.Phase == corev1.NamespaceTerminating { - // if the namespace is terminating, don't try to renew the robot credentials - opLog.Info(fmt.Sprintf("Namespace %s is being terminated, aborting task pod pruner", ns.ObjectMeta.Name)) - return - } - opLog.Info(fmt.Sprintf("Checking Lagoon task pods in namespace %s", ns.ObjectMeta.Name)) - taskPods := &corev1.PodList{} - listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ - client.InNamespace(ns.ObjectMeta.Name), - client.MatchingLabels(map[string]string{ - "lagoon.sh/jobType": "task", - "lagoon.sh/controller": p.ControllerNamespace, // created by this controller - }), - }) - if err := p.Client.List(context.Background(), taskPods, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list Lagoon task pods, there may be none or something went wrong")) - return - } - // sort the build pods by creation timestamp - sort.Slice(taskPods.Items, func(i, j int) bool { - return taskPods.Items[i].ObjectMeta.CreationTimestamp.After(taskPods.Items[j].ObjectMeta.CreationTimestamp.Time) - }) - if len(taskPods.Items) > p.TaskPodsToKeep { - for idx, pod := range taskPods.Items { - if idx >= p.TaskPodsToKeep { - if pod.Status.Phase == corev1.PodFailed || - pod.Status.Phase == corev1.PodSucceeded { - opLog.Info(fmt.Sprintf("Cleaning up pod %s", pod.ObjectMeta.Name)) - if err := p.Client.Delete(context.Background(), &pod); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to delete pod")) - break - } - } - } - } - } - } - return -} diff --git a/main.go b/main.go index 7238b1ab..9aef681d 100644 --- a/main.go +++ b/main.go @@ -45,6 +45,7 @@ import ( "github.com/hashicorp/golang-lru/v2/expirable" k8upv1 "github.com/k8up-io/k8up/v2/api/v1" lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" + harborctrl "github.com/uselagoon/remote-controller/controllers/harbor" lagoonv1beta1ctrl "github.com/uselagoon/remote-controller/controllers/v1beta1" "github.com/uselagoon/remote-controller/internal/messenger" k8upv1alpha1 "github.com/vshn/k8up/api/v1alpha1" @@ -85,7 +86,6 @@ func main() { var enableLeaderElection bool var enableMQ bool var leaderElectionID string - var pendingMessageCron string var mqWorkers int var rabbitRetryInterval int var startupConnectionAttempts int @@ -191,8 +191,8 @@ func main() { "The retry interval for rabbitmq.") flag.StringVar(&leaderElectionID, "leader-election-id", "lagoon-builddeploy-leader-election-helper", "The ID to use for leader election.") - flag.StringVar(&pendingMessageCron, "pending-message-cron", "15,45 * * * *", - "The cron definition for pending messages.") + flag.String("pending-message-cron", "", + "This feature has been deprecated, this flag will be removed in a future version.") flag.IntVar(&startupConnectionAttempts, "startup-connection-attempts", 10, "The number of startup attempts before exiting.") flag.IntVar(&startupConnectionInterval, "startup-connection-interval-seconds", 30, @@ -375,7 +375,6 @@ func main() { mqPass = helpers.GetEnv("RABBITMQ_PASSWORD", mqPass) mqHost = helpers.GetEnv("RABBITMQ_HOSTNAME", mqHost) lagoonTargetName = helpers.GetEnv("LAGOON_TARGET_NAME", lagoonTargetName) - pendingMessageCron = helpers.GetEnv("PENDING_MESSAGE_CRON", pendingMessageCron) overrideBuildDeployImage = helpers.GetEnv("OVERRIDE_BUILD_DEPLOY_DIND_IMAGE", overrideBuildDeployImage) namespacePrefix = helpers.GetEnv("NAMESPACE_PREFIX", namespacePrefix) if len(namespacePrefix) > 8 { @@ -657,16 +656,9 @@ func main() { if enableMQ { setupLog.Info("starting messaging handler") go messaging.Consumer(lagoonTargetName) - - // use cron to run a pending message task - // this will check any `LagoonBuild` resources for the pendingMessages label - // and attempt to re-publish them - c.AddFunc(pendingMessageCron, func() { - messaging.GetPendingMessages() - }) } - buildQoSConfig := lagoonv1beta1ctrl.BuildQoS{ + buildQoSConfigv1beta1 := lagoonv1beta1ctrl.BuildQoS{ MaxBuilds: qosMaxBuilds, DefaultValue: qosDefaultValue, } @@ -688,7 +680,7 @@ func main() { // use cron to run a lagoonbuild cleanup task // this will check any Lagoon builds and attempt to delete them c.AddFunc(buildsCleanUpCron, func() { - resourceCleanup.LagoonBuildPruner() + lagoonv1beta1.LagoonBuildPruner(context.Background(), mgr.GetClient(), controllerNamespace, buildsToKeep) }) } // if the build pod cleanup is enabled, add the cronjob for it @@ -697,7 +689,7 @@ func main() { // use cron to run a build pod cleanup task // this will check any Lagoon build pods and attempt to delete them c.AddFunc(buildPodCleanUpCron, func() { - resourceCleanup.BuildPodPruner() + lagoonv1beta1.BuildPodPruner(context.Background(), mgr.GetClient(), controllerNamespace, buildPodsToKeep) }) } // if the lagoontask cleanup is enabled, add the cronjob for it @@ -706,7 +698,7 @@ func main() { // use cron to run a lagoontask cleanup task // this will check any Lagoon tasks and attempt to delete them c.AddFunc(taskCleanUpCron, func() { - resourceCleanup.LagoonTaskPruner() + lagoonv1beta1.LagoonTaskPruner(context.Background(), mgr.GetClient(), controllerNamespace, tasksToKeep) }) } // if the task pod cleanup is enabled, add the cronjob for it @@ -715,7 +707,7 @@ func main() { // use cron to run a task pod cleanup task // this will check any Lagoon task pods and attempt to delete them c.AddFunc(taskPodCleanUpCron, func() { - resourceCleanup.TaskPodPruner() + lagoonv1beta1.TaskPodPruner(context.Background(), mgr.GetClient(), controllerNamespace, taskPodsToKeep) }) } // if harbor is enabled, add the cronjob for credential rotation @@ -748,6 +740,7 @@ func main() { setupLog.Info("starting controllers") + // v1beta1 is deprecated, these controllers will eventually be removed if err = (&lagoonv1beta1ctrl.LagoonBuildReconciler{ Client: mgr.GetClient(), Log: ctrl.Log.WithName("v1beta1").WithName("LagoonBuild"), @@ -789,7 +782,7 @@ func main() { LFFHarborEnabled: lffHarborEnabled, Harbor: harborConfig, LFFQoSEnabled: lffQoSEnabled, - BuildQoS: buildQoSConfig, + BuildQoS: buildQoSConfigv1beta1, NativeCronPodMinFrequency: nativeCronPodMinFrequency, LagoonTargetName: lagoonTargetName, LagoonFeatureFlags: helpers.GetLagoonFeatureFlags(), @@ -821,7 +814,7 @@ func main() { EnableDebug: enableDebug, LagoonTargetName: lagoonTargetName, LFFQoSEnabled: lffQoSEnabled, - BuildQoS: buildQoSConfig, + BuildQoS: buildQoSConfigv1beta1, Cache: cache, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "LagoonMonitor") @@ -855,9 +848,9 @@ func main() { // for now the namespace reconciler only needs to run if harbor is enabled so that we can watch the namespace for rotation label events if lffHarborEnabled { - if err = (&lagoonv1beta1ctrl.HarborCredentialReconciler{ + if err = (&harborctrl.HarborCredentialReconciler{ Client: mgr.GetClient(), - Log: ctrl.Log.WithName("v1beta1").WithName("HarborCredentialReconciler"), + Log: ctrl.Log.WithName("harbor").WithName("HarborCredentialReconciler"), Scheme: mgr.GetScheme(), LFFHarborEnabled: lffHarborEnabled, ControllerNamespace: controllerNamespace, From e7e1823d9cd2fa4077caa07a550230fea8e7a21c Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Tue, 9 Jan 2024 19:33:02 +1100 Subject: [PATCH 05/17] feat: add v1beta2 api --- Makefile | 2 +- PROJECT | 6 + apis/lagoon/v1beta1/lagoonbuild_types.go | 43 +- apis/lagoon/v1beta1/lagoontask_types.go | 43 +- apis/lagoon/v1beta2/groupversion_info.go | 35 + apis/lagoon/v1beta2/helpers_test.go | 105 ++ apis/lagoon/v1beta2/lagoonbuild_helpers.go | 444 +++++++ apis/lagoon/v1beta2/lagoonbuild_types.go | 180 +++ apis/lagoon/v1beta2/lagoontask_helpers.go | 292 +++++ apis/lagoon/v1beta2/lagoontask_types.go | 205 ++++ apis/lagoon/v1beta2/zz_generated.deepcopy.go | 539 +++++++++ .../crd/bases/crd.lagoon.sh_lagoonbuilds.yaml | 192 ++- .../crd/bases/crd.lagoon.sh_lagoontasks.yaml | 174 ++- controller-test.sh | 16 +- controllers/v1beta1/build_helpers.go | 11 - .../v1beta1/podmonitor_buildhandlers.go | 7 - controllers/v1beta1/podmonitor_controller.go | 10 +- .../v1beta1/podmonitor_taskhandlers.go | 7 - controllers/v1beta1/predicates.go | 56 +- controllers/v1beta1/task_controller.go | 11 - controllers/v1beta2/build_controller.go | 342 ++++++ controllers/v1beta2/build_deletionhandlers.go | 514 ++++++++ controllers/v1beta2/build_helpers.go | 1061 +++++++++++++++++ controllers/v1beta2/build_helpers_test.go | 233 ++++ controllers/v1beta2/build_qoshandler.go | 169 +++ controllers/v1beta2/build_standardhandler.go | 73 ++ controllers/{v1beta1 => v1beta2}/metrics.go | 2 +- .../v1beta2/podmonitor_buildhandlers.go | 623 ++++++++++ controllers/v1beta2/podmonitor_controller.go | 225 ++++ .../podmonitor_metrics.go | 2 +- .../v1beta2/podmonitor_taskhandlers.go | 401 +++++++ controllers/v1beta2/predicates.go | 259 ++++ controllers/v1beta2/suite_test.go | 80 ++ controllers/v1beta2/task_controller.go | 678 +++++++++++ controllers/v1beta2/task_helpers.go | 59 + internal/harbor/harbor_credentialrotation.go | 4 +- internal/messenger/consumer.go | 93 +- internal/messenger/tasks_handler.go | 18 +- internal/messenger/tasks_restore.go | 8 +- internal/utilities/deletions/process.go | 7 + main.go | 118 ++ .../dynamic-secret-in-task-project1.yaml | 3 +- test-resources/example-project1.yaml | 4 +- test-resources/example-project2.yaml | 4 +- test-resources/example-project3.yaml | 32 + 45 files changed, 7228 insertions(+), 162 deletions(-) create mode 100644 apis/lagoon/v1beta2/groupversion_info.go create mode 100644 apis/lagoon/v1beta2/helpers_test.go create mode 100644 apis/lagoon/v1beta2/lagoonbuild_helpers.go create mode 100644 apis/lagoon/v1beta2/lagoonbuild_types.go create mode 100644 apis/lagoon/v1beta2/lagoontask_helpers.go create mode 100644 apis/lagoon/v1beta2/lagoontask_types.go create mode 100644 apis/lagoon/v1beta2/zz_generated.deepcopy.go create mode 100644 controllers/v1beta2/build_controller.go create mode 100644 controllers/v1beta2/build_deletionhandlers.go create mode 100644 controllers/v1beta2/build_helpers.go create mode 100644 controllers/v1beta2/build_helpers_test.go create mode 100644 controllers/v1beta2/build_qoshandler.go create mode 100644 controllers/v1beta2/build_standardhandler.go rename controllers/{v1beta1 => v1beta2}/metrics.go (99%) create mode 100644 controllers/v1beta2/podmonitor_buildhandlers.go create mode 100644 controllers/v1beta2/podmonitor_controller.go rename controllers/{v1beta1 => v1beta2}/podmonitor_metrics.go (98%) create mode 100644 controllers/v1beta2/podmonitor_taskhandlers.go create mode 100644 controllers/v1beta2/predicates.go create mode 100644 controllers/v1beta2/suite_test.go create mode 100644 controllers/v1beta2/task_controller.go create mode 100644 controllers/v1beta2/task_helpers.go create mode 100644 test-resources/example-project3.yaml diff --git a/Makefile b/Makefile index 9a4c9409..f85f09ac 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ # Image URL to use all building/pushing image targets IMG ?= controller:latest # Produce CRDs that work back to Kubernetes 1.11 (no version conversion) -CRD_OPTIONS ?= "crd:trivialVersions=false" +CRD_OPTIONS ?= "crd:trivialVersions=true,preserveUnknownFields=false" CONTROLLER_NAMESPACE ?= lagoon-builddeploy diff --git a/PROJECT b/PROJECT index 7b02f074..6bcbae60 100644 --- a/PROJECT +++ b/PROJECT @@ -2,6 +2,12 @@ domain: lagoon.sh multigroup: true repo: github.com/uselagoon/remote-controller resources: +- group: crd + kind: LagoonBuild + version: v1beta2 +- group: crd + kind: LagoonTask + version: v1beta2 - group: crd kind: LagoonBuild version: v1beta1 diff --git a/apis/lagoon/v1beta1/lagoonbuild_types.go b/apis/lagoon/v1beta1/lagoonbuild_types.go index c901ef22..ed3aa281 100644 --- a/apis/lagoon/v1beta1/lagoonbuild_types.go +++ b/apis/lagoon/v1beta1/lagoonbuild_types.go @@ -22,6 +22,28 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// +kubebuilder:object:root=true +// +kubebuilder:deprecatedversion:warning="use lagoonbuilds.crd.lagoon.sh/v1beta2" + +// LagoonBuild is the Schema for the lagoonbuilds API +type LagoonBuild struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec LagoonBuildSpec `json:"spec,omitempty"` + Status LagoonBuildStatus `json:"status,omitempty"` + StatusMessages *LagoonStatusMessages `json:"statusMessages,omitempty"` +} + +// +kubebuilder:object:root=true + +// LagoonBuildList contains a list of LagoonBuild +type LagoonBuildList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []LagoonBuild `json:"items"` +} + // BuildStatusType const for the status type type BuildStatusType string @@ -83,27 +105,6 @@ type LagoonBuildConditions struct { // Condition string `json:"condition"` } -// +kubebuilder:object:root=true - -// LagoonBuild is the Schema for the lagoonbuilds API -type LagoonBuild struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec LagoonBuildSpec `json:"spec,omitempty"` - Status LagoonBuildStatus `json:"status,omitempty"` - StatusMessages *LagoonStatusMessages `json:"statusMessages,omitempty"` -} - -// +kubebuilder:object:root=true - -// LagoonBuildList contains a list of LagoonBuild -type LagoonBuildList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []LagoonBuild `json:"items"` -} - func init() { SchemeBuilder.Register(&LagoonBuild{}, &LagoonBuildList{}) } diff --git a/apis/lagoon/v1beta1/lagoontask_types.go b/apis/lagoon/v1beta1/lagoontask_types.go index bb48276d..20393163 100644 --- a/apis/lagoon/v1beta1/lagoontask_types.go +++ b/apis/lagoon/v1beta1/lagoontask_types.go @@ -30,6 +30,28 @@ import ( // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! // NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. +// +kubebuilder:object:root=true +// +kubebuilder:deprecatedversion:warning="use lagoontasks.crd.lagoon.sh/v1beta2" + +// LagoonTask is the Schema for the lagoontasks API +type LagoonTask struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec LagoonTaskSpec `json:"spec,omitempty"` + Status LagoonTaskStatus `json:"status,omitempty"` + StatusMessages *LagoonStatusMessages `json:"statusMessages,omitempty"` +} + +// +kubebuilder:object:root=true + +// LagoonTaskList contains a list of LagoonTask +type LagoonTaskList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []LagoonTask `json:"items"` +} + // TaskStatusType const for the status type type TaskStatusType string @@ -152,27 +174,6 @@ type LagoonTaskConditions struct { // Condition string `json:"condition"` } -// +kubebuilder:object:root=true - -// LagoonTask is the Schema for the lagoontasks API -type LagoonTask struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec LagoonTaskSpec `json:"spec,omitempty"` - Status LagoonTaskStatus `json:"status,omitempty"` - StatusMessages *LagoonStatusMessages `json:"statusMessages,omitempty"` -} - -// +kubebuilder:object:root=true - -// LagoonTaskList contains a list of LagoonTask -type LagoonTaskList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []LagoonTask `json:"items"` -} - func init() { SchemeBuilder.Register(&LagoonTask{}, &LagoonTaskList{}) } diff --git a/apis/lagoon/v1beta2/groupversion_info.go b/apis/lagoon/v1beta2/groupversion_info.go new file mode 100644 index 00000000..19bc86ce --- /dev/null +++ b/apis/lagoon/v1beta2/groupversion_info.go @@ -0,0 +1,35 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1beta1 contains API Schema definitions for the lagoon v1beta1 API group +// +kubebuilder:object:generate=true +// +groupName=crd.lagoon.sh +package v1beta2 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "crd.lagoon.sh", Version: "v1beta2"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/apis/lagoon/v1beta2/helpers_test.go b/apis/lagoon/v1beta2/helpers_test.go new file mode 100644 index 00000000..86a51639 --- /dev/null +++ b/apis/lagoon/v1beta2/helpers_test.go @@ -0,0 +1,105 @@ +package v1beta2 + +import ( + "testing" +) + +func TestCheckLagoonVersion(t *testing.T) { + type args struct { + build *LagoonBuild + checkVersion string + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "test1", + args: args{ + build: &LagoonBuild{ + Spec: LagoonBuildSpec{ + Project: Project{ + Variables: LagoonVariables{ + Project: []byte(`[{"name":"LAGOON_SYSTEM_CORE_VERSION","value":"v2.12.0","scope":"internal_system"}]`), + }, + }, + }, + }, + checkVersion: "2.12.0", + }, + want: true, + }, + { + name: "test2", + args: args{ + build: &LagoonBuild{ + Spec: LagoonBuildSpec{ + Project: Project{ + Variables: LagoonVariables{ + Project: []byte(`[{"name":"LAGOON_SYSTEM_CORE_VERSION","value":"v2.11.0","scope":"internal_system"}]`), + }, + }, + }, + }, + checkVersion: "2.12.0", + }, + want: false, + }, + { + name: "test3", + args: args{ + build: &LagoonBuild{ + Spec: LagoonBuildSpec{ + Project: Project{ + Variables: LagoonVariables{ + Project: []byte(`[]`), + }, + }, + }, + }, + checkVersion: "2.12.0", + }, + want: false, + }, + { + name: "test4", + args: args{ + build: &LagoonBuild{ + Spec: LagoonBuildSpec{ + Project: Project{ + Variables: LagoonVariables{ + Project: []byte(`[{"name":"LAGOON_SYSTEM_CORE_VERSION","value":"v2.12.0","scope":"internal_system"}]`), + }, + }, + }, + }, + checkVersion: "v2.12.0", + }, + want: true, + }, + { + name: "test5", + args: args{ + build: &LagoonBuild{ + Spec: LagoonBuildSpec{ + Project: Project{ + Variables: LagoonVariables{ + Project: []byte(`[{"name":"LAGOON_SYSTEM_CORE_VERSION","value":"v2.11.0","scope":"internal_system"}]`), + }, + }, + }, + }, + checkVersion: "v2.12.0", + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := CheckLagoonVersion(tt.args.build, tt.args.checkVersion); got != tt.want { + t.Errorf("CheckLagoonVersion() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/apis/lagoon/v1beta2/lagoonbuild_helpers.go b/apis/lagoon/v1beta2/lagoonbuild_helpers.go new file mode 100644 index 00000000..7dd7f337 --- /dev/null +++ b/apis/lagoon/v1beta2/lagoonbuild_helpers.go @@ -0,0 +1,444 @@ +package v1beta2 + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + "github.com/go-logr/logr" + "github.com/hashicorp/go-version" + "github.com/uselagoon/machinery/api/schema" + "github.com/uselagoon/remote-controller/internal/helpers" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var ( + // BuildRunningPendingStatus . + BuildRunningPendingStatus = []string{ + BuildStatusPending.String(), + BuildStatusQueued.String(), + BuildStatusRunning.String(), + } + // BuildCompletedCancelledFailedStatus . + BuildCompletedCancelledFailedStatus = []string{ + BuildStatusFailed.String(), + BuildStatusComplete.String(), + BuildStatusCancelled.String(), + } +) + +// BuildContainsStatus . +func BuildContainsStatus(slice []LagoonBuildConditions, s LagoonBuildConditions) bool { + for _, item := range slice { + if item == s { + return true + } + } + return false +} + +// RemoveBuild remove a LagoonBuild from a slice of LagoonBuilds +func RemoveBuild(slice []LagoonBuild, s LagoonBuild) []LagoonBuild { + result := []LagoonBuild{} + for _, item := range slice { + if item.ObjectMeta.Name == s.ObjectMeta.Name { + continue + } + result = append(result, item) + } + return result +} + +// Check if the version of lagoon provided in the internal_system scope variable is greater than or equal to the checked version +func CheckLagoonVersion(build *LagoonBuild, checkVersion string) bool { + lagoonProjectVariables := &[]helpers.LagoonEnvironmentVariable{} + json.Unmarshal(build.Spec.Project.Variables.Project, lagoonProjectVariables) + lagoonVersion, err := helpers.GetLagoonVariable("LAGOON_SYSTEM_CORE_VERSION", []string{"internal_system"}, *lagoonProjectVariables) + if err != nil { + return false + } + aVer, err := version.NewSemver(lagoonVersion.Value) + if err != nil { + return false + } + bVer, err := version.NewSemver(checkVersion) + if err != nil { + return false + } + return aVer.GreaterThanOrEqual(bVer) +} + +// CancelExtraBuilds cancels extra builds. +func CancelExtraBuilds(ctx context.Context, r client.Client, opLog logr.Logger, ns string, status string) error { + pendingBuilds := &LagoonBuildList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(ns), + client.MatchingLabels(map[string]string{"lagoon.sh/buildStatus": BuildStatusPending.String()}), + }) + if err := r.List(ctx, pendingBuilds, listOption); err != nil { + return fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err) + } + if len(pendingBuilds.Items) > 0 { + // opLog.Info(fmt.Sprintf("There are %v pending builds", len(pendingBuilds.Items))) + // if we have any pending builds, then grab the latest one and make it running + // if there are any other pending builds, cancel them so only the latest one runs + sort.Slice(pendingBuilds.Items, func(i, j int) bool { + return pendingBuilds.Items[i].ObjectMeta.CreationTimestamp.After(pendingBuilds.Items[j].ObjectMeta.CreationTimestamp.Time) + }) + for idx, pBuild := range pendingBuilds.Items { + pendingBuild := pBuild.DeepCopy() + if idx == 0 { + pendingBuild.Labels["lagoon.sh/buildStatus"] = status + } else { + // cancel any other pending builds + opLog.Info(fmt.Sprintf("Setting build %s as cancelled", pendingBuild.ObjectMeta.Name)) + pendingBuild.Labels["lagoon.sh/buildStatus"] = BuildStatusCancelled.String() + pendingBuild.Labels["lagoon.sh/cancelledByNewBuild"] = "true" + } + if err := r.Update(ctx, pendingBuild); err != nil { + return err + } + } + } + return nil +} + +func GetBuildConditionFromPod(phase corev1.PodPhase) BuildStatusType { + var buildCondition BuildStatusType + switch phase { + case corev1.PodFailed: + buildCondition = BuildStatusFailed + case corev1.PodSucceeded: + buildCondition = BuildStatusComplete + case corev1.PodPending: + buildCondition = BuildStatusPending + case corev1.PodRunning: + buildCondition = BuildStatusRunning + } + return buildCondition +} + +func GetTaskConditionFromPod(phase corev1.PodPhase) TaskStatusType { + var taskCondition TaskStatusType + switch phase { + case corev1.PodFailed: + taskCondition = TaskStatusFailed + case corev1.PodSucceeded: + taskCondition = TaskStatusComplete + case corev1.PodPending: + taskCondition = TaskStatusPending + case corev1.PodRunning: + taskCondition = TaskStatusRunning + } + return taskCondition +} + +func CheckRunningBuilds(ctx context.Context, cns string, opLog logr.Logger, cl client.Client, ns corev1.Namespace) bool { + lagoonBuilds := &LagoonBuildList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(ns.ObjectMeta.Name), + client.MatchingLabels(map[string]string{ + "lagoon.sh/controller": cns, // created by this controller + }), + }) + if err := cl.List(ctx, lagoonBuilds, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list Lagoon build pods, there may be none or something went wrong")) + return false + } + runningBuilds := false + sort.Slice(lagoonBuilds.Items, func(i, j int) bool { + return lagoonBuilds.Items[i].ObjectMeta.CreationTimestamp.After(lagoonBuilds.Items[j].ObjectMeta.CreationTimestamp.Time) + }) + // if there are any builds pending or running, don't try and refresh the credentials as this + // could break the build + if len(lagoonBuilds.Items) > 0 { + if helpers.ContainsString( + BuildRunningPendingStatus, + lagoonBuilds.Items[0].Labels["lagoon.sh/buildStatus"], + ) { + runningBuilds = true + } + } + return runningBuilds +} + +// DeleteLagoonBuilds will delete any lagoon builds from the namespace. +func DeleteLagoonBuilds(ctx context.Context, opLog logr.Logger, cl client.Client, ns, project, environment string) bool { + lagoonBuilds := &LagoonBuildList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(ns), + }) + if err := cl.List(ctx, lagoonBuilds, listOption); err != nil { + opLog.Error(err, + fmt.Sprintf( + "Unable to list lagoon build in namespace %s for project %s, environment %s", + ns, + project, + environment, + ), + ) + return false + } + for _, lagoonBuild := range lagoonBuilds.Items { + if err := cl.Delete(ctx, &lagoonBuild); helpers.IgnoreNotFound(err) != nil { + opLog.Error(err, + fmt.Sprintf( + "Unable to delete lagoon build %s in %s for project %s, environment %s", + lagoonBuild.ObjectMeta.Name, + ns, + project, + environment, + ), + ) + return false + } + opLog.Info( + fmt.Sprintf( + "Deleted lagoon build %s in %s for project %s, environment %s", + lagoonBuild.ObjectMeta.Name, + ns, + project, + environment, + ), + ) + } + return true +} + +func LagoonBuildPruner(ctx context.Context, cl client.Client, cns string, buildsToKeep int) { + opLog := ctrl.Log.WithName("utilities").WithName("LagoonBuildPruner") + namespaces := &corev1.NamespaceList{} + labelRequirements, _ := labels.NewRequirement("lagoon.sh/environmentType", selection.Exists, nil) + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.MatchingLabelsSelector{ + Selector: labels.NewSelector().Add(*labelRequirements), + }, + }) + if err := cl.List(ctx, namespaces, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list namespaces created by Lagoon, there may be none or something went wrong")) + return + } + for _, ns := range namespaces.Items { + if ns.Status.Phase == corev1.NamespaceTerminating { + // if the namespace is terminating, don't try to renew the robot credentials + opLog.Info(fmt.Sprintf("Namespace %s is being terminated, aborting build pruner", ns.ObjectMeta.Name)) + continue + } + opLog.Info(fmt.Sprintf("Checking LagoonBuilds in namespace %s", ns.ObjectMeta.Name)) + lagoonBuilds := &LagoonBuildList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(ns.ObjectMeta.Name), + client.MatchingLabels(map[string]string{ + "lagoon.sh/controller": cns, // created by this controller + }), + }) + if err := cl.List(ctx, lagoonBuilds, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list LagoonBuild resources, there may be none or something went wrong")) + continue + } + // sort the build pods by creation timestamp + sort.Slice(lagoonBuilds.Items, func(i, j int) bool { + return lagoonBuilds.Items[i].ObjectMeta.CreationTimestamp.After(lagoonBuilds.Items[j].ObjectMeta.CreationTimestamp.Time) + }) + if len(lagoonBuilds.Items) > buildsToKeep { + for idx, lagoonBuild := range lagoonBuilds.Items { + if idx >= buildsToKeep { + if helpers.ContainsString( + BuildCompletedCancelledFailedStatus, + lagoonBuild.ObjectMeta.Labels["lagoon.sh/buildStatus"], + ) { + opLog.Info(fmt.Sprintf("Cleaning up LagoonBuild %s", lagoonBuild.ObjectMeta.Name)) + if err := cl.Delete(ctx, &lagoonBuild); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to update status condition")) + break + } + } + } + } + } + } + return +} + +// BuildPodPruner will prune any build pods that are hanging around. +func BuildPodPruner(ctx context.Context, cl client.Client, cns string, buildPodsToKeep int) { + opLog := ctrl.Log.WithName("utilities").WithName("BuildPodPruner") + namespaces := &corev1.NamespaceList{} + labelRequirements, _ := labels.NewRequirement("lagoon.sh/environmentType", selection.Exists, nil) + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.MatchingLabelsSelector{ + Selector: labels.NewSelector().Add(*labelRequirements), + }, + }) + if err := cl.List(ctx, namespaces, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list namespaces created by Lagoon, there may be none or something went wrong")) + return + } + for _, ns := range namespaces.Items { + if ns.Status.Phase == corev1.NamespaceTerminating { + // if the namespace is terminating, don't try to renew the robot credentials + opLog.Info(fmt.Sprintf("Namespace %s is being terminated, aborting build pod pruner", ns.ObjectMeta.Name)) + return + } + opLog.Info(fmt.Sprintf("Checking Lagoon build pods in namespace %s", ns.ObjectMeta.Name)) + buildPods := &corev1.PodList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(ns.ObjectMeta.Name), + client.MatchingLabels(map[string]string{ + "lagoon.sh/jobType": "build", + "lagoon.sh/controller": cns, // created by this controller + }), + }) + if err := cl.List(ctx, buildPods, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list Lagoon build pods, there may be none or something went wrong")) + return + } + // sort the build pods by creation timestamp + sort.Slice(buildPods.Items, func(i, j int) bool { + return buildPods.Items[i].ObjectMeta.CreationTimestamp.After(buildPods.Items[j].ObjectMeta.CreationTimestamp.Time) + }) + if len(buildPods.Items) > buildPodsToKeep { + for idx, pod := range buildPods.Items { + if idx >= buildPodsToKeep { + if pod.Status.Phase == corev1.PodFailed || + pod.Status.Phase == corev1.PodSucceeded { + opLog.Info(fmt.Sprintf("Cleaning up pod %s", pod.ObjectMeta.Name)) + if err := cl.Delete(ctx, &pod); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to update status condition")) + break + } + } + } + } + } + } + return +} + +func updateLagoonBuild(opLog logr.Logger, namespace string, jobSpec LagoonTaskSpec, lagoonBuild *LagoonBuild) ([]byte, error) { + // if the build isn't found by the controller + // then publish a response back to controllerhandler to tell it to update the build to cancelled + // this allows us to update builds in the API that may have gone stale or not updated from `New`, `Pending`, or `Running` status + buildCondition := "cancelled" + if lagoonBuild != nil { + if val, ok := lagoonBuild.ObjectMeta.Labels["lagoon.sh/buildStatus"]; ok { + // if the build isnt running,pending,queued, then set the buildcondition to the value failed/complete/cancelled + if !helpers.ContainsString(BuildRunningPendingStatus, val) { + buildCondition = strings.ToLower(val) + } + } + } + msg := schema.LagoonMessage{ + Type: "build", + Namespace: namespace, + Meta: &schema.LagoonLogMeta{ + Environment: jobSpec.Environment.Name, + Project: jobSpec.Project.Name, + BuildStatus: buildCondition, + BuildName: jobSpec.Misc.Name, + }, + } + // set the start/end time to be now as the default + // to stop the duration counter in the ui + msg.Meta.StartTime = time.Now().UTC().Format("2006-01-02 15:04:05") + msg.Meta.EndTime = time.Now().UTC().Format("2006-01-02 15:04:05") + + // if possible, get the start and end times from the build resource, these will be sent back to lagoon to update the api + if lagoonBuild != nil && lagoonBuild.Status.Conditions != nil { + conditions := lagoonBuild.Status.Conditions + // sort the build conditions by time so the first and last can be extracted + sort.Slice(conditions, func(i, j int) bool { + iTime, _ := time.Parse("2006-01-02T15:04:05Z", conditions[i].LastTransitionTime) + jTime, _ := time.Parse("2006-01-02T15:04:05Z", conditions[j].LastTransitionTime) + return iTime.Before(jTime) + }) + // get the starting time, or fallback to default + sTime, err := time.Parse("2006-01-02T15:04:05Z", conditions[0].LastTransitionTime) + if err == nil { + msg.Meta.StartTime = sTime.Format("2006-01-02 15:04:05") + } + // get the ending time, or fallback to default + eTime, err := time.Parse("2006-01-02T15:04:05Z", conditions[len(conditions)-1].LastTransitionTime) + if err == nil { + msg.Meta.EndTime = eTime.Format("2006-01-02 15:04:05") + } + } + msgBytes, err := json.Marshal(msg) + if err != nil { + return nil, fmt.Errorf("Unable to encode message as JSON: %v", err) + } + return msgBytes, nil +} + +// CancelBuild handles cancelling builds or handling if a build no longer exists. +func CancelBuild(ctx context.Context, cl client.Client, namespace string, body []byte) (bool, []byte, error) { + opLog := ctrl.Log.WithName("handlers").WithName("LagoonTasks") + jobSpec := &LagoonTaskSpec{} + json.Unmarshal(body, jobSpec) + var jobPod corev1.Pod + if err := cl.Get(ctx, types.NamespacedName{ + Name: jobSpec.Misc.Name, + Namespace: namespace, + }, &jobPod); err != nil { + opLog.Info(fmt.Sprintf( + "Unable to find build pod %s to cancel it. Checking to see if LagoonBuild exists.", + jobSpec.Misc.Name, + )) + // since there was no build pod, check for the lagoon build resource + var lagoonBuild LagoonBuild + if err := cl.Get(ctx, types.NamespacedName{ + Name: jobSpec.Misc.Name, + Namespace: namespace, + }, &lagoonBuild); err != nil { + opLog.Info(fmt.Sprintf( + "Unable to find build %s to cancel it. Sending response to Lagoon to update the build to cancelled.", + jobSpec.Misc.Name, + )) + // if there is no pod or build, update the build in Lagoon to cancelled, assume completely cancelled with no other information + // and then send the response back to lagoon to say it was cancelled. + b, err := updateLagoonBuild(opLog, namespace, *jobSpec, nil) + return false, b, err + } + // as there is no build pod, but there is a lagoon build resource + // update it to cancelled so that the controller doesn't try to run it + // check if the build has existing status or not though to consume it + if helpers.ContainsString( + BuildRunningPendingStatus, + lagoonBuild.ObjectMeta.Labels["lagoon.sh/buildStatus"], + ) { + lagoonBuild.ObjectMeta.Labels["lagoon.sh/buildStatus"] = BuildStatusCancelled.String() + } + lagoonBuild.ObjectMeta.Labels["lagoon.sh/cancelBuildNoPod"] = "true" + if err := cl.Update(ctx, &lagoonBuild); err != nil { + opLog.Error(err, + fmt.Sprintf( + "Unable to update build %s to cancel it.", + jobSpec.Misc.Name, + ), + ) + return false, nil, err + } + // and then send the response back to lagoon to say it was cancelled. + b, err := updateLagoonBuild(opLog, namespace, *jobSpec, &lagoonBuild) + return true, b, err + } + jobPod.ObjectMeta.Labels["lagoon.sh/cancelBuild"] = "true" + if err := cl.Update(ctx, &jobPod); err != nil { + opLog.Error(err, + fmt.Sprintf( + "Unable to update build %s to cancel it.", + jobSpec.Misc.Name, + ), + ) + return false, nil, err + } + return false, nil, nil +} diff --git a/apis/lagoon/v1beta2/lagoonbuild_types.go b/apis/lagoon/v1beta2/lagoonbuild_types.go new file mode 100644 index 00000000..fcbcc7b5 --- /dev/null +++ b/apis/lagoon/v1beta2/lagoonbuild_types.go @@ -0,0 +1,180 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta2 + +import ( + "strings" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// +kubebuilder:object:root=true +//+kubebuilder:storageversion + +// LagoonBuild is the Schema for the lagoonbuilds API +type LagoonBuild struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec LagoonBuildSpec `json:"spec,omitempty"` + Status LagoonBuildStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// LagoonBuildList contains a list of LagoonBuild +type LagoonBuildList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []LagoonBuild `json:"items"` +} + +// BuildStatusType const for the status type +type BuildStatusType string + +// These are valid conditions of a job. +const ( + // BuildStatusPending means the build is pending. + BuildStatusPending BuildStatusType = "Pending" + // BuildStatusQueued means the build is queued. + BuildStatusQueued BuildStatusType = "Queued" + // BuildStatusRunning means the build is running. + BuildStatusRunning BuildStatusType = "Running" + // BuildStatusComplete means the build has completed its execution. + BuildStatusComplete BuildStatusType = "Complete" + // BuildStatusFailed means the job has failed its execution. + BuildStatusFailed BuildStatusType = "Failed" + // BuildStatusCancelled means the job been cancelled. + BuildStatusCancelled BuildStatusType = "Cancelled" +) + +func (b BuildStatusType) String() string { + return string(b) +} + +func (b BuildStatusType) ToLower() string { + return strings.ToLower(b.String()) +} + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// LagoonBuildSpec defines the desired state of LagoonBuild +type LagoonBuildSpec struct { + // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + // Important: Run "make" to regenerate code after modifying this file + + Build Build `json:"build"` + Project Project `json:"project"` + Branch Branch `json:"branch,omitempty"` + Pullrequest Pullrequest `json:"pullrequest,omitempty"` + Promote Promote `json:"promote,omitempty"` + GitReference string `json:"gitReference"` +} + +// LagoonBuildStatus defines the observed state of LagoonBuild +type LagoonBuildStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file + Conditions []LagoonBuildConditions `json:"conditions,omitempty"` + Log []byte `json:"log,omitempty"` +} + +// LagoonBuildConditions defines the observed conditions of build pods. +type LagoonBuildConditions struct { + LastTransitionTime string `json:"lastTransitionTime"` + Status corev1.ConditionStatus `json:"status"` + Type BuildStatusType `json:"type"` + // Condition string `json:"condition"` +} + +func init() { + SchemeBuilder.Register(&LagoonBuild{}, &LagoonBuildList{}) +} + +// Build contains the type of build, and the image to use for the builder. +type Build struct { + CI string `json:"ci,omitempty"` + Image string `json:"image,omitempty"` + Type string `json:"type"` + Priority *int `json:"priority,omitempty"` + BulkID string `json:"bulkId,omitempty"` +} + +// Project contains the project information from lagoon. +type Project struct { + ID *uint `json:"id,omitempty"` + Name string `json:"name"` + Environment string `json:"environment"` + EnvironmentID *uint `json:"environmentId,omitempty"` + UILink string `json:"uiLink,omitempty"` + GitURL string `json:"gitUrl"` + NamespacePattern string `json:"namespacePattern,omitempty"` + RouterPattern string `json:"routerPattern,omitempty"` + EnvironmentType string `json:"environmentType"` + ProductionEnvironment string `json:"productionEnvironment"` + StandbyEnvironment string `json:"standbyEnvironment"` + DeployTarget string `json:"deployTarget"` + ProjectSecret string `json:"projectSecret"` + SubFolder string `json:"subfolder,omitempty"` + Key []byte `json:"key"` + Monitoring Monitoring `json:"monitoring"` + Variables LagoonVariables `json:"variables"` + Registry string `json:"registry,omitempty"` + EnvironmentIdling *int `json:"environmentIdling,omitempty"` + ProjectIdling *int `json:"projectIdling,omitempty"` + StorageCalculator *int `json:"storageCalculator,omitempty"` + Organization *Organization `json:"organization,omitempty"` +} + +type Organization struct { + ID *uint `json:"id,omitempty"` + Name string `json:"name,omitempty"` +} + +// Variables contains the project and environment variables from lagoon. +type LagoonVariables struct { + Project []byte `json:"project,omitempty"` + Environment []byte `json:"environment,omitempty"` +} + +// Branch contains the branch name used for a branch deployment. +type Branch struct { + Name string `json:"name,omitempty"` +} + +// Pullrequest contains the information for a pullrequest deployment. +type Pullrequest struct { + HeadBranch string `json:"headBranch,omitempty"` + HeadSha string `json:"headSha,omitempty"` + BaseBranch string `json:"baseBranch,omitempty"` + BaseSha string `json:"baseSha,omitempty"` + Title string `json:"title,omitempty"` + Number string `json:"number,omitempty"` +} + +// Promote contains the information for a promote deployment. +type Promote struct { + SourceEnvironment string `json:"sourceEnvironment,omitempty"` + SourceProject string `json:"sourceProject,omitempty"` +} + +// Monitoring contains the monitoring information for the project in Lagoon. +type Monitoring struct { + Contact string `json:"contact,omitempty"` + StatuspageID string `json:"statuspageID,omitempty"` +} diff --git a/apis/lagoon/v1beta2/lagoontask_helpers.go b/apis/lagoon/v1beta2/lagoontask_helpers.go new file mode 100644 index 00000000..b8964272 --- /dev/null +++ b/apis/lagoon/v1beta2/lagoontask_helpers.go @@ -0,0 +1,292 @@ +package v1beta2 + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "time" + + "github.com/go-logr/logr" + "github.com/uselagoon/machinery/api/schema" + "github.com/uselagoon/remote-controller/internal/helpers" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var ( + // TaskRunningPendingStatus . + TaskRunningPendingStatus = []string{ + TaskStatusPending.String(), + TaskStatusQueued.String(), + TaskStatusRunning.String(), + } + // TaskCompletedCancelledFailedStatus . + TaskCompletedCancelledFailedStatus = []string{ + TaskStatusFailed.String(), + TaskStatusComplete.String(), + TaskStatusCancelled.String(), + } +) + +// TaskContainsStatus . +func TaskContainsStatus(slice []LagoonTaskConditions, s LagoonTaskConditions) bool { + for _, item := range slice { + if item == s { + return true + } + } + return false +} + +// DeleteLagoonTasks will delete any lagoon tasks from the namespace. +func DeleteLagoonTasks(ctx context.Context, opLog logr.Logger, cl client.Client, ns, project, environment string) bool { + lagoonTasks := &LagoonTaskList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(ns), + }) + if err := cl.List(ctx, lagoonTasks, listOption); err != nil { + opLog.Error(err, + fmt.Sprintf( + "Unable to list lagoon task in namespace %s for project %s, environment %s", + ns, + project, + environment, + ), + ) + return false + } + for _, lagoonTask := range lagoonTasks.Items { + if err := cl.Delete(ctx, &lagoonTask); helpers.IgnoreNotFound(err) != nil { + opLog.Error(err, + fmt.Sprintf( + "Unable to delete lagoon task %s in %s for project %s, environment %s", + lagoonTask.ObjectMeta.Name, + ns, + project, + environment, + ), + ) + return false + } + opLog.Info( + fmt.Sprintf( + "Deleted lagoon task %s in %s for project %s, environment %s", + lagoonTask.ObjectMeta.Name, + ns, + project, + environment, + ), + ) + } + return true +} + +// LagoonTaskPruner will prune any build crds that are hanging around. +func LagoonTaskPruner(ctx context.Context, cl client.Client, cns string, tasksToKeep int) { + opLog := ctrl.Log.WithName("utilities").WithName("LagoonTaskPruner") + namespaces := &corev1.NamespaceList{} + labelRequirements, _ := labels.NewRequirement("lagoon.sh/environmentType", selection.Exists, nil) + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.MatchingLabelsSelector{ + Selector: labels.NewSelector().Add(*labelRequirements), + }, + }) + if err := cl.List(ctx, namespaces, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list namespaces created by Lagoon, there may be none or something went wrong")) + return + } + for _, ns := range namespaces.Items { + if ns.Status.Phase == corev1.NamespaceTerminating { + // if the namespace is terminating, don't try to renew the robot credentials + opLog.Info(fmt.Sprintf("Namespace %s is being terminated, aborting task pruner", ns.ObjectMeta.Name)) + continue + } + opLog.Info(fmt.Sprintf("Checking LagoonTasks in namespace %s", ns.ObjectMeta.Name)) + lagoonTasks := &LagoonTaskList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(ns.ObjectMeta.Name), + client.MatchingLabels(map[string]string{ + "lagoon.sh/controller": cns, // created by this controller + }), + }) + if err := cl.List(ctx, lagoonTasks, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list LagoonTask resources, there may be none or something went wrong")) + continue + } + // sort the build pods by creation timestamp + sort.Slice(lagoonTasks.Items, func(i, j int) bool { + return lagoonTasks.Items[i].ObjectMeta.CreationTimestamp.After(lagoonTasks.Items[j].ObjectMeta.CreationTimestamp.Time) + }) + if len(lagoonTasks.Items) > tasksToKeep { + for idx, lagoonTask := range lagoonTasks.Items { + if idx >= tasksToKeep { + if helpers.ContainsString( + TaskCompletedCancelledFailedStatus, + lagoonTask.ObjectMeta.Labels["lagoon.sh/taskStatus"], + ) { + opLog.Info(fmt.Sprintf("Cleaning up LagoonTask %s", lagoonTask.ObjectMeta.Name)) + if err := cl.Delete(ctx, &lagoonTask); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to update status condition")) + break + } + } + } + } + } + } + return +} + +// TaskPodPruner will prune any task pods that are hanging around. +func TaskPodPruner(ctx context.Context, cl client.Client, cns string, taskPodsToKeep int) { + opLog := ctrl.Log.WithName("utilities").WithName("TaskPodPruner") + namespaces := &corev1.NamespaceList{} + labelRequirements, _ := labels.NewRequirement("lagoon.sh/environmentType", selection.Exists, nil) + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.MatchingLabelsSelector{ + Selector: labels.NewSelector().Add(*labelRequirements), + }, + }) + if err := cl.List(ctx, namespaces, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list namespaces created by Lagoon, there may be none or something went wrong")) + return + } + for _, ns := range namespaces.Items { + if ns.Status.Phase == corev1.NamespaceTerminating { + // if the namespace is terminating, don't try to renew the robot credentials + opLog.Info(fmt.Sprintf("Namespace %s is being terminated, aborting task pod pruner", ns.ObjectMeta.Name)) + return + } + opLog.Info(fmt.Sprintf("Checking Lagoon task pods in namespace %s", ns.ObjectMeta.Name)) + taskPods := &corev1.PodList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(ns.ObjectMeta.Name), + client.MatchingLabels(map[string]string{ + "lagoon.sh/jobType": "task", + "lagoon.sh/controller": cns, // created by this controller + }), + }) + if err := cl.List(ctx, taskPods, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list Lagoon task pods, there may be none or something went wrong")) + return + } + // sort the build pods by creation timestamp + sort.Slice(taskPods.Items, func(i, j int) bool { + return taskPods.Items[i].ObjectMeta.CreationTimestamp.After(taskPods.Items[j].ObjectMeta.CreationTimestamp.Time) + }) + if len(taskPods.Items) > taskPodsToKeep { + for idx, pod := range taskPods.Items { + if idx >= taskPodsToKeep { + if pod.Status.Phase == corev1.PodFailed || + pod.Status.Phase == corev1.PodSucceeded { + opLog.Info(fmt.Sprintf("Cleaning up pod %s", pod.ObjectMeta.Name)) + if err := cl.Delete(ctx, &pod); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to delete pod")) + break + } + } + } + } + } + } + return +} + +func updateLagoonTask(opLog logr.Logger, namespace string, taskSpec LagoonTaskSpec) ([]byte, error) { + //@TODO: use `taskName` in the future only + taskName := fmt.Sprintf("lagoon-task-%s-%s", taskSpec.Task.ID, helpers.HashString(taskSpec.Task.ID)[0:6]) + if taskSpec.Task.TaskName != "" { + taskName = taskSpec.Task.TaskName + } + // if the task isn't found by the controller + // then publish a response back to controllerhandler to tell it to update the task to cancelled + // this allows us to update tasks in the API that may have gone stale or not updated from `New`, `Pending`, or `Running` status + msg := schema.LagoonMessage{ + Type: "task", + Namespace: namespace, + Meta: &schema.LagoonLogMeta{ + Environment: taskSpec.Environment.Name, + Project: taskSpec.Project.Name, + JobName: taskName, + JobStatus: "cancelled", + Task: &schema.LagoonTaskInfo{ + TaskName: taskSpec.Task.TaskName, + ID: taskSpec.Task.ID, + Name: taskSpec.Task.Name, + Service: taskSpec.Task.Service, + }, + }, + } + // if the task isn't found at all, then set the start/end time to be now + // to stop the duration counter in the ui + msg.Meta.StartTime = time.Now().UTC().Format("2006-01-02 15:04:05") + msg.Meta.EndTime = time.Now().UTC().Format("2006-01-02 15:04:05") + msgBytes, err := json.Marshal(msg) + if err != nil { + return nil, fmt.Errorf("Unable to encode message as JSON: %v", err) + } + return msgBytes, nil +} + +// CancelTask handles cancelling tasks or handling if a tasks no longer exists. +func CancelTask(ctx context.Context, cl client.Client, namespace string, body []byte) (bool, []byte, error) { + opLog := ctrl.Log.WithName("handlers").WithName("LagoonTasks") + jobSpec := &LagoonTaskSpec{} + json.Unmarshal(body, jobSpec) + var jobPod corev1.Pod + //@TODO: use `taskName` in the future only + taskName := fmt.Sprintf("lagoon-task-%s-%s", jobSpec.Task.ID, helpers.HashString(jobSpec.Task.ID)[0:6]) + if jobSpec.Task.TaskName != "" { + taskName = jobSpec.Task.TaskName + } + if err := cl.Get(ctx, types.NamespacedName{ + Name: taskName, + Namespace: namespace, + }, &jobPod); err != nil { + // since there was no task pod, check for the lagoon task resource + var lagoonTask LagoonTask + if err := cl.Get(ctx, types.NamespacedName{ + Name: taskName, + Namespace: namespace, + }, &lagoonTask); err != nil { + opLog.Info(fmt.Sprintf( + "Unable to find task %s to cancel it. Sending response to Lagoon to update the task to cancelled.", + taskName, + )) + // if there is no pod or task, update the task in Lagoon to cancelled + b, err := updateLagoonTask(opLog, namespace, *jobSpec) + return false, b, err + } + // as there is no task pod, but there is a lagoon task resource + // update it to cancelled so that the controller doesn't try to run it + lagoonTask.ObjectMeta.Labels["lagoon.sh/taskStatus"] = TaskStatusCancelled.String() + if err := cl.Update(ctx, &lagoonTask); err != nil { + opLog.Error(err, + fmt.Sprintf( + "Unable to update task %s to cancel it.", + taskName, + ), + ) + return false, nil, err + } + // and then send the response back to lagoon to say it was cancelled. + b, err := updateLagoonTask(opLog, namespace, *jobSpec) + return true, b, err + } + jobPod.ObjectMeta.Labels["lagoon.sh/cancelTask"] = "true" + if err := cl.Update(ctx, &jobPod); err != nil { + opLog.Error(err, + fmt.Sprintf( + "Unable to update task %s to cancel it.", + jobSpec.Misc.Name, + ), + ) + return false, nil, err + } + return false, nil, nil +} diff --git a/apis/lagoon/v1beta2/lagoontask_types.go b/apis/lagoon/v1beta2/lagoontask_types.go new file mode 100644 index 00000000..cc1d18a2 --- /dev/null +++ b/apis/lagoon/v1beta2/lagoontask_types.go @@ -0,0 +1,205 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta2 + +import ( + "encoding/json" + "fmt" + "reflect" + "strconv" + "strings" + + "github.com/uselagoon/machinery/api/schema" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// +kubebuilder:object:root=true +//+kubebuilder:storageversion + +// LagoonTask is the Schema for the lagoontasks API +type LagoonTask struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec LagoonTaskSpec `json:"spec,omitempty"` + Status LagoonTaskStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// LagoonTaskList contains a list of LagoonTask +type LagoonTaskList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []LagoonTask `json:"items"` +} + +// TaskStatusType const for the status type +type TaskStatusType string + +// These are valid conditions of a job. +const ( + // TaskStatusPending means the job is pending. + TaskStatusPending TaskStatusType = "Pending" + // TaskStatusQueued means the job is queued. + TaskStatusQueued TaskStatusType = "Queued" + // TaskStatusRunning means the job is running. + TaskStatusRunning TaskStatusType = "Running" + // TaskStatusComplete means the job has completed its execution. + TaskStatusComplete TaskStatusType = "Complete" + // TaskStatusFailed means the job has failed its execution. + TaskStatusFailed TaskStatusType = "Failed" + // TaskStatusCancelled means the job been cancelled. + TaskStatusCancelled TaskStatusType = "Cancelled" +) + +func (b TaskStatusType) String() string { + return string(b) +} + +func (b TaskStatusType) ToLower() string { + return strings.ToLower(b.String()) +} + +// TaskType const for the status type +type TaskType string + +// These are valid conditions of a job. +const ( + // TaskTypeStandard means the task is a standard task. + TaskTypeStandard TaskType = "standard" + // TaskTypeAdvanced means the task is an advanced task. + TaskTypeAdvanced TaskType = "advanced" +) + +func (b TaskType) String() string { + return string(b) +} + +// LagoonTaskSpec defines the desired state of LagoonTask +type LagoonTaskSpec struct { + // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + // Important: Run "make" to regenerate code after modifying this file + Key string `json:"key,omitempty"` + Task schema.LagoonTaskInfo `json:"task,omitempty"` + Project LagoonTaskProject `json:"project,omitempty"` + Environment LagoonTaskEnvironment `json:"environment,omitempty"` + Misc *LagoonMiscInfo `json:"misc,omitempty"` + AdvancedTask *LagoonAdvancedTaskInfo `json:"advancedTask,omitempty"` +} + +// LagoonAdvancedTaskInfo defines what an advanced task can use for the creation of the pod. +type LagoonAdvancedTaskInfo struct { + RunnerImage string `json:"runnerImage,omitempty"` + JSONPayload string `json:"JSONPayload,omitempty"` + DeployerToken bool `json:"deployerToken,omitempty"` + SSHKey bool `json:"sshKey,omitempty"` +} + +// LagoonMiscInfo defines the resource or backup information for a misc task. +type LagoonMiscInfo struct { + ID string `json:"id"` // should be int, but the api sends it as a string :\ + Name string `json:"name,omitempty"` + Backup *LagoonMiscBackupInfo `json:"backup,omitempty"` + MiscResource []byte `json:"miscResource,omitempty"` +} + +// LagoonMiscBackupInfo defines the information for a backup. +type LagoonMiscBackupInfo struct { + ID string `json:"id"` // should be int, but the api sends it as a string :\ + Source string `json:"source"` + BackupID string `json:"backupId"` +} + +// LagoonTaskProject defines the lagoon project information. +type LagoonTaskProject struct { + ID string `json:"id"` // should be int, but the api sends it as a string :\ + Name string `json:"name"` + NamespacePattern string `json:"namespacePattern,omitempty"` + Variables LagoonVariables `json:"variables,omitempty"` + Organization *Organization `json:"organization,omitempty"` +} + +// LagoonTaskEnvironment defines the lagoon environment information. +type LagoonTaskEnvironment struct { + ID string `json:"id"` // should be int, but the api sends it as a string :\ + Name string `json:"name"` + Project string `json:"project"` // should be int, but the api sends it as a string :\ + EnvironmentType string `json:"environmentType"` +} + +// LagoonTaskStatus defines the observed state of LagoonTask +type LagoonTaskStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file + Conditions []LagoonTaskConditions `json:"conditions,omitempty"` + Log []byte `json:"log,omitempty"` +} + +// LagoonTaskConditions defines the observed conditions of task pods. +type LagoonTaskConditions struct { + LastTransitionTime string `json:"lastTransitionTime"` + Status corev1.ConditionStatus `json:"status"` + Type TaskStatusType `json:"type"` + // Condition string `json:"condition"` +} + +func init() { + SchemeBuilder.Register(&LagoonTask{}, &LagoonTaskList{}) +} + +// this is a custom unmarshal function that will check deployerToken and sshKey which come from Lagoon as `1|0` booleans because javascript +// this converts them from floats to bools +func (a *LagoonAdvancedTaskInfo) UnmarshalJSON(data []byte) error { + tmpMap := map[string]interface{}{} + json.Unmarshal(data, &tmpMap) + if value, ok := tmpMap["deployerToken"]; ok { + if reflect.TypeOf(value).Kind() == reflect.Float64 { + vBool, err := strconv.ParseBool(fmt.Sprintf("%v", value)) + if err == nil { + a.DeployerToken = vBool + } + } + if reflect.TypeOf(value).Kind() == reflect.Bool { + a.DeployerToken = value.(bool) + } + } + if value, ok := tmpMap["sshKey"]; ok { + if reflect.TypeOf(value).Kind() == reflect.Float64 { + vBool, err := strconv.ParseBool(fmt.Sprintf("%v", value)) + if err == nil { + a.SSHKey = vBool + } + } + if reflect.TypeOf(value).Kind() == reflect.Bool { + a.SSHKey = value.(bool) + } + } + if value, ok := tmpMap["RunnerImage"]; ok { + a.RunnerImage = value.(string) + } + if value, ok := tmpMap["runnerImage"]; ok { + a.RunnerImage = value.(string) + } + if value, ok := tmpMap["JSONPayload"]; ok { + a.JSONPayload = value.(string) + } + return nil +} diff --git a/apis/lagoon/v1beta2/zz_generated.deepcopy.go b/apis/lagoon/v1beta2/zz_generated.deepcopy.go new file mode 100644 index 00000000..54401dc6 --- /dev/null +++ b/apis/lagoon/v1beta2/zz_generated.deepcopy.go @@ -0,0 +1,539 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1beta2 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Branch) DeepCopyInto(out *Branch) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Branch. +func (in *Branch) DeepCopy() *Branch { + if in == nil { + return nil + } + out := new(Branch) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Build) DeepCopyInto(out *Build) { + *out = *in + if in.Priority != nil { + in, out := &in.Priority, &out.Priority + *out = new(int) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Build. +func (in *Build) DeepCopy() *Build { + if in == nil { + return nil + } + out := new(Build) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LagoonAdvancedTaskInfo) DeepCopyInto(out *LagoonAdvancedTaskInfo) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonAdvancedTaskInfo. +func (in *LagoonAdvancedTaskInfo) DeepCopy() *LagoonAdvancedTaskInfo { + if in == nil { + return nil + } + out := new(LagoonAdvancedTaskInfo) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LagoonBuild) DeepCopyInto(out *LagoonBuild) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonBuild. +func (in *LagoonBuild) DeepCopy() *LagoonBuild { + if in == nil { + return nil + } + out := new(LagoonBuild) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *LagoonBuild) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LagoonBuildConditions) DeepCopyInto(out *LagoonBuildConditions) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonBuildConditions. +func (in *LagoonBuildConditions) DeepCopy() *LagoonBuildConditions { + if in == nil { + return nil + } + out := new(LagoonBuildConditions) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LagoonBuildList) DeepCopyInto(out *LagoonBuildList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]LagoonBuild, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonBuildList. +func (in *LagoonBuildList) DeepCopy() *LagoonBuildList { + if in == nil { + return nil + } + out := new(LagoonBuildList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *LagoonBuildList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LagoonBuildSpec) DeepCopyInto(out *LagoonBuildSpec) { + *out = *in + in.Build.DeepCopyInto(&out.Build) + in.Project.DeepCopyInto(&out.Project) + out.Branch = in.Branch + out.Pullrequest = in.Pullrequest + out.Promote = in.Promote +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonBuildSpec. +func (in *LagoonBuildSpec) DeepCopy() *LagoonBuildSpec { + if in == nil { + return nil + } + out := new(LagoonBuildSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LagoonBuildStatus) DeepCopyInto(out *LagoonBuildStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]LagoonBuildConditions, len(*in)) + copy(*out, *in) + } + if in.Log != nil { + in, out := &in.Log, &out.Log + *out = make([]byte, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonBuildStatus. +func (in *LagoonBuildStatus) DeepCopy() *LagoonBuildStatus { + if in == nil { + return nil + } + out := new(LagoonBuildStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LagoonMiscBackupInfo) DeepCopyInto(out *LagoonMiscBackupInfo) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonMiscBackupInfo. +func (in *LagoonMiscBackupInfo) DeepCopy() *LagoonMiscBackupInfo { + if in == nil { + return nil + } + out := new(LagoonMiscBackupInfo) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LagoonMiscInfo) DeepCopyInto(out *LagoonMiscInfo) { + *out = *in + if in.Backup != nil { + in, out := &in.Backup, &out.Backup + *out = new(LagoonMiscBackupInfo) + **out = **in + } + if in.MiscResource != nil { + in, out := &in.MiscResource, &out.MiscResource + *out = make([]byte, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonMiscInfo. +func (in *LagoonMiscInfo) DeepCopy() *LagoonMiscInfo { + if in == nil { + return nil + } + out := new(LagoonMiscInfo) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LagoonTask) DeepCopyInto(out *LagoonTask) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonTask. +func (in *LagoonTask) DeepCopy() *LagoonTask { + if in == nil { + return nil + } + out := new(LagoonTask) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *LagoonTask) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LagoonTaskConditions) DeepCopyInto(out *LagoonTaskConditions) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonTaskConditions. +func (in *LagoonTaskConditions) DeepCopy() *LagoonTaskConditions { + if in == nil { + return nil + } + out := new(LagoonTaskConditions) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LagoonTaskEnvironment) DeepCopyInto(out *LagoonTaskEnvironment) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonTaskEnvironment. +func (in *LagoonTaskEnvironment) DeepCopy() *LagoonTaskEnvironment { + if in == nil { + return nil + } + out := new(LagoonTaskEnvironment) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LagoonTaskList) DeepCopyInto(out *LagoonTaskList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]LagoonTask, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonTaskList. +func (in *LagoonTaskList) DeepCopy() *LagoonTaskList { + if in == nil { + return nil + } + out := new(LagoonTaskList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *LagoonTaskList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LagoonTaskProject) DeepCopyInto(out *LagoonTaskProject) { + *out = *in + in.Variables.DeepCopyInto(&out.Variables) + if in.Organization != nil { + in, out := &in.Organization, &out.Organization + *out = new(Organization) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonTaskProject. +func (in *LagoonTaskProject) DeepCopy() *LagoonTaskProject { + if in == nil { + return nil + } + out := new(LagoonTaskProject) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LagoonTaskSpec) DeepCopyInto(out *LagoonTaskSpec) { + *out = *in + out.Task = in.Task + in.Project.DeepCopyInto(&out.Project) + out.Environment = in.Environment + if in.Misc != nil { + in, out := &in.Misc, &out.Misc + *out = new(LagoonMiscInfo) + (*in).DeepCopyInto(*out) + } + if in.AdvancedTask != nil { + in, out := &in.AdvancedTask, &out.AdvancedTask + *out = new(LagoonAdvancedTaskInfo) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonTaskSpec. +func (in *LagoonTaskSpec) DeepCopy() *LagoonTaskSpec { + if in == nil { + return nil + } + out := new(LagoonTaskSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LagoonTaskStatus) DeepCopyInto(out *LagoonTaskStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]LagoonTaskConditions, len(*in)) + copy(*out, *in) + } + if in.Log != nil { + in, out := &in.Log, &out.Log + *out = make([]byte, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonTaskStatus. +func (in *LagoonTaskStatus) DeepCopy() *LagoonTaskStatus { + if in == nil { + return nil + } + out := new(LagoonTaskStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LagoonVariables) DeepCopyInto(out *LagoonVariables) { + *out = *in + if in.Project != nil { + in, out := &in.Project, &out.Project + *out = make([]byte, len(*in)) + copy(*out, *in) + } + if in.Environment != nil { + in, out := &in.Environment, &out.Environment + *out = make([]byte, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonVariables. +func (in *LagoonVariables) DeepCopy() *LagoonVariables { + if in == nil { + return nil + } + out := new(LagoonVariables) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Monitoring) DeepCopyInto(out *Monitoring) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Monitoring. +func (in *Monitoring) DeepCopy() *Monitoring { + if in == nil { + return nil + } + out := new(Monitoring) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Organization) DeepCopyInto(out *Organization) { + *out = *in + if in.ID != nil { + in, out := &in.ID, &out.ID + *out = new(uint) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Organization. +func (in *Organization) DeepCopy() *Organization { + if in == nil { + return nil + } + out := new(Organization) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Project) DeepCopyInto(out *Project) { + *out = *in + if in.ID != nil { + in, out := &in.ID, &out.ID + *out = new(uint) + **out = **in + } + if in.EnvironmentID != nil { + in, out := &in.EnvironmentID, &out.EnvironmentID + *out = new(uint) + **out = **in + } + if in.Key != nil { + in, out := &in.Key, &out.Key + *out = make([]byte, len(*in)) + copy(*out, *in) + } + out.Monitoring = in.Monitoring + in.Variables.DeepCopyInto(&out.Variables) + if in.EnvironmentIdling != nil { + in, out := &in.EnvironmentIdling, &out.EnvironmentIdling + *out = new(int) + **out = **in + } + if in.ProjectIdling != nil { + in, out := &in.ProjectIdling, &out.ProjectIdling + *out = new(int) + **out = **in + } + if in.StorageCalculator != nil { + in, out := &in.StorageCalculator, &out.StorageCalculator + *out = new(int) + **out = **in + } + if in.Organization != nil { + in, out := &in.Organization, &out.Organization + *out = new(Organization) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Project. +func (in *Project) DeepCopy() *Project { + if in == nil { + return nil + } + out := new(Project) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Promote) DeepCopyInto(out *Promote) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Promote. +func (in *Promote) DeepCopy() *Promote { + if in == nil { + return nil + } + out := new(Promote) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Pullrequest) DeepCopyInto(out *Pullrequest) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Pullrequest. +func (in *Pullrequest) DeepCopy() *Pullrequest { + if in == nil { + return nil + } + out := new(Pullrequest) + in.DeepCopyInto(out) + return out +} diff --git a/config/crd/bases/crd.lagoon.sh_lagoonbuilds.yaml b/config/crd/bases/crd.lagoon.sh_lagoonbuilds.yaml index bbc2dc2d..4d8db47c 100644 --- a/config/crd/bases/crd.lagoon.sh_lagoonbuilds.yaml +++ b/config/crd/bases/crd.lagoon.sh_lagoonbuilds.yaml @@ -16,7 +16,9 @@ spec: singular: lagoonbuild scope: Namespaced versions: - - name: v1beta1 + - deprecated: true + deprecationWarning: use lagoonbuilds.crd.lagoon.sh/v1beta2 + name: v1beta1 schema: openAPIV3Schema: description: LagoonBuild is the Schema for the lagoonbuilds API @@ -574,6 +576,194 @@ spec: type: object type: object served: true + storage: false + - name: v1beta2 + schema: + openAPIV3Schema: + description: LagoonBuild is the Schema for the lagoonbuilds API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: LagoonBuildSpec defines the desired state of LagoonBuild + properties: + branch: + description: Branch contains the branch name used for a branch deployment. + properties: + name: + type: string + type: object + build: + description: Build contains the type of build, and the image to use + for the builder. + properties: + bulkId: + type: string + ci: + type: string + image: + type: string + priority: + type: integer + type: + type: string + required: + - type + type: object + gitReference: + type: string + project: + description: Project contains the project information from lagoon. + properties: + deployTarget: + type: string + environment: + type: string + environmentId: + type: integer + environmentIdling: + type: integer + environmentType: + type: string + gitUrl: + type: string + id: + type: integer + key: + format: byte + type: string + monitoring: + description: Monitoring contains the monitoring information for + the project in Lagoon. + properties: + contact: + type: string + statuspageID: + type: string + type: object + name: + type: string + namespacePattern: + type: string + organization: + properties: + id: + type: integer + name: + type: string + type: object + productionEnvironment: + type: string + projectIdling: + type: integer + projectSecret: + type: string + registry: + type: string + routerPattern: + type: string + standbyEnvironment: + type: string + storageCalculator: + type: integer + subfolder: + type: string + uiLink: + type: string + variables: + description: Variables contains the project and environment variables + from lagoon. + properties: + environment: + format: byte + type: string + project: + format: byte + type: string + type: object + required: + - deployTarget + - environment + - environmentType + - gitUrl + - key + - monitoring + - name + - productionEnvironment + - projectSecret + - standbyEnvironment + - variables + type: object + promote: + description: Promote contains the information for a promote deployment. + properties: + sourceEnvironment: + type: string + sourceProject: + type: string + type: object + pullrequest: + description: Pullrequest contains the information for a pullrequest + deployment. + properties: + baseBranch: + type: string + baseSha: + type: string + headBranch: + type: string + headSha: + type: string + number: + type: string + title: + type: string + type: object + required: + - build + - gitReference + - project + type: object + status: + description: LagoonBuildStatus defines the observed state of LagoonBuild + properties: + conditions: + description: 'INSERT ADDITIONAL STATUS FIELD - define observed state + of cluster Important: Run "make" to regenerate code after modifying + this file' + items: + description: LagoonBuildConditions defines the observed conditions + of build pods. + properties: + lastTransitionTime: + type: string + status: + type: string + type: + description: BuildStatusType const for the status type + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + log: + format: byte + type: string + type: object + type: object + served: true storage: true status: acceptedNames: diff --git a/config/crd/bases/crd.lagoon.sh_lagoontasks.yaml b/config/crd/bases/crd.lagoon.sh_lagoontasks.yaml index f1394f91..0f88b0bd 100644 --- a/config/crd/bases/crd.lagoon.sh_lagoontasks.yaml +++ b/config/crd/bases/crd.lagoon.sh_lagoontasks.yaml @@ -16,7 +16,9 @@ spec: singular: lagoontask scope: Namespaced versions: - - name: v1beta1 + - deprecated: true + deprecationWarning: use lagoontasks.crd.lagoon.sh/v1beta2 + name: v1beta1 schema: openAPIV3Schema: description: LagoonTask is the Schema for the lagoontasks API @@ -556,6 +558,176 @@ spec: type: object type: object served: true + storage: false + - name: v1beta2 + schema: + openAPIV3Schema: + description: LagoonTask is the Schema for the lagoontasks API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: LagoonTaskSpec defines the desired state of LagoonTask + properties: + advancedTask: + description: LagoonAdvancedTaskInfo defines what an advanced task + can use for the creation of the pod. + properties: + JSONPayload: + type: string + deployerToken: + type: boolean + runnerImage: + type: string + sshKey: + type: boolean + type: object + environment: + description: LagoonTaskEnvironment defines the lagoon environment + information. + properties: + environmentType: + type: string + id: + type: string + name: + type: string + project: + type: string + required: + - environmentType + - id + - name + - project + type: object + key: + description: 'INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + Important: Run "make" to regenerate code after modifying this file' + type: string + misc: + description: LagoonMiscInfo defines the resource or backup information + for a misc task. + properties: + backup: + description: LagoonMiscBackupInfo defines the information for + a backup. + properties: + backupId: + type: string + id: + type: string + source: + type: string + required: + - backupId + - id + - source + type: object + id: + type: string + miscResource: + format: byte + type: string + name: + type: string + required: + - id + type: object + project: + description: LagoonTaskProject defines the lagoon project information. + properties: + id: + type: string + name: + type: string + namespacePattern: + type: string + organization: + properties: + id: + type: integer + name: + type: string + type: object + variables: + description: Variables contains the project and environment variables + from lagoon. + properties: + environment: + format: byte + type: string + project: + format: byte + type: string + type: object + required: + - id + - name + type: object + task: + description: LagoonTaskInfo defines what a task can use to communicate + with Lagoon via SSH/API. + properties: + apiHost: + type: string + command: + type: string + id: + type: string + name: + type: string + service: + type: string + sshHost: + type: string + sshPort: + type: string + taskName: + type: string + required: + - id + type: object + type: object + status: + description: LagoonTaskStatus defines the observed state of LagoonTask + properties: + conditions: + description: 'INSERT ADDITIONAL STATUS FIELD - define observed state + of cluster Important: Run "make" to regenerate code after modifying + this file' + items: + description: LagoonTaskConditions defines the observed conditions + of task pods. + properties: + lastTransitionTime: + type: string + status: + type: string + type: + description: TaskStatusType const for the status type + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + log: + format: byte + type: string + type: object + type: object + served: true storage: true status: acceptedNames: diff --git a/controller-test.sh b/controller-test.sh index 014247c9..c51aa2f1 100755 --- a/controller-test.sh +++ b/controller-test.sh @@ -18,6 +18,9 @@ NS=nginx-example-main LBUILD=7m5zypx LBUILD2=8m5zypx LBUILD3=9m5zypx +LBUILD4=1m5zypx + +LATEST_CRD_VERSION=v1beta2 HARBOR_VERSION=${HARBOR_VERSION:-1.6.4} @@ -233,6 +236,15 @@ else echo "===> label exists" fi +echo "==> deprecated v1beta1 api: Trigger a lagoon build using kubectl apply" +kubectl -n $CONTROLLER_NAMESPACE apply -f test-resources/example-project3.yaml +# patch the resource with the controller namespace +kubectl -n $CONTROLLER_NAMESPACE patch lagoonbuilds.v1beta1.crd.lagoon.sh lagoon-build-${LBUILD4} --type=merge --patch '{"metadata":{"labels":{"lagoon.sh/controller":"'$CONTROLLER_NAMESPACE'"}}}' +# patch the resource with a random label to bump the controller event filter +kubectl -n $CONTROLLER_NAMESPACE patch lagoonbuilds.v1beta1.crd.lagoon.sh lagoon-build-${LBUILD4} --type=merge --patch '{"metadata":{"labels":{"bump":"bump"}}}' +sleep 10 +check_lagoon_build lagoon-build-${LBUILD4} + echo "==> Trigger a Task using kubectl apply to test dynamic secret mounting" kubectl -n $NS apply -f test-resources/dynamic-secret-in-task-project1-secret.yaml @@ -344,7 +356,9 @@ else fi done echo "==> Pod cleanup output (should only be 1 lagoon-build pod)" -POD_CLEANUP_OUTPUT=$(kubectl -n nginx-example-main get pods | grep "lagoon-build") + +# only check +POD_CLEANUP_OUTPUT=$(kubectl -n nginx-example-main get pods -l crd.lagoon.sh/version=${LATEST_CRD_VERSION} | grep "lagoon-build") echo "${POD_CLEANUP_OUTPUT}" POD_CLEANUP_COUNT=$(echo "${POD_CLEANUP_OUTPUT}" | wc -l | tr -d " ") if [ $POD_CLEANUP_COUNT -gt 1 ]; then diff --git a/controllers/v1beta1/build_helpers.go b/controllers/v1beta1/build_helpers.go index 0ff88268..5b89b5ff 100644 --- a/controllers/v1beta1/build_helpers.go +++ b/controllers/v1beta1/build_helpers.go @@ -18,7 +18,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "github.com/go-logr/logr" - "github.com/prometheus/client_golang/prometheus" lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" "github.com/uselagoon/remote-controller/internal/harbor" "github.com/uselagoon/remote-controller/internal/helpers" @@ -871,16 +870,6 @@ func (r *LagoonBuildReconciler) processBuild(ctx context.Context, opLog logr.Log // @TODO: should update the build to failed return nil } - buildRunningStatus.With(prometheus.Labels{ - "build_namespace": lagoonBuild.ObjectMeta.Namespace, - "build_name": lagoonBuild.ObjectMeta.Name, - }).Set(1) - buildStatus.With(prometheus.Labels{ - "build_namespace": lagoonBuild.ObjectMeta.Namespace, - "build_name": lagoonBuild.ObjectMeta.Name, - "build_step": "running", - }).Set(1) - buildsStartedCounter.Inc() // then break out of the build } opLog.Info(fmt.Sprintf("Build pod already running for: %s", lagoonBuild.ObjectMeta.Name)) diff --git a/controllers/v1beta1/podmonitor_buildhandlers.go b/controllers/v1beta1/podmonitor_buildhandlers.go index 8ddac801..496b18d9 100644 --- a/controllers/v1beta1/podmonitor_buildhandlers.go +++ b/controllers/v1beta1/podmonitor_buildhandlers.go @@ -11,7 +11,6 @@ import ( "time" "github.com/go-logr/logr" - "github.com/prometheus/client_golang/prometheus" "github.com/uselagoon/machinery/api/schema" lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" "github.com/uselagoon/remote-controller/internal/helpers" @@ -247,12 +246,6 @@ func (r *LagoonMonitorReconciler) updateDeploymentAndEnvironmentTask(ctx context buildStep = value } if condition == "failed" || condition == "complete" || condition == "cancelled" { - time.AfterFunc(31*time.Second, func() { - buildRunningStatus.Delete(prometheus.Labels{ - "build_namespace": lagoonBuild.ObjectMeta.Namespace, - "build_name": lagoonBuild.ObjectMeta.Name, - }) - }) time.Sleep(2 * time.Second) // smol sleep to reduce race of final messages with previous messages } envName := lagoonBuild.Spec.Project.Environment diff --git a/controllers/v1beta1/podmonitor_controller.go b/controllers/v1beta1/podmonitor_controller.go index a1ea2406..da169187 100644 --- a/controllers/v1beta1/podmonitor_controller.go +++ b/controllers/v1beta1/podmonitor_controller.go @@ -75,10 +75,6 @@ func (r *LagoonMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Reques // if this is a lagoon task, then run the handle task monitoring process if jobPod.ObjectMeta.Labels["lagoon.sh/jobType"] == "task" { - err := r.calculateTaskMetrics(ctx) - if err != nil { - opLog.Error(err, fmt.Sprintf("Unable to generate metrics.")) - } if jobPod.ObjectMeta.DeletionTimestamp.IsZero() { // pod is not being deleted return ctrl.Result{}, r.handleTaskMonitor(ctx, opLog, req, jobPod) @@ -100,10 +96,6 @@ func (r *LagoonMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Reques } // if this is a lagoon build, then run the handle build monitoring process if jobPod.ObjectMeta.Labels["lagoon.sh/jobType"] == "build" { - err := r.calculateBuildMetrics(ctx) - if err != nil { - opLog.Error(err, fmt.Sprintf("Unable to generate metrics.")) - } if jobPod.ObjectMeta.DeletionTimestamp.IsZero() { // pod is not being deleted return ctrl.Result{}, r.handleBuildMonitor(ctx, opLog, req, jobPod) @@ -113,7 +105,7 @@ func (r *LagoonMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Reques // first try and clean up the pod and capture the logs and update // the lagoonbuild that owns it with the status var lagoonBuild lagoonv1beta1.LagoonBuild - err = r.Get(ctx, types.NamespacedName{ + err := r.Get(ctx, types.NamespacedName{ Namespace: jobPod.ObjectMeta.Namespace, Name: jobPod.ObjectMeta.Labels["lagoon.sh/buildName"], }, &lagoonBuild) diff --git a/controllers/v1beta1/podmonitor_taskhandlers.go b/controllers/v1beta1/podmonitor_taskhandlers.go index c5e34c1f..0e25f851 100644 --- a/controllers/v1beta1/podmonitor_taskhandlers.go +++ b/controllers/v1beta1/podmonitor_taskhandlers.go @@ -11,7 +11,6 @@ import ( "time" "github.com/go-logr/logr" - "github.com/prometheus/client_golang/prometheus" "github.com/uselagoon/machinery/api/schema" lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" "github.com/uselagoon/remote-controller/internal/helpers" @@ -165,12 +164,6 @@ func (r *LagoonMonitorReconciler) updateLagoonTask(ctx context.Context, opLog lo ) (bool, schema.LagoonMessage) { if r.EnableMQ && lagoonTask != nil { if condition == "failed" || condition == "complete" || condition == "cancelled" { - time.AfterFunc(31*time.Second, func() { - taskRunningStatus.Delete(prometheus.Labels{ - "task_namespace": lagoonTask.ObjectMeta.Namespace, - "task_name": lagoonTask.ObjectMeta.Name, - }) - }) time.Sleep(2 * time.Second) // smol sleep to reduce race of final messages with previous messages } msg := schema.LagoonMessage{ diff --git a/controllers/v1beta1/predicates.go b/controllers/v1beta1/predicates.go index afb99a03..a1c4295d 100644 --- a/controllers/v1beta1/predicates.go +++ b/controllers/v1beta1/predicates.go @@ -4,9 +4,7 @@ package v1beta1 import ( "regexp" - "time" - "github.com/prometheus/client_golang/prometheus" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/predicate" ) @@ -70,28 +68,6 @@ func (p PodPredicates) Update(e event.UpdateEvent) bool { if value == crdVersion { if _, okOld := e.ObjectOld.GetLabels()["lagoon.sh/buildName"]; okOld { if value, ok := e.ObjectNew.GetLabels()["lagoon.sh/buildName"]; ok { - oldBuildStep := "running" - newBuildStep := "running" - if value, ok := e.ObjectNew.GetLabels()["lagoon.sh/buildStep"]; ok { - newBuildStep = value - } - if value, ok := e.ObjectOld.GetLabels()["lagoon.sh/buildStep"]; ok { - oldBuildStep = value - } - if newBuildStep != oldBuildStep { - buildStatus.With(prometheus.Labels{ - "build_namespace": e.ObjectOld.GetNamespace(), - "build_name": e.ObjectOld.GetName(), - "build_step": newBuildStep, - }).Set(1) - } - time.AfterFunc(31*time.Second, func() { - buildStatus.Delete(prometheus.Labels{ - "build_namespace": e.ObjectOld.GetNamespace(), - "build_name": e.ObjectOld.GetName(), - "build_step": oldBuildStep, - }) - }) match, _ := regexp.MatchString("^lagoon-build", value) return match } @@ -143,7 +119,9 @@ type BuildPredicates struct { func (b BuildPredicates) Create(e event.CreateEvent) bool { if controller, ok := e.Object.GetLabels()["lagoon.sh/controller"]; ok { if controller == b.ControllerNamespace { - return true + if _, ok := e.Object.GetLabels()["crd.lagoon.sh/version"]; !ok { + return true + } } } return false @@ -153,7 +131,9 @@ func (b BuildPredicates) Create(e event.CreateEvent) bool { func (b BuildPredicates) Delete(e event.DeleteEvent) bool { if controller, ok := e.Object.GetLabels()["lagoon.sh/controller"]; ok { if controller == b.ControllerNamespace { - return true + if _, ok := e.Object.GetLabels()["crd.lagoon.sh/version"]; !ok { + return true + } } } return false @@ -163,7 +143,9 @@ func (b BuildPredicates) Delete(e event.DeleteEvent) bool { func (b BuildPredicates) Update(e event.UpdateEvent) bool { if controller, ok := e.ObjectOld.GetLabels()["lagoon.sh/controller"]; ok { if controller == b.ControllerNamespace { - return true + if _, ok := e.ObjectNew.GetLabels()["crd.lagoon.sh/version"]; !ok { + return true + } } } return false @@ -173,7 +155,9 @@ func (b BuildPredicates) Update(e event.UpdateEvent) bool { func (b BuildPredicates) Generic(e event.GenericEvent) bool { if controller, ok := e.Object.GetLabels()["lagoon.sh/controller"]; ok { if controller == b.ControllerNamespace { - return true + if _, ok := e.Object.GetLabels()["crd.lagoon.sh/version"]; !ok { + return true + } } } return false @@ -190,7 +174,9 @@ type TaskPredicates struct { func (t TaskPredicates) Create(e event.CreateEvent) bool { if controller, ok := e.Object.GetLabels()["lagoon.sh/controller"]; ok { if controller == t.ControllerNamespace { - return true + if _, ok := e.Object.GetLabels()["crd.lagoon.sh/version"]; !ok { + return true + } } } return false @@ -200,7 +186,9 @@ func (t TaskPredicates) Create(e event.CreateEvent) bool { func (t TaskPredicates) Delete(e event.DeleteEvent) bool { if controller, ok := e.Object.GetLabels()["lagoon.sh/controller"]; ok { if controller == t.ControllerNamespace { - return true + if _, ok := e.Object.GetLabels()["crd.lagoon.sh/version"]; !ok { + return true + } } } return false @@ -210,7 +198,9 @@ func (t TaskPredicates) Delete(e event.DeleteEvent) bool { func (t TaskPredicates) Update(e event.UpdateEvent) bool { if controller, ok := e.ObjectOld.GetLabels()["lagoon.sh/controller"]; ok { if controller == t.ControllerNamespace { - return true + if _, ok := e.ObjectNew.GetLabels()["crd.lagoon.sh/version"]; !ok { + return true + } } } return false @@ -220,7 +210,9 @@ func (t TaskPredicates) Update(e event.UpdateEvent) bool { func (t TaskPredicates) Generic(e event.GenericEvent) bool { if controller, ok := e.Object.GetLabels()["lagoon.sh/controller"]; ok { if controller == t.ControllerNamespace { - return true + if _, ok := e.Object.GetLabels()["crd.lagoon.sh/version"]; !ok { + return true + } } } return false diff --git a/controllers/v1beta1/task_controller.go b/controllers/v1beta1/task_controller.go index 4119b475..7e34e126 100644 --- a/controllers/v1beta1/task_controller.go +++ b/controllers/v1beta1/task_controller.go @@ -23,7 +23,6 @@ import ( "strconv" "github.com/go-logr/logr" - "github.com/prometheus/client_golang/prometheus" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -332,11 +331,6 @@ func (r *LagoonTaskReconciler) createStandardTask(ctx context.Context, lagoonTas //@TODO: send msg back and update task to failed? return nil } - taskRunningStatus.With(prometheus.Labels{ - "task_namespace": lagoonTask.ObjectMeta.Namespace, - "task_name": lagoonTask.ObjectMeta.Name, - }).Set(1) - tasksStartedCounter.Inc() } else { opLog.Info(fmt.Sprintf("Task pod already running for: %s", lagoonTask.ObjectMeta.Name)) } @@ -623,11 +617,6 @@ func (r *LagoonTaskReconciler) createAdvancedTask(ctx context.Context, lagoonTas ) return err } - taskRunningStatus.With(prometheus.Labels{ - "task_namespace": lagoonTask.ObjectMeta.Namespace, - "task_name": lagoonTask.ObjectMeta.Name, - }).Set(1) - tasksStartedCounter.Inc() } else { opLog.Info(fmt.Sprintf("Advanced task pod already running for: %s", lagoonTask.ObjectMeta.Name)) } diff --git a/controllers/v1beta2/build_controller.go b/controllers/v1beta2/build_controller.go new file mode 100644 index 00000000..a5dc9481 --- /dev/null +++ b/controllers/v1beta2/build_controller.go @@ -0,0 +1,342 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta2 + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + lagooncrd "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" + "github.com/uselagoon/remote-controller/internal/harbor" + "github.com/uselagoon/remote-controller/internal/helpers" + "github.com/uselagoon/remote-controller/internal/messenger" +) + +// LagoonBuildReconciler reconciles a LagoonBuild object +type LagoonBuildReconciler struct { + client.Client + Log logr.Logger + Scheme *runtime.Scheme + EnableMQ bool + Messaging *messenger.Messenger + BuildImage string + NamespacePrefix string + RandomNamespacePrefix bool + ControllerNamespace string + EnableDebug bool + FastlyServiceID string + FastlyWatchStatus bool + // BuildPodRunAsUser sets the build pod securityContext.runAsUser value. + BuildPodRunAsUser int64 + // BuildPodRunAsGroup sets the build pod securityContext.runAsGroup value. + BuildPodRunAsGroup int64 + // BuildPodFSGroup sets the build pod securityContext.fsGroup value. + BuildPodFSGroup int64 + // Lagoon feature flags + LFFForceRootlessWorkload string + LFFDefaultRootlessWorkload string + LFFForceIsolationNetworkPolicy string + LFFDefaultIsolationNetworkPolicy string + LFFForceInsights string + LFFDefaultInsights string + LFFForceRWX2RWO string + LFFDefaultRWX2RWO string + LFFBackupWeeklyRandom bool + LFFRouterURL bool + LFFHarborEnabled bool + BackupConfig BackupConfig + Harbor harbor.Harbor + LFFQoSEnabled bool + BuildQoS BuildQoS + NativeCronPodMinFrequency int + LagoonTargetName string + LagoonFeatureFlags map[string]string + LagoonAPIConfiguration helpers.LagoonAPIConfiguration + ProxyConfig ProxyConfig +} + +// BackupConfig holds all the backup configuration settings +type BackupConfig struct { + BackupDefaultSchedule string + BackupDefaultMonthlyRetention int + BackupDefaultWeeklyRetention int + BackupDefaultDailyRetention int + BackupDefaultHourlyRetention int + + BackupDefaultDevelopmentSchedule string + BackupDefaultPullrequestSchedule string + BackupDefaultDevelopmentRetention string + BackupDefaultPullrequestRetention string +} + +// ProxyConfig is used for proxy configuration. +type ProxyConfig struct { + HTTPProxy string + HTTPSProxy string + NoProxy string +} + +var ( + buildFinalizer = "finalizer.lagoonbuild.crd.lagoon.sh/v1beta2" +) + +// +kubebuilder:rbac:groups=crd.lagoon.sh,resources=lagoonbuilds,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=crd.lagoon.sh,resources=lagoonbuilds/status,verbs=get;update;patch + +// @TODO: all the things for now, review later +// +kubebuilder:rbac:groups="*",resources="*",verbs="*" + +// Reconcile runs when a request comes through +func (r *LagoonBuildReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + opLog := r.Log.WithValues("lagoonbuild", req.NamespacedName) + + // your logic here + var lagoonBuild lagooncrd.LagoonBuild + if err := r.Get(ctx, req.NamespacedName, &lagoonBuild); err != nil { + return ctrl.Result{}, helpers.IgnoreNotFound(err) + } + + // examine DeletionTimestamp to determine if object is under deletion + if lagoonBuild.ObjectMeta.DeletionTimestamp.IsZero() { + // if the build isn't being deleted, but the status is cancelled + // then clean up the undeployable build + if value, ok := lagoonBuild.ObjectMeta.Labels["lagoon.sh/buildStatus"]; ok { + if value == lagooncrd.BuildStatusCancelled.String() { + if value, ok := lagoonBuild.ObjectMeta.Labels["lagoon.sh/cancelledByNewBuild"]; ok { + if value == "true" { + opLog.Info(fmt.Sprintf("Cleaning up build %s as cancelled by new build", lagoonBuild.ObjectMeta.Name)) + r.cleanUpUndeployableBuild(ctx, lagoonBuild, "This build was cancelled as a newer build was triggered.", opLog, true) + } else { + opLog.Info(fmt.Sprintf("Cleaning up build %s as cancelled", lagoonBuild.ObjectMeta.Name)) + r.cleanUpUndeployableBuild(ctx, lagoonBuild, "", opLog, false) + } + } + } + } + if r.LFFQoSEnabled { + // handle QoS builds here + // if we do have a `lagoon.sh/buildStatus` set as running, then process it + runningNSBuilds := &lagooncrd.LagoonBuildList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(req.Namespace), + client.MatchingLabels(map[string]string{ + "lagoon.sh/buildStatus": lagooncrd.BuildStatusRunning.String(), + "lagoon.sh/controller": r.ControllerNamespace, + }), + }) + // list any builds that are running + if err := r.List(ctx, runningNSBuilds, listOption); err != nil { + return ctrl.Result{}, fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err) + } + for _, runningBuild := range runningNSBuilds.Items { + // if the running build is the one from this request then process it + if lagoonBuild.ObjectMeta.Name == runningBuild.ObjectMeta.Name { + // actually process the build here + if _, ok := lagoonBuild.ObjectMeta.Labels["lagoon.sh/buildStarted"]; !ok { + if err := r.processBuild(ctx, opLog, lagoonBuild); err != nil { + return ctrl.Result{}, err + } + } + } // end check if running build is current LagoonBuild + } // end loop for running builds + // once running builds are processed, run the qos handler + return r.qosBuildProcessor(ctx, opLog, lagoonBuild, req) + } + // if qos is not enabled, just process it as a standard build + return r.standardBuildProcessor(ctx, opLog, lagoonBuild, req) + } + // The object is being deleted + if helpers.ContainsString(lagoonBuild.ObjectMeta.Finalizers, buildFinalizer) { + // our finalizer is present, so lets handle any external dependency + // first deleteExternalResources will try and check for any pending builds that it can + // can change to running to kick off the next pending build + if err := r.deleteExternalResources(ctx, + opLog, + &lagoonBuild, + req, + ); err != nil { + // if fail to delete the external dependency here, return with error + // so that it can be retried + opLog.Error(err, fmt.Sprintf("Unable to delete external resources")) + return ctrl.Result{}, err + } + // remove our finalizer from the list and update it. + lagoonBuild.ObjectMeta.Finalizers = helpers.RemoveString(lagoonBuild.ObjectMeta.Finalizers, buildFinalizer) + // use patches to avoid update errors + mergePatch, _ := json.Marshal(map[string]interface{}{ + "metadata": map[string]interface{}{ + "finalizers": lagoonBuild.ObjectMeta.Finalizers, + }, + }) + if err := r.Patch(ctx, &lagoonBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { + return ctrl.Result{}, helpers.IgnoreNotFound(err) + } + } + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the given manager +// and we set it to watch LagoonBuilds +func (r *LagoonBuildReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&lagooncrd.LagoonBuild{}). + WithEventFilter(BuildPredicates{ + ControllerNamespace: r.ControllerNamespace, + }). + Complete(r) +} + +func (r *LagoonBuildReconciler) createNamespaceBuild(ctx context.Context, + opLog logr.Logger, + lagoonBuild lagooncrd.LagoonBuild) (ctrl.Result, error) { + + namespace := &corev1.Namespace{} + if r.EnableDebug { + opLog.Info(fmt.Sprintf("Checking Namespace exists for: %s", lagoonBuild.ObjectMeta.Name)) + } + err := r.getOrCreateNamespace(ctx, namespace, lagoonBuild, opLog) + if err != nil { + return ctrl.Result{}, err + } + // create the `lagoon-deployer` ServiceAccount + if r.EnableDebug { + opLog.Info(fmt.Sprintf("Checking `lagoon-deployer` ServiceAccount exists: %s", lagoonBuild.ObjectMeta.Name)) + } + serviceAccount := &corev1.ServiceAccount{} + err = r.getOrCreateServiceAccount(ctx, serviceAccount, namespace.ObjectMeta.Name) + if err != nil { + return ctrl.Result{}, err + } + // ServiceAccount RoleBinding creation + if r.EnableDebug { + opLog.Info(fmt.Sprintf("Checking `lagoon-deployer-admin` RoleBinding exists: %s", lagoonBuild.ObjectMeta.Name)) + } + saRoleBinding := &rbacv1.RoleBinding{} + err = r.getOrCreateSARoleBinding(ctx, saRoleBinding, namespace.ObjectMeta.Name) + if err != nil { + return ctrl.Result{}, err + } + + // Get or create the lagoon-env configmap + lagoonEnvConfigMap := &corev1.ConfigMap{} + if r.EnableDebug { + opLog.Info("Checking `lagoon-env` configMap exists - creating if not") + } + err = r.getOrCreateConfigMap(ctx, "lagoon-env", lagoonEnvConfigMap, namespace.ObjectMeta.Name) + if err != nil { + return ctrl.Result{}, err + } + + // copy the build resource into a new resource and set the status to pending + // create the new resource and the controller will handle it via queue + opLog.Info(fmt.Sprintf("Creating LagoonBuild in Pending status: %s", lagoonBuild.ObjectMeta.Name)) + err = r.getOrCreateBuildResource(ctx, &lagoonBuild, namespace.ObjectMeta.Name) + if err != nil { + return ctrl.Result{}, err + } + + // if everything is all good controller will handle the new build resource that gets created as it will have + // the `lagoon.sh/buildStatus = Pending` now + err = lagooncrd.CancelExtraBuilds(ctx, r.Client, opLog, namespace.ObjectMeta.Name, lagooncrd.BuildStatusPending.String()) + if err != nil { + return ctrl.Result{}, err + } + + // as this is a new build coming through, check if there are any running builds in the namespace + // if there are, then check the status of that build. if the build pod is missing then the build running will block + // if the pod exists, attempt to get the status of it (only if its complete or failed) and ship the status + runningBuilds := &lagooncrd.LagoonBuildList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(namespace.ObjectMeta.Name), + client.MatchingLabels(map[string]string{"lagoon.sh/buildStatus": lagooncrd.BuildStatusRunning.String()}), + }) + // list all builds in the namespace that have the running buildstatus + if err := r.List(ctx, runningBuilds, listOption); err != nil { + return ctrl.Result{}, fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err) + } + // if there are running builds still, check if the pod exists or if the pod is complete/failed and attempt to get the status + for _, rBuild := range runningBuilds.Items { + runningBuild := rBuild.DeepCopy() + lagoonBuildPod := corev1.Pod{} + err := r.Get(ctx, types.NamespacedName{ + Namespace: rBuild.ObjectMeta.Namespace, + Name: rBuild.ObjectMeta.Name, + }, &lagoonBuildPod) + buildCondition := lagooncrd.BuildStatusCancelled + if err != nil { + // cancel the build as there is no pod available + opLog.Info(fmt.Sprintf("Setting build %s as cancelled", runningBuild.ObjectMeta.Name)) + runningBuild.Labels["lagoon.sh/buildStatus"] = buildCondition.String() + } else { + // get the status from the pod and update the build + if lagoonBuildPod.Status.Phase == corev1.PodFailed || lagoonBuildPod.Status.Phase == corev1.PodSucceeded { + buildCondition = lagooncrd.GetBuildConditionFromPod(lagoonBuildPod.Status.Phase) + opLog.Info(fmt.Sprintf("Setting build %s as %s", runningBuild.ObjectMeta.Name, buildCondition.String())) + runningBuild.Labels["lagoon.sh/buildStatus"] = buildCondition.String() + } else { + // drop out, don't do anything else + continue + } + } + if err := r.Update(ctx, runningBuild); err != nil { + // log the error and drop out + opLog.Error(err, fmt.Sprintf("Error setting build %s as cancelled", runningBuild.ObjectMeta.Name)) + continue + } + // send the status change to lagoon + r.updateDeploymentAndEnvironmentTask(ctx, opLog, runningBuild, nil, buildCondition, "cancelled") + continue + } + // handle processing running but no pod/failed pod builds + return ctrl.Result{}, nil +} + +// getOrCreateBuildResource will deepcopy the lagoon build into a new resource and push it to the new namespace +// then clean up the old one. +func (r *LagoonBuildReconciler) getOrCreateBuildResource(ctx context.Context, lagoonBuild *lagooncrd.LagoonBuild, ns string) error { + newBuild := lagoonBuild.DeepCopy() + newBuild.SetNamespace(ns) + newBuild.SetResourceVersion("") + newBuild.SetLabels( + map[string]string{ + "lagoon.sh/buildStatus": lagooncrd.BuildStatusPending.String(), + "lagoon.sh/controller": r.ControllerNamespace, + }, + ) + err := r.Get(ctx, types.NamespacedName{ + Namespace: ns, + Name: newBuild.ObjectMeta.Name, + }, newBuild) + if err != nil { + if err := r.Create(ctx, newBuild); err != nil { + return err + } + } + err = r.Delete(ctx, lagoonBuild) + if err != nil { + return err + } + return nil +} diff --git a/controllers/v1beta2/build_deletionhandlers.go b/controllers/v1beta2/build_deletionhandlers.go new file mode 100644 index 00000000..bfe90ccb --- /dev/null +++ b/controllers/v1beta2/build_deletionhandlers.go @@ -0,0 +1,514 @@ +package v1beta2 + +// this file is used by the `lagoonbuild` controller + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + "github.com/go-logr/logr" + "github.com/uselagoon/machinery/api/schema" + lagooncrd "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" + "github.com/uselagoon/remote-controller/internal/helpers" + "gopkg.in/matryer/try.v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// handle deleting any external resources here +func (r *LagoonBuildReconciler) deleteExternalResources( + ctx context.Context, + opLog logr.Logger, + lagoonBuild *lagooncrd.LagoonBuild, + req ctrl.Request, +) error { + // get any running pods that this build may have already created + lagoonBuildPod := corev1.Pod{} + err := r.Get(ctx, types.NamespacedName{ + Namespace: lagoonBuild.ObjectMeta.Namespace, + Name: lagoonBuild.ObjectMeta.Name, + }, &lagoonBuildPod) + if err != nil { + opLog.Info(fmt.Sprintf("Unable to find a build pod for %s, continuing to process build deletion", lagoonBuild.ObjectMeta.Name)) + // handle updating lagoon for a deleted build with no running pod + // only do it if the build status is Pending or Running though + err = r.updateCancelledDeploymentWithLogs(ctx, req, *lagoonBuild) + if err != nil { + opLog.Error(err, fmt.Sprintf("Unable to update the lagoon with LagoonBuild result")) + } + } else { + opLog.Info(fmt.Sprintf("Found build pod for %s, deleting it", lagoonBuild.ObjectMeta.Name)) + // handle updating lagoon for a deleted build with a running pod + // only do it if the build status is Pending or Running though + // delete the pod, let the pod deletion handler deal with the cleanup there + if err := r.Delete(ctx, &lagoonBuildPod); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to delete the the LagoonBuild pod %s", lagoonBuild.ObjectMeta.Name)) + } + // check that the pod is deleted before continuing, this allows the pod deletion to happen + // and the pod deletion process in the LagoonMonitor controller to be able to send what it needs back to lagoon + // this 1 minute timeout will just hold up the deletion of `LagoonBuild` resources only if a build pod exists + // if the 1 minute timeout is reached the worst that happens is a deployment will show as running + // but cancelling the deployment in lagoon will force it to go to a cancelling state in the lagoon api + // @TODO: we could use finalizers on the build pods, but to avoid holding up other processes we can just give up after waiting for a minute + try.MaxRetries = 12 + err = try.Do(func(attempt int) (bool, error) { + var podErr error + err := r.Get(ctx, types.NamespacedName{ + Namespace: lagoonBuild.ObjectMeta.Namespace, + Name: lagoonBuild.ObjectMeta.Name, + }, &lagoonBuildPod) + if err != nil { + // the pod doesn't exist anymore, so exit the retry + podErr = nil + opLog.Info(fmt.Sprintf("Pod %s deleted", lagoonBuild.ObjectMeta.Name)) + } else { + // if the pod still exists wait 5 seconds before trying again + time.Sleep(5 * time.Second) + podErr = fmt.Errorf("pod %s still exists", lagoonBuild.ObjectMeta.Name) + opLog.Info(fmt.Sprintf("Pod %s still exists", lagoonBuild.ObjectMeta.Name)) + } + return attempt < 12, podErr + }) + if err != nil { + return err + } + } + + // if the LagoonBuild is deleted, then check if the only running build is the one being deleted + // or if there are any pending builds that can be started + runningBuilds := &lagooncrd.LagoonBuildList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(lagoonBuild.ObjectMeta.Namespace), + client.MatchingLabels(map[string]string{ + "lagoon.sh/buildStatus": lagooncrd.BuildStatusRunning.String(), + "lagoon.sh/controller": r.ControllerNamespace, + }), + }) + // list any builds that are running + if err := r.List(ctx, runningBuilds, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list builds in the namespace, there may be none or something went wrong")) + // just return nil so the deletion of the resource isn't held up + return nil + } + newRunningBuilds := runningBuilds.Items + for _, runningBuild := range runningBuilds.Items { + // if there are any running builds, check if it is the one currently being deleted + if lagoonBuild.ObjectMeta.Name == runningBuild.ObjectMeta.Name { + // if the one being deleted is a running one, remove it from the list of running builds + newRunningBuilds = lagooncrd.RemoveBuild(newRunningBuilds, runningBuild) + } + } + // if the number of runningBuilds is 0 (excluding the one being deleted) + if len(newRunningBuilds) == 0 { + pendingBuilds := &lagooncrd.LagoonBuildList{} + listOption = (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(lagoonBuild.ObjectMeta.Namespace), + client.MatchingLabels(map[string]string{ + "lagoon.sh/buildStatus": lagooncrd.BuildStatusPending.String(), + "lagoon.sh/controller": r.ControllerNamespace, + }), + }) + if err := r.List(ctx, pendingBuilds, listOption); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to list builds in the namespace, there may be none or something went wrong")) + // just return nil so the deletion of the resource isn't held up + return nil + } + newPendingBuilds := pendingBuilds.Items + for _, pendingBuild := range pendingBuilds.Items { + // if there are any pending builds, check if it is the one currently being deleted + if lagoonBuild.ObjectMeta.Name == pendingBuild.ObjectMeta.Name { + // if the one being deleted a the pending one, remove it from the list of pending builds + newPendingBuilds = lagooncrd.RemoveBuild(newPendingBuilds, pendingBuild) + } + } + // sort the pending builds by creation timestamp + sort.Slice(newPendingBuilds, func(i, j int) bool { + return newPendingBuilds[i].ObjectMeta.CreationTimestamp.Before(&newPendingBuilds[j].ObjectMeta.CreationTimestamp) + }) + // if there are more than 1 pending builds (excluding the one being deleted), update the oldest one to running + if len(newPendingBuilds) > 0 { + pendingBuild := pendingBuilds.Items[0].DeepCopy() + mergePatch, _ := json.Marshal(map[string]interface{}{ + "metadata": map[string]interface{}{ + "labels": map[string]interface{}{ + "lagoon.sh/buildStatus": lagooncrd.BuildStatusRunning.String(), + }, + }, + }) + if err := r.Patch(ctx, pendingBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to update pending build to running status")) + return nil + } + } else { + opLog.Info(fmt.Sprintf("No pending builds")) + } + } + return nil +} + +func (r *LagoonBuildReconciler) updateCancelledDeploymentWithLogs( + ctx context.Context, + req ctrl.Request, + lagoonBuild lagooncrd.LagoonBuild, +) error { + opLog := r.Log.WithValues("lagoonbuild", req.NamespacedName) + // if the build status is Pending or Running, + // then the buildCondition will be set to cancelled when we tell lagoon + // this is because we are deleting it, so we are basically cancelling it + // if it was already Failed or Completed, lagoon probably already knows + // so we don't have to do anything else. + if helpers.ContainsString( + lagooncrd.BuildRunningPendingStatus, + lagoonBuild.Labels["lagoon.sh/buildStatus"], + ) { + opLog.Info( + fmt.Sprintf( + "Updating build status for %s to %v", + lagoonBuild.ObjectMeta.Name, + lagoonBuild.Labels["lagoon.sh/buildStatus"], + ), + ) + + var allContainerLogs []byte + // if we get this handler, then it is likely that the build was in a pending or running state with no actual running pod + // so just set the logs to be cancellation message + allContainerLogs = []byte(fmt.Sprintf(` +======================================== +Build cancelled +========================================`)) + var buildCondition lagooncrd.BuildStatusType + buildCondition = lagooncrd.BuildStatusCancelled + lagoonBuild.Labels["lagoon.sh/buildStatus"] = buildCondition.String() + mergePatch, _ := json.Marshal(map[string]interface{}{ + "metadata": map[string]interface{}{ + "labels": map[string]interface{}{ + "lagoon.sh/buildStatus": buildCondition.String(), + }, + }, + }) + if err := r.Patch(ctx, &lagoonBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to update build status")) + } + // get the configmap for lagoon-env so we can use it for updating the deployment in lagoon + var lagoonEnv corev1.ConfigMap + err := r.Get(ctx, types.NamespacedName{ + Namespace: lagoonBuild.ObjectMeta.Namespace, + Name: "lagoon-env", + }, + &lagoonEnv, + ) + if err != nil { + // if there isn't a configmap, just info it and move on + // the updatedeployment function will see it as nil and not bother doing the bits that require the configmap + if r.EnableDebug { + opLog.Info(fmt.Sprintf("There is no configmap %s in namespace %s ", "lagoon-env", lagoonBuild.ObjectMeta.Namespace)) + } + } + // send any messages to lagoon message queues + // update the deployment with the status of cancelled in lagoon + r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &lagoonEnv, lagooncrd.BuildStatusCancelled, "cancelled") + r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagooncrd.BuildStatusCancelled, "cancelled") + r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, allContainerLogs, lagooncrd.BuildStatusCancelled) + } + return nil +} + +// buildLogsToLagoonLogs sends the build logs to the lagoon-logs message queue +// it contains the actual pod log output that is sent to elasticsearch, it is what eventually is displayed in the UI +func (r *LagoonBuildReconciler) buildLogsToLagoonLogs(ctx context.Context, + opLog logr.Logger, + lagoonBuild *lagooncrd.LagoonBuild, + logs []byte, + buildCondition lagooncrd.BuildStatusType, +) { + if r.EnableMQ { + condition := buildCondition + buildStep := "queued" + if condition == lagooncrd.BuildStatusCancelled { + buildStep = "cancelled" + } + msg := schema.LagoonLog{ + Severity: "info", + Project: lagoonBuild.Spec.Project.Name, + Event: "build-logs:builddeploy-kubernetes:" + lagoonBuild.ObjectMeta.Name, + Meta: &schema.LagoonLogMeta{ + JobName: lagoonBuild.ObjectMeta.Name, // @TODO: remove once lagoon is corrected in controller-handler + BuildName: lagoonBuild.ObjectMeta.Name, + BuildStatus: buildCondition.ToLower(), // same as buildstatus label + BuildStep: buildStep, + BranchName: lagoonBuild.Spec.Project.Environment, + RemoteID: string(lagoonBuild.ObjectMeta.UID), + LogLink: lagoonBuild.Spec.Project.UILink, + Cluster: r.LagoonTargetName, + }, + } + // add the actual build log message + msg.Message = fmt.Sprintf("%s", logs) + msgBytes, err := json.Marshal(msg) + if err != nil { + opLog.Error(err, "Unable to encode message as JSON") + } + // @TODO: if we can't publish the message because we are deleting the resource, then should we even + // bother to patch the resource?? + // leave it for now cause the resource will just be deleted anyway + if err := r.Messaging.Publish("lagoon-logs", msgBytes); err != nil { + // if we can't publish the message, set it as a pending message + // overwrite whatever is there as these are just current state messages so it doesn't + // really matter if we don't smootly transition in what we send back to lagoon + r.updateBuildLogMessage(ctx, lagoonBuild, msg) + return + } + // if we are able to publish the message, then we need to remove any pending messages from the resource + // and make sure we don't try and publish again + r.removeBuildPendingMessageStatus(ctx, lagoonBuild) + } +} + +// updateDeploymentAndEnvironmentTask sends the status of the build and deployment to the controllerhandler message queue in lagoon, +// this is for the handler in lagoon to process. +func (r *LagoonBuildReconciler) updateDeploymentAndEnvironmentTask(ctx context.Context, + opLog logr.Logger, + lagoonBuild *lagooncrd.LagoonBuild, + lagoonEnv *corev1.ConfigMap, + buildCondition lagooncrd.BuildStatusType, + buildStep string, +) { + namespace := helpers.GenerateNamespaceName( + lagoonBuild.Spec.Project.NamespacePattern, // the namespace pattern or `openshiftProjectPattern` from Lagoon is never received by the controller + lagoonBuild.Spec.Project.Environment, + lagoonBuild.Spec.Project.Name, + r.NamespacePrefix, + r.ControllerNamespace, + r.RandomNamespacePrefix, + ) + if r.EnableMQ { + msg := schema.LagoonMessage{ + Type: "build", + Namespace: namespace, + Meta: &schema.LagoonLogMeta{ + Environment: lagoonBuild.Spec.Project.Environment, + Project: lagoonBuild.Spec.Project.Name, + BuildStatus: buildCondition.ToLower(), + BuildStep: buildStep, + BuildName: lagoonBuild.ObjectMeta.Name, + LogLink: lagoonBuild.Spec.Project.UILink, + RemoteID: string(lagoonBuild.ObjectMeta.UID), + Cluster: r.LagoonTargetName, + }, + } + labelRequirements1, _ := labels.NewRequirement("lagoon.sh/service", selection.NotIn, []string{"faketest"}) + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(lagoonBuild.ObjectMeta.Namespace), + client.MatchingLabelsSelector{ + Selector: labels.NewSelector().Add(*labelRequirements1), + }, + }) + podList := &corev1.PodList{} + serviceNames := []string{} + if err := r.List(context.TODO(), podList, listOption); err == nil { + // generate the list of services to add to the environment + for _, pod := range podList.Items { + if _, ok := pod.ObjectMeta.Labels["lagoon.sh/service"]; ok { + for _, container := range pod.Spec.Containers { + serviceNames = append(serviceNames, container.Name) + } + } + if _, ok := pod.ObjectMeta.Labels["service"]; ok { + for _, container := range pod.Spec.Containers { + serviceNames = append(serviceNames, container.Name) + } + } + } + msg.Meta.Services = serviceNames + } + // if we aren't being provided the lagoon config, we can skip adding the routes etc + if lagoonEnv != nil { + msg.Meta.Route = "" + if route, ok := lagoonEnv.Data["LAGOON_ROUTE"]; ok { + msg.Meta.Route = route + } + msg.Meta.Routes = []string{} + if routes, ok := lagoonEnv.Data["LAGOON_ROUTES"]; ok { + msg.Meta.Routes = strings.Split(routes, ",") + } + } + if buildCondition.ToLower() == "failed" || buildCondition.ToLower() == "complete" || buildCondition.ToLower() == "cancelled" { + msg.Meta.EndTime = time.Now().UTC().Format("2006-01-02 15:04:05") + } + msgBytes, err := json.Marshal(msg) + if err != nil { + opLog.Error(err, "Unable to encode message as JSON") + } + // @TODO: if we can't publish the message because we are deleting the resource, then should we even + // bother to patch the resource?? + // leave it for now cause the resource will just be deleted anyway + if err := r.Messaging.Publish("lagoon-tasks:controller", msgBytes); err != nil { + // if we can't publish the message, set it as a pending message + // overwrite whatever is there as these are just current state messages so it doesn't + // really matter if we don't smootly transition in what we send back to lagoon + r.updateEnvironmentMessage(ctx, lagoonBuild, msg) + return + } + // if we are able to publish the message, then we need to remove any pending messages from the resource + // and make sure we don't try and publish again + r.removeBuildPendingMessageStatus(ctx, lagoonBuild) + } +} + +// buildStatusLogsToLagoonLogs sends the logs to lagoon-logs message queue, used for general messaging +func (r *LagoonBuildReconciler) buildStatusLogsToLagoonLogs(ctx context.Context, + opLog logr.Logger, + lagoonBuild *lagooncrd.LagoonBuild, + lagoonEnv *corev1.ConfigMap, + buildCondition lagooncrd.BuildStatusType, + buildStep string, +) { + if r.EnableMQ { + msg := schema.LagoonLog{ + Severity: "info", + Project: lagoonBuild.Spec.Project.Name, + Event: "task:builddeploy-kubernetes:" + buildCondition.ToLower(), //@TODO: this probably needs to be changed to a new task event for the controller + Meta: &schema.LagoonLogMeta{ + ProjectName: lagoonBuild.Spec.Project.Name, + BranchName: lagoonBuild.Spec.Project.Environment, + BuildStatus: buildCondition.ToLower(), // same as buildstatus label + BuildName: lagoonBuild.ObjectMeta.Name, + BuildStep: buildStep, + LogLink: lagoonBuild.Spec.Project.UILink, + Cluster: r.LagoonTargetName, + }, + Message: fmt.Sprintf("*[%s]* %s Build `%s` %s", + lagoonBuild.Spec.Project.Name, + lagoonBuild.Spec.Project.Environment, + lagoonBuild.ObjectMeta.Name, + buildCondition.ToLower(), + ), + } + // if we aren't being provided the lagoon config, we can skip adding the routes etc + if lagoonEnv != nil { + msg.Meta.Route = "" + if route, ok := lagoonEnv.Data["LAGOON_ROUTE"]; ok { + msg.Meta.Route = route + } + msg.Meta.Routes = []string{} + if routes, ok := lagoonEnv.Data["LAGOON_ROUTES"]; ok { + msg.Meta.Routes = strings.Split(routes, ",") + } + } + msgBytes, err := json.Marshal(msg) + if err != nil { + opLog.Error(err, "Unable to encode message as JSON") + } + // @TODO: if we can't publish the message because we are deleting the resource, then should we even + // bother to patch the resource?? + // leave it for now cause the resource will just be deleted anyway + if err := r.Messaging.Publish("lagoon-logs", msgBytes); err != nil { + // if we can't publish the message, set it as a pending message + // overwrite whatever is there as these are just current state messages so it doesn't + // really matter if we don't smootly transition in what we send back to lagoon + r.updateBuildStatusMessage(ctx, lagoonBuild, msg) + return + } + // if we are able to publish the message, then we need to remove any pending messages from the resource + // and make sure we don't try and publish again + r.removeBuildPendingMessageStatus(ctx, lagoonBuild) + } +} + +// updateEnvironmentMessage this is called if the message queue is unavailable, it stores the message that would be sent in the lagoon build +func (r *LagoonBuildReconciler) updateEnvironmentMessage(ctx context.Context, + lagoonBuild *lagooncrd.LagoonBuild, + envMessage schema.LagoonMessage, +) error { + // set the transition time + mergePatch, _ := json.Marshal(map[string]interface{}{ + "metadata": map[string]interface{}{ + "labels": map[string]interface{}{ + "lagoon.sh/pendingMessages": "true", + }, + }, + "statusMessages": map[string]interface{}{ + "environmentMessage": envMessage, + }, + }) + if err := r.Patch(ctx, lagoonBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { + return fmt.Errorf("Unable to update status condition: %v", err) + } + return nil +} + +// updateBuildStatusMessage this is called if the message queue is unavailable, it stores the message that would be sent in the lagoon build +func (r *LagoonBuildReconciler) updateBuildStatusMessage(ctx context.Context, + lagoonBuild *lagooncrd.LagoonBuild, + statusMessage schema.LagoonLog, +) error { + // set the transition time + mergePatch, _ := json.Marshal(map[string]interface{}{ + "metadata": map[string]interface{}{ + "labels": map[string]interface{}{ + "lagoon.sh/pendingMessages": "true", + }, + }, + "statusMessages": map[string]interface{}{ + "statusMessage": statusMessage, + }, + }) + if err := r.Patch(ctx, lagoonBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { + return fmt.Errorf("Unable to update status condition: %v", err) + } + return nil +} + +// removeBuildPendingMessageStatus purges the status messages from the resource once they are successfully re-sent +func (r *LagoonBuildReconciler) removeBuildPendingMessageStatus(ctx context.Context, + lagoonBuild *lagooncrd.LagoonBuild, +) error { + // if we have the pending messages label as true, then we want to remove this label and any pending statusmessages + // so we can avoid double handling, or an old pending message from being sent after a new pending message + if val, ok := lagoonBuild.ObjectMeta.Labels["lagoon.sh/pendingMessages"]; !ok { + if val == "true" { + mergePatch, _ := json.Marshal(map[string]interface{}{ + "metadata": map[string]interface{}{ + "labels": map[string]interface{}{ + "lagoon.sh/pendingMessages": "false", + }, + }, + "statusMessages": nil, + }) + if err := r.Patch(ctx, lagoonBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { + return fmt.Errorf("Unable to update status condition: %v", err) + } + } + } + return nil +} + +// updateBuildLogMessage this is called if the message queue is unavailable, it stores the message that would be sent in the lagoon build +func (r *LagoonBuildReconciler) updateBuildLogMessage(ctx context.Context, + lagoonBuild *lagooncrd.LagoonBuild, + buildMessage schema.LagoonLog, +) error { + // set the transition time + mergePatch, _ := json.Marshal(map[string]interface{}{ + "metadata": map[string]interface{}{ + "labels": map[string]interface{}{ + "lagoon.sh/pendingMessages": "true", + }, + }, + "statusMessages": map[string]interface{}{ + "buildLogMessage": buildMessage, + }, + }) + if err := r.Patch(ctx, lagoonBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { + return fmt.Errorf("Unable to update status condition: %v", err) + } + return nil +} diff --git a/controllers/v1beta2/build_helpers.go b/controllers/v1beta2/build_helpers.go new file mode 100644 index 00000000..983bc0f3 --- /dev/null +++ b/controllers/v1beta2/build_helpers.go @@ -0,0 +1,1061 @@ +package v1beta2 + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "regexp" + "sort" + "strconv" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/go-logr/logr" + "github.com/prometheus/client_golang/prometheus" + lagooncrd "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" + "github.com/uselagoon/remote-controller/internal/harbor" + "github.com/uselagoon/remote-controller/internal/helpers" +) + +var ( + crdVersion string = "v1beta2" +) + +const ( + // NotOwnedByControllerMessage is used to describe an error where the controller was unable to start the build because + // the `lagoon.sh/controller` label does not match this controllers name + NotOwnedByControllerMessage = `Build was cancelled due to an issue with the build controller. +This issue is related to the deployment system, not the repository or code base changes. +Contact your Lagoon support team for help` + // MissingLabelsMessage is used to describe an error where the controller was unable to start the build because + // the `lagoon.sh/controller` label is missing + MissingLabelsMessage = `"Build was cancelled due to namespace configuration issue. A label or labels are missing on the namespace. +This issue is related to the deployment system, not the repository or code base changes. +Contact your Lagoon support team for help` +) + +// getOrCreateServiceAccount will create the lagoon-deployer service account if it doesn't exist. +func (r *LagoonBuildReconciler) getOrCreateServiceAccount(ctx context.Context, serviceAccount *corev1.ServiceAccount, ns string) error { + serviceAccount.ObjectMeta = metav1.ObjectMeta{ + Name: "lagoon-deployer", + Namespace: ns, + } + err := r.Get(ctx, types.NamespacedName{ + Namespace: ns, + Name: "lagoon-deployer", + }, serviceAccount) + if err != nil { + if err := r.Create(ctx, serviceAccount); err != nil { + return fmt.Errorf("There was an error creating the lagoon-deployer servicea account. Error was: %v", err) + } + } + return nil +} + +// getOrCreateSARoleBinding will create the rolebinding for the lagoon-deployer if it doesn't exist. +func (r *LagoonBuildReconciler) getOrCreateSARoleBinding(ctx context.Context, saRoleBinding *rbacv1.RoleBinding, ns string) error { + saRoleBinding.ObjectMeta = metav1.ObjectMeta{ + Name: "lagoon-deployer-admin", + Namespace: ns, + } + saRoleBinding.RoleRef = rbacv1.RoleRef{ + Name: "admin", + Kind: "ClusterRole", + APIGroup: "rbac.authorization.k8s.io", + } + saRoleBinding.Subjects = []rbacv1.Subject{ + { + Name: "lagoon-deployer", + Kind: "ServiceAccount", + Namespace: ns, + }, + } + err := r.Get(ctx, types.NamespacedName{ + Namespace: ns, + Name: "lagoon-deployer-admin", + }, saRoleBinding) + if err != nil { + if err := r.Create(ctx, saRoleBinding); err != nil { + return fmt.Errorf("There was an error creating the lagoon-deployer-admin role binding. Error was: %v", err) + } + } + return nil +} + +// getOrCreateNamespace will create the namespace if it doesn't exist. +func (r *LagoonBuildReconciler) getOrCreateNamespace(ctx context.Context, namespace *corev1.Namespace, lagoonBuild lagooncrd.LagoonBuild, opLog logr.Logger) error { + // parse the project/env through the project pattern, or use the default + ns := helpers.GenerateNamespaceName( + lagoonBuild.Spec.Project.NamespacePattern, // the namespace pattern or `openshiftProjectPattern` from Lagoon is never received by the controller + lagoonBuild.Spec.Project.Environment, + lagoonBuild.Spec.Project.Name, + r.NamespacePrefix, + r.ControllerNamespace, + r.RandomNamespacePrefix, + ) + nsLabels := map[string]string{ + "lagoon.sh/project": lagoonBuild.Spec.Project.Name, + "lagoon.sh/environment": lagoonBuild.Spec.Project.Environment, + "lagoon.sh/environmentType": lagoonBuild.Spec.Project.EnvironmentType, + "lagoon.sh/controller": r.ControllerNamespace, + } + if lagoonBuild.Spec.Project.Organization != nil { + nsLabels["organization.lagoon.sh/id"] = fmt.Sprintf("%d", *lagoonBuild.Spec.Project.Organization.ID) + nsLabels["organization.lagoon.sh/name"] = lagoonBuild.Spec.Project.Organization.Name + } + if lagoonBuild.Spec.Project.ID != nil { + nsLabels["lagoon.sh/projectId"] = fmt.Sprintf("%d", *lagoonBuild.Spec.Project.ID) + } + if lagoonBuild.Spec.Project.EnvironmentID != nil { + nsLabels["lagoon.sh/environmentId"] = fmt.Sprintf("%d", *lagoonBuild.Spec.Project.EnvironmentID) + } + // set the auto idling values if they are defined + if lagoonBuild.Spec.Project.EnvironmentIdling != nil { + // eventually deprecate 'lagoon.sh/environmentAutoIdle' for 'lagoon.sh/environmentIdlingEnabled' + nsLabels["lagoon.sh/environmentAutoIdle"] = fmt.Sprintf("%d", *lagoonBuild.Spec.Project.EnvironmentIdling) + if *lagoonBuild.Spec.Project.EnvironmentIdling == 1 { + nsLabels["lagoon.sh/environmentIdlingEnabled"] = "true" + } else { + nsLabels["lagoon.sh/environmentIdlingEnabled"] = "false" + } + } + if lagoonBuild.Spec.Project.ProjectIdling != nil { + // eventually deprecate 'lagoon.sh/projectAutoIdle' for 'lagoon.sh/projectIdlingEnabled' + nsLabels["lagoon.sh/projectAutoIdle"] = fmt.Sprintf("%d", *lagoonBuild.Spec.Project.ProjectIdling) + if *lagoonBuild.Spec.Project.ProjectIdling == 1 { + nsLabels["lagoon.sh/projectIdlingEnabled"] = "true" + } else { + nsLabels["lagoon.sh/projectIdlingEnabled"] = "false" + } + } + if lagoonBuild.Spec.Project.StorageCalculator != nil { + if *lagoonBuild.Spec.Project.StorageCalculator == 1 { + nsLabels["lagoon.sh/storageCalculatorEnabled"] = "true" + } else { + nsLabels["lagoon.sh/storageCalculatorEnabled"] = "false" + } + } + // add the required lagoon labels to the namespace when creating + namespace.ObjectMeta = metav1.ObjectMeta{ + Name: ns, + Labels: nsLabels, + } + + if err := r.Get(ctx, types.NamespacedName{Name: ns}, namespace); err != nil { + if helpers.IgnoreNotFound(err) != nil { + return fmt.Errorf("There was an error getting the namespace. Error was: %v", err) + } + } + if namespace.Status.Phase == corev1.NamespaceTerminating { + opLog.Info(fmt.Sprintf("Cleaning up build %s as cancelled, the namespace is stuck in terminating state", lagoonBuild.ObjectMeta.Name)) + r.cleanUpUndeployableBuild(ctx, lagoonBuild, "Namespace is currently in terminating status - contact your Lagoon support team for help", opLog, true) + return fmt.Errorf("%s is currently terminating, aborting build", ns) + } + + // if the namespace exists, check that the controller label exists and matches this controllers namespace name + if namespace.Status.Phase == corev1.NamespaceActive { + if value, ok := namespace.ObjectMeta.Labels["lagoon.sh/controller"]; ok { + if value != r.ControllerNamespace { + // if the namespace is deployed by a different controller, fail the build + opLog.Info(fmt.Sprintf("Cleaning up build %s as cancelled, the namespace is owned by a different remote-controller", lagoonBuild.ObjectMeta.Name)) + r.cleanUpUndeployableBuild(ctx, lagoonBuild, NotOwnedByControllerMessage, opLog, true) + return fmt.Errorf("%s is owned by a different remote-controller, aborting build", ns) + } + } else { + // if the label doesn't exist at all, fail the build + opLog.Info(fmt.Sprintf("Cleaning up build %s as cancelled, the namespace is not a Lagoon project/environment", lagoonBuild.ObjectMeta.Name)) + r.cleanUpUndeployableBuild(ctx, lagoonBuild, MissingLabelsMessage, opLog, true) + return fmt.Errorf("%s is not a Lagoon project/environment, aborting build", ns) + } + } + + // if kubernetes, just create it if it doesn't exist + if err := r.Get(ctx, types.NamespacedName{Name: ns}, namespace); err != nil { + if err := r.Create(ctx, namespace); err != nil { + return fmt.Errorf("There was an error creating the namespace. Error was: %v", err) + } + } + + // once the namespace exists, then we can patch it with our labels + // this means the labels will always get added or updated if we need to change them or add new labels + // after the namespace has been created + mergePatch, _ := json.Marshal(map[string]interface{}{ + "metadata": map[string]interface{}{ + "labels": nsLabels, + }, + }) + if err := r.Patch(ctx, namespace, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { + return fmt.Errorf("There was an error patching the namespace. Error was: %v", err) + } + if err := r.Get(ctx, types.NamespacedName{Name: ns}, namespace); err != nil { + return fmt.Errorf("There was an error getting the namespace. Error was: %v", err) + } + + // if local/regional harbor is enabled + if r.LFFHarborEnabled { + // create the harbor client + lagoonHarbor, err := harbor.New(r.Harbor) + if err != nil { + return fmt.Errorf("Error creating harbor client, check your harbor configuration. Error was: %v", err) + } + // create the project in harbor + robotCreds := &helpers.RegistryCredentials{} + curVer, err := lagoonHarbor.GetHarborVersion(ctx) + if err != nil { + return fmt.Errorf("Error getting harbor version, check your harbor configuration. Error was: %v", err) + } + if lagoonHarbor.UseV2Functions(curVer) { + hProject, err := lagoonHarbor.CreateProjectV2(ctx, lagoonBuild.Spec.Project.Name) + if err != nil { + return fmt.Errorf("Error creating harbor project: %v", err) + } + // create or refresh the robot credentials + robotCreds, err = lagoonHarbor.CreateOrRefreshRobotV2(ctx, + r.Client, + hProject, + lagoonBuild.Spec.Project.Environment, + ns, + lagoonHarbor.RobotAccountExpiry, + false) + if err != nil { + return fmt.Errorf("Error creating harbor robot account: %v", err) + } + } else { + hProject, err := lagoonHarbor.CreateProject(ctx, lagoonBuild.Spec.Project.Name) + if err != nil { + return fmt.Errorf("Error creating harbor project: %v", err) + } + // create or refresh the robot credentials + robotCreds, err = lagoonHarbor.CreateOrRefreshRobot(ctx, + r.Client, + hProject, + lagoonBuild.Spec.Project.Environment, + ns, + time.Now().Add(lagoonHarbor.RobotAccountExpiry).Unix(), + false) + if err != nil { + return fmt.Errorf("Error creating harbor robot account: %v", err) + } + } + // if we have robotcredentials to create, do that here + _, err = lagoonHarbor.UpsertHarborSecret(ctx, + r.Client, + ns, + "lagoon-internal-registry-secret", + robotCreds) + if err != nil { + return fmt.Errorf("Error upserting harbor robot account secret: %v", err) + } + } + return nil +} + +// getCreateOrUpdateSSHKeySecret will create or update the ssh key. +func (r *LagoonBuildReconciler) getCreateOrUpdateSSHKeySecret(ctx context.Context, + sshKey *corev1.Secret, + spec lagooncrd.LagoonBuildSpec, + ns string) error { + sshKey.ObjectMeta = metav1.ObjectMeta{ + Name: "lagoon-sshkey", + Namespace: ns, + } + sshKey.Type = "kubernetes.io/ssh-auth" + sshKey.Data = map[string][]byte{ + "ssh-privatekey": spec.Project.Key, + } + err := r.Get(ctx, types.NamespacedName{ + Namespace: ns, + Name: "lagoon-sshkey", + }, sshKey) + if err != nil { + if err := r.Create(ctx, sshKey); err != nil { + return fmt.Errorf("There was an error creating the lagoon-sshkey. Error was: %v", err) + } + } + // if the keys are different, then load in the new key from the spec + if bytes.Compare(sshKey.Data["ssh-privatekey"], spec.Project.Key) != 0 { + sshKey.Data = map[string][]byte{ + "ssh-privatekey": spec.Project.Key, + } + if err := r.Update(ctx, sshKey); err != nil { + return fmt.Errorf("There was an error updating the lagoon-sshkey. Error was: %v", err) + } + } + return nil +} + +func (r *LagoonBuildReconciler) getOrCreateConfigMap(ctx context.Context, cmName string, configMap *corev1.ConfigMap, ns string) error { + err := r.Get(ctx, types.NamespacedName{ + Namespace: ns, + Name: cmName, + }, configMap) + if err != nil { + configMap.SetNamespace(ns) + configMap.SetName(cmName) + //we create it + if err = r.Create(ctx, configMap); err != nil { + return fmt.Errorf("There was an error creating the configmap '%v'. Error was: %v", cmName, err) + } + } + return nil +} + +// processBuild will actually process the build. +func (r *LagoonBuildReconciler) processBuild(ctx context.Context, opLog logr.Logger, lagoonBuild lagooncrd.LagoonBuild) error { + // we run these steps again just to be sure that it gets updated/created if it hasn't already + opLog.Info(fmt.Sprintf("Checking and preparing namespace and associated resources for build: %s", lagoonBuild.ObjectMeta.Name)) + // create the lagoon-sshkey secret + sshKey := &corev1.Secret{} + if r.EnableDebug { + opLog.Info(fmt.Sprintf("Checking `lagoon-sshkey` Secret exists: %s", lagoonBuild.ObjectMeta.Name)) + } + err := r.getCreateOrUpdateSSHKeySecret(ctx, sshKey, lagoonBuild.Spec, lagoonBuild.ObjectMeta.Namespace) + if err != nil { + return err + } + + // create the `lagoon-deployer` ServiceAccount + if r.EnableDebug { + opLog.Info(fmt.Sprintf("Checking `lagoon-deployer` ServiceAccount exists: %s", lagoonBuild.ObjectMeta.Name)) + } + serviceAccount := &corev1.ServiceAccount{} + err = r.getOrCreateServiceAccount(ctx, serviceAccount, lagoonBuild.ObjectMeta.Namespace) + if err != nil { + return err + } + + // ServiceAccount RoleBinding creation + if r.EnableDebug { + opLog.Info(fmt.Sprintf("Checking `lagoon-deployer-admin` RoleBinding exists: %s", lagoonBuild.ObjectMeta.Name)) + } + saRoleBinding := &rbacv1.RoleBinding{} + err = r.getOrCreateSARoleBinding(ctx, saRoleBinding, lagoonBuild.ObjectMeta.Namespace) + if err != nil { + return err + } + + if r.EnableDebug { + opLog.Info(fmt.Sprintf("Checking `lagoon-deployer` Token exists: %s", lagoonBuild.ObjectMeta.Name)) + } + + var serviceaccountTokenSecret string + for _, secret := range serviceAccount.Secrets { + match, _ := regexp.MatchString("^lagoon-deployer-token", secret.Name) + if match { + serviceaccountTokenSecret = secret.Name + break + } + } + + // create the Pod that will do the work + podEnvs := []corev1.EnvVar{ + { + Name: "SOURCE_REPOSITORY", + Value: lagoonBuild.Spec.Project.GitURL, + }, + { + Name: "GIT_REF", + Value: lagoonBuild.Spec.GitReference, + }, + { + Name: "SUBFOLDER", + Value: lagoonBuild.Spec.Project.SubFolder, + }, + { + Name: "BRANCH", + Value: lagoonBuild.Spec.Branch.Name, + }, + { + Name: "PROJECT", + Value: lagoonBuild.Spec.Project.Name, + }, + { + Name: "ENVIRONMENT_TYPE", + Value: lagoonBuild.Spec.Project.EnvironmentType, + }, + { + Name: "ACTIVE_ENVIRONMENT", + Value: lagoonBuild.Spec.Project.ProductionEnvironment, + }, + { + Name: "STANDBY_ENVIRONMENT", + Value: lagoonBuild.Spec.Project.StandbyEnvironment, + }, + { + Name: "PROJECT_SECRET", + Value: lagoonBuild.Spec.Project.ProjectSecret, + }, + { + Name: "MONITORING_ALERTCONTACT", + Value: lagoonBuild.Spec.Project.Monitoring.Contact, + }, + { + Name: "DEFAULT_BACKUP_SCHEDULE", + Value: r.BackupConfig.BackupDefaultSchedule, + }, + { + Name: "MONTHLY_BACKUP_DEFAULT_RETENTION", + Value: strconv.Itoa(r.BackupConfig.BackupDefaultMonthlyRetention), + }, + { + Name: "WEEKLY_BACKUP_DEFAULT_RETENTION", + Value: strconv.Itoa(r.BackupConfig.BackupDefaultWeeklyRetention), + }, + { + Name: "DAILY_BACKUP_DEFAULT_RETENTION", + Value: strconv.Itoa(r.BackupConfig.BackupDefaultDailyRetention), + }, + { + Name: "HOURLY_BACKUP_DEFAULT_RETENTION", + Value: strconv.Itoa(r.BackupConfig.BackupDefaultHourlyRetention), + }, + { + Name: "LAGOON_FEATURE_BACKUP_DEV_SCHEDULE", + Value: r.BackupConfig.BackupDefaultDevelopmentSchedule, + }, + { + Name: "LAGOON_FEATURE_BACKUP_PR_SCHEDULE", + Value: r.BackupConfig.BackupDefaultPullrequestSchedule, + }, + { + Name: "LAGOON_FEATURE_BACKUP_DEV_RETENTION", + Value: r.BackupConfig.BackupDefaultDevelopmentRetention, + }, + { + Name: "LAGOON_FEATURE_BACKUP_PR_RETENTION", + Value: r.BackupConfig.BackupDefaultPullrequestRetention, + }, + { + Name: "K8UP_WEEKLY_RANDOM_FEATURE_FLAG", + Value: strconv.FormatBool(r.LFFBackupWeeklyRandom), + }, + { + Name: "NATIVE_CRON_POD_MINIMUM_FREQUENCY", + Value: strconv.Itoa(r.NativeCronPodMinFrequency), + }, + // add the API and SSH endpoint configuration to environments + { + Name: "LAGOON_CONFIG_API_HOST", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "LAGOON_CONFIG_API_HOST"), + }, + { + Name: "LAGOON_CONFIG_TOKEN_HOST", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "LAGOON_CONFIG_TOKEN_HOST"), + }, + { + Name: "LAGOON_CONFIG_TOKEN_PORT", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "LAGOON_CONFIG_TOKEN_PORT"), + }, + { + Name: "LAGOON_CONFIG_SSH_HOST", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "LAGOON_CONFIG_SSH_HOST"), + }, + { + Name: "LAGOON_CONFIG_SSH_PORT", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "LAGOON_CONFIG_SSH_PORT"), + }, + } + // add proxy variables to builds if they are defined + if r.ProxyConfig.HTTPProxy != "" { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "HTTP_PROXY", + Value: r.ProxyConfig.HTTPProxy, + }) + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "http_proxy", + Value: r.ProxyConfig.HTTPProxy, + }) + } + if r.ProxyConfig.HTTPSProxy != "" { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "HTTPS_PROXY", + Value: r.ProxyConfig.HTTPSProxy, + }) + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "https_proxy", + Value: r.ProxyConfig.HTTPSProxy, + }) + } + if r.ProxyConfig.NoProxy != "" { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "NO_PROXY", + Value: r.ProxyConfig.NoProxy, + }) + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "no_proxy", + Value: r.ProxyConfig.NoProxy, + }) + } + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "BUILD_TYPE", + Value: lagoonBuild.Spec.Build.Type, + }) + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "ENVIRONMENT", + Value: lagoonBuild.Spec.Project.Environment, + }) + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "KUBERNETES", + Value: lagoonBuild.Spec.Project.DeployTarget, + }) + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "REGISTRY", + Value: lagoonBuild.Spec.Project.Registry, + }) + // this is enabled by default for now + // eventually will be disabled by default because support for the generation/modification of this will + // be handled by lagoon or the builds themselves + if r.LFFRouterURL { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "ROUTER_URL", + Value: strings.ToLower( + strings.Replace( + strings.Replace( + lagoonBuild.Spec.Project.RouterPattern, + "${environment}", + lagoonBuild.Spec.Project.Environment, + -1, + ), + "${project}", + lagoonBuild.Spec.Project.Name, + -1, + ), + ), + }) + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "SHORT_ROUTER_URL", + Value: strings.ToLower( + strings.Replace( + strings.Replace( + lagoonBuild.Spec.Project.RouterPattern, + "${environment}", + helpers.ShortName(lagoonBuild.Spec.Project.Environment), + -1, + ), + "${project}", + helpers.ShortName(lagoonBuild.Spec.Project.Name), + -1, + ), + ), + }) + } + if lagoonBuild.Spec.Build.CI != "" { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "CI", + Value: lagoonBuild.Spec.Build.CI, + }) + } + if lagoonBuild.Spec.Build.Type == "pullrequest" { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "PR_HEAD_BRANCH", + Value: lagoonBuild.Spec.Pullrequest.HeadBranch, + }) + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "PR_HEAD_SHA", + Value: lagoonBuild.Spec.Pullrequest.HeadSha, + }) + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "PR_BASE_BRANCH", + Value: lagoonBuild.Spec.Pullrequest.BaseBranch, + }) + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "PR_BASE_SHA", + Value: lagoonBuild.Spec.Pullrequest.BaseSha, + }) + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "PR_TITLE", + Value: lagoonBuild.Spec.Pullrequest.Title, + }) + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "PR_NUMBER", + Value: string(lagoonBuild.Spec.Pullrequest.Number), + }) + } + if lagoonBuild.Spec.Build.Type == "promote" { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "PROMOTION_SOURCE_ENVIRONMENT", + Value: lagoonBuild.Spec.Promote.SourceEnvironment, + }) + } + // if local/regional harbor is enabled + if r.LFFHarborEnabled { + // unmarshal the project variables + lagoonProjectVariables := &[]helpers.LagoonEnvironmentVariable{} + lagoonEnvironmentVariables := &[]helpers.LagoonEnvironmentVariable{} + json.Unmarshal(lagoonBuild.Spec.Project.Variables.Project, lagoonProjectVariables) + json.Unmarshal(lagoonBuild.Spec.Project.Variables.Environment, lagoonEnvironmentVariables) + // check if INTERNAL_REGISTRY_SOURCE_LAGOON is defined, and if it isn't true + // if this value is true, then we want to use what is provided by Lagoon + // if it is false, or not set, then we use what is provided by this controller + // this allows us to make it so a specific environment or the project entirely + // can still use whats provided by lagoon + if !helpers.VariableExists(lagoonProjectVariables, "INTERNAL_REGISTRY_SOURCE_LAGOON", "true") || + !helpers.VariableExists(lagoonEnvironmentVariables, "INTERNAL_REGISTRY_SOURCE_LAGOON", "true") { + // source the robot credential, and inject it into the lagoon project variables + // this will overwrite what is provided by lagoon (if lagoon has provided them) + // or it will add them. + robotCredential := &corev1.Secret{} + if err = r.Get(ctx, types.NamespacedName{ + Namespace: lagoonBuild.ObjectMeta.Namespace, + Name: "lagoon-internal-registry-secret", + }, robotCredential); err != nil { + return fmt.Errorf("Could not find Harbor RobotAccount credential") + } + auths := helpers.Auths{} + if secretData, ok := robotCredential.Data[".dockerconfigjson"]; ok { + if err := json.Unmarshal(secretData, &auths); err != nil { + return fmt.Errorf("Could not unmarshal Harbor RobotAccount credential") + } + // if the defined regional harbor key exists using the hostname + if creds, ok := auths.Registries[r.Harbor.URL]; ok { + // use the regional harbor in the build + helpers.ReplaceOrAddVariable(lagoonProjectVariables, "INTERNAL_REGISTRY_URL", r.Harbor.URL, "internal_container_registry") + helpers.ReplaceOrAddVariable(lagoonProjectVariables, "INTERNAL_REGISTRY_USERNAME", creds.Username, "internal_container_registry") + helpers.ReplaceOrAddVariable(lagoonProjectVariables, "INTERNAL_REGISTRY_PASSWORD", creds.Password, "internal_container_registry") + } + if creds, ok := auths.Registries[r.Harbor.Hostname]; ok { + // use the regional harbor in the build + helpers.ReplaceOrAddVariable(lagoonProjectVariables, "INTERNAL_REGISTRY_URL", r.Harbor.Hostname, "internal_container_registry") + helpers.ReplaceOrAddVariable(lagoonProjectVariables, "INTERNAL_REGISTRY_USERNAME", creds.Username, "internal_container_registry") + helpers.ReplaceOrAddVariable(lagoonProjectVariables, "INTERNAL_REGISTRY_PASSWORD", creds.Password, "internal_container_registry") + } + } + // marshal any changes into the project spec on the fly, don't save the spec though + // these values are being overwritten and injected directly into the build pod to be consumed + // by the build pod image + lagoonBuild.Spec.Project.Variables.Project, _ = json.Marshal(lagoonProjectVariables) + } + } + if lagoonBuild.Spec.Project.Variables.Project != nil { + // if this is 2 bytes long, then it means its just an empty json array + // we only want to add it if it is more than 2 bytes + if len(lagoonBuild.Spec.Project.Variables.Project) > 2 { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "LAGOON_PROJECT_VARIABLES", + Value: string(lagoonBuild.Spec.Project.Variables.Project), + }) + } + } + if lagoonBuild.Spec.Project.Variables.Environment != nil { + // if this is 2 bytes long, then it means its just an empty json array + // we only want to add it if it is more than 2 bytes + if len(lagoonBuild.Spec.Project.Variables.Environment) > 2 { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "LAGOON_ENVIRONMENT_VARIABLES", + Value: string(lagoonBuild.Spec.Project.Variables.Environment), + }) + } + } + if lagoonBuild.Spec.Project.Monitoring.StatuspageID != "" { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "MONITORING_STATUSPAGEID", + Value: lagoonBuild.Spec.Project.Monitoring.StatuspageID, + }) + } + // if the fastly watch status is set on the controller, inject the fastly service ID into the build pod to be consumed + // by the build-depoy-dind image + if r.FastlyWatchStatus { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "LAGOON_FASTLY_NOCACHE_SERVICE_ID", + Value: r.FastlyServiceID, + }) + } + // Set any defined Lagoon feature flags in the build environment. + if r.LFFForceRootlessWorkload != "" { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "LAGOON_FEATURE_FLAG_FORCE_ROOTLESS_WORKLOAD", + Value: r.LFFForceRootlessWorkload, + }) + } + if r.LFFDefaultRootlessWorkload != "" { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "LAGOON_FEATURE_FLAG_DEFAULT_ROOTLESS_WORKLOAD", + Value: r.LFFDefaultRootlessWorkload, + }) + } + if r.LFFForceIsolationNetworkPolicy != "" { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "LAGOON_FEATURE_FLAG_FORCE_ISOLATION_NETWORK_POLICY", + Value: r.LFFForceIsolationNetworkPolicy, + }) + } + if r.LFFDefaultIsolationNetworkPolicy != "" { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "LAGOON_FEATURE_FLAG_DEFAULT_ISOLATION_NETWORK_POLICY", + Value: r.LFFDefaultIsolationNetworkPolicy, + }) + } + if r.LFFForceInsights != "" { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "LAGOON_FEATURE_FLAG_FORCE_INSIGHTS", + Value: r.LFFForceInsights, + }) + } + if r.LFFDefaultInsights != "" { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "LAGOON_FEATURE_FLAG_DEFAULT_INSIGHTS", + Value: r.LFFDefaultInsights, + }) + } + if r.LFFForceRWX2RWO != "" { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "LAGOON_FEATURE_FLAG_FORCE_RWX_TO_RWO", + Value: r.LFFForceRWX2RWO, + }) + } + if r.LFFDefaultRWX2RWO != "" { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "LAGOON_FEATURE_FLAG_DEFAULT_RWX_TO_RWO", + Value: r.LFFDefaultRWX2RWO, + }) + } + + // set the organization variables into the build pod so they're available to builds for consumption + if lagoonBuild.Spec.Project.Organization != nil { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "LAGOON_ORGANIZATION_ID", + Value: fmt.Sprintf("%d", *lagoonBuild.Spec.Project.Organization.ID), + }) + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "LAGOON_ORGANIZATION_NAME", + Value: lagoonBuild.Spec.Project.Organization.Name, + }) + } + + // add any LAGOON_FEATURE_FLAG_ variables in the controller into the build pods + for fName, fValue := range r.LagoonFeatureFlags { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: fName, + Value: fValue, + }) + } + // Use the build image in the controller definition + buildImage := r.BuildImage + if lagoonBuild.Spec.Build.Image != "" { + // otherwise if the build spec contains an image definition, use it instead. + buildImage = lagoonBuild.Spec.Build.Image + } + volumes := []corev1.Volume{ + { + Name: "lagoon-sshkey", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "lagoon-sshkey", + DefaultMode: helpers.Int32Ptr(420), + }, + }, + }, + } + volumeMounts := []corev1.VolumeMount{ + { + Name: "lagoon-sshkey", + ReadOnly: true, + MountPath: "/var/run/secrets/lagoon/ssh", + }, + } + + // if the existing token exists, mount it + if serviceaccountTokenSecret != "" { + volumes = append(volumes, corev1.Volume{ + Name: serviceaccountTokenSecret, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: serviceaccountTokenSecret, + DefaultMode: helpers.Int32Ptr(420), + }, + }, + }) + // legacy tokens are mounted /var/run/secrets/lagoon/deployer + // new tokens using volume projection are mounted /var/run/secrets/kubernetes.io/serviceaccount/token + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: serviceaccountTokenSecret, + ReadOnly: true, + MountPath: "/var/run/secrets/lagoon/deployer", + }) + } + newPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: lagoonBuild.ObjectMeta.Name, + Namespace: lagoonBuild.ObjectMeta.Namespace, + Labels: map[string]string{ + "lagoon.sh/jobType": "build", + "lagoon.sh/buildName": lagoonBuild.ObjectMeta.Name, + "lagoon.sh/controller": r.ControllerNamespace, + "crd.lagoon.sh/version": crdVersion, + "lagoon.sh/buildRemoteID": string(lagoonBuild.ObjectMeta.UID), + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: fmt.Sprintf("%v", lagooncrd.GroupVersion), + Kind: "LagoonBuild", + Name: lagoonBuild.ObjectMeta.Name, + UID: lagoonBuild.UID, + }, + }, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: "lagoon-deployer", + RestartPolicy: "Never", + Volumes: volumes, + Tolerations: []corev1.Toleration{ + { + Key: "lagoon/build", + Effect: "NoSchedule", + Operator: "Exists", + }, + { + Key: "lagoon/build", + Effect: "PreferNoSchedule", + Operator: "Exists", + }, + { + Key: "lagoon.sh/build", + Effect: "NoSchedule", + Operator: "Exists", + }, + { + Key: "lagoon.sh/build", + Effect: "PreferNoSchedule", + Operator: "Exists", + }, + }, + Containers: []corev1.Container{ + { + Name: "lagoon-build", + Image: buildImage, + ImagePullPolicy: "Always", + Env: podEnvs, + VolumeMounts: volumeMounts, + }, + }, + }, + } + + // set the organization labels on build pods + if lagoonBuild.Spec.Project.Organization != nil { + newPod.ObjectMeta.Labels["organization.lagoon.sh/id"] = fmt.Sprintf("%d", *lagoonBuild.Spec.Project.Organization.ID) + newPod.ObjectMeta.Labels["organization.lagoon.sh/name"] = lagoonBuild.Spec.Project.Organization.Name + } + + // set the pod security context, if defined to a non-default value + if r.BuildPodRunAsUser != 0 || r.BuildPodRunAsGroup != 0 || + r.BuildPodFSGroup != 0 { + newPod.Spec.SecurityContext = &corev1.PodSecurityContext{ + RunAsUser: &r.BuildPodRunAsUser, + RunAsGroup: &r.BuildPodRunAsGroup, + FSGroup: &r.BuildPodFSGroup, + } + } + + if r.EnableDebug { + opLog.Info(fmt.Sprintf("Checking build pod for: %s", lagoonBuild.ObjectMeta.Name)) + } + // once the pod spec has been defined, check if it isn't already created + err = r.Get(ctx, types.NamespacedName{ + Namespace: lagoonBuild.ObjectMeta.Namespace, + Name: newPod.ObjectMeta.Name, + }, newPod) + if err != nil { + // if it doesn't exist, then create the build pod + opLog.Info(fmt.Sprintf("Creating build pod for: %s", lagoonBuild.ObjectMeta.Name)) + if err := r.Create(ctx, newPod); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to create build pod")) + // log the error and just exit, don't continue to try and do anything + // @TODO: should update the build to failed + return nil + } + buildRunningStatus.With(prometheus.Labels{ + "build_namespace": lagoonBuild.ObjectMeta.Namespace, + "build_name": lagoonBuild.ObjectMeta.Name, + }).Set(1) + buildStatus.With(prometheus.Labels{ + "build_namespace": lagoonBuild.ObjectMeta.Namespace, + "build_name": lagoonBuild.ObjectMeta.Name, + "build_step": "running", + }).Set(1) + buildsStartedCounter.Inc() + // then break out of the build + } + opLog.Info(fmt.Sprintf("Build pod already running for: %s", lagoonBuild.ObjectMeta.Name)) + return nil +} + +// updateQueuedBuild will update a build if it is queued +func (r *LagoonBuildReconciler) updateQueuedBuild( + ctx context.Context, + lagoonBuild lagooncrd.LagoonBuild, + queuePosition, queueLength int, + opLog logr.Logger, +) error { + if r.EnableDebug { + opLog.Info(fmt.Sprintf("Updating build %s to queued: %s", lagoonBuild.ObjectMeta.Name, fmt.Sprintf("This build is currently queued in position %v/%v", queuePosition, queueLength))) + } + var allContainerLogs []byte + // if we get this handler, then it is likely that the build was in a pending or running state with no actual running pod + // so just set the logs to be cancellation message + allContainerLogs = []byte(fmt.Sprintf(` +======================================== +%s +======================================== +`, fmt.Sprintf("This build is currently queued in position %v/%v", queuePosition, queueLength))) + // get the configmap for lagoon-env so we can use it for updating the deployment in lagoon + var lagoonEnv corev1.ConfigMap + err := r.Get(ctx, types.NamespacedName{ + Namespace: lagoonBuild.ObjectMeta.Namespace, + Name: "lagoon-env", + }, + &lagoonEnv, + ) + if err != nil { + // if there isn't a configmap, just info it and move on + // the updatedeployment function will see it as nil and not bother doing the bits that require the configmap + if r.EnableDebug { + opLog.Info(fmt.Sprintf("There is no configmap %s in namespace %s ", "lagoon-env", lagoonBuild.ObjectMeta.Namespace)) + } + } + // send any messages to lagoon message queues + // update the deployment with the status, lagoon v2.12.0 supports queued status, otherwise use pending + if lagooncrd.CheckLagoonVersion(&lagoonBuild, "2.12.0") { + r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &lagoonEnv, lagooncrd.BuildStatusQueued, fmt.Sprintf("queued %v/%v", queuePosition, queueLength)) + r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagooncrd.BuildStatusQueued, fmt.Sprintf("queued %v/%v", queuePosition, queueLength)) + r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, allContainerLogs, lagooncrd.BuildStatusQueued) + } else { + r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &lagoonEnv, lagooncrd.BuildStatusPending, fmt.Sprintf("queued %v/%v", queuePosition, queueLength)) + r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagooncrd.BuildStatusPending, fmt.Sprintf("queued %v/%v", queuePosition, queueLength)) + r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, allContainerLogs, lagooncrd.BuildStatusPending) + + } + return nil +} + +// cleanUpUndeployableBuild will clean up a build if the namespace is being terminated, or some other reason that it can't deploy (or create the pod, pending in queue) +func (r *LagoonBuildReconciler) cleanUpUndeployableBuild( + ctx context.Context, + lagoonBuild lagooncrd.LagoonBuild, + message string, + opLog logr.Logger, + cancelled bool, +) error { + var allContainerLogs []byte + if cancelled { + // if we get this handler, then it is likely that the build was in a pending or running state with no actual running pod + // so just set the logs to be cancellation message + allContainerLogs = []byte(fmt.Sprintf(` +======================================== +Build cancelled +======================================== +%s`, message)) + var buildCondition lagooncrd.BuildStatusType + buildCondition = lagooncrd.BuildStatusCancelled + lagoonBuild.Labels["lagoon.sh/buildStatus"] = buildCondition.String() + mergePatch, _ := json.Marshal(map[string]interface{}{ + "metadata": map[string]interface{}{ + "labels": map[string]interface{}{ + "lagoon.sh/buildStatus": buildCondition.String(), + }, + }, + }) + if err := r.Patch(ctx, &lagoonBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to update build status")) + } + } + // get the configmap for lagoon-env so we can use it for updating the deployment in lagoon + var lagoonEnv corev1.ConfigMap + err := r.Get(ctx, types.NamespacedName{ + Namespace: lagoonBuild.ObjectMeta.Namespace, + Name: "lagoon-env", + }, + &lagoonEnv, + ) + if err != nil { + // if there isn't a configmap, just info it and move on + // the updatedeployment function will see it as nil and not bother doing the bits that require the configmap + if r.EnableDebug { + opLog.Info(fmt.Sprintf("There is no configmap %s in namespace %s ", "lagoon-env", lagoonBuild.ObjectMeta.Namespace)) + } + } + // send any messages to lagoon message queues + // update the deployment with the status of cancelled in lagoon + r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &lagoonEnv, lagooncrd.BuildStatusCancelled, "cancelled") + r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagooncrd.BuildStatusCancelled, "cancelled") + if cancelled { + r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, allContainerLogs, lagooncrd.BuildStatusCancelled) + } + // delete the build from the lagoon namespace in kubernetes entirely + err = r.Delete(ctx, &lagoonBuild) + if err != nil { + return fmt.Errorf("There was an error deleting the lagoon build. Error was: %v", err) + } + return nil +} + +func (r *LagoonBuildReconciler) cancelExtraBuilds(ctx context.Context, opLog logr.Logger, pendingBuilds *lagooncrd.LagoonBuildList, ns string, status string) error { + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(ns), + client.MatchingLabels(map[string]string{"lagoon.sh/buildStatus": lagooncrd.BuildStatusPending.String()}), + }) + if err := r.List(ctx, pendingBuilds, listOption); err != nil { + return fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err) + } + if len(pendingBuilds.Items) > 0 { + if r.EnableDebug { + opLog.Info(fmt.Sprintf("There are %v pending builds", len(pendingBuilds.Items))) + } + // if we have any pending builds, then grab the latest one and make it running + // if there are any other pending builds, cancel them so only the latest one runs + sort.Slice(pendingBuilds.Items, func(i, j int) bool { + return pendingBuilds.Items[i].ObjectMeta.CreationTimestamp.After(pendingBuilds.Items[j].ObjectMeta.CreationTimestamp.Time) + }) + for idx, pBuild := range pendingBuilds.Items { + pendingBuild := pBuild.DeepCopy() + if idx == 0 { + pendingBuild.Labels["lagoon.sh/buildStatus"] = status + } else { + // cancel any other pending builds + opLog.Info(fmt.Sprintf("Attempting to cancel build %s", pendingBuild.ObjectMeta.Name)) + pendingBuild.Labels["lagoon.sh/buildStatus"] = lagooncrd.BuildStatusCancelled.String() + } + if err := r.Update(ctx, pendingBuild); err != nil { + return fmt.Errorf("There was an error updating the pending build. Error was: %v", err) + } + var lagoonBuild lagooncrd.LagoonBuild + if err := r.Get(ctx, types.NamespacedName{ + Namespace: pendingBuild.ObjectMeta.Namespace, + Name: pendingBuild.ObjectMeta.Name, + }, &lagoonBuild); err != nil { + return helpers.IgnoreNotFound(err) + } + opLog.Info(fmt.Sprintf("Cleaning up build %s as cancelled extra build", lagoonBuild.ObjectMeta.Name)) + r.cleanUpUndeployableBuild(ctx, lagoonBuild, "This build was cancelled as a newer build was triggered.", opLog, true) + } + } + return nil +} + +func sortBuilds(defaultPriority int, pendingBuilds *lagooncrd.LagoonBuildList) { + sort.Slice(pendingBuilds.Items, func(i, j int) bool { + // sort by priority, then creation timestamp + iPriority := defaultPriority + jPriority := defaultPriority + if ok := pendingBuilds.Items[i].Spec.Build.Priority; ok != nil { + iPriority = *pendingBuilds.Items[i].Spec.Build.Priority + } + if ok := pendingBuilds.Items[j].Spec.Build.Priority; ok != nil { + jPriority = *pendingBuilds.Items[j].Spec.Build.Priority + } + // better sorting based on priority then creation timestamp + switch { + case iPriority != jPriority: + return iPriority < jPriority + default: + return pendingBuilds.Items[i].ObjectMeta.CreationTimestamp.Before(&pendingBuilds.Items[j].ObjectMeta.CreationTimestamp) + } + }) +} diff --git a/controllers/v1beta2/build_helpers_test.go b/controllers/v1beta2/build_helpers_test.go new file mode 100644 index 00000000..ac8909f6 --- /dev/null +++ b/controllers/v1beta2/build_helpers_test.go @@ -0,0 +1,233 @@ +package v1beta2 + +import ( + "testing" + "time" + + "github.com/google/go-cmp/cmp" + lagooncrd "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" + "github.com/uselagoon/remote-controller/internal/helpers" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func timeFromString(s string) time.Time { + time, _ := time.Parse("2006-01-02T15:04:05.000Z", s) + return time +} + +func Test_sortBuilds(t *testing.T) { + type args struct { + defaultPriority int + pendingBuilds *lagooncrd.LagoonBuildList + } + tests := []struct { + name string + args args + wantBuilds *lagooncrd.LagoonBuildList + }{ + { + name: "test1 - 5 and 6 same time order by priority", + args: args{ + defaultPriority: 5, + pendingBuilds: &lagooncrd.LagoonBuildList{ + Items: []lagooncrd.LagoonBuild{ + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-abcdefg", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:45:00.000Z")), + }, + Spec: lagooncrd.LagoonBuildSpec{ + Build: lagooncrd.Build{ + Priority: helpers.IntPtr(5), + }, + }, + }, + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-1234567", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:45:00.000Z")), + }, + Spec: lagooncrd.LagoonBuildSpec{ + Build: lagooncrd.Build{ + Priority: helpers.IntPtr(6), + }, + }, + }, + }, + }, + }, + wantBuilds: &lagooncrd.LagoonBuildList{ + Items: []lagooncrd.LagoonBuild{ + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-abcdefg", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:45:00.000Z")), + }, + Spec: lagooncrd.LagoonBuildSpec{ + Build: lagooncrd.Build{ + Priority: helpers.IntPtr(5), + }, + }, + }, + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-1234567", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:45:00.000Z")), + }, + Spec: lagooncrd.LagoonBuildSpec{ + Build: lagooncrd.Build{ + Priority: helpers.IntPtr(6), + }, + }, + }, + }, + }, + }, + { + name: "test2 - 2x5 sorted by time", + args: args{ + defaultPriority: 5, + pendingBuilds: &lagooncrd.LagoonBuildList{ + Items: []lagooncrd.LagoonBuild{ + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-abcdefg", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:50:00.000Z")), + }, + Spec: lagooncrd.LagoonBuildSpec{ + Build: lagooncrd.Build{ + Priority: helpers.IntPtr(5), + }, + }, + }, + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-1234567", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:45:00.000Z")), + }, + Spec: lagooncrd.LagoonBuildSpec{ + Build: lagooncrd.Build{ + Priority: helpers.IntPtr(5), + }, + }, + }, + }, + }, + }, + wantBuilds: &lagooncrd.LagoonBuildList{ + Items: []lagooncrd.LagoonBuild{ + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-1234567", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:45:00.000Z")), + }, + Spec: lagooncrd.LagoonBuildSpec{ + Build: lagooncrd.Build{ + Priority: helpers.IntPtr(5), + }, + }, + }, + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-abcdefg", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:50:00.000Z")), + }, + Spec: lagooncrd.LagoonBuildSpec{ + Build: lagooncrd.Build{ + Priority: helpers.IntPtr(5), + }, + }, + }, + }, + }, + }, + { + name: "test3 - 2x5 and 1x6 sorted by priority then time", + args: args{ + defaultPriority: 5, + pendingBuilds: &lagooncrd.LagoonBuildList{ + Items: []lagooncrd.LagoonBuild{ + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-abcdefg", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:50:00.000Z")), + }, + Spec: lagooncrd.LagoonBuildSpec{ + Build: lagooncrd.Build{ + Priority: helpers.IntPtr(5), + }, + }, + }, + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-abc1234", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:46:00.000Z")), + }, + Spec: lagooncrd.LagoonBuildSpec{ + Build: lagooncrd.Build{ + Priority: helpers.IntPtr(6), + }, + }, + }, + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-1234567", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:45:00.000Z")), + }, + Spec: lagooncrd.LagoonBuildSpec{ + Build: lagooncrd.Build{ + Priority: helpers.IntPtr(5), + }, + }, + }, + }, + }, + }, + wantBuilds: &lagooncrd.LagoonBuildList{ + Items: []lagooncrd.LagoonBuild{ + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-1234567", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:45:00.000Z")), + }, + Spec: lagooncrd.LagoonBuildSpec{ + Build: lagooncrd.Build{ + Priority: helpers.IntPtr(5), + }, + }, + }, + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-abcdefg", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:50:00.000Z")), + }, + Spec: lagooncrd.LagoonBuildSpec{ + Build: lagooncrd.Build{ + Priority: helpers.IntPtr(5), + }, + }, + }, + { + ObjectMeta: v1.ObjectMeta{ + Name: "lagoon-build-abc1234", + CreationTimestamp: v1.NewTime(timeFromString("2023-09-18T11:46:00.000Z")), + }, + Spec: lagooncrd.LagoonBuildSpec{ + Build: lagooncrd.Build{ + Priority: helpers.IntPtr(6), + }, + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sortBuilds(tt.args.defaultPriority, tt.args.pendingBuilds) + if !cmp.Equal(tt.args.pendingBuilds, tt.wantBuilds) { + t.Errorf("sortBuilds() = %v, want %v", tt.args.pendingBuilds, tt.wantBuilds) + } + }) + } +} diff --git a/controllers/v1beta2/build_qoshandler.go b/controllers/v1beta2/build_qoshandler.go new file mode 100644 index 00000000..6cddf05c --- /dev/null +++ b/controllers/v1beta2/build_qoshandler.go @@ -0,0 +1,169 @@ +package v1beta2 + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/go-logr/logr" + lagooncrd "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" + "github.com/uselagoon/remote-controller/internal/helpers" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// BuildQoS is use for the quality of service configuration for lagoon builds. +type BuildQoS struct { + MaxBuilds int + DefaultValue int +} + +func (r *LagoonBuildReconciler) qosBuildProcessor(ctx context.Context, + opLog logr.Logger, + lagoonBuild lagooncrd.LagoonBuild, + req ctrl.Request) (ctrl.Result, error) { + // check if we get a lagoonbuild that hasn't got any buildstatus + // this means it was created by the message queue handler + // so we should do the steps required for a lagoon build and then copy the build + // into the created namespace + if _, ok := lagoonBuild.ObjectMeta.Labels["lagoon.sh/buildStatus"]; !ok { + if r.EnableDebug { + opLog.Info(fmt.Sprintf("Creating new build %s from message queue", lagoonBuild.ObjectMeta.Name)) + } + return r.createNamespaceBuild(ctx, opLog, lagoonBuild) + } + if r.EnableDebug { + opLog.Info(fmt.Sprintf("Checking which build next")) + } + // handle the QoS build process here + return ctrl.Result{}, r.whichBuildNext(ctx, opLog) +} + +func (r *LagoonBuildReconciler) whichBuildNext(ctx context.Context, opLog logr.Logger) error { + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.MatchingLabels(map[string]string{ + "lagoon.sh/buildStatus": lagooncrd.BuildStatusRunning.String(), + "lagoon.sh/controller": r.ControllerNamespace, + }), + }) + runningBuilds := &lagooncrd.LagoonBuildList{} + if err := r.List(ctx, runningBuilds, listOption); err != nil { + return fmt.Errorf("Unable to list builds in the cluster, there may be none or something went wrong: %v", err) + } + buildsToStart := r.BuildQoS.MaxBuilds - len(runningBuilds.Items) + if len(runningBuilds.Items) >= r.BuildQoS.MaxBuilds { + // if the maximum number of builds is hit, then drop out and try again next time + if r.EnableDebug { + opLog.Info(fmt.Sprintf("Currently %v running builds, no room for new builds to be started", len(runningBuilds.Items))) + } + go r.processQueue(ctx, opLog, buildsToStart, true) + return nil + } + if buildsToStart > 0 { + opLog.Info(fmt.Sprintf("Currently %v running builds, room for %v builds to be started", len(runningBuilds.Items), buildsToStart)) + // if there are any free slots to start a build, do that here + go r.processQueue(ctx, opLog, buildsToStart, false) + } + return nil +} + +var runningProcessQueue bool + +// this is a processor for any builds that are currently `queued` status. all normal build activity will still be performed +// this just allows the controller to update any builds that are in the queue periodically +// if this ran on every single event, it would flood the queue with messages, so it is restricted using `runningProcessQueue` global +// to only run the process at any one time til it is complete +// buildsToStart is the number of builds that can be started at the time the process is called +// limitHit is used to determine if the build limit has been hit, this is used to prevent new builds from being started inside this process +func (r *LagoonBuildReconciler) processQueue(ctx context.Context, opLog logr.Logger, buildsToStart int, limitHit bool) error { + // this should only ever be able to run one instance of at a time within a single controller + // this is because this process is quite heavy when it goes to submit the queue messages to the api + // the downside of this is that there can be delays with the messages it sends to the actual + // status of the builds, but build complete/fail/cancel will always win out on the lagoon-core side + // so this isn't that much of an issue if there are some delays in the messages + opLog = opLog.WithName("QueueProcessor") + if !runningProcessQueue { + runningProcessQueue = true + if r.EnableDebug { + opLog.Info(fmt.Sprintf("Processing queue")) + } + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.MatchingLabels(map[string]string{ + "lagoon.sh/buildStatus": lagooncrd.BuildStatusPending.String(), + "lagoon.sh/controller": r.ControllerNamespace, + }), + }) + pendingBuilds := &lagooncrd.LagoonBuildList{} + if err := r.List(ctx, pendingBuilds, listOption); err != nil { + runningProcessQueue = false + return fmt.Errorf("Unable to list builds in the cluster, there may be none or something went wrong: %v", err) + } + if len(pendingBuilds.Items) > 0 { + if r.EnableDebug { + opLog.Info(fmt.Sprintf("There are %v pending builds", len(pendingBuilds.Items))) + } + // if we have any pending builds, then grab the latest one and make it running + // if there are any other pending builds, cancel them so only the latest one runs + sortBuilds(r.BuildQoS.DefaultValue, pendingBuilds) + for idx, pBuild := range pendingBuilds.Items { + // need to +1 to index because 0 + if idx+1 <= buildsToStart && !limitHit { + if r.EnableDebug { + opLog.Info(fmt.Sprintf("Checking if build %s can be started", pBuild.ObjectMeta.Name)) + } + // if we do have a `lagoon.sh/buildStatus` set, then process as normal + runningNSBuilds := &lagooncrd.LagoonBuildList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(pBuild.ObjectMeta.Namespace), + client.MatchingLabels(map[string]string{ + "lagoon.sh/buildStatus": lagooncrd.BuildStatusRunning.String(), + "lagoon.sh/controller": r.ControllerNamespace, + }), + }) + // list any builds that are running + if err := r.List(ctx, runningNSBuilds, listOption); err != nil { + runningProcessQueue = false + return fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err) + } + // if there are no running builds, check if there are any pending builds that can be started + if len(runningNSBuilds.Items) == 0 { + if err := lagooncrd.CancelExtraBuilds(ctx, r.Client, opLog, pBuild.ObjectMeta.Namespace, "Running"); err != nil { + // only return if there is an error doing this operation + // continue on otherwise to allow the queued status updater to run + runningProcessQueue = false + return err + } + // don't handle the queued process for this build, continue to next in the list + continue + } + // The object is not being deleted, so if it does not have our finalizer, + // then lets add the finalizer and update the object. This is equivalent + // registering our finalizer. + if !helpers.ContainsString(pBuild.ObjectMeta.Finalizers, buildFinalizer) { + pBuild.ObjectMeta.Finalizers = append(pBuild.ObjectMeta.Finalizers, buildFinalizer) + // use patches to avoid update errors + mergePatch, _ := json.Marshal(map[string]interface{}{ + "metadata": map[string]interface{}{ + "finalizers": pBuild.ObjectMeta.Finalizers, + }, + }) + if err := r.Patch(ctx, &pBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { + runningProcessQueue = false + return err + } + } + } + // update the build to be queued, and add a log message with the build log with the current position in the queue + // this position will update as builds are created/processed, so the position of a build could change depending on + // higher or lower priority builds being created + if err := r.updateQueuedBuild(ctx, pBuild, (idx + 1), len(pendingBuilds.Items), opLog); err != nil { + runningProcessQueue = false + return nil + } + } + } + runningProcessQueue = false + } + return nil +} diff --git a/controllers/v1beta2/build_standardhandler.go b/controllers/v1beta2/build_standardhandler.go new file mode 100644 index 00000000..5315fd89 --- /dev/null +++ b/controllers/v1beta2/build_standardhandler.go @@ -0,0 +1,73 @@ +package v1beta2 + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/go-logr/logr" + lagooncrd "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" + "github.com/uselagoon/remote-controller/internal/helpers" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func (r *LagoonBuildReconciler) standardBuildProcessor(ctx context.Context, + opLog logr.Logger, + lagoonBuild lagooncrd.LagoonBuild, + req ctrl.Request) (ctrl.Result, error) { + // check if we get a lagoonbuild that hasn't got any buildstatus + // this means it was created by the message queue handler + // so we should do the steps required for a lagoon build and then copy the build + // into the created namespace + if _, ok := lagoonBuild.ObjectMeta.Labels["lagoon.sh/buildStatus"]; !ok { + return r.createNamespaceBuild(ctx, opLog, lagoonBuild) + } + + // if we do have a `lagoon.sh/buildStatus` set, then process as normal + runningBuilds := &lagooncrd.LagoonBuildList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(req.Namespace), + client.MatchingLabels(map[string]string{ + "lagoon.sh/buildStatus": lagooncrd.BuildStatusRunning.String(), + "lagoon.sh/controller": r.ControllerNamespace, + }), + }) + // list any builds that are running + if err := r.List(ctx, runningBuilds, listOption); err != nil { + return ctrl.Result{}, fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err) + } + for _, runningBuild := range runningBuilds.Items { + // if the running build is the one from this request then process it + if lagoonBuild.ObjectMeta.Name == runningBuild.ObjectMeta.Name { + // actually process the build here + if _, ok := lagoonBuild.ObjectMeta.Labels["lagoon.sh/buildStarted"]; !ok { + if err := r.processBuild(ctx, opLog, lagoonBuild); err != nil { + return ctrl.Result{}, err + } + } + } // end check if running build is current LagoonBuild + } // end loop for running builds + + // if there are no running builds, check if there are any pending builds that can be started + if len(runningBuilds.Items) == 0 { + return ctrl.Result{}, lagooncrd.CancelExtraBuilds(ctx, r.Client, opLog, req.Namespace, "Running") + } + // The object is not being deleted, so if it does not have our finalizer, + // then lets add the finalizer and update the object. This is equivalent + // registering our finalizer. + if !helpers.ContainsString(lagoonBuild.ObjectMeta.Finalizers, buildFinalizer) { + lagoonBuild.ObjectMeta.Finalizers = append(lagoonBuild.ObjectMeta.Finalizers, buildFinalizer) + // use patches to avoid update errors + mergePatch, _ := json.Marshal(map[string]interface{}{ + "metadata": map[string]interface{}{ + "finalizers": lagoonBuild.ObjectMeta.Finalizers, + }, + }) + if err := r.Patch(ctx, &lagoonBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { + return ctrl.Result{}, err + } + } + return ctrl.Result{}, nil +} diff --git a/controllers/v1beta1/metrics.go b/controllers/v1beta2/metrics.go similarity index 99% rename from controllers/v1beta1/metrics.go rename to controllers/v1beta2/metrics.go index 1958b2eb..900ca39b 100644 --- a/controllers/v1beta1/metrics.go +++ b/controllers/v1beta2/metrics.go @@ -1,4 +1,4 @@ -package v1beta1 +package v1beta2 import ( "github.com/prometheus/client_golang/prometheus" diff --git a/controllers/v1beta2/podmonitor_buildhandlers.go b/controllers/v1beta2/podmonitor_buildhandlers.go new file mode 100644 index 00000000..0bacdc78 --- /dev/null +++ b/controllers/v1beta2/podmonitor_buildhandlers.go @@ -0,0 +1,623 @@ +package v1beta2 + +// this file is used by the `lagoonmonitor` controller + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "strings" + "time" + + "github.com/go-logr/logr" + "github.com/prometheus/client_golang/prometheus" + "github.com/uselagoon/machinery/api/schema" + lagooncrd "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" + "github.com/uselagoon/remote-controller/internal/helpers" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func (r *LagoonMonitorReconciler) handleBuildMonitor(ctx context.Context, + opLog logr.Logger, + req ctrl.Request, + jobPod corev1.Pod, +) error { + // get the build associated to this pod, we wil need update it at some point + var lagoonBuild lagooncrd.LagoonBuild + err := r.Get(ctx, types.NamespacedName{ + Namespace: jobPod.ObjectMeta.Namespace, + Name: jobPod.ObjectMeta.Labels["lagoon.sh/buildName"], + }, &lagoonBuild) + if err != nil { + return err + } + cancel := false + if cancelBuild, ok := jobPod.ObjectMeta.Labels["lagoon.sh/cancelBuild"]; ok { + cancel, _ = strconv.ParseBool(cancelBuild) + } + _, ok := r.Cache.Get(lagoonBuild.ObjectMeta.Name) + if ok { + opLog.Info(fmt.Sprintf("Cached cancellation exists for: %s", lagoonBuild.ObjectMeta.Name)) + // this object exists in the cache meaning the task has been cancelled, set cancel to true and remove from cache + r.Cache.Remove(lagoonBuild.ObjectMeta.Name) + cancel = true + } + if cancel { + opLog.Info(fmt.Sprintf("Attempting to cancel build %s", lagoonBuild.ObjectMeta.Name)) + return r.updateDeploymentWithLogs(ctx, req, lagoonBuild, jobPod, nil, cancel) + } + // check if the build pod is in pending, a container in the pod could be failed in this state + if jobPod.Status.Phase == corev1.PodPending { + // check each container in the pod + for _, container := range jobPod.Status.ContainerStatuses { + // if the container is a lagoon-build container + // which currently it will be as only one container is spawned in a build + if container.Name == "lagoon-build" { + // check if the state of the pod is one of our failure states + if container.State.Waiting != nil && helpers.ContainsString(failureStates, container.State.Waiting.Reason) { + // if we have a failure state, then fail the build and get the logs from the container + opLog.Info(fmt.Sprintf("Build failed, container exit reason was: %v", container.State.Waiting.Reason)) + lagoonBuild.Labels["lagoon.sh/buildStatus"] = lagooncrd.BuildStatusFailed.String() + if err := r.Update(ctx, &lagoonBuild); err != nil { + return err + } + opLog.Info(fmt.Sprintf("Marked build %s as %s", lagoonBuild.ObjectMeta.Name, lagooncrd.BuildStatusFailed.String())) + if err := r.Delete(ctx, &jobPod); err != nil { + return err + } + opLog.Info(fmt.Sprintf("Deleted failed build pod: %s", jobPod.ObjectMeta.Name)) + // update the status to failed on the deleted pod + // and set the terminate time to now, it is used when we update the deployment and environment + jobPod.Status.Phase = corev1.PodFailed + state := corev1.ContainerStatus{ + State: corev1.ContainerState{ + Terminated: &corev1.ContainerStateTerminated{ + FinishedAt: metav1.Time{Time: time.Now().UTC()}, + }, + }, + } + jobPod.Status.ContainerStatuses[0] = state + + // get the configmap for lagoon-env so we can use it for updating the deployment in lagoon + var lagoonEnv corev1.ConfigMap + err := r.Get(ctx, types.NamespacedName{Namespace: jobPod.ObjectMeta.Namespace, Name: "lagoon-env"}, &lagoonEnv) + if err != nil { + // if there isn't a configmap, just info it and move on + // the updatedeployment function will see it as nil and not bother doing the bits that require the configmap + if r.EnableDebug { + opLog.Info(fmt.Sprintf("There is no configmap %s in namespace %s ", "lagoon-env", jobPod.ObjectMeta.Namespace)) + } + } + // send any messages to lagoon message queues + logMsg := fmt.Sprintf("%v: %v", container.State.Waiting.Reason, container.State.Waiting.Message) + return r.updateDeploymentWithLogs(ctx, req, lagoonBuild, jobPod, []byte(logMsg), false) + } + } + } + return r.updateDeploymentWithLogs(ctx, req, lagoonBuild, jobPod, nil, false) + } else if jobPod.Status.Phase == corev1.PodRunning { + // if the pod is running and detects a change to the pod (eg, detecting an updated lagoon.sh/buildStep label) + // then ship or store the logs + // get the build associated to this pod, the information in the resource is used for shipping the logs + var lagoonBuild lagooncrd.LagoonBuild + err := r.Get(ctx, + types.NamespacedName{ + Namespace: jobPod.ObjectMeta.Namespace, + Name: jobPod.ObjectMeta.Labels["lagoon.sh/buildName"], + }, &lagoonBuild) + if err != nil { + return err + } + } + // if the buildpod status is failed or succeeded + // mark the build accordingly and ship the information back to lagoon + if jobPod.Status.Phase == corev1.PodFailed || jobPod.Status.Phase == corev1.PodSucceeded { + // get the build associated to this pod, we wil need update it at some point + var lagoonBuild lagooncrd.LagoonBuild + err := r.Get(ctx, + types.NamespacedName{ + Namespace: jobPod.ObjectMeta.Namespace, + Name: jobPod.ObjectMeta.Labels["lagoon.sh/buildName"], + }, &lagoonBuild) + if err != nil { + return err + } + } + // send any messages to lagoon message queues + return r.updateDeploymentWithLogs(ctx, req, lagoonBuild, jobPod, nil, false) +} + +// buildLogsToLagoonLogs sends the build logs to the lagoon-logs message queue +// it contains the actual pod log output that is sent to elasticsearch, it is what eventually is displayed in the UI +func (r *LagoonMonitorReconciler) buildLogsToLagoonLogs(ctx context.Context, + opLog logr.Logger, + lagoonBuild *lagooncrd.LagoonBuild, + jobPod *corev1.Pod, + namespace *corev1.Namespace, + condition string, + logs []byte, +) (bool, schema.LagoonLog) { + if r.EnableMQ { + buildStep := "running" + if condition == "failed" || condition == "complete" || condition == "cancelled" { + // set build step to anything other than running if the condition isn't running + buildStep = condition + } + // then check the resource to see if the buildstep exists, this bit we care about so we can see where it maybe failed if its available + if value, ok := jobPod.Labels["lagoon.sh/buildStep"]; ok { + buildStep = value + } + envName := lagoonBuild.Spec.Project.Environment + envID := lagoonBuild.Spec.Project.EnvironmentID + projectName := lagoonBuild.Spec.Project.Name + projectID := lagoonBuild.Spec.Project.ID + if lagoonBuild == nil { + envName = namespace.ObjectMeta.Labels["lagoon.sh/environment"] + eID, _ := strconv.Atoi(namespace.ObjectMeta.Labels["lagoon.sh/environment"]) + envID = helpers.UintPtr(uint(eID)) + projectName = namespace.ObjectMeta.Labels["lagoon.sh/environment"] + pID, _ := strconv.Atoi(namespace.ObjectMeta.Labels["lagoon.sh/environment"]) + projectID = helpers.UintPtr(uint(pID)) + } + remoteId := string(jobPod.ObjectMeta.UID) + if value, ok := jobPod.Labels["lagoon.sh/buildRemoteID"]; ok { + remoteId = value + } + msg := schema.LagoonLog{ + Severity: "info", + Project: projectName, + Event: "build-logs:builddeploy-kubernetes:" + jobPod.ObjectMeta.Name, + Meta: &schema.LagoonLogMeta{ + EnvironmentID: envID, + ProjectID: projectID, + BuildName: jobPod.ObjectMeta.Name, + BranchName: envName, + BuildStatus: condition, // same as buildstatus label + BuildStep: buildStep, + RemoteID: remoteId, + LogLink: lagoonBuild.Spec.Project.UILink, + Cluster: r.LagoonTargetName, + }, + } + // add the actual build log message + if jobPod.Spec.NodeName != "" { + msg.Message = fmt.Sprintf(`================================================================================ +Logs on pod %s, assigned to node %s on cluster %s +================================================================================ +%s`, jobPod.ObjectMeta.Name, jobPod.Spec.NodeName, r.LagoonTargetName, logs) + } else { + msg.Message = fmt.Sprintf(`======================================== +Logs on pod %s, assigned to cluster %s +======================================== +%s`, jobPod.ObjectMeta.Name, r.LagoonTargetName, logs) + } + msgBytes, err := json.Marshal(msg) + if err != nil { + opLog.Error(err, "Unable to encode message as JSON") + } + if err := r.Messaging.Publish("lagoon-logs", msgBytes); err != nil { + // if we can't publish the message, set it as a pending message + // overwrite whatever is there as these are just current state messages so it doesn't + // really matter if we don't smootly transition in what we send back to lagoon + // r.updateBuildLogMessage(ctx, lagoonBuild, msg) + return true, msg + } + if r.EnableDebug { + opLog.Info( + fmt.Sprintf( + "Published event %s for %s to lagoon-logs exchange", + fmt.Sprintf("build-logs:builddeploy-kubernetes:%s", jobPod.ObjectMeta.Name), + jobPod.ObjectMeta.Name, + ), + ) + } + // if we are able to publish the message, then we need to remove any pending messages from the resource + // and make sure we don't try and publish again + } + return false, schema.LagoonLog{} +} + +// updateDeploymentAndEnvironmentTask sends the status of the build and deployment to the controllerhandler message queue in lagoon, +// this is for the handler in lagoon to process. +func (r *LagoonMonitorReconciler) updateDeploymentAndEnvironmentTask(ctx context.Context, + opLog logr.Logger, + lagoonBuild *lagooncrd.LagoonBuild, + jobPod *corev1.Pod, + lagoonEnv *corev1.ConfigMap, + namespace *corev1.Namespace, + condition string, +) (bool, schema.LagoonMessage) { + if r.EnableMQ { + buildStep := "running" + if condition == "failed" || condition == "complete" || condition == "cancelled" { + // set build step to anything other than running if the condition isn't running + buildStep = condition + } + // then check the resource to see if the buildstep exists, this bit we care about so we can see where it maybe failed if its available + if value, ok := jobPod.Labels["lagoon.sh/buildStep"]; ok { + buildStep = value + } + if condition == "failed" || condition == "complete" || condition == "cancelled" { + time.AfterFunc(31*time.Second, func() { + buildRunningStatus.Delete(prometheus.Labels{ + "build_namespace": lagoonBuild.ObjectMeta.Namespace, + "build_name": lagoonBuild.ObjectMeta.Name, + }) + }) + time.Sleep(2 * time.Second) // smol sleep to reduce race of final messages with previous messages + } + envName := lagoonBuild.Spec.Project.Environment + envID := lagoonBuild.Spec.Project.EnvironmentID + projectName := lagoonBuild.Spec.Project.Name + projectID := lagoonBuild.Spec.Project.ID + if lagoonBuild == nil { + envName = namespace.ObjectMeta.Labels["lagoon.sh/environment"] + eID, _ := strconv.Atoi(namespace.ObjectMeta.Labels["lagoon.sh/environment"]) + envID = helpers.UintPtr(uint(eID)) + projectName = namespace.ObjectMeta.Labels["lagoon.sh/environment"] + pID, _ := strconv.Atoi(namespace.ObjectMeta.Labels["lagoon.sh/environment"]) + projectID = helpers.UintPtr(uint(pID)) + } + remoteId := string(jobPod.ObjectMeta.UID) + if value, ok := jobPod.Labels["lagoon.sh/buildRemoteID"]; ok { + remoteId = value + } + msg := schema.LagoonMessage{ + Type: "build", + Namespace: namespace.ObjectMeta.Name, + Meta: &schema.LagoonLogMeta{ + Environment: envName, + EnvironmentID: envID, + Project: projectName, + ProjectID: projectID, + BuildName: jobPod.ObjectMeta.Name, + BuildStatus: condition, // same as buildstatus label + BuildStep: buildStep, + LogLink: lagoonBuild.Spec.Project.UILink, + RemoteID: remoteId, + Cluster: r.LagoonTargetName, + }, + } + labelRequirements1, _ := labels.NewRequirement("lagoon.sh/service", selection.NotIn, []string{"faketest"}) + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(jobPod.ObjectMeta.Namespace), + client.MatchingLabelsSelector{ + Selector: labels.NewSelector().Add(*labelRequirements1), + }, + }) + depList := &appsv1.DeploymentList{} + serviceNames := []string{} + if err := r.List(context.TODO(), depList, listOption); err == nil { + // generate the list of services to add to the environment + for _, deployment := range depList.Items { + if _, ok := deployment.ObjectMeta.Labels["lagoon.sh/service"]; ok { + for _, container := range deployment.Spec.Template.Spec.Containers { + serviceNames = append(serviceNames, container.Name) + } + } + if _, ok := deployment.ObjectMeta.Labels["service"]; ok { + for _, container := range deployment.Spec.Template.Spec.Containers { + serviceNames = append(serviceNames, container.Name) + } + } + } + msg.Meta.Services = serviceNames + } + // if we aren't being provided the lagoon config, we can skip adding the routes etc + if lagoonEnv != nil { + msg.Meta.Route = "" + if route, ok := lagoonEnv.Data["LAGOON_ROUTE"]; ok { + msg.Meta.Route = route + } + msg.Meta.Routes = []string{} + if routes, ok := lagoonEnv.Data["LAGOON_ROUTES"]; ok { + msg.Meta.Routes = strings.Split(routes, ",") + } + } + // we can add the build start time here + if jobPod.Status.StartTime != nil { + msg.Meta.StartTime = jobPod.Status.StartTime.Time.UTC().Format("2006-01-02 15:04:05") + } + if condition == "cancelled" { + // if the build has been canclled, the pod termination time may not exist yet. + // use the current time first, it will get overwritten if there is a pod termination time later. + msg.Meta.EndTime = time.Now().UTC().Format("2006-01-02 15:04:05") + } + // and then once the pod is terminated we can add the terminated time here + if jobPod.Status.ContainerStatuses != nil { + if jobPod.Status.ContainerStatuses[0].State.Terminated != nil { + msg.Meta.EndTime = jobPod.Status.ContainerStatuses[0].State.Terminated.FinishedAt.Time.UTC().Format("2006-01-02 15:04:05") + } + } + msgBytes, err := json.Marshal(msg) + if err != nil { + opLog.Error(err, "Unable to encode message as JSON") + } + if err := r.Messaging.Publish("lagoon-tasks:controller", msgBytes); err != nil { + // if we can't publish the message, set it as a pending message + // overwrite whatever is there as these are just current state messages so it doesn't + // really matter if we don't smootly transition in what we send back to lagoon + return true, msg + } + if r.EnableDebug { + opLog.Info( + fmt.Sprintf( + "Published build update message for %s to lagoon-tasks:controller queue", + jobPod.ObjectMeta.Name, + ), + ) + } + // if we are able to publish the message, then we need to remove any pending messages from the resource + // and make sure we don't try and publish again + } + return false, schema.LagoonMessage{} +} + +// buildStatusLogsToLagoonLogs sends the logs to lagoon-logs message queue, used for general messaging +func (r *LagoonMonitorReconciler) buildStatusLogsToLagoonLogs(ctx context.Context, + opLog logr.Logger, + lagoonBuild *lagooncrd.LagoonBuild, + jobPod *corev1.Pod, + lagoonEnv *corev1.ConfigMap, + namespace *corev1.Namespace, + condition string, +) (bool, schema.LagoonLog) { + if r.EnableMQ { + buildStep := "running" + if condition == "failed" || condition == "complete" || condition == "cancelled" { + // set build step to anything other than running if the condition isn't running + buildStep = condition + } + // then check the resource to see if the buildstep exists, this bit we care about so we can see where it maybe failed if its available + if value, ok := jobPod.Labels["lagoon.sh/buildStep"]; ok { + buildStep = value + } + envName := lagoonBuild.Spec.Project.Environment + envID := lagoonBuild.Spec.Project.EnvironmentID + projectName := lagoonBuild.Spec.Project.Name + projectID := lagoonBuild.Spec.Project.ID + if lagoonBuild == nil { + envName = namespace.ObjectMeta.Labels["lagoon.sh/environment"] + eID, _ := strconv.Atoi(namespace.ObjectMeta.Labels["lagoon.sh/environment"]) + envID = helpers.UintPtr(uint(eID)) + projectName = namespace.ObjectMeta.Labels["lagoon.sh/environment"] + pID, _ := strconv.Atoi(namespace.ObjectMeta.Labels["lagoon.sh/environment"]) + projectID = helpers.UintPtr(uint(pID)) + } + msg := schema.LagoonLog{ + Severity: "info", + Project: projectName, + Event: "task:builddeploy-kubernetes:" + condition, //@TODO: this probably needs to be changed to a new task event for the controller + Meta: &schema.LagoonLogMeta{ + EnvironmentID: envID, + ProjectID: projectID, + ProjectName: projectName, + BranchName: envName, + BuildName: jobPod.ObjectMeta.Name, + BuildStatus: condition, // same as buildstatus label + BuildStep: buildStep, + LogLink: lagoonBuild.Spec.Project.UILink, + Cluster: r.LagoonTargetName, + }, + } + // if we aren't being provided the lagoon config, we can skip adding the routes etc + var addRoute, addRoutes string + if lagoonEnv != nil { + msg.Meta.Route = "" + if route, ok := lagoonEnv.Data["LAGOON_ROUTE"]; ok { + msg.Meta.Route = route + addRoute = fmt.Sprintf("\n%s", route) + } + msg.Meta.Routes = []string{} + if routes, ok := lagoonEnv.Data["LAGOON_ROUTES"]; ok { + msg.Meta.Routes = strings.Split(routes, ",") + addRoutes = fmt.Sprintf("\n%s", strings.Join(strings.Split(routes, ","), "\n")) + } + } + msg.Message = fmt.Sprintf("*[%s]* `%s` Build `%s` %s <%s|Logs>%s%s", + projectName, + envName, + jobPod.ObjectMeta.Name, + string(jobPod.Status.Phase), + lagoonBuild.Spec.Project.UILink, + addRoute, + addRoutes, + ) + msgBytes, err := json.Marshal(msg) + if err != nil { + opLog.Error(err, "Unable to encode message as JSON") + } + if err := r.Messaging.Publish("lagoon-logs", msgBytes); err != nil { + // if we can't publish the message, set it as a pending message + // overwrite whatever is there as these are just current state messages so it doesn't + // really matter if we don't smootly transition in what we send back to lagoon + return true, msg + } + if r.EnableDebug { + opLog.Info( + fmt.Sprintf( + "Published event %s for %s to lagoon-logs exchange", + fmt.Sprintf("task:builddeploy-kubernetes:%s", condition), + jobPod.ObjectMeta.Name, + ), + ) + } + // if we are able to publish the message, then we need to remove any pending messages from the resource + // and make sure we don't try and publish again + } + return false, schema.LagoonLog{} +} + +// updateDeploymentWithLogs collects logs from the build containers and ships or stores them +func (r *LagoonMonitorReconciler) updateDeploymentWithLogs( + ctx context.Context, + req ctrl.Request, + lagoonBuild lagooncrd.LagoonBuild, + jobPod corev1.Pod, + logs []byte, + cancel bool, +) error { + opLog := r.Log.WithValues("lagoonmonitor", req.NamespacedName) + buildCondition := lagooncrd.GetBuildConditionFromPod(jobPod.Status.Phase) + collectLogs := true + if cancel { + // only set the status to cancelled if the pod is running/pending/queued + // otherwise send the existing status of complete/failed/cancelled + if helpers.ContainsString( + lagooncrd.BuildRunningPendingStatus, + lagoonBuild.Labels["lagoon.sh/buildStatus"], + ) { + buildCondition = lagooncrd.BuildStatusCancelled + } + if _, ok := lagoonBuild.ObjectMeta.Labels["lagoon.sh/cancelBuildNoPod"]; ok { + collectLogs = false + } + } + buildStep := "running" + if value, ok := jobPod.Labels["lagoon.sh/buildStep"]; ok { + buildStep = value + } + + namespace := &corev1.Namespace{} + if err := r.Get(ctx, types.NamespacedName{Name: jobPod.ObjectMeta.Namespace}, namespace); err != nil { + if helpers.IgnoreNotFound(err) != nil { + return err + } + } + // if the buildstatus is pending or running, or the cancel flag is provided + // send the update status to lagoon + if helpers.ContainsString( + lagooncrd.BuildRunningPendingStatus, + lagoonBuild.Labels["lagoon.sh/buildStatus"], + ) || cancel { + opLog.Info( + fmt.Sprintf( + "Updating build status for %s to %v/%v", + jobPod.ObjectMeta.Labels["lagoon.sh/buildName"], + buildCondition, + buildStep, + ), + ) + var allContainerLogs []byte + var err error + if logs == nil { + if collectLogs { + allContainerLogs, err = r.collectLogs(ctx, req, jobPod) + if err == nil { + if cancel { + cancellationMessage := "Build cancelled" + if cancellationDetails, ok := jobPod.GetAnnotations()["lagoon.sh/cancelReason"]; ok { + cancellationMessage = fmt.Sprintf("%v : %v", cancellationMessage, cancellationDetails) + } + allContainerLogs = append(allContainerLogs, []byte(fmt.Sprintf(` +======================================== +%v +========================================`, cancellationMessage))...) + } + } else { + allContainerLogs = []byte(fmt.Sprintf(` +======================================== +Build %s +========================================`, buildCondition)) + } + } + } else { + allContainerLogs = logs + } + + mergeMap := map[string]interface{}{ + "metadata": map[string]interface{}{ + "labels": map[string]interface{}{ + "lagoon.sh/buildStatus": buildCondition.String(), + "lagoon.sh/buildStarted": "true", + }, + }, + "statusMessages": map[string]interface{}{}, + } + + condition := lagooncrd.LagoonBuildConditions{ + Type: buildCondition, + Status: corev1.ConditionTrue, + LastTransitionTime: time.Now().UTC().Format(time.RFC3339), + } + if !lagooncrd.BuildContainsStatus(lagoonBuild.Status.Conditions, condition) { + lagoonBuild.Status.Conditions = append(lagoonBuild.Status.Conditions, condition) + mergeMap["status"] = map[string]interface{}{ + "conditions": lagoonBuild.Status.Conditions, + // don't save build logs in resource anymore + } + } + + // get the configmap for lagoon-env so we can use it for updating the deployment in lagoon + var lagoonEnv corev1.ConfigMap + if err := r.Get(ctx, types.NamespacedName{ + Namespace: jobPod.ObjectMeta.Namespace, + Name: "lagoon-env", + }, + &lagoonEnv, + ); err != nil { + // if there isn't a configmap, just info it and move on + // the updatedeployment function will see it as nil and not bother doing the bits that require the configmap + if r.EnableDebug { + opLog.Info(fmt.Sprintf("There is no configmap %s in namespace %s ", "lagoon-env", jobPod.ObjectMeta.Namespace)) + } + } + + // do any message publishing here, and update any pending messages if needed + pendingStatus, pendingStatusMessage := r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &jobPod, &lagoonEnv, namespace, buildCondition.ToLower()) + pendingEnvironment, pendingEnvironmentMessage := r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &jobPod, &lagoonEnv, namespace, buildCondition.ToLower()) + var pendingBuildLog bool + var pendingBuildLogMessage schema.LagoonLog + // if the container logs can't be retrieved, we don't want to send any build logs back, as this will nuke + // any previously received logs + if !strings.Contains(string(allContainerLogs), "unable to retrieve container logs for containerd") { + pendingBuildLog, pendingBuildLogMessage = r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &jobPod, namespace, buildCondition.ToLower(), allContainerLogs) + } + if pendingStatus || pendingEnvironment || pendingBuildLog { + mergeMap["metadata"].(map[string]interface{})["labels"].(map[string]interface{})["lagoon.sh/pendingMessages"] = "true" + if pendingStatus { + mergeMap["statusMessages"].(map[string]interface{})["statusMessage"] = pendingStatusMessage + } + if pendingEnvironment { + mergeMap["statusMessages"].(map[string]interface{})["environmentMessage"] = pendingEnvironmentMessage + } + // if the build log message is too long, don't save it + if pendingBuildLog && len(pendingBuildLogMessage.Message) > 1048576 { + mergeMap["statusMessages"].(map[string]interface{})["buildLogMessage"] = pendingBuildLogMessage + } + } + if !pendingStatus && !pendingEnvironment && !pendingBuildLog { + mergeMap["metadata"].(map[string]interface{})["labels"].(map[string]interface{})["lagoon.sh/pendingMessages"] = nil + mergeMap["statusMessages"] = nil + } + mergePatch, _ := json.Marshal(mergeMap) + // check if the build exists + if err := r.Get(ctx, req.NamespacedName, &lagoonBuild); err == nil { + // if it does, try to patch it + if err := r.Patch(ctx, &lagoonBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to update resource")) + } + } + // just delete the pod + // maybe if we move away from using BASH for the kubectl-build-deploy-dind scripts we could handle cancellations better + if cancel { + if err := r.Get(ctx, req.NamespacedName, &jobPod); err == nil { + if r.EnableDebug { + opLog.Info(fmt.Sprintf("Build pod exists %s", jobPod.ObjectMeta.Name)) + } + if err := r.Delete(ctx, &jobPod); err != nil { + return err + } + } + } + } + return nil +} diff --git a/controllers/v1beta2/podmonitor_controller.go b/controllers/v1beta2/podmonitor_controller.go new file mode 100644 index 00000000..9ae9f340 --- /dev/null +++ b/controllers/v1beta2/podmonitor_controller.go @@ -0,0 +1,225 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta2 + +import ( + "bytes" + "context" + "fmt" + "io" + "strconv" + + "github.com/go-logr/logr" + "github.com/hashicorp/golang-lru/v2/expirable" + lagooncrd "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" + "github.com/uselagoon/remote-controller/internal/helpers" + "github.com/uselagoon/remote-controller/internal/messenger" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/config" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +// LagoonMonitorReconciler reconciles a LagoonBuild object +type LagoonMonitorReconciler struct { + client.Client + Log logr.Logger + Scheme *runtime.Scheme + EnableMQ bool + Messaging *messenger.Messenger + ControllerNamespace string + NamespacePrefix string + RandomNamespacePrefix bool + EnableDebug bool + LagoonTargetName string + LFFQoSEnabled bool + BuildQoS BuildQoS + Cache *expirable.LRU[string, string] +} + +// slice of the different failure states of pods that we care about +// if we observe these on a pending pod, fail the build and get the logs +var failureStates = []string{ + "CrashLoopBackOff", + "ImagePullBackOff", +} + +// @TODO: all the things for now, review later +// +kubebuilder:rbac:groups="*",resources="*",verbs="*" + +// Reconcile runs when a request comes through +func (r *LagoonMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + opLog := r.Log.WithValues("lagoonmonitor", req.NamespacedName) + + var jobPod corev1.Pod + if err := r.Get(ctx, req.NamespacedName, &jobPod); err != nil { + return ctrl.Result{}, helpers.IgnoreNotFound(err) + } + + // if this is a lagoon task, then run the handle task monitoring process + if jobPod.ObjectMeta.Labels["lagoon.sh/jobType"] == "task" { + err := r.calculateTaskMetrics(ctx) + if err != nil { + opLog.Error(err, fmt.Sprintf("Unable to generate metrics.")) + } + if jobPod.ObjectMeta.DeletionTimestamp.IsZero() { + // pod is not being deleted + return ctrl.Result{}, r.handleTaskMonitor(ctx, opLog, req, jobPod) + } + // pod deletion request came through, check if this is an activestandby task, if it is, delete the activestandby role + if value, ok := jobPod.ObjectMeta.Labels["lagoon.sh/activeStandby"]; ok { + isActiveStandby, _ := strconv.ParseBool(value) + if isActiveStandby { + var destinationNamespace string + if value, ok := jobPod.ObjectMeta.Labels["lagoon.sh/activeStandbyDestinationNamespace"]; ok { + destinationNamespace = value + } + err := r.deleteActiveStandbyRole(ctx, destinationNamespace) + if err != nil { + return ctrl.Result{}, err + } + } + } + } + // if this is a lagoon build, then run the handle build monitoring process + if jobPod.ObjectMeta.Labels["lagoon.sh/jobType"] == "build" { + err := r.calculateBuildMetrics(ctx) + if err != nil { + opLog.Error(err, fmt.Sprintf("Unable to generate metrics.")) + } + if jobPod.ObjectMeta.DeletionTimestamp.IsZero() { + // pod is not being deleted + return ctrl.Result{}, r.handleBuildMonitor(ctx, opLog, req, jobPod) + } + + // a pod deletion request came through + // first try and clean up the pod and capture the logs and update + // the lagoonbuild that owns it with the status + var lagoonBuild lagooncrd.LagoonBuild + err = r.Get(ctx, types.NamespacedName{ + Namespace: jobPod.ObjectMeta.Namespace, + Name: jobPod.ObjectMeta.Labels["lagoon.sh/buildName"], + }, &lagoonBuild) + if err != nil { + opLog.Info(fmt.Sprintf("The build that started this pod may have been deleted or not started yet, continuing with cancellation if required.")) + err = r.updateDeploymentWithLogs(ctx, req, lagoonBuild, jobPod, nil, true) + if err != nil { + opLog.Error(err, fmt.Sprintf("Unable to update the LagoonBuild.")) + } + } else { + if helpers.ContainsString( + lagooncrd.BuildRunningPendingStatus, + lagoonBuild.Labels["lagoon.sh/buildStatus"], + ) { + opLog.Info(fmt.Sprintf("Attempting to update the LagoonBuild with cancellation if required.")) + // this will update the deployment back to lagoon if it can do so + // and should only update if the LagoonBuild is Pending or Running + err = r.updateDeploymentWithLogs(ctx, req, lagoonBuild, jobPod, nil, true) + if err != nil { + opLog.Error(err, fmt.Sprintf("Unable to update the LagoonBuild.")) + } + } + } + // if the update is successful or not, it will just continue on to check for pending builds + // in the event pending builds are not processed and the build pod itself has been deleted + // then manually patching the `LagoonBuild` with the label + // "lagoon.sh/buildStatus=Cancelled" + // should be enough to get things rolling again if no pending builds are being picked up + + // if we got any pending builds come through while one is running + // they will be processed here when any pods are cleaned up + // we check all `LagoonBuild` in the requested namespace + // if there are no running jobs, we check for any pending jobs + // sorted by their creation timestamp and set the first to running + if !r.LFFQoSEnabled { + // if qos is not enabled, then handle the check for pending builds here + opLog.Info(fmt.Sprintf("Checking for any pending builds.")) + runningBuilds := &lagooncrd.LagoonBuildList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(req.Namespace), + client.MatchingLabels(map[string]string{"lagoon.sh/buildStatus": lagooncrd.BuildStatusRunning.String()}), + }) + // list all builds in the namespace that have the running buildstatus + if err := r.List(ctx, runningBuilds, listOption); err != nil { + return ctrl.Result{}, fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err) + } + // if we have no running builds, then check for any pending builds + if len(runningBuilds.Items) == 0 { + return ctrl.Result{}, lagooncrd.CancelExtraBuilds(ctx, r.Client, opLog, req.Namespace, "Running") + } + } else { + // since qos handles pending build checks as part of its own operations, we can skip the running pod check step with no-op + if r.EnableDebug { + opLog.Info(fmt.Sprintf("No pending build check in namespaces when QoS is enabled")) + } + } + } + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the given manager +// and we set it to watch Pods with an event filter that contains our build label +func (r *LagoonMonitorReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&corev1.Pod{}). + WithEventFilter(PodPredicates{ + ControllerNamespace: r.ControllerNamespace, + }). + Complete(r) +} + +// getContainerLogs grabs the logs from a given container +func getContainerLogs(ctx context.Context, containerName string, request ctrl.Request) ([]byte, error) { + restCfg, err := config.GetConfig() + if err != nil { + return nil, fmt.Errorf("unable to get config: %v", err) + } + clientset, err := kubernetes.NewForConfig(restCfg) + if err != nil { + return nil, fmt.Errorf("unable to create client: %v", err) + } + req := clientset.CoreV1().Pods(request.Namespace).GetLogs(request.Name, &corev1.PodLogOptions{Container: containerName}) + podLogs, err := req.Stream(ctx) + if err != nil { + return nil, fmt.Errorf("error in opening stream: %v", err) + } + defer podLogs.Close() + buf := new(bytes.Buffer) + _, err = io.Copy(buf, podLogs) + if err != nil { + return nil, fmt.Errorf("error in copy information from podLogs to buffer: %v", err) + } + return buf.Bytes(), nil +} + +func (r *LagoonMonitorReconciler) collectLogs(ctx context.Context, req reconcile.Request, jobPod corev1.Pod) ([]byte, error) { + var allContainerLogs []byte + // grab all the logs from the containers in the task pod and just merge them all together + // we only have 1 container at the moment in a taskpod anyway so it doesn't matter + // if we do move to multi container tasks, then worry about it + for _, container := range jobPod.Spec.Containers { + cLogs, err := getContainerLogs(ctx, container.Name, req) + if err != nil { + return nil, fmt.Errorf("Unable to retrieve logs from pod: %v", err) + } + allContainerLogs = append(allContainerLogs, cLogs...) + } + return allContainerLogs, nil +} diff --git a/controllers/v1beta1/podmonitor_metrics.go b/controllers/v1beta2/podmonitor_metrics.go similarity index 98% rename from controllers/v1beta1/podmonitor_metrics.go rename to controllers/v1beta2/podmonitor_metrics.go index afd1751b..24520f51 100644 --- a/controllers/v1beta1/podmonitor_metrics.go +++ b/controllers/v1beta2/podmonitor_metrics.go @@ -1,4 +1,4 @@ -package v1beta1 +package v1beta2 import ( "context" diff --git a/controllers/v1beta2/podmonitor_taskhandlers.go b/controllers/v1beta2/podmonitor_taskhandlers.go new file mode 100644 index 00000000..5868e543 --- /dev/null +++ b/controllers/v1beta2/podmonitor_taskhandlers.go @@ -0,0 +1,401 @@ +package v1beta2 + +// this file is used by the `lagoonmonitor` controller + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "strings" + "time" + + "github.com/go-logr/logr" + "github.com/prometheus/client_golang/prometheus" + "github.com/uselagoon/machinery/api/schema" + lagooncrd "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" + "github.com/uselagoon/remote-controller/internal/helpers" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func (r *LagoonMonitorReconciler) handleTaskMonitor(ctx context.Context, opLog logr.Logger, req ctrl.Request, jobPod corev1.Pod) error { + // get the task associated to this pod, we wil need update it at some point + var lagoonTask lagooncrd.LagoonTask + err := r.Get(ctx, types.NamespacedName{ + Namespace: jobPod.ObjectMeta.Namespace, + Name: jobPod.ObjectMeta.Labels["lagoon.sh/taskName"], + }, &lagoonTask) + if err != nil { + return err + } + cancel := false + if cancelTask, ok := jobPod.ObjectMeta.Labels["lagoon.sh/cancelTask"]; ok { + cancel, _ = strconv.ParseBool(cancelTask) + } + _, ok := r.Cache.Get(lagoonTask.ObjectMeta.Name) + if ok { + opLog.Info(fmt.Sprintf("Cached cancellation exists for: %s", lagoonTask.ObjectMeta.Name)) + // this object exists in the cache meaning the task has been cancelled, set cancel to true and remove from cache + r.Cache.Remove(lagoonTask.ObjectMeta.Name) + cancel = true + } + if cancel { + opLog.Info(fmt.Sprintf("Attempting to cancel task %s", lagoonTask.ObjectMeta.Name)) + return r.updateTaskWithLogs(ctx, req, lagoonTask, jobPod, nil, cancel) + } + if jobPod.Status.Phase == corev1.PodPending { + for _, container := range jobPod.Status.ContainerStatuses { + if container.State.Waiting != nil && helpers.ContainsString(failureStates, container.State.Waiting.Reason) { + // if we have a failure state, then fail the task and get the logs from the container + opLog.Info(fmt.Sprintf("Task failed, container exit reason was: %v", container.State.Waiting.Reason)) + lagoonTask.Labels["lagoon.sh/taskStatus"] = lagooncrd.TaskStatusFailed.String() + if err := r.Update(ctx, &lagoonTask); err != nil { + return err + } + opLog.Info(fmt.Sprintf("Marked task %s as %s", lagoonTask.ObjectMeta.Name, lagooncrd.TaskStatusFailed.String())) + if err := r.Delete(ctx, &jobPod); err != nil { + return err + } + opLog.Info(fmt.Sprintf("Deleted failed task pod: %s", jobPod.ObjectMeta.Name)) + // update the status to failed on the deleted pod + // and set the terminate time to now, it is used when we update the deployment and environment + jobPod.Status.Phase = corev1.PodFailed + state := corev1.ContainerStatus{ + State: corev1.ContainerState{ + Terminated: &corev1.ContainerStateTerminated{ + FinishedAt: metav1.Time{Time: time.Now().UTC()}, + }, + }, + } + jobPod.Status.ContainerStatuses[0] = state + logMsg := fmt.Sprintf("%v: %v", container.State.Waiting.Reason, container.State.Waiting.Message) + return r.updateTaskWithLogs(ctx, req, lagoonTask, jobPod, []byte(logMsg), false) + } + } + return r.updateTaskWithLogs(ctx, req, lagoonTask, jobPod, nil, false) + } else if jobPod.Status.Phase == corev1.PodRunning { + // if the pod is running and detects a change to the pod (eg, detecting an updated lagoon.sh/taskStep label) + // then ship or store the logs + // get the task associated to this pod, the information in the resource is used for shipping the logs + var lagoonTask lagooncrd.LagoonTask + err := r.Get(ctx, + types.NamespacedName{ + Namespace: jobPod.ObjectMeta.Namespace, + Name: jobPod.ObjectMeta.Labels["lagoon.sh/taskName"], + }, &lagoonTask) + if err != nil { + return err + } + } + if jobPod.Status.Phase == corev1.PodFailed || jobPod.Status.Phase == corev1.PodSucceeded { + // get the task associated to this pod, we wil need update it at some point + var lagoonTask lagooncrd.LagoonTask + err := r.Get(ctx, types.NamespacedName{ + Namespace: jobPod.ObjectMeta.Namespace, + Name: jobPod.ObjectMeta.Labels["lagoon.sh/taskName"], + }, &lagoonTask) + if err != nil { + return err + } + } + // if it isn't pending, failed, or complete, it will be running, we should tell lagoon + return r.updateTaskWithLogs(ctx, req, lagoonTask, jobPod, nil, false) +} + +// taskLogsToLagoonLogs sends the task logs to the lagoon-logs message queue +// it contains the actual pod log output that is sent to elasticsearch, it is what eventually is displayed in the UI +func (r *LagoonMonitorReconciler) taskLogsToLagoonLogs(opLog logr.Logger, + lagoonTask *lagooncrd.LagoonTask, + jobPod *corev1.Pod, + condition string, + logs []byte, +) (bool, schema.LagoonLog) { + if r.EnableMQ && lagoonTask != nil { + msg := schema.LagoonLog{ + Severity: "info", + Project: lagoonTask.Spec.Project.Name, + Event: "task-logs:job-kubernetes:" + lagoonTask.ObjectMeta.Name, + Meta: &schema.LagoonLogMeta{ + Task: &lagoonTask.Spec.Task, + Environment: lagoonTask.Spec.Environment.Name, + JobName: lagoonTask.ObjectMeta.Name, + JobStatus: condition, + RemoteID: string(jobPod.ObjectMeta.UID), + Key: lagoonTask.Spec.Key, + Cluster: r.LagoonTargetName, + }, + } + if jobPod.Spec.NodeName != "" { + msg.Message = fmt.Sprintf(`================================================================================ +Logs on pod %s, assigned to node %s on cluster %s +================================================================================ +%s`, jobPod.ObjectMeta.Name, jobPod.Spec.NodeName, r.LagoonTargetName, logs) + } else { + msg.Message = fmt.Sprintf(`================================================================================ +Logs on pod %s, assigned to cluster %s +================================================================================ +%s`, jobPod.ObjectMeta.Name, r.LagoonTargetName, logs) + } + msgBytes, err := json.Marshal(msg) + if err != nil { + opLog.Error(err, "Unable to encode message as JSON") + } + if err := r.Messaging.Publish("lagoon-logs", msgBytes); err != nil { + // if we can't publish the message, set it as a pending message + // overwrite whatever is there as these are just current state messages so it doesn't + // really matter if we don't smootly transition in what we send back to lagoon + return true, msg + } + // if we are able to publish the message, then we need to remove any pending messages from the resource + // and make sure we don't try and publish again + } + return false, schema.LagoonLog{} +} + +// updateLagoonTask sends the status of the task and deployment to the controllerhandler message queue in lagoon, +// this is for the handler in lagoon to process. +func (r *LagoonMonitorReconciler) updateLagoonTask(ctx context.Context, opLog logr.Logger, + lagoonTask *lagooncrd.LagoonTask, + jobPod *corev1.Pod, + condition string, +) (bool, schema.LagoonMessage) { + if r.EnableMQ && lagoonTask != nil { + if condition == "failed" || condition == "complete" || condition == "cancelled" { + time.AfterFunc(31*time.Second, func() { + taskRunningStatus.Delete(prometheus.Labels{ + "task_namespace": lagoonTask.ObjectMeta.Namespace, + "task_name": lagoonTask.ObjectMeta.Name, + }) + }) + time.Sleep(2 * time.Second) // smol sleep to reduce race of final messages with previous messages + } + msg := schema.LagoonMessage{ + Type: "task", + Namespace: lagoonTask.ObjectMeta.Namespace, + Meta: &schema.LagoonLogMeta{ + Task: &lagoonTask.Spec.Task, + Environment: lagoonTask.Spec.Environment.Name, + Project: lagoonTask.Spec.Project.Name, + EnvironmentID: helpers.StringToUintPtr(lagoonTask.Spec.Environment.ID), + ProjectID: helpers.StringToUintPtr(lagoonTask.Spec.Project.ID), + JobName: lagoonTask.ObjectMeta.Name, + JobStatus: condition, + RemoteID: string(jobPod.ObjectMeta.UID), + Key: lagoonTask.Spec.Key, + Cluster: r.LagoonTargetName, + }, + } + if _, ok := jobPod.ObjectMeta.Annotations["lagoon.sh/taskData"]; ok { + // if the task contains `taskData` annotation, this is used to send data back to lagoon + // lagoon will use the data to perform an action against the api or something else + // the data in taskData should be base64 encoded + msg.Meta.AdvancedData = jobPod.ObjectMeta.Annotations["lagoon.sh/taskData"] + } + // we can add the task start time here + if jobPod.Status.StartTime != nil { + msg.Meta.StartTime = jobPod.Status.StartTime.Time.UTC().Format("2006-01-02 15:04:05") + } + // and then once the pod is terminated we can add the terminated time here + if jobPod.Status.ContainerStatuses != nil { + if jobPod.Status.ContainerStatuses[0].State.Terminated != nil { + msg.Meta.EndTime = jobPod.Status.ContainerStatuses[0].State.Terminated.FinishedAt.Time.UTC().Format("2006-01-02 15:04:05") + } + } + msgBytes, err := json.Marshal(msg) + if err != nil { + opLog.Error(err, "Unable to encode message as JSON") + } + if err := r.Messaging.Publish("lagoon-tasks:controller", msgBytes); err != nil { + // if we can't publish the message, set it as a pending message + // overwrite whatever is there as these are just current state messages so it doesn't + // really matter if we don't smootly transition in what we send back to lagoon + return true, msg + } + // if we are able to publish the message, then we need to remove any pending messages from the resource + // and make sure we don't try and publish again + } + return false, schema.LagoonMessage{} +} + +// taskStatusLogsToLagoonLogs sends the logs to lagoon-logs message queue, used for general messaging +func (r *LagoonMonitorReconciler) taskStatusLogsToLagoonLogs(opLog logr.Logger, + lagoonTask *lagooncrd.LagoonTask, + jobPod *corev1.Pod, + condition string, +) (bool, schema.LagoonLog) { + if r.EnableMQ && lagoonTask != nil { + msg := schema.LagoonLog{ + Severity: "info", + Project: lagoonTask.Spec.Project.Name, + Event: "task:job-kubernetes:" + condition, //@TODO: this probably needs to be changed to a new task event for the controller + Meta: &schema.LagoonLogMeta{ + Task: &lagoonTask.Spec.Task, + ProjectName: lagoonTask.Spec.Project.Name, + Environment: lagoonTask.Spec.Environment.Name, + EnvironmentID: helpers.StringToUintPtr(lagoonTask.Spec.Environment.ID), + ProjectID: helpers.StringToUintPtr(lagoonTask.Spec.Project.ID), + JobName: lagoonTask.ObjectMeta.Name, + JobStatus: condition, + RemoteID: string(jobPod.ObjectMeta.UID), + Key: lagoonTask.Spec.Key, + Cluster: r.LagoonTargetName, + }, + Message: fmt.Sprintf("*[%s]* Task `%s` *%s* %s", + lagoonTask.Spec.Project.Name, + lagoonTask.Spec.Task.ID, + lagoonTask.Spec.Task.Name, + condition, + ), + } + msgBytes, err := json.Marshal(msg) + if err != nil { + opLog.Error(err, "Unable to encode message as JSON") + } + if err := r.Messaging.Publish("lagoon-logs", msgBytes); err != nil { + // if we can't publish the message, set it as a pending message + // overwrite whatever is there as these are just current state messages so it doesn't + // really matter if we don't smootly transition in what we send back to lagoon + return true, msg + } + // if we are able to publish the message, then we need to remove any pending messages from the resource + // and make sure we don't try and publish again + } + return false, schema.LagoonLog{} +} + +// updateTaskWithLogs collects logs from the task containers and ships or stores them +func (r *LagoonMonitorReconciler) updateTaskWithLogs( + ctx context.Context, + req ctrl.Request, + lagoonTask lagooncrd.LagoonTask, + jobPod corev1.Pod, + logs []byte, + cancel bool, +) error { + opLog := r.Log.WithValues("lagoonmonitor", req.NamespacedName) + taskCondition := lagooncrd.GetTaskConditionFromPod(jobPod.Status.Phase) + collectLogs := true + if cancel { + taskCondition = lagooncrd.TaskStatusCancelled + } + // if the task status is Pending or Running + // then the taskCondition is Failed, Complete, or Cancelled + // then update the task to reflect the current pod status + // we do this so we don't update the status of the task again + if helpers.ContainsString( + lagooncrd.TaskRunningPendingStatus, + lagoonTask.Labels["lagoon.sh/taskStatus"], + ) || cancel { + opLog.Info( + fmt.Sprintf( + "Updating task status for %s to %v", + jobPod.ObjectMeta.Name, + taskCondition, + ), + ) + // grab all the logs from the containers in the task pod and just merge them all together + // we only have 1 container at the moment in a taskpod anyway so it doesn't matter + // if we do move to multi container tasks, then worry about it + var allContainerLogs []byte + var err error + if logs == nil { + if collectLogs { + allContainerLogs, err = r.collectLogs(ctx, req, jobPod) + if err == nil { + if cancel { + cancellationMessage := "Task cancelled" + if cancellationDetails, ok := jobPod.GetAnnotations()["lagoon.sh/cancelReason"]; ok { + cancellationMessage = fmt.Sprintf("%v : %v", cancellationMessage, cancellationDetails) + } + allContainerLogs = append(allContainerLogs, []byte(fmt.Sprintf(` +======================================== +%v +========================================`, cancellationMessage))...) + } + } else { + allContainerLogs = []byte(fmt.Sprintf(` +======================================== +Task %s +========================================`, taskCondition)) + } + } + } else { + allContainerLogs = logs + } + + mergeMap := map[string]interface{}{ + "metadata": map[string]interface{}{ + "labels": map[string]interface{}{ + "lagoon.sh/taskStatus": taskCondition.String(), + }, + }, + "statusMessages": map[string]interface{}{}, + } + + condition := lagooncrd.LagoonTaskConditions{ + Type: taskCondition, + Status: corev1.ConditionTrue, + LastTransitionTime: time.Now().UTC().Format(time.RFC3339), + } + if !lagooncrd.TaskContainsStatus(lagoonTask.Status.Conditions, condition) { + lagoonTask.Status.Conditions = append(lagoonTask.Status.Conditions, condition) + mergeMap["status"] = map[string]interface{}{ + "conditions": lagoonTask.Status.Conditions, + // don't save build logs in resource anymore + } + } + + // send any messages to lagoon message queues + // update the deployment with the status + pendingStatus, pendingStatusMessage := r.taskStatusLogsToLagoonLogs(opLog, &lagoonTask, &jobPod, taskCondition.ToLower()) + pendingEnvironment, pendingEnvironmentMessage := r.updateLagoonTask(ctx, opLog, &lagoonTask, &jobPod, taskCondition.ToLower()) + var pendingTaskLog bool + var pendingTaskLogMessage schema.LagoonLog + // if the container logs can't be retrieved, we don't want to send any task logs back, as this will nuke + // any previously received logs + if !strings.Contains(string(allContainerLogs), "unable to retrieve container logs for containerd") { + pendingTaskLog, pendingTaskLogMessage = r.taskLogsToLagoonLogs(opLog, &lagoonTask, &jobPod, taskCondition.ToLower(), allContainerLogs) + } + + if pendingStatus || pendingEnvironment || pendingTaskLog { + mergeMap["metadata"].(map[string]interface{})["labels"].(map[string]interface{})["lagoon.sh/pendingMessages"] = "true" + if pendingStatus { + mergeMap["statusMessages"].(map[string]interface{})["statusMessage"] = pendingStatusMessage + } + if pendingEnvironment { + mergeMap["statusMessages"].(map[string]interface{})["environmentMessage"] = pendingEnvironmentMessage + } + if pendingTaskLog { + mergeMap["statusMessages"].(map[string]interface{})["taskLogMessage"] = pendingTaskLogMessage + } + } + if !pendingStatus && !pendingEnvironment && !pendingTaskLog { + mergeMap["metadata"].(map[string]interface{})["labels"].(map[string]interface{})["lagoon.sh/pendingMessages"] = nil + mergeMap["statusMessages"] = nil + } + mergePatch, _ := json.Marshal(mergeMap) + // check if the task exists + if err := r.Get(ctx, req.NamespacedName, &lagoonTask); err == nil { + // if it does, try to patch it + if err := r.Patch(ctx, &lagoonTask, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { + opLog.Error(err, fmt.Sprintf("Unable to update resource")) + } + } + // just delete the pod + if cancel { + if err := r.Get(ctx, req.NamespacedName, &jobPod); err == nil { + if r.EnableDebug { + opLog.Info(fmt.Sprintf("Task pod exists %s", jobPod.ObjectMeta.Name)) + } + if err := r.Delete(ctx, &jobPod); err != nil { + return err + } + } + } + } + return nil +} diff --git a/controllers/v1beta2/predicates.go b/controllers/v1beta2/predicates.go new file mode 100644 index 00000000..0e4185aa --- /dev/null +++ b/controllers/v1beta2/predicates.go @@ -0,0 +1,259 @@ +package v1beta2 + +// contains all the event watch conditions for secret and ingresses + +import ( + "regexp" + "time" + + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +// PodPredicates is used by the filter for the monitor controller to make sure the correct resources +// are acted on. +type PodPredicates struct { + predicate.Funcs + ControllerNamespace string +} + +// Create is used when a creation event is received by the controller. +func (p PodPredicates) Create(e event.CreateEvent) bool { + if controller, ok := e.Object.GetLabels()["lagoon.sh/controller"]; ok { + if controller == p.ControllerNamespace { + if value, ok := e.Object.GetLabels()["crd.lagoon.sh/version"]; ok { + if value == crdVersion { + if value, ok := e.Object.GetLabels()["lagoon.sh/buildName"]; ok { + match, _ := regexp.MatchString("^lagoon-build", value) + return match + } + if value, ok := e.Object.GetLabels()["lagoon.sh/jobType"]; ok { + if value == "task" { + return true + } + } + } + } + } + } + return false +} + +// Delete is used when a deletion event is received by the controller. +func (p PodPredicates) Delete(e event.DeleteEvent) bool { + if controller, ok := e.Object.GetLabels()["lagoon.sh/controller"]; ok { + if controller == p.ControllerNamespace { + if value, ok := e.Object.GetLabels()["crd.lagoon.sh/version"]; ok { + if value == crdVersion { + if value, ok := e.Object.GetLabels()["lagoon.sh/buildName"]; ok { + match, _ := regexp.MatchString("^lagoon-build", value) + return match + } + if value, ok := e.Object.GetLabels()["lagoon.sh/jobType"]; ok { + if value == "task" { + return true + } + } + } + } + } + } + return false +} + +// Update is used when an update event is received by the controller. +func (p PodPredicates) Update(e event.UpdateEvent) bool { + if controller, ok := e.ObjectOld.GetLabels()["lagoon.sh/controller"]; ok { + if controller == p.ControllerNamespace { + if value, ok := e.ObjectNew.GetLabels()["crd.lagoon.sh/version"]; ok { + if value == crdVersion { + if _, okOld := e.ObjectOld.GetLabels()["lagoon.sh/buildName"]; okOld { + if value, ok := e.ObjectNew.GetLabels()["lagoon.sh/buildName"]; ok { + oldBuildStep := "running" + newBuildStep := "running" + if value, ok := e.ObjectNew.GetLabels()["lagoon.sh/buildStep"]; ok { + newBuildStep = value + } + if value, ok := e.ObjectOld.GetLabels()["lagoon.sh/buildStep"]; ok { + oldBuildStep = value + } + if newBuildStep != oldBuildStep { + buildStatus.With(prometheus.Labels{ + "build_namespace": e.ObjectOld.GetNamespace(), + "build_name": e.ObjectOld.GetName(), + "build_step": newBuildStep, + }).Set(1) + } + time.AfterFunc(31*time.Second, func() { + buildStatus.Delete(prometheus.Labels{ + "build_namespace": e.ObjectOld.GetNamespace(), + "build_name": e.ObjectOld.GetName(), + "build_step": oldBuildStep, + }) + }) + match, _ := regexp.MatchString("^lagoon-build", value) + return match + } + } + if _, ok := e.ObjectOld.GetLabels()["lagoon.sh/jobType"]; ok { + if value, ok := e.ObjectNew.GetLabels()["lagoon.sh/jobType"]; ok { + if value == "task" { + return true + } + } + } + } + } + } + } + return false +} + +// Generic is used when any other event is received by the controller. +func (p PodPredicates) Generic(e event.GenericEvent) bool { + if controller, ok := e.Object.GetLabels()["lagoon.sh/controller"]; ok { + if controller == p.ControllerNamespace { + if value, ok := e.Object.GetLabels()["crd.lagoon.sh/version"]; ok { + if value == crdVersion { + if value, ok := e.Object.GetLabels()["lagoon.sh/buildName"]; ok { + match, _ := regexp.MatchString("^lagoon-build", value) + return match + } + if value, ok := e.Object.GetLabels()["lagoon.sh/jobType"]; ok { + if value == "task" { + return true + } + } + } + } + } + } + return false +} + +// BuildPredicates is used by the filter for the build controller to make sure the correct resources +// are acted on. +type BuildPredicates struct { + predicate.Funcs + ControllerNamespace string +} + +// Create is used when a creation event is received by the controller. +func (b BuildPredicates) Create(e event.CreateEvent) bool { + if controller, ok := e.Object.GetLabels()["lagoon.sh/controller"]; ok { + if controller == b.ControllerNamespace { + if value, ok := e.Object.GetLabels()["crd.lagoon.sh/version"]; ok { + if value == crdVersion { + return true + } + } + } + } + return false +} + +// Delete is used when a deletion event is received by the controller. +func (b BuildPredicates) Delete(e event.DeleteEvent) bool { + if controller, ok := e.Object.GetLabels()["lagoon.sh/controller"]; ok { + if controller == b.ControllerNamespace { + if value, ok := e.Object.GetLabels()["crd.lagoon.sh/version"]; ok { + if value == crdVersion { + return true + } + } + } + } + return false +} + +// Update is used when an update event is received by the controller. +func (b BuildPredicates) Update(e event.UpdateEvent) bool { + if controller, ok := e.ObjectOld.GetLabels()["lagoon.sh/controller"]; ok { + if controller == b.ControllerNamespace { + if value, ok := e.ObjectNew.GetLabels()["crd.lagoon.sh/version"]; ok { + if value == crdVersion { + return true + } + } + } + } + return false +} + +// Generic is used when any other event is received by the controller. +func (b BuildPredicates) Generic(e event.GenericEvent) bool { + if controller, ok := e.Object.GetLabels()["lagoon.sh/controller"]; ok { + if controller == b.ControllerNamespace { + if value, ok := e.Object.GetLabels()["crd.lagoon.sh/version"]; ok { + if value == crdVersion { + return true + } + } + } + } + return false +} + +// TaskPredicates is used by the filter for the task controller to make sure the correct resources +// are acted on. +type TaskPredicates struct { + predicate.Funcs + ControllerNamespace string +} + +// Create is used when a creation event is received by the controller. +func (t TaskPredicates) Create(e event.CreateEvent) bool { + if controller, ok := e.Object.GetLabels()["lagoon.sh/controller"]; ok { + if controller == t.ControllerNamespace { + if value, ok := e.Object.GetLabels()["crd.lagoon.sh/version"]; ok { + if value == crdVersion { + return true + } + } + } + } + return false +} + +// Delete is used when a deletion event is received by the controller. +func (t TaskPredicates) Delete(e event.DeleteEvent) bool { + if controller, ok := e.Object.GetLabels()["lagoon.sh/controller"]; ok { + if controller == t.ControllerNamespace { + if value, ok := e.Object.GetLabels()["crd.lagoon.sh/version"]; ok { + if value == crdVersion { + return true + } + } + } + } + return false +} + +// Update is used when an update event is received by the controller. +func (t TaskPredicates) Update(e event.UpdateEvent) bool { + if controller, ok := e.ObjectOld.GetLabels()["lagoon.sh/controller"]; ok { + if controller == t.ControllerNamespace { + if value, ok := e.ObjectNew.GetLabels()["crd.lagoon.sh/version"]; ok { + if value == crdVersion { + return true + } + } + } + } + return false +} + +// Generic is used when any other event is received by the controller. +func (t TaskPredicates) Generic(e event.GenericEvent) bool { + if controller, ok := e.Object.GetLabels()["lagoon.sh/controller"]; ok { + if controller == t.ControllerNamespace { + if value, ok := e.Object.GetLabels()["crd.lagoon.sh/version"]; ok { + if value == crdVersion { + return true + } + } + } + } + return false +} diff --git a/controllers/v1beta2/suite_test.go b/controllers/v1beta2/suite_test.go new file mode 100644 index 00000000..6d04f49a --- /dev/null +++ b/controllers/v1beta2/suite_test.go @@ -0,0 +1,80 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta2 + +import ( + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + lagooncrd "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + // +kubebuilder:scaffold:imports +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var cfg *rest.Config +var k8sClient client.Client +var testEnv *envtest.Environment + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Controller Suite") +} + +var _ = BeforeSuite(func(done Done) { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, + } + + var err error + cfg, err = testEnv.Start() + Expect(err).ToNot(HaveOccurred()) + Expect(cfg).ToNot(BeNil()) + + err = lagooncrd.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + err = lagooncrd.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + // +kubebuilder:scaffold:scheme + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).ToNot(HaveOccurred()) + Expect(k8sClient).ToNot(BeNil()) + + close(done) +}, 60) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + err := testEnv.Stop() + Expect(err).ToNot(HaveOccurred()) +}) diff --git a/controllers/v1beta2/task_controller.go b/controllers/v1beta2/task_controller.go new file mode 100644 index 00000000..e5796e1e --- /dev/null +++ b/controllers/v1beta2/task_controller.go @@ -0,0 +1,678 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta2 + +import ( + "context" + "encoding/json" + "fmt" + "regexp" + "strconv" + + "github.com/go-logr/logr" + "github.com/prometheus/client_golang/prometheus" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + lagooncrd "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" + "github.com/uselagoon/remote-controller/internal/helpers" +) + +// LagoonTaskReconciler reconciles a LagoonTask object +type LagoonTaskReconciler struct { + client.Client + Log logr.Logger + Scheme *runtime.Scheme + ControllerNamespace string + NamespacePrefix string + RandomNamespacePrefix bool + LagoonAPIConfiguration helpers.LagoonAPIConfiguration + EnableDebug bool + LagoonTargetName string + ProxyConfig ProxyConfig +} + +var ( + taskFinalizer = "finalizer.lagoontask.crd.lagoon.sh/v1beta2" +) + +// +kubebuilder:rbac:groups=crd.lagoon.sh,resources=lagoontasks,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=crd.lagoon.sh,resources=lagoontasks/status,verbs=get;update;patch + +// Reconcile runs when a request comes through +func (r *LagoonTaskReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + opLog := r.Log.WithValues("lagoontask", req.NamespacedName) + + // your logic here + var lagoonTask lagooncrd.LagoonTask + if err := r.Get(ctx, req.NamespacedName, &lagoonTask); err != nil { + return ctrl.Result{}, helpers.IgnoreNotFound(err) + } + + // examine DeletionTimestamp to determine if object is under deletion + if lagoonTask.ObjectMeta.DeletionTimestamp.IsZero() { + // check if the task that has been recieved is a standard or advanced task + if lagoonTask.ObjectMeta.Labels["lagoon.sh/taskStatus"] == lagooncrd.TaskStatusPending.String() && + lagoonTask.ObjectMeta.Labels["lagoon.sh/taskType"] == lagooncrd.TaskTypeStandard.String() { + return ctrl.Result{}, r.createStandardTask(ctx, &lagoonTask, opLog) + } + if lagoonTask.ObjectMeta.Labels["lagoon.sh/taskStatus"] == lagooncrd.TaskStatusPending.String() && + lagoonTask.ObjectMeta.Labels["lagoon.sh/taskType"] == lagooncrd.TaskTypeAdvanced.String() { + return ctrl.Result{}, r.createAdvancedTask(ctx, &lagoonTask, opLog) + } + } else { + // The object is being deleted + if helpers.ContainsString(lagoonTask.ObjectMeta.Finalizers, taskFinalizer) { + // our finalizer is present, so lets handle any external dependency + if err := r.deleteExternalResources(ctx, &lagoonTask, req.NamespacedName.Namespace); err != nil { + // if fail to delete the external dependency here, return with error + // so that it can be retried + return ctrl.Result{}, err + } + // remove our finalizer from the list and update it. + lagoonTask.ObjectMeta.Finalizers = helpers.RemoveString(lagoonTask.ObjectMeta.Finalizers, taskFinalizer) + // use patches to avoid update errors + mergePatch, _ := json.Marshal(map[string]interface{}{ + "metadata": map[string]interface{}{ + "finalizers": lagoonTask.ObjectMeta.Finalizers, + }, + }) + if err := r.Patch(ctx, &lagoonTask, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { + return ctrl.Result{}, err + } + } + } + + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the given manager +func (r *LagoonTaskReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&lagooncrd.LagoonTask{}). + WithEventFilter(TaskPredicates{ + ControllerNamespace: r.ControllerNamespace, + }). + Complete(r) +} + +func (r *LagoonTaskReconciler) deleteExternalResources(ctx context.Context, lagoonTask *lagooncrd.LagoonTask, namespace string) error { + // delete any external resources if required + return nil +} + +// get the task pod information for kubernetes +func (r *LagoonTaskReconciler) getTaskPodDeployment(ctx context.Context, lagoonTask *lagooncrd.LagoonTask) (*corev1.Pod, error) { + deployments := &appsv1.DeploymentList{} + listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ + client.InNamespace(lagoonTask.ObjectMeta.Namespace), + }) + err := r.List(ctx, deployments, listOption) + if err != nil { + return nil, fmt.Errorf( + "Unable to get deployments for project %s, environment %s: %v", + lagoonTask.Spec.Project.Name, + lagoonTask.Spec.Environment.Name, + err, + ) + } + if len(deployments.Items) > 0 { + hasService := false + for _, dep := range deployments.Items { + // grab the deployment that contains the task service we want to use + if dep.ObjectMeta.Name == lagoonTask.Spec.Task.Service { + hasService = true + // grab the container + for idx, depCon := range dep.Spec.Template.Spec.Containers { + // --- deprecate these at some point in favor of the `LAGOON_CONFIG_X` variants + dep.Spec.Template.Spec.Containers[idx].Env = append(dep.Spec.Template.Spec.Containers[idx].Env, corev1.EnvVar{ + Name: "TASK_API_HOST", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "TASK_API_HOST"), + }) + dep.Spec.Template.Spec.Containers[idx].Env = append(dep.Spec.Template.Spec.Containers[idx].Env, corev1.EnvVar{ + Name: "TASK_SSH_HOST", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "TASK_SSH_HOST"), + }) + dep.Spec.Template.Spec.Containers[idx].Env = append(dep.Spec.Template.Spec.Containers[idx].Env, corev1.EnvVar{ + Name: "TASK_SSH_PORT", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "TASK_SSH_PORT"), + }) + // ^^ --- + // add the API and SSH endpoint configuration to environments + dep.Spec.Template.Spec.Containers[idx].Env = append(dep.Spec.Template.Spec.Containers[idx].Env, corev1.EnvVar{ + Name: "LAGOON_CONFIG_API_HOST", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "LAGOON_CONFIG_API_HOST"), + }) + dep.Spec.Template.Spec.Containers[idx].Env = append(dep.Spec.Template.Spec.Containers[idx].Env, corev1.EnvVar{ + Name: "LAGOON_CONFIG_TOKEN_HOST", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "LAGOON_CONFIG_TOKEN_HOST"), + }) + dep.Spec.Template.Spec.Containers[idx].Env = append(dep.Spec.Template.Spec.Containers[idx].Env, corev1.EnvVar{ + Name: "LAGOON_CONFIG_TOKEN_PORT", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "LAGOON_CONFIG_TOKEN_PORT"), + }) + dep.Spec.Template.Spec.Containers[idx].Env = append(dep.Spec.Template.Spec.Containers[idx].Env, corev1.EnvVar{ + Name: "LAGOON_CONFIG_SSH_HOST", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "LAGOON_CONFIG_SSH_HOST"), + }) + dep.Spec.Template.Spec.Containers[idx].Env = append(dep.Spec.Template.Spec.Containers[idx].Env, corev1.EnvVar{ + Name: "LAGOON_CONFIG_SSH_PORT", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "LAGOON_CONFIG_SSH_PORT"), + }) + dep.Spec.Template.Spec.Containers[idx].Env = append(dep.Spec.Template.Spec.Containers[idx].Env, corev1.EnvVar{ + Name: "TASK_DATA_ID", + Value: lagoonTask.Spec.Task.ID, + }) + // add proxy variables to builds if they are defined + if r.ProxyConfig.HTTPProxy != "" { + dep.Spec.Template.Spec.Containers[idx].Env = append(dep.Spec.Template.Spec.Containers[idx].Env, corev1.EnvVar{ + Name: "HTTP_PROXY", + Value: r.ProxyConfig.HTTPProxy, + }) + dep.Spec.Template.Spec.Containers[idx].Env = append(dep.Spec.Template.Spec.Containers[idx].Env, corev1.EnvVar{ + Name: "http_proxy", + Value: r.ProxyConfig.HTTPProxy, + }) + } + if r.ProxyConfig.HTTPSProxy != "" { + dep.Spec.Template.Spec.Containers[idx].Env = append(dep.Spec.Template.Spec.Containers[idx].Env, corev1.EnvVar{ + Name: "HTTPS_PROXY", + Value: r.ProxyConfig.HTTPSProxy, + }) + dep.Spec.Template.Spec.Containers[idx].Env = append(dep.Spec.Template.Spec.Containers[idx].Env, corev1.EnvVar{ + Name: "https_proxy", + Value: r.ProxyConfig.HTTPSProxy, + }) + } + if r.ProxyConfig.NoProxy != "" { + dep.Spec.Template.Spec.Containers[idx].Env = append(dep.Spec.Template.Spec.Containers[idx].Env, corev1.EnvVar{ + Name: "NO_PROXY", + Value: r.ProxyConfig.NoProxy, + }) + dep.Spec.Template.Spec.Containers[idx].Env = append(dep.Spec.Template.Spec.Containers[idx].Env, corev1.EnvVar{ + Name: "no_proxy", + Value: r.ProxyConfig.NoProxy, + }) + } + if lagoonTask.Spec.Project.Variables.Project != nil { + // if this is 2 bytes long, then it means its just an empty json array + // we only want to add it if it is more than 2 bytes + if len(lagoonTask.Spec.Project.Variables.Project) > 2 { + dep.Spec.Template.Spec.Containers[idx].Env = append(dep.Spec.Template.Spec.Containers[idx].Env, corev1.EnvVar{ + Name: "LAGOON_PROJECT_VARIABLES", + Value: string(lagoonTask.Spec.Project.Variables.Project), + }) + } + } + if lagoonTask.Spec.Project.Variables.Environment != nil { + // if this is 2 bytes long, then it means its just an empty json array + // we only want to add it if it is more than 2 bytes + if len(lagoonTask.Spec.Project.Variables.Environment) > 2 { + dep.Spec.Template.Spec.Containers[idx].Env = append(dep.Spec.Template.Spec.Containers[idx].Env, corev1.EnvVar{ + Name: "LAGOON_ENVIRONMENT_VARIABLES", + Value: string(lagoonTask.Spec.Project.Variables.Environment), + }) + } + } + for idx2, env := range depCon.Env { + // remove any cronjobs from the envvars + if env.Name == "CRONJOBS" { + // Shift left one index. + copy(dep.Spec.Template.Spec.Containers[idx].Env[idx2:], dep.Spec.Template.Spec.Containers[idx].Env[idx2+1:]) + // Erase last element (write zero value). + dep.Spec.Template.Spec.Containers[idx].Env[len(dep.Spec.Template.Spec.Containers[idx].Env)-1] = corev1.EnvVar{} + dep.Spec.Template.Spec.Containers[idx].Env = dep.Spec.Template.Spec.Containers[idx].Env[:len(dep.Spec.Template.Spec.Containers[idx].Env)-1] + } + } + dep.Spec.Template.Spec.Containers[idx].Command = []string{"/sbin/tini", + "--", + "/lagoon/entrypoints.sh", + "/bin/sh", + "-c", + lagoonTask.Spec.Task.Command, + } + dep.Spec.Template.Spec.RestartPolicy = "Never" + taskPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: lagoonTask.ObjectMeta.Name, + Namespace: lagoonTask.ObjectMeta.Namespace, + Labels: map[string]string{ + "lagoon.sh/jobType": "task", + "lagoon.sh/taskName": lagoonTask.ObjectMeta.Name, + "crd.lagoon.sh/version": crdVersion, + "lagoon.sh/controller": r.ControllerNamespace, + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: fmt.Sprintf("%v", lagooncrd.GroupVersion), + Kind: "LagoonTask", + Name: lagoonTask.ObjectMeta.Name, + UID: lagoonTask.UID, + }, + }, + }, + Spec: dep.Spec.Template.Spec, + } + // set the organization labels on task pods + if lagoonTask.Spec.Project.Organization != nil { + taskPod.ObjectMeta.Labels["organization.lagoon.sh/id"] = fmt.Sprintf("%d", *lagoonTask.Spec.Project.Organization.ID) + taskPod.ObjectMeta.Labels["organization.lagoon.sh/name"] = lagoonTask.Spec.Project.Organization.Name + } + return taskPod, nil + } + } + } + if !hasService { + return nil, fmt.Errorf( + "No matching service %s for project %s, environment %s: %v", + lagoonTask.Spec.Task.Service, + lagoonTask.Spec.Project.Name, + lagoonTask.Spec.Environment.Name, + err, + ) + } + } + // no deployments found return error + return nil, fmt.Errorf( + "No deployments %s for project %s, environment %s: %v", + lagoonTask.ObjectMeta.Namespace, + lagoonTask.Spec.Project.Name, + lagoonTask.Spec.Environment.Name, + err, + ) +} + +func (r *LagoonTaskReconciler) createStandardTask(ctx context.Context, lagoonTask *lagooncrd.LagoonTask, opLog logr.Logger) error { + newTaskPod := &corev1.Pod{} + var err error + + newTaskPod, err = r.getTaskPodDeployment(ctx, lagoonTask) + if err != nil { + opLog.Info(fmt.Sprintf("%v", err)) + //@TODO: send msg back and update task to failed? + return nil + } + opLog.Info(fmt.Sprintf("Checking task pod for: %s", lagoonTask.ObjectMeta.Name)) + // once the pod spec has been defined, check if it isn't already created + err = r.Get(ctx, types.NamespacedName{ + Namespace: lagoonTask.ObjectMeta.Namespace, + Name: newTaskPod.ObjectMeta.Name, + }, newTaskPod) + if err != nil { + // if it doesn't exist, then create the task pod + opLog.Info(fmt.Sprintf("Creating task pod for: %s", lagoonTask.ObjectMeta.Name)) + // create the task pod + if err := r.Create(ctx, newTaskPod); err != nil { + opLog.Info( + fmt.Sprintf( + "Unable to create task pod for project %s, environment %s: %v", + lagoonTask.Spec.Project.Name, + lagoonTask.Spec.Environment.Name, + err, + ), + ) + //@TODO: send msg back and update task to failed? + return nil + } + taskRunningStatus.With(prometheus.Labels{ + "task_namespace": lagoonTask.ObjectMeta.Namespace, + "task_name": lagoonTask.ObjectMeta.Name, + }).Set(1) + tasksStartedCounter.Inc() + } else { + opLog.Info(fmt.Sprintf("Task pod already running for: %s", lagoonTask.ObjectMeta.Name)) + } + // The object is not being deleted, so if it does not have our finalizer, + // then lets add the finalizer and update the object. This is equivalent + // registering our finalizer. + if !helpers.ContainsString(lagoonTask.ObjectMeta.Finalizers, taskFinalizer) { + lagoonTask.ObjectMeta.Finalizers = append(lagoonTask.ObjectMeta.Finalizers, taskFinalizer) + // use patches to avoid update errors + mergePatch, _ := json.Marshal(map[string]interface{}{ + "metadata": map[string]interface{}{ + "finalizers": lagoonTask.ObjectMeta.Finalizers, + }, + }) + if err := r.Patch(ctx, lagoonTask, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { + return err + } + } + return nil +} + +// createAdvancedTask allows running of more advanced tasks than the standard lagoon tasks +// see notes in the docs for infomration about advanced tasks +func (r *LagoonTaskReconciler) createAdvancedTask(ctx context.Context, lagoonTask *lagooncrd.LagoonTask, opLog logr.Logger) error { + additionalLabels := map[string]string{} + + // check if this is an activestandby task, if it is, create the activestandby role + if value, ok := lagoonTask.ObjectMeta.Labels["lagoon.sh/activeStandby"]; ok { + isActiveStandby, _ := strconv.ParseBool(value) + if isActiveStandby { + var sourceNamespace, destinationNamespace string + if value, ok := lagoonTask.ObjectMeta.Labels["lagoon.sh/activeStandbySourceNamespace"]; ok { + sourceNamespace = value + } + if value, ok := lagoonTask.ObjectMeta.Labels["lagoon.sh/activeStandbyDestinationNamespace"]; ok { + destinationNamespace = value + } + // create the role + binding to allow the service account to interact with both namespaces + err := r.createActiveStandbyRole(ctx, sourceNamespace, destinationNamespace) + if err != nil { + return err + } + additionalLabels["lagoon.sh/activeStandby"] = "true" + additionalLabels["lagoon.sh/activeStandbySourceNamespace"] = sourceNamespace + additionalLabels["lagoon.sh/activeStandbyDestinationNamespace"] = destinationNamespace + } + } + + // handle the volumes for sshkey + sshKeyVolume := corev1.Volume{ + Name: "lagoon-sshkey", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "lagoon-sshkey", + DefaultMode: helpers.Int32Ptr(420), + }, + }, + } + sshKeyVolumeMount := corev1.VolumeMount{ + Name: "lagoon-sshkey", + ReadOnly: true, + MountPath: "/var/run/secrets/lagoon/ssh", + } + volumes := []corev1.Volume{} + volumeMounts := []corev1.VolumeMount{} + if lagoonTask.Spec.AdvancedTask.SSHKey { + volumes = append(volumes, sshKeyVolume) + volumeMounts = append(volumeMounts, sshKeyVolumeMount) + } + if lagoonTask.Spec.AdvancedTask.DeployerToken { + // if this advanced task can access kubernetes, mount the token in + serviceAccount := &corev1.ServiceAccount{} + err := r.getServiceAccount(ctx, serviceAccount, lagoonTask.ObjectMeta.Namespace) + if err != nil { + return err + } + var serviceaccountTokenSecret string + for _, secret := range serviceAccount.Secrets { + match, _ := regexp.MatchString("^lagoon-deployer-token", secret.Name) + if match { + serviceaccountTokenSecret = secret.Name + break + } + } + // if the existing token exists, mount it + if serviceaccountTokenSecret != "" { + volumes = append(volumes, corev1.Volume{ + Name: serviceaccountTokenSecret, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: serviceaccountTokenSecret, + DefaultMode: helpers.Int32Ptr(420), + }, + }, + }) + // legacy tokens are mounted /var/run/secrets/lagoon/deployer + // new tokens using volume projection are mounted /var/run/secrets/kubernetes.io/serviceaccount/token + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: serviceaccountTokenSecret, + ReadOnly: true, + MountPath: "/var/run/secrets/lagoon/deployer", + }) + } + } + opLog.Info(fmt.Sprintf("Checking advanced task pod for: %s", lagoonTask.ObjectMeta.Name)) + // once the pod spec has been defined, check if it isn't already created + podEnvs := []corev1.EnvVar{ + { + Name: "JSON_PAYLOAD", + Value: string(lagoonTask.Spec.AdvancedTask.JSONPayload), + }, + { + Name: "NAMESPACE", + Value: lagoonTask.ObjectMeta.Namespace, + }, + { + Name: "PODNAME", + Value: lagoonTask.ObjectMeta.Name, + }, + { + Name: "LAGOON_PROJECT", + Value: lagoonTask.Spec.Project.Name, + }, + { + Name: "LAGOON_GIT_BRANCH", + Value: lagoonTask.Spec.Environment.Name, + }, + { + Name: "TASK_API_HOST", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "TASK_API_HOST"), + }, + { + Name: "TASK_SSH_HOST", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "TASK_SSH_HOST"), + }, + { + Name: "TASK_SSH_PORT", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "TASK_SSH_PORT"), + }, + { + Name: "LAGOON_CONFIG_API_HOST", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "LAGOON_CONFIG_API_HOST"), + }, + { + Name: "LAGOON_CONFIG_SSH_HOST", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "LAGOON_CONFIG_SSH_HOST"), + }, + { + Name: "LAGOON_CONFIG_SSH_PORT", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "LAGOON_CONFIG_SSH_PORT"), + }, + { + Name: "LAGOON_CONFIG_TOKEN_HOST", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "LAGOON_CONFIG_TOKEN_HOST"), + }, + { + Name: "LAGOON_CONFIG_TOKEN_PORT", + Value: helpers.GetAPIValues(r.LagoonAPIConfiguration, "LAGOON_CONFIG_TOKEN_PORT"), + }, + { + Name: "TASK_DATA_ID", + Value: lagoonTask.Spec.Task.ID, + }, + } + if lagoonTask.Spec.Project.Variables.Project != nil { + // if this is 2 bytes long, then it means its just an empty json array + // we only want to add it if it is more than 2 bytes + if len(lagoonTask.Spec.Project.Variables.Project) > 2 { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "LAGOON_PROJECT_VARIABLES", + Value: string(lagoonTask.Spec.Project.Variables.Project), + }) + } + } + if lagoonTask.Spec.Project.Variables.Environment != nil { + // if this is 2 bytes long, then it means its just an empty json array + // we only want to add it if it is more than 2 bytes + if len(lagoonTask.Spec.Project.Variables.Environment) > 2 { + podEnvs = append(podEnvs, corev1.EnvVar{ + Name: "LAGOON_ENVIRONMENT_VARIABLES", + Value: string(lagoonTask.Spec.Project.Variables.Environment), + }) + } + } + newPod := &corev1.Pod{} + err := r.Get(ctx, types.NamespacedName{ + Namespace: lagoonTask.ObjectMeta.Namespace, + Name: lagoonTask.ObjectMeta.Name, + }, newPod) + if err != nil { + newPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: lagoonTask.ObjectMeta.Name, + Namespace: lagoonTask.ObjectMeta.Namespace, + Labels: map[string]string{ + "lagoon.sh/jobType": "task", + "lagoon.sh/taskName": lagoonTask.ObjectMeta.Name, + "crd.lagoon.sh/version": crdVersion, + "lagoon.sh/controller": r.ControllerNamespace, + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: fmt.Sprintf("%v", lagooncrd.GroupVersion), + Kind: "LagoonTask", + Name: lagoonTask.ObjectMeta.Name, + UID: lagoonTask.UID, + }, + }, + }, + Spec: corev1.PodSpec{ + RestartPolicy: "Never", + Volumes: volumes, + Containers: []corev1.Container{ + { + Name: "lagoon-task", + Image: lagoonTask.Spec.AdvancedTask.RunnerImage, + ImagePullPolicy: "Always", + EnvFrom: []corev1.EnvFromSource{ + { + ConfigMapRef: &corev1.ConfigMapEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "lagoon-env", + }, + }, + }, + }, + Env: podEnvs, + VolumeMounts: volumeMounts, + }, + }, + }, + } + if lagoonTask.Spec.Project.Organization != nil { + newPod.ObjectMeta.Labels["organization.lagoon.sh/id"] = fmt.Sprintf("%d", *lagoonTask.Spec.Project.Organization.ID) + newPod.ObjectMeta.Labels["organization.lagoon.sh/name"] = lagoonTask.Spec.Project.Organization.Name + } + if lagoonTask.Spec.AdvancedTask.DeployerToken { + // start this with the serviceaccount so that it gets the token mounted into it + newPod.Spec.ServiceAccountName = "lagoon-deployer" + } + opLog.Info(fmt.Sprintf("Creating advanced task pod for: %s", lagoonTask.ObjectMeta.Name)) + + //Decorate the pod spec with additional details + + //dynamic secrets + secrets, err := getSecretsForNamespace(r.Client, lagoonTask.Namespace) + secrets = filterDynamicSecrets(secrets) + if err != nil { + return err + } + + const dynamicSecretVolumeNamePrefex = "dynamic-" + for _, secret := range secrets { + volumeMountName := dynamicSecretVolumeNamePrefex + secret.Name + v := corev1.Volume{ + Name: volumeMountName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: secret.Name, + DefaultMode: helpers.Int32Ptr(444), + }, + }, + } + newPod.Spec.Volumes = append(newPod.Spec.Volumes, v) + + //now add the volume mount + vm := corev1.VolumeMount{ + Name: volumeMountName, + ReadOnly: true, + MountPath: "/var/run/secrets/lagoon/dynamic/" + secret.Name, + } + + newPod.Spec.Containers[0].VolumeMounts = append(newPod.Spec.Containers[0].VolumeMounts, vm) + } + + if err := r.Create(ctx, newPod); err != nil { + opLog.Info( + fmt.Sprintf( + "Unable to create advanced task pod for project %s, environment %s: %v", + lagoonTask.Spec.Project.Name, + lagoonTask.Spec.Environment.Name, + err, + ), + ) + return err + } + taskRunningStatus.With(prometheus.Labels{ + "task_namespace": lagoonTask.ObjectMeta.Namespace, + "task_name": lagoonTask.ObjectMeta.Name, + }).Set(1) + tasksStartedCounter.Inc() + } else { + opLog.Info(fmt.Sprintf("Advanced task pod already running for: %s", lagoonTask.ObjectMeta.Name)) + } + return nil +} + +// getSecretsForNamespace is a convenience function to pull a list of secrets for a given namespace +func getSecretsForNamespace(k8sClient client.Client, namespace string) (map[string]corev1.Secret, error) { + secretList := &corev1.SecretList{} + err := k8sClient.List(context.Background(), secretList, &client.ListOptions{Namespace: namespace}) + if err != nil { + return nil, err + } + + secrets := map[string]corev1.Secret{} + for _, secret := range secretList.Items { + secrets[secret.Name] = secret + } + + return secrets, nil +} + +// filterDynamicSecrets will, given a map of secrets, filter those that match the dynamic secret label +func filterDynamicSecrets(secrets map[string]corev1.Secret) map[string]corev1.Secret { + filteredSecrets := map[string]corev1.Secret{} + for secretName, secret := range secrets { + if _, ok := secret.Labels["lagoon.sh/dynamic-secret"]; ok { + filteredSecrets[secretName] = secret + } + } + return filteredSecrets +} + +// getServiceAccount will get the service account if it exists +func (r *LagoonTaskReconciler) getServiceAccount(ctx context.Context, serviceAccount *corev1.ServiceAccount, ns string) error { + serviceAccount.ObjectMeta = metav1.ObjectMeta{ + Name: "lagoon-deployer", + Namespace: ns, + } + err := r.Get(ctx, types.NamespacedName{ + Namespace: ns, + Name: "lagoon-deployer", + }, serviceAccount) + if err != nil { + return err + } + return nil +} diff --git a/controllers/v1beta2/task_helpers.go b/controllers/v1beta2/task_helpers.go new file mode 100644 index 00000000..017857dc --- /dev/null +++ b/controllers/v1beta2/task_helpers.go @@ -0,0 +1,59 @@ +package v1beta2 + +import ( + "context" + "fmt" + + "github.com/uselagoon/remote-controller/internal/helpers" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +// createActiveStandbyRole will create the rolebinding for allowing lagoon-deployer to talk between namespaces for active/standby functionality +func (r *LagoonTaskReconciler) createActiveStandbyRole(ctx context.Context, sourceNamespace, destinationNamespace string) error { + activeStandbyRoleBinding := &rbacv1.RoleBinding{} + activeStandbyRoleBinding.ObjectMeta = metav1.ObjectMeta{ + Name: "lagoon-deployer-activestandby", + Namespace: destinationNamespace, + } + activeStandbyRoleBinding.RoleRef = rbacv1.RoleRef{ + Name: "admin", + Kind: "ClusterRole", + APIGroup: "rbac.authorization.k8s.io", + } + activeStandbyRoleBinding.Subjects = []rbacv1.Subject{ + { + Name: "lagoon-deployer", + Kind: "ServiceAccount", + Namespace: sourceNamespace, + }, + } + err := r.Get(ctx, types.NamespacedName{ + Namespace: destinationNamespace, + Name: "lagoon-deployer-activestandby", + }, activeStandbyRoleBinding) + if err != nil { + if err := r.Create(ctx, activeStandbyRoleBinding); err != nil { + return fmt.Errorf("There was an error creating the lagoon-deployer-activestandby role binding. Error was: %v", err) + } + } + return nil +} + +// deleteActiveStandbyRole +func (r *LagoonMonitorReconciler) deleteActiveStandbyRole(ctx context.Context, destinationNamespace string) error { + activeStandbyRoleBinding := &rbacv1.RoleBinding{} + err := r.Get(ctx, types.NamespacedName{ + Namespace: destinationNamespace, + Name: "lagoon-deployer-activestandby", + }, activeStandbyRoleBinding) + if err != nil { + helpers.IgnoreNotFound(err) + } + err = r.Delete(ctx, activeStandbyRoleBinding) + if err != nil { + return fmt.Errorf("Unable to delete lagoon-deployer-activestandby role binding") + } + return nil +} diff --git a/internal/harbor/harbor_credentialrotation.go b/internal/harbor/harbor_credentialrotation.go index 7a3bc4e7..43c97d90 100644 --- a/internal/harbor/harbor_credentialrotation.go +++ b/internal/harbor/harbor_credentialrotation.go @@ -7,6 +7,7 @@ import ( "time" lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" + lagoonv1beta2 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" "github.com/uselagoon/remote-controller/internal/helpers" corev1 "k8s.io/api/core/v1" @@ -45,7 +46,8 @@ func (h *Harbor) RotateRobotCredentials(ctx context.Context, cl client.Client) { opLog.Info(fmt.Sprintf("Checking if %s needs robot credentials rotated", ns.ObjectMeta.Name)) // check for running builds! runningBuildsv1beta1 := lagoonv1beta1.CheckRunningBuilds(ctx, h.ControllerNamespace, opLog, cl, ns) - if !runningBuildsv1beta1 { + runningBuildsv1beta2 := lagoonv1beta2.CheckRunningBuilds(ctx, h.ControllerNamespace, opLog, cl, ns) + if !runningBuildsv1beta1 && !runningBuildsv1beta2 { rotated, err := h.RotateRobotCredential(ctx, cl, ns, false) if err != nil { opLog.Error(err, "error") diff --git a/internal/messenger/consumer.go b/internal/messenger/consumer.go index 51173b37..8ad7c8d7 100644 --- a/internal/messenger/consumer.go +++ b/internal/messenger/consumer.go @@ -11,6 +11,7 @@ import ( "github.com/cheshir/go-mq/v2" "github.com/uselagoon/machinery/api/schema" lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" + lagoonv1beta2 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" "github.com/uselagoon/remote-controller/internal/helpers" "gopkg.in/matryer/try.v1" corev1 "k8s.io/api/core/v1" @@ -67,7 +68,7 @@ func (m *Messenger) Consumer(targetName string) { //error { err = messageQueue.SetConsumerHandler("builddeploy-queue", func(message mq.Message) { if err == nil { // unmarshal the body into a lagoonbuild - newBuild := &lagoonv1beta1.LagoonBuild{} + newBuild := &lagoonv1beta2.LagoonBuild{} json.Unmarshal(message.Body(), newBuild) // new builds that come in should initially get created in the controllers own // namespace before being handled and re-created in the correct namespace @@ -75,7 +76,8 @@ func (m *Messenger) Consumer(targetName string) { //error { newBuild.ObjectMeta.Namespace = m.ControllerNamespace newBuild.SetLabels( map[string]string{ - "lagoon.sh/controller": m.ControllerNamespace, + "lagoon.sh/controller": m.ControllerNamespace, + "crd.lagoon.sh/version": "v1beta2", }, ) opLog.Info( @@ -234,7 +236,7 @@ func (m *Messenger) Consumer(targetName string) { //error { err = messageQueue.SetConsumerHandler("jobs-queue", func(message mq.Message) { if err == nil { // unmarshall the message into a remove task to be processed - jobSpec := &lagoonv1beta1.LagoonTaskSpec{} + jobSpec := &lagoonv1beta2.LagoonTaskSpec{} json.Unmarshal(message.Body(), jobSpec) namespace := helpers.GenerateNamespaceName( jobSpec.Project.NamespacePattern, // the namespace pattern or `openshiftProjectPattern` from Lagoon is never received by the controller @@ -252,14 +254,14 @@ func (m *Messenger) Consumer(targetName string) { //error { namespace, ), ) - job := &lagoonv1beta1.LagoonTask{} + job := &lagoonv1beta2.LagoonTask{} job.Spec = *jobSpec // set the namespace to the `openshiftProjectName` from the environment job.ObjectMeta.Namespace = namespace job.SetLabels( map[string]string{ - "lagoon.sh/taskType": lagoonv1beta1.TaskTypeStandard.String(), - "lagoon.sh/taskStatus": lagoonv1beta1.TaskStatusPending.String(), + "lagoon.sh/taskType": lagoonv1beta2.TaskTypeStandard.String(), + "lagoon.sh/taskStatus": lagoonv1beta2.TaskStatusPending.String(), "lagoon.sh/controller": m.ControllerNamespace, }, ) @@ -292,7 +294,7 @@ func (m *Messenger) Consumer(targetName string) { //error { if err == nil { opLog := ctrl.Log.WithName("handlers").WithName("LagoonTasks") // unmarshall the message into a remove task to be processed - jobSpec := &lagoonv1beta1.LagoonTaskSpec{} + jobSpec := &lagoonv1beta2.LagoonTaskSpec{} json.Unmarshal(message.Body(), jobSpec) // check which key has been received namespace := helpers.GenerateNamespaceName( @@ -314,18 +316,43 @@ func (m *Messenger) Consumer(targetName string) { //error { ), ) m.Cache.Add(jobSpec.Misc.Name, jobSpec.Project.Name) - _, v1beta1Bytes, err := lagoonv1beta1.CancelBuild(ctx, m.Client, namespace, message.Body()) + // check if there is a v1beta2 task to cancel + hasv1beta2Build, v1beta2Bytes, err := lagoonv1beta2.CancelBuild(ctx, m.Client, namespace, message.Body()) + if err != nil { + //@TODO: send msg back to lagoon and update task to failed? + message.Ack(false) // ack to remove from queue + return + } + hasv1beta1Build, v1beta1Bytes, err := lagoonv1beta1.CancelBuild(ctx, m.Client, namespace, message.Body()) if err != nil { //@TODO: send msg back to lagoon and update build to failed? message.Ack(false) // ack to remove from queue return } - if v1beta1Bytes != nil { - // if v1beta1 has a build, send its response - if err := m.Publish("lagoon-tasks:controller", v1beta1Bytes); err != nil { - opLog.Error(err, "Unable to publish message.") - message.Ack(false) // ack to remove from queue - return + if v1beta1Bytes != nil && v1beta2Bytes != nil { + // check if either v1beta2 or v1beta1 have a matching build + if hasv1beta2Build && !hasv1beta1Build { + // if v1beta2 has a build, send its response + if err := m.Publish("lagoon-tasks:controller", v1beta2Bytes); err != nil { + opLog.Error(err, "Unable to publish message.") + message.Ack(false) // ack to remove from queue + return + } + } else if !hasv1beta2Build && hasv1beta1Build { + // if v1beta1 has a build, send its response + if err := m.Publish("lagoon-tasks:controller", v1beta1Bytes); err != nil { + opLog.Error(err, "Unable to publish message.") + message.Ack(false) // ack to remove from queue + return + } + } else { + // if both v1beta2 and v1beta1 say there is no associated build + // then respond with the v1beta2 response + if err := m.Publish("lagoon-tasks:controller", v1beta2Bytes); err != nil { + opLog.Error(err, "Unable to publish message.") + message.Ack(false) // ack to remove from queue + return + } } } case "deploytarget:task:cancel", "kubernetes:task:cancel": @@ -338,17 +365,43 @@ func (m *Messenger) Consumer(targetName string) { //error { ), ) m.Cache.Add(jobSpec.Task.TaskName, jobSpec.Project.Name) - _, v1beta1Bytes, err := lagoonv1beta1.CancelTask(ctx, m.Client, namespace, message.Body()) + // check if there is a v1beta2 task to cancel + hasv1beta2Task, v1beta2Bytes, err := lagoonv1beta2.CancelTask(ctx, m.Client, namespace, message.Body()) if err != nil { //@TODO: send msg back to lagoon and update task to failed? message.Ack(false) // ack to remove from queue return } - if v1beta1Bytes != nil { - if err := m.Publish("lagoon-tasks:controller", v1beta1Bytes); err != nil { - opLog.Error(err, "Unable to publish message.") - message.Ack(false) // ack to remove from queue - return + hasv1beta1Task, v1beta1Bytes, err := lagoonv1beta1.CancelTask(ctx, m.Client, namespace, message.Body()) + if err != nil { + //@TODO: send msg back to lagoon and update task to failed? + message.Ack(false) // ack to remove from queue + return + } + if v1beta1Bytes != nil && v1beta2Bytes != nil { + // check if either v1beta2 or v1beta1 have a matching task + if hasv1beta2Task && !hasv1beta1Task { + // if v1beta2 has a task, send its response + if err := m.Publish("lagoon-tasks:controller", v1beta2Bytes); err != nil { + opLog.Error(err, "Unable to publish message.") + message.Ack(false) // ack to remove from queue + return + } + } else if !hasv1beta2Task && hasv1beta1Task { + // if v1beta1 has a task, send its response + if err := m.Publish("lagoon-tasks:controller", v1beta1Bytes); err != nil { + opLog.Error(err, "Unable to publish message.") + message.Ack(false) // ack to remove from queue + return + } + } else { + // if both v1beta2 and v1beta1 say there is no associated task + // then respond with the v1beta2 response + if err := m.Publish("lagoon-tasks:controller", v1beta2Bytes); err != nil { + opLog.Error(err, "Unable to publish message.") + message.Ack(false) // ack to remove from queue + return + } } } case "deploytarget:restic:backup:restore", "kubernetes:restic:backup:restore": diff --git a/internal/messenger/tasks_handler.go b/internal/messenger/tasks_handler.go index 2b8b68a0..3a88b464 100644 --- a/internal/messenger/tasks_handler.go +++ b/internal/messenger/tasks_handler.go @@ -6,7 +6,7 @@ import ( "encoding/json" "fmt" - lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" + lagoonv1beta2 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" "github.com/uselagoon/remote-controller/internal/helpers" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" @@ -18,7 +18,7 @@ type ActiveStandbyPayload struct { } // IngressRouteMigration handles running the ingress migrations. -func (m *Messenger) IngressRouteMigration(namespace string, jobSpec *lagoonv1beta1.LagoonTaskSpec) error { +func (m *Messenger) IngressRouteMigration(namespace string, jobSpec *lagoonv1beta2.LagoonTaskSpec) error { // always set these to true for ingress migration tasks jobSpec.AdvancedTask.DeployerToken = true jobSpec.AdvancedTask.SSHKey = true @@ -26,7 +26,7 @@ func (m *Messenger) IngressRouteMigration(namespace string, jobSpec *lagoonv1bet } // ActiveStandbySwitch handles running the active standby switch setup advanced task. -func (m *Messenger) ActiveStandbySwitch(namespace string, jobSpec *lagoonv1beta1.LagoonTaskSpec) error { +func (m *Messenger) ActiveStandbySwitch(namespace string, jobSpec *lagoonv1beta2.LagoonTaskSpec) error { // always set these to true for ingress migration tasks jobSpec.AdvancedTask.DeployerToken = true jobSpec.AdvancedTask.SSHKey = true @@ -47,30 +47,30 @@ func (m *Messenger) ActiveStandbySwitch(namespace string, jobSpec *lagoonv1beta1 } // AdvancedTask handles running the ingress migrations. -func (m *Messenger) AdvancedTask(namespace string, jobSpec *lagoonv1beta1.LagoonTaskSpec) error { +func (m *Messenger) AdvancedTask(namespace string, jobSpec *lagoonv1beta2.LagoonTaskSpec) error { return m.createAdvancedTask(namespace, jobSpec, nil) } // CreateAdvancedTask takes care of creating actual advanced tasks -func (m *Messenger) createAdvancedTask(namespace string, jobSpec *lagoonv1beta1.LagoonTaskSpec, additionalLabels map[string]string) error { +func (m *Messenger) createAdvancedTask(namespace string, jobSpec *lagoonv1beta2.LagoonTaskSpec, additionalLabels map[string]string) error { return createAdvancedTask(namespace, jobSpec, m, additionalLabels) } // CreateAdvancedTask takes care of creating actual advanced tasks -func createAdvancedTask(namespace string, jobSpec *lagoonv1beta1.LagoonTaskSpec, m *Messenger, additionalLabels map[string]string) error { +func createAdvancedTask(namespace string, jobSpec *lagoonv1beta2.LagoonTaskSpec, m *Messenger, additionalLabels map[string]string) error { opLog := ctrl.Log.WithName("handlers").WithName("LagoonTasks") // create the advanced task taskName := fmt.Sprintf("lagoon-advanced-task-%s", helpers.RandString(6)) if jobSpec.Task.TaskName != "" { taskName = jobSpec.Task.TaskName } - task := lagoonv1beta1.LagoonTask{ + task := lagoonv1beta2.LagoonTask{ ObjectMeta: metav1.ObjectMeta{ Name: taskName, Namespace: namespace, Labels: map[string]string{ - "lagoon.sh/taskType": lagoonv1beta1.TaskTypeAdvanced.String(), - "lagoon.sh/taskStatus": lagoonv1beta1.TaskStatusPending.String(), + "lagoon.sh/taskType": lagoonv1beta2.TaskTypeAdvanced.String(), + "lagoon.sh/taskStatus": lagoonv1beta2.TaskStatusPending.String(), "lagoon.sh/controller": m.ControllerNamespace, }, }, diff --git a/internal/messenger/tasks_restore.go b/internal/messenger/tasks_restore.go index 315ddb4d..a380c4a1 100644 --- a/internal/messenger/tasks_restore.go +++ b/internal/messenger/tasks_restore.go @@ -6,7 +6,7 @@ import ( "fmt" "github.com/go-logr/logr" - lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" + lagoonv1beta2 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" "github.com/uselagoon/remote-controller/internal/helpers" ctrl "sigs.k8s.io/controller-runtime" @@ -17,7 +17,7 @@ import ( ) // ResticRestore handles creating the restic restore jobs. -func (m *Messenger) ResticRestore(namespace string, jobSpec *lagoonv1beta1.LagoonTaskSpec) error { +func (m *Messenger) ResticRestore(namespace string, jobSpec *lagoonv1beta2.LagoonTaskSpec) error { opLog := ctrl.Log.WithName("handlers").WithName("LagoonTasks") vers, err := checkRestoreVersionFromCore(jobSpec.Misc.MiscResource) if err != nil { @@ -97,7 +97,7 @@ func checkRestoreVersionFromCore(resource []byte) (string, error) { } // createv1alpha1Restore will create a restore task using the restores.backup.appuio.ch v1alpha1 api (k8up v1) -func (m *Messenger) createv1alpha1Restore(opLog logr.Logger, namespace string, jobSpec *lagoonv1beta1.LagoonTaskSpec) error { +func (m *Messenger) createv1alpha1Restore(opLog logr.Logger, namespace string, jobSpec *lagoonv1beta2.LagoonTaskSpec) error { restorev1alpha1 := &k8upv1alpha1.Restore{} if err := json.Unmarshal(jobSpec.Misc.MiscResource, restorev1alpha1); err != nil { opLog.Error(err, @@ -122,7 +122,7 @@ func (m *Messenger) createv1alpha1Restore(opLog logr.Logger, namespace string, j } // createv1Restore will create a restore task using the restores.k8up.io v1 api (k8up v2) -func (m *Messenger) createv1Restore(opLog logr.Logger, namespace string, jobSpec *lagoonv1beta1.LagoonTaskSpec) error { +func (m *Messenger) createv1Restore(opLog logr.Logger, namespace string, jobSpec *lagoonv1beta2.LagoonTaskSpec) error { restorev1 := &k8upv1.Restore{} if err := json.Unmarshal(jobSpec.Misc.MiscResource, restorev1); err != nil { opLog.Error(err, diff --git a/internal/utilities/deletions/process.go b/internal/utilities/deletions/process.go index c8521a3a..997868d1 100644 --- a/internal/utilities/deletions/process.go +++ b/internal/utilities/deletions/process.go @@ -10,6 +10,7 @@ import ( "k8s.io/apimachinery/pkg/types" lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" + lagoonv1beta2 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" "github.com/uselagoon/remote-controller/internal/harbor" ) @@ -101,9 +102,15 @@ func (d *Deletions) ProcessDeletion(ctx context.Context, opLog logr.Logger, name if del := lagoonv1beta1.DeleteLagoonTasks(ctx, opLog.WithName("DeleteLagoonTasks"), d.Client, namespace.ObjectMeta.Name, project, environment); del == false { return fmt.Errorf("error deleting tasks") } + if del := lagoonv1beta2.DeleteLagoonTasks(ctx, opLog.WithName("DeleteLagoonTasks"), d.Client, namespace.ObjectMeta.Name, project, environment); del == false { + return fmt.Errorf("error deleting tasks") + } if del := lagoonv1beta1.DeleteLagoonBuilds(ctx, opLog.WithName("DeleteLagoonBuilds"), d.Client, namespace.ObjectMeta.Name, project, environment); del == false { return fmt.Errorf("error deleting builds") } + if del := lagoonv1beta2.DeleteLagoonBuilds(ctx, opLog.WithName("DeleteLagoonBuilds"), d.Client, namespace.ObjectMeta.Name, project, environment); del == false { + return fmt.Errorf("error deleting builds") + } if del := d.DeleteDeployments(ctx, opLog.WithName("DeleteDeployments"), namespace.ObjectMeta.Name, project, environment); del == false { return fmt.Errorf("error deleting deployments") } diff --git a/main.go b/main.go index 9aef681d..1e66c0cf 100644 --- a/main.go +++ b/main.go @@ -45,8 +45,10 @@ import ( "github.com/hashicorp/golang-lru/v2/expirable" k8upv1 "github.com/k8up-io/k8up/v2/api/v1" lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1" + lagoonv1beta2 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" harborctrl "github.com/uselagoon/remote-controller/controllers/harbor" lagoonv1beta1ctrl "github.com/uselagoon/remote-controller/controllers/v1beta1" + lagoonv1beta2ctrl "github.com/uselagoon/remote-controller/controllers/v1beta2" "github.com/uselagoon/remote-controller/internal/messenger" k8upv1alpha1 "github.com/vshn/k8up/api/v1alpha1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" @@ -75,6 +77,7 @@ func init() { _ = clientgoscheme.AddToScheme(scheme) _ = lagoonv1beta1.AddToScheme(scheme) + _ = lagoonv1beta2.AddToScheme(scheme) _ = k8upv1.AddToScheme(scheme) _ = k8upv1alpha1.AddToScheme(scheme) _ = apiextensionsv1.AddToScheme(scheme) @@ -663,6 +666,11 @@ func main() { DefaultValue: qosDefaultValue, } + buildQoSConfigv1beta2 := lagoonv1beta2ctrl.BuildQoS{ + MaxBuilds: qosMaxBuilds, + DefaultValue: qosDefaultValue, + } + resourceCleanup := pruner.New(mgr.GetClient(), buildsToKeep, buildPodsToKeep, @@ -680,6 +688,7 @@ func main() { // use cron to run a lagoonbuild cleanup task // this will check any Lagoon builds and attempt to delete them c.AddFunc(buildsCleanUpCron, func() { + lagoonv1beta2.LagoonBuildPruner(context.Background(), mgr.GetClient(), controllerNamespace, buildsToKeep) lagoonv1beta1.LagoonBuildPruner(context.Background(), mgr.GetClient(), controllerNamespace, buildsToKeep) }) } @@ -689,6 +698,7 @@ func main() { // use cron to run a build pod cleanup task // this will check any Lagoon build pods and attempt to delete them c.AddFunc(buildPodCleanUpCron, func() { + lagoonv1beta2.BuildPodPruner(context.Background(), mgr.GetClient(), controllerNamespace, buildPodsToKeep) lagoonv1beta1.BuildPodPruner(context.Background(), mgr.GetClient(), controllerNamespace, buildPodsToKeep) }) } @@ -698,6 +708,7 @@ func main() { // use cron to run a lagoontask cleanup task // this will check any Lagoon tasks and attempt to delete them c.AddFunc(taskCleanUpCron, func() { + lagoonv1beta2.LagoonTaskPruner(context.Background(), mgr.GetClient(), controllerNamespace, tasksToKeep) lagoonv1beta1.LagoonTaskPruner(context.Background(), mgr.GetClient(), controllerNamespace, tasksToKeep) }) } @@ -707,6 +718,7 @@ func main() { // use cron to run a task pod cleanup task // this will check any Lagoon task pods and attempt to delete them c.AddFunc(taskPodCleanUpCron, func() { + lagoonv1beta2.TaskPodPruner(context.Background(), mgr.GetClient(), controllerNamespace, taskPodsToKeep) lagoonv1beta1.TaskPodPruner(context.Background(), mgr.GetClient(), controllerNamespace, taskPodsToKeep) }) } @@ -846,6 +858,112 @@ func main() { os.Exit(1) } + // v1beta2 is the latest version + if err = (&lagoonv1beta2ctrl.LagoonBuildReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("v1beta2").WithName("LagoonBuild"), + Scheme: mgr.GetScheme(), + EnableMQ: enableMQ, + BuildImage: overrideBuildDeployImage, + Messaging: messaging, + NamespacePrefix: namespacePrefix, + RandomNamespacePrefix: randomPrefix, + ControllerNamespace: controllerNamespace, + EnableDebug: enableDebug, + FastlyServiceID: fastlyServiceID, + FastlyWatchStatus: fastlyWatchStatus, + BuildPodRunAsUser: int64(buildPodRunAsUser), + BuildPodRunAsGroup: int64(buildPodRunAsGroup), + BuildPodFSGroup: int64(buildPodFSGroup), + BackupConfig: lagoonv1beta2ctrl.BackupConfig{ + BackupDefaultSchedule: backupDefaultSchedule, + BackupDefaultDevelopmentSchedule: backupDefaultDevelopmentSchedule, + BackupDefaultPullrequestSchedule: backupDefaultPullrequestSchedule, + BackupDefaultDevelopmentRetention: backupDefaultDevelopmentRetention, + BackupDefaultPullrequestRetention: backupDefaultPullrequestRetention, + BackupDefaultMonthlyRetention: backupDefaultMonthlyRetention, + BackupDefaultWeeklyRetention: backupDefaultWeeklyRetention, + BackupDefaultDailyRetention: backupDefaultDailyRetention, + BackupDefaultHourlyRetention: backupDefaultHourlyRetention, + }, + // Lagoon feature flags + LFFForceRootlessWorkload: lffForceRootlessWorkload, + LFFDefaultRootlessWorkload: lffDefaultRootlessWorkload, + LFFForceIsolationNetworkPolicy: lffForceIsolationNetworkPolicy, + LFFDefaultIsolationNetworkPolicy: lffDefaultIsolationNetworkPolicy, + LFFForceInsights: lffForceInsights, + LFFDefaultInsights: lffDefaultInsights, + LFFForceRWX2RWO: lffForceRWX2RWO, + LFFDefaultRWX2RWO: lffDefaultRWX2RWO, + LFFBackupWeeklyRandom: lffBackupWeeklyRandom, + LFFRouterURL: lffRouterURL, + LFFHarborEnabled: lffHarborEnabled, + Harbor: harborConfig, + LFFQoSEnabled: lffQoSEnabled, + BuildQoS: buildQoSConfigv1beta2, + NativeCronPodMinFrequency: nativeCronPodMinFrequency, + LagoonTargetName: lagoonTargetName, + LagoonFeatureFlags: helpers.GetLagoonFeatureFlags(), + LagoonAPIConfiguration: helpers.LagoonAPIConfiguration{ + APIHost: lagoonAPIHost, + TokenHost: lagoonTokenHost, + TokenPort: lagoonTokenPort, + SSHHost: lagoonSSHHost, + SSHPort: lagoonSSHPort, + }, + ProxyConfig: lagoonv1beta2ctrl.ProxyConfig{ + HTTPProxy: httpProxy, + HTTPSProxy: httpsProxy, + NoProxy: noProxy, + }, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "LagoonBuild") + os.Exit(1) + } + if err = (&lagoonv1beta2ctrl.LagoonMonitorReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("v1beta2").WithName("LagoonMonitor"), + Scheme: mgr.GetScheme(), + EnableMQ: enableMQ, + Messaging: messaging, + ControllerNamespace: controllerNamespace, + NamespacePrefix: namespacePrefix, + RandomNamespacePrefix: randomPrefix, + EnableDebug: enableDebug, + LagoonTargetName: lagoonTargetName, + LFFQoSEnabled: lffQoSEnabled, + BuildQoS: buildQoSConfigv1beta2, + Cache: cache, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "LagoonMonitor") + os.Exit(1) + } + if err = (&lagoonv1beta2ctrl.LagoonTaskReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("v1beta2").WithName("LagoonTask"), + Scheme: mgr.GetScheme(), + ControllerNamespace: controllerNamespace, + NamespacePrefix: namespacePrefix, + RandomNamespacePrefix: randomPrefix, + LagoonAPIConfiguration: helpers.LagoonAPIConfiguration{ + APIHost: lagoonAPIHost, + TokenHost: lagoonTokenHost, + TokenPort: lagoonTokenPort, + SSHHost: lagoonSSHHost, + SSHPort: lagoonSSHPort, + }, + EnableDebug: enableDebug, + LagoonTargetName: lagoonTargetName, + ProxyConfig: lagoonv1beta2ctrl.ProxyConfig{ + HTTPProxy: httpProxy, + HTTPSProxy: httpsProxy, + NoProxy: noProxy, + }, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "LagoonTask") + os.Exit(1) + } + // for now the namespace reconciler only needs to run if harbor is enabled so that we can watch the namespace for rotation label events if lffHarborEnabled { if err = (&harborctrl.HarborCredentialReconciler{ diff --git a/test-resources/dynamic-secret-in-task-project1.yaml b/test-resources/dynamic-secret-in-task-project1.yaml index e242a0a6..079582ef 100644 --- a/test-resources/dynamic-secret-in-task-project1.yaml +++ b/test-resources/dynamic-secret-in-task-project1.yaml @@ -1,4 +1,4 @@ -apiVersion: crd.lagoon.sh/v1beta1 +apiVersion: crd.lagoon.sh/v1beta2 kind: LagoonTask metadata: name: "lagoon-advanced-task-example-task-project-1" @@ -6,6 +6,7 @@ metadata: lagoon.sh/controller: lagoon lagoon.sh/taskStatus: Pending lagoon.sh/taskType: advanced + crd.lagoon.sh/version: v1beta2 spec: advancedTask: JSONPayload: e30= diff --git a/test-resources/example-project1.yaml b/test-resources/example-project1.yaml index 355f8fb8..8b620e46 100644 --- a/test-resources/example-project1.yaml +++ b/test-resources/example-project1.yaml @@ -1,7 +1,9 @@ kind: LagoonBuild -apiVersion: crd.lagoon.sh/v1beta1 +apiVersion: crd.lagoon.sh/v1beta2 metadata: name: lagoon-build-7m5zypx + labels: + crd.lagoon.sh/version: v1beta2 spec: build: ci: 'true' #to make sure that readwritemany is changed to readwriteonce diff --git a/test-resources/example-project2.yaml b/test-resources/example-project2.yaml index 67451af2..904528df 100644 --- a/test-resources/example-project2.yaml +++ b/test-resources/example-project2.yaml @@ -1,7 +1,9 @@ kind: LagoonBuild -apiVersion: crd.lagoon.sh/v1beta1 +apiVersion: crd.lagoon.sh/v1beta2 metadata: name: lagoon-build-8m5zypx + labels: + crd.lagoon.sh/version: v1beta2 spec: build: ci: 'true' #to make sure that readwritemany is changed to readwriteonce diff --git a/test-resources/example-project3.yaml b/test-resources/example-project3.yaml new file mode 100644 index 00000000..57cba070 --- /dev/null +++ b/test-resources/example-project3.yaml @@ -0,0 +1,32 @@ +kind: LagoonBuild +apiVersion: crd.lagoon.sh/v1beta1 +metadata: + name: lagoon-build-1m5zypx +spec: + build: + ci: 'true' #to make sure that readwritemany is changed to readwriteonce + type: branch + gitReference: origin/main + project: + name: nginx-example + environment: main + organization: + id: 123 + name: test-org + uiLink: https://dashboard.amazeeio.cloud/projects/project/project-environment/deployments/lagoon-build-7m5zypx + routerPattern: 'main-nginx-example' + environmentType: production + productionEnvironment: main + standbyEnvironment: master + gitUrl: https://github.com/shreddedbacon/lagoon-nginx-example.git + deployTarget: kind + projectSecret: 4d6e7dd0f013a75d62a0680139fa82d350c2a1285f43f867535bad1143f228b1 + key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlDWFFJQkFBS0JnUUNjc1g2RG5KNXpNb0RqQ2R6a1JFOEg2TEh2TDQzaUhsekJLTWo4T1VNV05ZZG5YekdqCkR5Mkp1anQ3ZDNlMTVLeC8zOFo5UzJLdHNnVFVtWi9lUlRQSTdabE1idHRJK250UmtyblZLblBWNzhEeEFKNW8KTGZtQndmdWE2MnlVYnl0cnpYQ2pwVVJrQUlBMEZiR2VqS2Rvd3cxcnZGMzJoZFUzQ3ZIcG5rKzE2d0lEQVFBQgpBb0dCQUkrV0dyL1NDbVMzdCtIVkRPVGtMNk9vdVR6Y1QrRVFQNkVGbGIrRFhaV0JjZFhwSnB3c2NXZFBEK2poCkhnTEJUTTFWS3hkdnVEcEE4aW83cUlMTzJWYm1MeGpNWGk4TUdwY212dXJFNVJydTZTMXJzRDl2R0c5TGxoR3UKK0pUSmViMVdaZFduWFZ2am5LbExrWEV1eUthbXR2Z253Um5xNld5V05OazJ6SktoQWtFQThFenpxYnowcFVuTApLc241K2k0NUdoRGVpRTQvajRtamo1b1FHVzJxbUZWT2pHaHR1UGpaM2lwTis0RGlTRkFyMkl0b2VlK085d1pyCkRINHBkdU5YOFFKQkFLYnVOQ3dXK29sYXA4R2pUSk1TQjV1MW8wMVRHWFdFOGhVZG1leFBBdjl0cTBBT0gzUUQKUTIrM0RsaVY0ektoTlMra2xaSkVjNndzS0YyQmJIby81NXNDUVFETlBJd24vdERja3loSkJYVFJyc1RxZEZuOApCUWpZYVhBZTZEQ3o1eXg3S3ZFSmp1K1h1a01xTXV1ajBUSnpITFkySHVzK3FkSnJQVG9VMDNSS3JHV2hBa0JFCnB3aXI3Vk5pYy9jMFN2MnVLcWNZWWM1a2ViMnB1R0I3VUs1Q0lvaWdGakZzNmFJRDYyZXJwVVJ3S0V6RlFNbUgKNjQ5Y0ZXemhMVlA0aU1iZFREVHJBa0FFMTZXU1A3WXBWOHV1eFVGMGV0L3lFR3dURVpVU2R1OEppSTBHN0tqagpqcVR6RjQ3YkJZc0pIYTRYcWpVb2E3TXgwcS9FSUtRWkJ2NGFvQm42bGFOQwotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQ== + monitoring: + contact: '1234' + statuspageID: '1234' + variables: + project: W3sibmFtZSI6IkxBR09PTl9TWVNURU1fUk9VVEVSX1BBVFRFUk4iLCJ2YWx1ZSI6IiR7ZW52aXJvbm1lbnR9LiR7cHJvamVjdH0uZXhhbXBsZS5jb20iLCJzY29wZSI6ImludGVybmFsX3N5c3RlbSJ9XQ== + environment: W10= + branch: + name: main \ No newline at end of file From f7e12fd55dfba9a605359a2cec3252fb16607b97 Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Wed, 24 Jan 2024 12:05:40 +1100 Subject: [PATCH 06/17] refactor: remove resources after theyve completed reconciling --- controllers/v1beta1/build_controller.go | 18 ++++++++++++++++++ controllers/v1beta1/task_controller.go | 16 ++++++++++++++++ controllers/v1beta2/build_controller.go | 1 + 3 files changed, 35 insertions(+) diff --git a/controllers/v1beta1/build_controller.go b/controllers/v1beta1/build_controller.go index db1f608c..736c9e84 100644 --- a/controllers/v1beta1/build_controller.go +++ b/controllers/v1beta1/build_controller.go @@ -135,6 +135,24 @@ func (r *LagoonBuildReconciler) Reconcile(ctx context.Context, req ctrl.Request) } } } + + // with the introduction of v1beta2, this will let any existing pending/qeued/running builds continue through + // but once the build is completed or failed and has processed anything else it needs to do, it should delete the resource + if _, ok := lagoonBuild.Labels["lagoon.sh/buildStatus"]; ok { + if helpers.ContainsString( + lagoonv1beta1.BuildCompletedCancelledFailedStatus, + lagoonBuild.Labels["lagoon.sh/buildStatus"], + ) { + opLog.Info(fmt.Sprintf("%s found in namespace %s is no longer required, removing it. v1beta1 is deprecated in favor of v1beta2", + lagoonBuild.ObjectMeta.Name, + req.NamespacedName.Namespace, + )) + if err := r.Delete(ctx, &lagoonBuild); err != nil { + return ctrl.Result{}, err + } + } + } + if r.LFFQoSEnabled { // handle QoS builds here // if we do have a `lagoon.sh/buildStatus` set as running, then process it diff --git a/controllers/v1beta1/task_controller.go b/controllers/v1beta1/task_controller.go index 7e34e126..bcb0eb41 100644 --- a/controllers/v1beta1/task_controller.go +++ b/controllers/v1beta1/task_controller.go @@ -68,6 +68,22 @@ func (r *LagoonTaskReconciler) Reconcile(ctx context.Context, req ctrl.Request) // examine DeletionTimestamp to determine if object is under deletion if lagoonTask.ObjectMeta.DeletionTimestamp.IsZero() { + // with the introduction of v1beta2, this will let any existing tasks continue through + // but once the task is done + if _, ok := lagoonTask.Labels["lagoon.sh/taskStatus"]; ok { + if helpers.ContainsString( + lagoonv1beta1.TaskCompletedCancelledFailedStatus, + lagoonTask.Labels["lagoon.sh/taskStatus"], + ) { + opLog.Info(fmt.Sprintf("%s found in namespace %s is no longer required, removing it. v1alpha1 is deprecated in favor of v1beta1", + lagoonTask.ObjectMeta.Name, + req.NamespacedName.Namespace, + )) + if err := r.Delete(ctx, &lagoonTask); err != nil { + return ctrl.Result{}, err + } + } + } // check if the task that has been recieved is a standard or advanced task if lagoonTask.ObjectMeta.Labels["lagoon.sh/taskStatus"] == lagoonv1beta1.TaskStatusPending.String() && lagoonTask.ObjectMeta.Labels["lagoon.sh/taskType"] == lagoonv1beta1.TaskTypeStandard.String() { diff --git a/controllers/v1beta2/build_controller.go b/controllers/v1beta2/build_controller.go index a5dc9481..de1ecab6 100644 --- a/controllers/v1beta2/build_controller.go +++ b/controllers/v1beta2/build_controller.go @@ -323,6 +323,7 @@ func (r *LagoonBuildReconciler) getOrCreateBuildResource(ctx context.Context, la map[string]string{ "lagoon.sh/buildStatus": lagooncrd.BuildStatusPending.String(), "lagoon.sh/controller": r.ControllerNamespace, + "crd.lagoon.sh/version": crdVersion, }, ) err := r.Get(ctx, types.NamespacedName{ From b273daa1f942f8510e2fb85cfe488b0ecda59b4a Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Fri, 15 Mar 2024 14:32:08 +1100 Subject: [PATCH 07/17] refactor: tidy up fmt and make metrics more available --- apis/lagoon/v1beta1/lagoonbuild_helpers.go | 26 +++-- apis/lagoon/v1beta1/lagoontask_helpers.go | 22 ++-- apis/lagoon/v1beta2/lagoonbuild_helpers.go | 40 +++---- apis/lagoon/v1beta2/lagoontask_helpers.go | 32 +++--- .../harbor/harborcredential_controller.go | 6 +- controllers/v1beta1/build_controller.go | 8 +- controllers/v1beta1/build_deletionhandlers.go | 40 ++++--- controllers/v1beta1/build_helpers.go | 93 +++++----------- controllers/v1beta1/build_qoshandler.go | 13 ++- controllers/v1beta1/build_standardhandler.go | 2 +- .../v1beta1/podmonitor_buildhandlers.go | 14 +-- controllers/v1beta1/podmonitor_controller.go | 16 +-- .../v1beta1/podmonitor_taskhandlers.go | 6 +- controllers/v1beta1/task_controller.go | 6 +- controllers/v1beta1/task_helpers.go | 4 +- controllers/v1beta2/build_controller.go | 8 +- controllers/v1beta2/build_deletionhandlers.go | 30 +++--- controllers/v1beta2/build_helpers.go | 100 +++++------------- controllers/v1beta2/build_qoshandler.go | 15 +-- controllers/v1beta2/build_standardhandler.go | 2 +- controllers/v1beta2/metrics.go | 92 ---------------- .../v1beta2/podmonitor_buildhandlers.go | 17 +-- controllers/v1beta2/podmonitor_controller.go | 20 ++-- controllers/v1beta2/podmonitor_metrics.go | 9 +- .../v1beta2/podmonitor_taskhandlers.go | 9 +- controllers/v1beta2/predicates.go | 5 +- controllers/v1beta2/task_controller.go | 15 +-- controllers/v1beta2/task_helpers.go | 4 +- internal/harbor/harbor21x.go | 4 +- internal/harbor/harbor22x.go | 4 +- internal/harbor/harbor_credentialrotation.go | 2 +- internal/harbor/harbor_helpers.go | 11 +- internal/harbor/harbor_helpers_test.go | 2 +- internal/helpers/helpers.go | 14 +-- internal/messenger/consumer.go | 4 +- internal/messenger/messenger.go | 6 -- internal/messenger/publish.go | 2 +- internal/messenger/tasks_handler.go | 4 +- internal/metrics/metrics.go | 88 +++++++++++++++ internal/utilities/deletions/process.go | 26 ++--- internal/utilities/pruner/namespace_pruner.go | 10 +- .../utilities/pruner/old_process_pruner.go | 8 +- internal/utilities/pruner/pruner.go | 6 -- 43 files changed, 365 insertions(+), 480 deletions(-) delete mode 100644 controllers/v1beta2/metrics.go diff --git a/apis/lagoon/v1beta1/lagoonbuild_helpers.go b/apis/lagoon/v1beta1/lagoonbuild_helpers.go index f32a45ad..a0ea66dd 100644 --- a/apis/lagoon/v1beta1/lagoonbuild_helpers.go +++ b/apis/lagoon/v1beta1/lagoonbuild_helpers.go @@ -84,7 +84,7 @@ func CancelExtraBuilds(ctx context.Context, r client.Client, opLog logr.Logger, client.MatchingLabels(map[string]string{"lagoon.sh/buildStatus": BuildStatusPending.String()}), }) if err := r.List(ctx, pendingBuilds, listOption); err != nil { - return fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err) + return fmt.Errorf("unable to list builds in the namespace, there may be none or something went wrong: %v", err) } if len(pendingBuilds.Items) > 0 { // opLog.Info(fmt.Sprintf("There are %v pending builds", len(pendingBuilds.Items))) @@ -150,7 +150,7 @@ func CheckRunningBuilds(ctx context.Context, cns string, opLog logr.Logger, cl c }), }) if err := cl.List(context.Background(), lagoonBuilds, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list Lagoon build pods, there may be none or something went wrong")) + opLog.Error(err, "Unable to list Lagoon build pods, there may be none or something went wrong") return false } runningBuilds := false @@ -223,7 +223,7 @@ func LagoonBuildPruner(ctx context.Context, cl client.Client, cns string, builds }, }) if err := cl.List(ctx, namespaces, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list namespaces created by Lagoon, there may be none or something went wrong")) + opLog.Error(err, "Unable to list namespaces created by Lagoon, there may be none or something went wrong") return } for _, ns := range namespaces.Items { @@ -241,7 +241,7 @@ func LagoonBuildPruner(ctx context.Context, cl client.Client, cns string, builds }), }) if err := cl.List(ctx, lagoonBuilds, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list LagoonBuild resources, there may be none or something went wrong")) + opLog.Error(err, "Unable to list LagoonBuild resources, there may be none or something went wrong") continue } // sort the build pods by creation timestamp @@ -257,7 +257,7 @@ func LagoonBuildPruner(ctx context.Context, cl client.Client, cns string, builds ) { opLog.Info(fmt.Sprintf("Cleaning up LagoonBuild %s", lagoonBuild.ObjectMeta.Name)) if err := cl.Delete(ctx, &lagoonBuild); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to update status condition")) + opLog.Error(err, "Unable to update status condition") break } } @@ -265,7 +265,6 @@ func LagoonBuildPruner(ctx context.Context, cl client.Client, cns string, builds } } } - return } // BuildPodPruner will prune any build pods that are hanging around. @@ -279,7 +278,7 @@ func BuildPodPruner(ctx context.Context, cl client.Client, cns string, buildPods }, }) if err := cl.List(ctx, namespaces, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list namespaces created by Lagoon, there may be none or something went wrong")) + opLog.Error(err, "Unable to list namespaces created by Lagoon, there may be none or something went wrong") return } for _, ns := range namespaces.Items { @@ -298,7 +297,7 @@ func BuildPodPruner(ctx context.Context, cl client.Client, cns string, buildPods }), }) if err := cl.List(ctx, buildPods, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list Lagoon build pods, there may be none or something went wrong")) + opLog.Error(err, "Unable to list Lagoon build pods, there may be none or something went wrong") return } // sort the build pods by creation timestamp @@ -312,7 +311,7 @@ func BuildPodPruner(ctx context.Context, cl client.Client, cns string, buildPods pod.Status.Phase == corev1.PodSucceeded { opLog.Info(fmt.Sprintf("Cleaning up pod %s", pod.ObjectMeta.Name)) if err := cl.Delete(ctx, &pod); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to update status condition")) + opLog.Error(err, "Unable to update status condition") break } } @@ -320,10 +319,9 @@ func BuildPodPruner(ctx context.Context, cl client.Client, cns string, buildPods } } } - return } -func updateLagoonBuild(opLog logr.Logger, namespace string, jobSpec LagoonTaskSpec, lagoonBuild *LagoonBuild) ([]byte, error) { +func updateLagoonBuild(namespace string, jobSpec LagoonTaskSpec, lagoonBuild *LagoonBuild) ([]byte, error) { // if the build isn't found by the controller // then publish a response back to controllerhandler to tell it to update the build to cancelled // this allows us to update builds in the API that may have gone stale or not updated from `New`, `Pending`, or `Running` status @@ -373,7 +371,7 @@ func updateLagoonBuild(opLog logr.Logger, namespace string, jobSpec LagoonTaskSp } msgBytes, err := json.Marshal(msg) if err != nil { - return nil, fmt.Errorf("Unable to encode message as JSON: %v", err) + return nil, fmt.Errorf("unable to encode message as JSON: %v", err) } return msgBytes, nil } @@ -404,7 +402,7 @@ func CancelBuild(ctx context.Context, cl client.Client, namespace string, body [ )) // if there is no pod or build, update the build in Lagoon to cancelled, assume completely cancelled with no other information // and then send the response back to lagoon to say it was cancelled. - b, err := updateLagoonBuild(opLog, namespace, *jobSpec, nil) + b, err := updateLagoonBuild(namespace, *jobSpec, nil) return false, b, err } // as there is no build pod, but there is a lagoon build resource @@ -427,7 +425,7 @@ func CancelBuild(ctx context.Context, cl client.Client, namespace string, body [ return false, nil, err } // and then send the response back to lagoon to say it was cancelled. - b, err := updateLagoonBuild(opLog, namespace, *jobSpec, &lagoonBuild) + b, err := updateLagoonBuild(namespace, *jobSpec, &lagoonBuild) return true, b, err } jobPod.ObjectMeta.Labels["lagoon.sh/cancelBuild"] = "true" diff --git a/apis/lagoon/v1beta1/lagoontask_helpers.go b/apis/lagoon/v1beta1/lagoontask_helpers.go index ba30f3c6..c6b90ee1 100644 --- a/apis/lagoon/v1beta1/lagoontask_helpers.go +++ b/apis/lagoon/v1beta1/lagoontask_helpers.go @@ -97,7 +97,7 @@ func LagoonTaskPruner(ctx context.Context, cl client.Client, cns string, tasksTo }, }) if err := cl.List(ctx, namespaces, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list namespaces created by Lagoon, there may be none or something went wrong")) + opLog.Error(err, "Unable to list namespaces created by Lagoon, there may be none or something went wrong") return } for _, ns := range namespaces.Items { @@ -115,7 +115,7 @@ func LagoonTaskPruner(ctx context.Context, cl client.Client, cns string, tasksTo }), }) if err := cl.List(ctx, lagoonTasks, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list LagoonTask resources, there may be none or something went wrong")) + opLog.Error(err, "Unable to list LagoonTask resources, there may be none or something went wrong") continue } // sort the build pods by creation timestamp @@ -131,7 +131,7 @@ func LagoonTaskPruner(ctx context.Context, cl client.Client, cns string, tasksTo ) { opLog.Info(fmt.Sprintf("Cleaning up LagoonTask %s", lagoonTask.ObjectMeta.Name)) if err := cl.Delete(ctx, &lagoonTask); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to update status condition")) + opLog.Error(err, "Unable to update status condition") break } } @@ -139,7 +139,6 @@ func LagoonTaskPruner(ctx context.Context, cl client.Client, cns string, tasksTo } } } - return } // TaskPodPruner will prune any task pods that are hanging around. @@ -153,7 +152,7 @@ func TaskPodPruner(ctx context.Context, cl client.Client, cns string, taskPodsTo }, }) if err := cl.List(ctx, namespaces, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list namespaces created by Lagoon, there may be none or something went wrong")) + opLog.Error(err, "Unable to list namespaces created by Lagoon, there may be none or something went wrong") return } for _, ns := range namespaces.Items { @@ -172,7 +171,7 @@ func TaskPodPruner(ctx context.Context, cl client.Client, cns string, taskPodsTo }), }) if err := cl.List(ctx, taskPods, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list Lagoon task pods, there may be none or something went wrong")) + opLog.Error(err, "Unable to list Lagoon task pods, there may be none or something went wrong") return } // sort the build pods by creation timestamp @@ -186,7 +185,7 @@ func TaskPodPruner(ctx context.Context, cl client.Client, cns string, taskPodsTo pod.Status.Phase == corev1.PodSucceeded { opLog.Info(fmt.Sprintf("Cleaning up pod %s", pod.ObjectMeta.Name)) if err := cl.Delete(ctx, &pod); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to delete pod")) + opLog.Error(err, "Unable to delete pod") break } } @@ -194,10 +193,9 @@ func TaskPodPruner(ctx context.Context, cl client.Client, cns string, taskPodsTo } } } - return } -func updateLagoonTask(opLog logr.Logger, namespace string, taskSpec LagoonTaskSpec) ([]byte, error) { +func updateLagoonTask(namespace string, taskSpec LagoonTaskSpec) ([]byte, error) { //@TODO: use `taskName` in the future only taskName := fmt.Sprintf("lagoon-task-%s-%s", taskSpec.Task.ID, helpers.HashString(taskSpec.Task.ID)[0:6]) if taskSpec.Task.TaskName != "" { @@ -228,7 +226,7 @@ func updateLagoonTask(opLog logr.Logger, namespace string, taskSpec LagoonTaskSp msg.Meta.EndTime = time.Now().UTC().Format("2006-01-02 15:04:05") msgBytes, err := json.Marshal(msg) if err != nil { - return nil, fmt.Errorf("Unable to encode message as JSON: %v", err) + return nil, fmt.Errorf("unable to encode message as JSON: %v", err) } return msgBytes, nil } @@ -259,7 +257,7 @@ func CancelTask(ctx context.Context, cl client.Client, namespace string, body [] taskName, )) // if there is no pod or task, update the task in Lagoon to cancelled - b, err := updateLagoonTask(opLog, namespace, *jobSpec) + b, err := updateLagoonTask(namespace, *jobSpec) return false, b, err } // as there is no task pod, but there is a lagoon task resource @@ -275,7 +273,7 @@ func CancelTask(ctx context.Context, cl client.Client, namespace string, body [] return false, nil, err } // and then send the response back to lagoon to say it was cancelled. - b, err := updateLagoonTask(opLog, namespace, *jobSpec) + b, err := updateLagoonTask(namespace, *jobSpec) return true, b, err } jobPod.ObjectMeta.Labels["lagoon.sh/cancelTask"] = "true" diff --git a/apis/lagoon/v1beta2/lagoonbuild_helpers.go b/apis/lagoon/v1beta2/lagoonbuild_helpers.go index 7dd7f337..89dde125 100644 --- a/apis/lagoon/v1beta2/lagoonbuild_helpers.go +++ b/apis/lagoon/v1beta2/lagoonbuild_helpers.go @@ -12,6 +12,7 @@ import ( "github.com/hashicorp/go-version" "github.com/uselagoon/machinery/api/schema" "github.com/uselagoon/remote-controller/internal/helpers" + "github.com/uselagoon/remote-controller/internal/metrics" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/selection" @@ -84,8 +85,9 @@ func CancelExtraBuilds(ctx context.Context, r client.Client, opLog logr.Logger, client.MatchingLabels(map[string]string{"lagoon.sh/buildStatus": BuildStatusPending.String()}), }) if err := r.List(ctx, pendingBuilds, listOption); err != nil { - return fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err) + return fmt.Errorf("unable to list builds in the namespace, there may be none or something went wrong: %v", err) } + metrics.BuildsPendingGauge.Set(float64(len(pendingBuilds.Items))) if len(pendingBuilds.Items) > 0 { // opLog.Info(fmt.Sprintf("There are %v pending builds", len(pendingBuilds.Items))) // if we have any pending builds, then grab the latest one and make it running @@ -150,7 +152,7 @@ func CheckRunningBuilds(ctx context.Context, cns string, opLog logr.Logger, cl c }), }) if err := cl.List(ctx, lagoonBuilds, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list Lagoon build pods, there may be none or something went wrong")) + opLog.Error(err, "unable to list Lagoon build pods, there may be none or something went wrong") return false } runningBuilds := false @@ -179,7 +181,7 @@ func DeleteLagoonBuilds(ctx context.Context, opLog logr.Logger, cl client.Client if err := cl.List(ctx, lagoonBuilds, listOption); err != nil { opLog.Error(err, fmt.Sprintf( - "Unable to list lagoon build in namespace %s for project %s, environment %s", + "unable to list lagoon build in namespace %s for project %s, environment %s", ns, project, environment, @@ -191,7 +193,7 @@ func DeleteLagoonBuilds(ctx context.Context, opLog logr.Logger, cl client.Client if err := cl.Delete(ctx, &lagoonBuild); helpers.IgnoreNotFound(err) != nil { opLog.Error(err, fmt.Sprintf( - "Unable to delete lagoon build %s in %s for project %s, environment %s", + "unable to delete lagoon build %s in %s for project %s, environment %s", lagoonBuild.ObjectMeta.Name, ns, project, @@ -223,7 +225,7 @@ func LagoonBuildPruner(ctx context.Context, cl client.Client, cns string, builds }, }) if err := cl.List(ctx, namespaces, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list namespaces created by Lagoon, there may be none or something went wrong")) + opLog.Error(err, "unable to list namespaces created by Lagoon, there may be none or something went wrong") return } for _, ns := range namespaces.Items { @@ -241,7 +243,7 @@ func LagoonBuildPruner(ctx context.Context, cl client.Client, cns string, builds }), }) if err := cl.List(ctx, lagoonBuilds, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list LagoonBuild resources, there may be none or something went wrong")) + opLog.Error(err, "unable to list LagoonBuild resources, there may be none or something went wrong") continue } // sort the build pods by creation timestamp @@ -257,7 +259,7 @@ func LagoonBuildPruner(ctx context.Context, cl client.Client, cns string, builds ) { opLog.Info(fmt.Sprintf("Cleaning up LagoonBuild %s", lagoonBuild.ObjectMeta.Name)) if err := cl.Delete(ctx, &lagoonBuild); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to update status condition")) + opLog.Error(err, "unable to update status condition") break } } @@ -265,7 +267,6 @@ func LagoonBuildPruner(ctx context.Context, cl client.Client, cns string, builds } } } - return } // BuildPodPruner will prune any build pods that are hanging around. @@ -279,7 +280,7 @@ func BuildPodPruner(ctx context.Context, cl client.Client, cns string, buildPods }, }) if err := cl.List(ctx, namespaces, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list namespaces created by Lagoon, there may be none or something went wrong")) + opLog.Error(err, "unable to list namespaces created by Lagoon, there may be none or something went wrong") return } for _, ns := range namespaces.Items { @@ -298,7 +299,7 @@ func BuildPodPruner(ctx context.Context, cl client.Client, cns string, buildPods }), }) if err := cl.List(ctx, buildPods, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list Lagoon build pods, there may be none or something went wrong")) + opLog.Error(err, "unable to list Lagoon build pods, there may be none or something went wrong") return } // sort the build pods by creation timestamp @@ -312,7 +313,7 @@ func BuildPodPruner(ctx context.Context, cl client.Client, cns string, buildPods pod.Status.Phase == corev1.PodSucceeded { opLog.Info(fmt.Sprintf("Cleaning up pod %s", pod.ObjectMeta.Name)) if err := cl.Delete(ctx, &pod); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to update status condition")) + opLog.Error(err, "unable to update status condition") break } } @@ -320,10 +321,9 @@ func BuildPodPruner(ctx context.Context, cl client.Client, cns string, buildPods } } } - return } -func updateLagoonBuild(opLog logr.Logger, namespace string, jobSpec LagoonTaskSpec, lagoonBuild *LagoonBuild) ([]byte, error) { +func updateLagoonBuild(namespace string, jobSpec LagoonTaskSpec, lagoonBuild *LagoonBuild) ([]byte, error) { // if the build isn't found by the controller // then publish a response back to controllerhandler to tell it to update the build to cancelled // this allows us to update builds in the API that may have gone stale or not updated from `New`, `Pending`, or `Running` status @@ -373,7 +373,7 @@ func updateLagoonBuild(opLog logr.Logger, namespace string, jobSpec LagoonTaskSp } msgBytes, err := json.Marshal(msg) if err != nil { - return nil, fmt.Errorf("Unable to encode message as JSON: %v", err) + return nil, fmt.Errorf("unable to encode message as JSON: %v", err) } return msgBytes, nil } @@ -389,7 +389,7 @@ func CancelBuild(ctx context.Context, cl client.Client, namespace string, body [ Namespace: namespace, }, &jobPod); err != nil { opLog.Info(fmt.Sprintf( - "Unable to find build pod %s to cancel it. Checking to see if LagoonBuild exists.", + "unable to find build pod %s to cancel it. Checking to see if LagoonBuild exists.", jobSpec.Misc.Name, )) // since there was no build pod, check for the lagoon build resource @@ -399,12 +399,12 @@ func CancelBuild(ctx context.Context, cl client.Client, namespace string, body [ Namespace: namespace, }, &lagoonBuild); err != nil { opLog.Info(fmt.Sprintf( - "Unable to find build %s to cancel it. Sending response to Lagoon to update the build to cancelled.", + "unable to find build %s to cancel it. Sending response to Lagoon to update the build to cancelled.", jobSpec.Misc.Name, )) // if there is no pod or build, update the build in Lagoon to cancelled, assume completely cancelled with no other information // and then send the response back to lagoon to say it was cancelled. - b, err := updateLagoonBuild(opLog, namespace, *jobSpec, nil) + b, err := updateLagoonBuild(namespace, *jobSpec, nil) return false, b, err } // as there is no build pod, but there is a lagoon build resource @@ -420,21 +420,21 @@ func CancelBuild(ctx context.Context, cl client.Client, namespace string, body [ if err := cl.Update(ctx, &lagoonBuild); err != nil { opLog.Error(err, fmt.Sprintf( - "Unable to update build %s to cancel it.", + "unable to update build %s to cancel it.", jobSpec.Misc.Name, ), ) return false, nil, err } // and then send the response back to lagoon to say it was cancelled. - b, err := updateLagoonBuild(opLog, namespace, *jobSpec, &lagoonBuild) + b, err := updateLagoonBuild(namespace, *jobSpec, &lagoonBuild) return true, b, err } jobPod.ObjectMeta.Labels["lagoon.sh/cancelBuild"] = "true" if err := cl.Update(ctx, &jobPod); err != nil { opLog.Error(err, fmt.Sprintf( - "Unable to update build %s to cancel it.", + "unable to update build %s to cancel it.", jobSpec.Misc.Name, ), ) diff --git a/apis/lagoon/v1beta2/lagoontask_helpers.go b/apis/lagoon/v1beta2/lagoontask_helpers.go index b8964272..ec4f538a 100644 --- a/apis/lagoon/v1beta2/lagoontask_helpers.go +++ b/apis/lagoon/v1beta2/lagoontask_helpers.go @@ -52,7 +52,7 @@ func DeleteLagoonTasks(ctx context.Context, opLog logr.Logger, cl client.Client, if err := cl.List(ctx, lagoonTasks, listOption); err != nil { opLog.Error(err, fmt.Sprintf( - "Unable to list lagoon task in namespace %s for project %s, environment %s", + "unable to list lagoon task in namespace %s for project %s, environment %s", ns, project, environment, @@ -64,7 +64,7 @@ func DeleteLagoonTasks(ctx context.Context, opLog logr.Logger, cl client.Client, if err := cl.Delete(ctx, &lagoonTask); helpers.IgnoreNotFound(err) != nil { opLog.Error(err, fmt.Sprintf( - "Unable to delete lagoon task %s in %s for project %s, environment %s", + "unable to delete lagoon task %s in %s for project %s, environment %s", lagoonTask.ObjectMeta.Name, ns, project, @@ -97,7 +97,7 @@ func LagoonTaskPruner(ctx context.Context, cl client.Client, cns string, tasksTo }, }) if err := cl.List(ctx, namespaces, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list namespaces created by Lagoon, there may be none or something went wrong")) + opLog.Error(err, "unable to list namespaces created by Lagoon, there may be none or something went wrong") return } for _, ns := range namespaces.Items { @@ -115,7 +115,7 @@ func LagoonTaskPruner(ctx context.Context, cl client.Client, cns string, tasksTo }), }) if err := cl.List(ctx, lagoonTasks, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list LagoonTask resources, there may be none or something went wrong")) + opLog.Error(err, "unable to list LagoonTask resources, there may be none or something went wrong") continue } // sort the build pods by creation timestamp @@ -131,7 +131,7 @@ func LagoonTaskPruner(ctx context.Context, cl client.Client, cns string, tasksTo ) { opLog.Info(fmt.Sprintf("Cleaning up LagoonTask %s", lagoonTask.ObjectMeta.Name)) if err := cl.Delete(ctx, &lagoonTask); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to update status condition")) + opLog.Error(err, "unable to update status condition") break } } @@ -139,7 +139,6 @@ func LagoonTaskPruner(ctx context.Context, cl client.Client, cns string, tasksTo } } } - return } // TaskPodPruner will prune any task pods that are hanging around. @@ -153,7 +152,7 @@ func TaskPodPruner(ctx context.Context, cl client.Client, cns string, taskPodsTo }, }) if err := cl.List(ctx, namespaces, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list namespaces created by Lagoon, there may be none or something went wrong")) + opLog.Error(err, "unable to list namespaces created by Lagoon, there may be none or something went wrong") return } for _, ns := range namespaces.Items { @@ -172,7 +171,7 @@ func TaskPodPruner(ctx context.Context, cl client.Client, cns string, taskPodsTo }), }) if err := cl.List(ctx, taskPods, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list Lagoon task pods, there may be none or something went wrong")) + opLog.Error(err, "unable to list Lagoon task pods, there may be none or something went wrong") return } // sort the build pods by creation timestamp @@ -186,7 +185,7 @@ func TaskPodPruner(ctx context.Context, cl client.Client, cns string, taskPodsTo pod.Status.Phase == corev1.PodSucceeded { opLog.Info(fmt.Sprintf("Cleaning up pod %s", pod.ObjectMeta.Name)) if err := cl.Delete(ctx, &pod); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to delete pod")) + opLog.Error(err, "unable to delete pod") break } } @@ -194,10 +193,9 @@ func TaskPodPruner(ctx context.Context, cl client.Client, cns string, taskPodsTo } } } - return } -func updateLagoonTask(opLog logr.Logger, namespace string, taskSpec LagoonTaskSpec) ([]byte, error) { +func updateLagoonTask(namespace string, taskSpec LagoonTaskSpec) ([]byte, error) { //@TODO: use `taskName` in the future only taskName := fmt.Sprintf("lagoon-task-%s-%s", taskSpec.Task.ID, helpers.HashString(taskSpec.Task.ID)[0:6]) if taskSpec.Task.TaskName != "" { @@ -228,7 +226,7 @@ func updateLagoonTask(opLog logr.Logger, namespace string, taskSpec LagoonTaskSp msg.Meta.EndTime = time.Now().UTC().Format("2006-01-02 15:04:05") msgBytes, err := json.Marshal(msg) if err != nil { - return nil, fmt.Errorf("Unable to encode message as JSON: %v", err) + return nil, fmt.Errorf("unable to encode message as JSON: %v", err) } return msgBytes, nil } @@ -255,11 +253,11 @@ func CancelTask(ctx context.Context, cl client.Client, namespace string, body [] Namespace: namespace, }, &lagoonTask); err != nil { opLog.Info(fmt.Sprintf( - "Unable to find task %s to cancel it. Sending response to Lagoon to update the task to cancelled.", + "unable to find task %s to cancel it. Sending response to Lagoon to update the task to cancelled.", taskName, )) // if there is no pod or task, update the task in Lagoon to cancelled - b, err := updateLagoonTask(opLog, namespace, *jobSpec) + b, err := updateLagoonTask(namespace, *jobSpec) return false, b, err } // as there is no task pod, but there is a lagoon task resource @@ -268,21 +266,21 @@ func CancelTask(ctx context.Context, cl client.Client, namespace string, body [] if err := cl.Update(ctx, &lagoonTask); err != nil { opLog.Error(err, fmt.Sprintf( - "Unable to update task %s to cancel it.", + "unable to update task %s to cancel it.", taskName, ), ) return false, nil, err } // and then send the response back to lagoon to say it was cancelled. - b, err := updateLagoonTask(opLog, namespace, *jobSpec) + b, err := updateLagoonTask(namespace, *jobSpec) return true, b, err } jobPod.ObjectMeta.Labels["lagoon.sh/cancelTask"] = "true" if err := cl.Update(ctx, &jobPod); err != nil { opLog.Error(err, fmt.Sprintf( - "Unable to update task %s to cancel it.", + "unable to update task %s to cancel it.", jobSpec.Misc.Name, ), ) diff --git a/controllers/harbor/harborcredential_controller.go b/controllers/harbor/harborcredential_controller.go index 7ca07f92..12a0284d 100644 --- a/controllers/harbor/harborcredential_controller.go +++ b/controllers/harbor/harborcredential_controller.go @@ -43,7 +43,7 @@ func (r *HarborCredentialReconciler) Reconcile(ctx context.Context, req ctrl.Req opLog.Info(fmt.Sprintf("Rotating harbor credentials for namespace %s", harborSecret.ObjectMeta.Namespace)) lagoonHarbor, err := harbor.New(r.Harbor) if err != nil { - return ctrl.Result{}, fmt.Errorf("Error creating harbor client, check your harbor configuration. Error was: %v", err) + return ctrl.Result{}, fmt.Errorf("error creating harbor client, check your harbor configuration. Error was: %v", err) } var ns corev1.Namespace if err := r.Get(ctx, types.NamespacedName{Name: harborSecret.ObjectMeta.Namespace}, &ns); err != nil { @@ -52,7 +52,7 @@ func (r *HarborCredentialReconciler) Reconcile(ctx context.Context, req ctrl.Req rotated, err := lagoonHarbor.RotateRobotCredential(ctx, r.Client, ns, true) if err != nil { // @TODO: resource unknown - return ctrl.Result{}, fmt.Errorf("Error was: %v", err) + return ctrl.Result{}, fmt.Errorf("error was: %v", err) } if rotated { opLog.Info(fmt.Sprintf("Robot credentials rotated for %s", ns.ObjectMeta.Name)) @@ -65,7 +65,7 @@ func (r *HarborCredentialReconciler) Reconcile(ctx context.Context, req ctrl.Req }, }) if err := r.Patch(ctx, &harborSecret, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { - return ctrl.Result{}, fmt.Errorf("There was an error patching the harbor secret. Error was: %v", err) + return ctrl.Result{}, fmt.Errorf("there was an error patching the harbor secret. Error was: %v", err) } } diff --git a/controllers/v1beta1/build_controller.go b/controllers/v1beta1/build_controller.go index 736c9e84..ac56f5a2 100644 --- a/controllers/v1beta1/build_controller.go +++ b/controllers/v1beta1/build_controller.go @@ -166,7 +166,7 @@ func (r *LagoonBuildReconciler) Reconcile(ctx context.Context, req ctrl.Request) }) // list any builds that are running if err := r.List(ctx, runningNSBuilds, listOption); err != nil { - return ctrl.Result{}, fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err) + return ctrl.Result{}, fmt.Errorf("unable to list builds in the namespace, there may be none or something went wrong: %v", err) } for _, runningBuild := range runningNSBuilds.Items { // if the running build is the one from this request then process it @@ -180,7 +180,7 @@ func (r *LagoonBuildReconciler) Reconcile(ctx context.Context, req ctrl.Request) } // end check if running build is current LagoonBuild } // end loop for running builds // once running builds are processed, run the qos handler - return r.qosBuildProcessor(ctx, opLog, lagoonBuild, req) + return r.qosBuildProcessor(ctx, opLog, lagoonBuild) } // if qos is not enabled, just process it as a standard build return r.standardBuildProcessor(ctx, opLog, lagoonBuild, req) @@ -197,7 +197,7 @@ func (r *LagoonBuildReconciler) Reconcile(ctx context.Context, req ctrl.Request) ); err != nil { // if fail to delete the external dependency here, return with error // so that it can be retried - opLog.Error(err, fmt.Sprintf("Unable to delete external resources")) + opLog.Error(err, "Unable to delete external resources") return ctrl.Result{}, err } // remove our finalizer from the list and update it. @@ -292,7 +292,7 @@ func (r *LagoonBuildReconciler) createNamespaceBuild(ctx context.Context, }) // list all builds in the namespace that have the running buildstatus if err := r.List(ctx, runningBuilds, listOption); err != nil { - return ctrl.Result{}, fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err) + return ctrl.Result{}, fmt.Errorf("unable to list builds in the namespace, there may be none or something went wrong: %v", err) } // if there are running builds still, check if the pod exists or if the pod is complete/failed and attempt to get the status for _, rBuild := range runningBuilds.Items { diff --git a/controllers/v1beta1/build_deletionhandlers.go b/controllers/v1beta1/build_deletionhandlers.go index 9af24062..3d61bf78 100644 --- a/controllers/v1beta1/build_deletionhandlers.go +++ b/controllers/v1beta1/build_deletionhandlers.go @@ -37,12 +37,12 @@ func (r *LagoonBuildReconciler) deleteExternalResources( Name: lagoonBuild.ObjectMeta.Name, }, &lagoonBuildPod) if err != nil { - opLog.Info(fmt.Sprintf("Unable to find a build pod for %s, continuing to process build deletion", lagoonBuild.ObjectMeta.Name)) + opLog.Info(fmt.Sprintf("unable to find a build pod for %s, continuing to process build deletion", lagoonBuild.ObjectMeta.Name)) // handle updating lagoon for a deleted build with no running pod // only do it if the build status is Pending or Running though err = r.updateCancelledDeploymentWithLogs(ctx, req, *lagoonBuild) if err != nil { - opLog.Error(err, fmt.Sprintf("Unable to update the lagoon with LagoonBuild result")) + opLog.Error(err, "unable to update the lagoon with LagoonBuild result") } } else { opLog.Info(fmt.Sprintf("Found build pod for %s, deleting it", lagoonBuild.ObjectMeta.Name)) @@ -50,7 +50,7 @@ func (r *LagoonBuildReconciler) deleteExternalResources( // only do it if the build status is Pending or Running though // delete the pod, let the pod deletion handler deal with the cleanup there if err := r.Delete(ctx, &lagoonBuildPod); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to delete the the LagoonBuild pod %s", lagoonBuild.ObjectMeta.Name)) + opLog.Error(err, fmt.Sprintf("unable to delete the the LagoonBuild pod %s", lagoonBuild.ObjectMeta.Name)) } // check that the pod is deleted before continuing, this allows the pod deletion to happen // and the pod deletion process in the LagoonMonitor controller to be able to send what it needs back to lagoon @@ -94,7 +94,7 @@ func (r *LagoonBuildReconciler) deleteExternalResources( }) // list any builds that are running if err := r.List(ctx, runningBuilds, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list builds in the namespace, there may be none or something went wrong")) + opLog.Error(err, "unable to list builds in the namespace, there may be none or something went wrong") // just return nil so the deletion of the resource isn't held up return nil } @@ -117,7 +117,7 @@ func (r *LagoonBuildReconciler) deleteExternalResources( }), }) if err := r.List(ctx, pendingBuilds, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list builds in the namespace, there may be none or something went wrong")) + opLog.Error(err, "unable to list builds in the namespace, there may be none or something went wrong") // just return nil so the deletion of the resource isn't held up return nil } @@ -144,11 +144,11 @@ func (r *LagoonBuildReconciler) deleteExternalResources( }, }) if err := r.Patch(ctx, pendingBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to update pending build to running status")) + opLog.Error(err, "unable to update pending build to running status") return nil } } else { - opLog.Info(fmt.Sprintf("No pending builds")) + opLog.Info("No pending builds") } } return nil @@ -177,15 +177,13 @@ func (r *LagoonBuildReconciler) updateCancelledDeploymentWithLogs( ), ) - var allContainerLogs []byte // if we get this handler, then it is likely that the build was in a pending or running state with no actual running pod // so just set the logs to be cancellation message - allContainerLogs = []byte(fmt.Sprintf(` + allContainerLogs := []byte(` ======================================== Build cancelled -========================================`)) - var buildCondition lagoonv1beta1.BuildStatusType - buildCondition = lagoonv1beta1.BuildStatusCancelled +========================================`) + buildCondition := lagoonv1beta1.BuildStatusCancelled lagoonBuild.Labels["lagoon.sh/buildStatus"] = buildCondition.String() mergePatch, _ := json.Marshal(map[string]interface{}{ "metadata": map[string]interface{}{ @@ -195,7 +193,7 @@ Build cancelled }, }) if err := r.Patch(ctx, &lagoonBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to update build status")) + opLog.Error(err, "unable to update build status") } // get the configmap for lagoon-env so we can use it for updating the deployment in lagoon var lagoonEnv corev1.ConfigMap @@ -252,10 +250,10 @@ func (r *LagoonBuildReconciler) buildLogsToLagoonLogs(ctx context.Context, }, } // add the actual build log message - msg.Message = fmt.Sprintf("%s", logs) + msg.Message = string(logs) msgBytes, err := json.Marshal(msg) if err != nil { - opLog.Error(err, "Unable to encode message as JSON") + opLog.Error(err, "unable to encode message as JSON") } // @TODO: if we can't publish the message because we are deleting the resource, then should we even // bother to patch the resource?? @@ -346,7 +344,7 @@ func (r *LagoonBuildReconciler) updateDeploymentAndEnvironmentTask(ctx context.C } msgBytes, err := json.Marshal(msg) if err != nil { - opLog.Error(err, "Unable to encode message as JSON") + opLog.Error(err, "unable to encode message as JSON") } // @TODO: if we can't publish the message because we are deleting the resource, then should we even // bother to patch the resource?? @@ -406,7 +404,7 @@ func (r *LagoonBuildReconciler) buildStatusLogsToLagoonLogs(ctx context.Context, } msgBytes, err := json.Marshal(msg) if err != nil { - opLog.Error(err, "Unable to encode message as JSON") + opLog.Error(err, "unable to encode message as JSON") } // @TODO: if we can't publish the message because we are deleting the resource, then should we even // bother to patch the resource?? @@ -441,7 +439,7 @@ func (r *LagoonBuildReconciler) updateEnvironmentMessage(ctx context.Context, }, }) if err := r.Patch(ctx, lagoonBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { - return fmt.Errorf("Unable to update status condition: %v", err) + return fmt.Errorf("unable to update status condition: %v", err) } return nil } @@ -463,7 +461,7 @@ func (r *LagoonBuildReconciler) updateBuildStatusMessage(ctx context.Context, }, }) if err := r.Patch(ctx, lagoonBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { - return fmt.Errorf("Unable to update status condition: %v", err) + return fmt.Errorf("unable to update status condition: %v", err) } return nil } @@ -485,7 +483,7 @@ func (r *LagoonBuildReconciler) removeBuildPendingMessageStatus(ctx context.Cont "statusMessages": nil, }) if err := r.Patch(ctx, lagoonBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { - return fmt.Errorf("Unable to update status condition: %v", err) + return fmt.Errorf("unable to update status condition: %v", err) } } } @@ -509,7 +507,7 @@ func (r *LagoonBuildReconciler) updateBuildLogMessage(ctx context.Context, }, }) if err := r.Patch(ctx, lagoonBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { - return fmt.Errorf("Unable to update status condition: %v", err) + return fmt.Errorf("unable to update status condition: %v", err) } return nil } diff --git a/controllers/v1beta1/build_helpers.go b/controllers/v1beta1/build_helpers.go index 5b89b5ff..8c8d2c5d 100644 --- a/controllers/v1beta1/build_helpers.go +++ b/controllers/v1beta1/build_helpers.go @@ -52,7 +52,7 @@ func (r *LagoonBuildReconciler) getOrCreateServiceAccount(ctx context.Context, s }, serviceAccount) if err != nil { if err := r.Create(ctx, serviceAccount); err != nil { - return fmt.Errorf("There was an error creating the lagoon-deployer servicea account. Error was: %v", err) + return fmt.Errorf("there was an error creating the lagoon-deployer servicea account. Error was: %v", err) } } return nil @@ -82,7 +82,7 @@ func (r *LagoonBuildReconciler) getOrCreateSARoleBinding(ctx context.Context, sa }, saRoleBinding) if err != nil { if err := r.Create(ctx, saRoleBinding); err != nil { - return fmt.Errorf("There was an error creating the lagoon-deployer-admin role binding. Error was: %v", err) + return fmt.Errorf("there was an error creating the lagoon-deployer-admin role binding. Error was: %v", err) } } return nil @@ -149,7 +149,7 @@ func (r *LagoonBuildReconciler) getOrCreateNamespace(ctx context.Context, namesp if err := r.Get(ctx, types.NamespacedName{Name: ns}, namespace); err != nil { if helpers.IgnoreNotFound(err) != nil { - return fmt.Errorf("There was an error getting the namespace. Error was: %v", err) + return fmt.Errorf("there was an error getting the namespace. Error was: %v", err) } } if namespace.Status.Phase == corev1.NamespaceTerminating { @@ -178,7 +178,7 @@ func (r *LagoonBuildReconciler) getOrCreateNamespace(ctx context.Context, namesp // if kubernetes, just create it if it doesn't exist if err := r.Get(ctx, types.NamespacedName{Name: ns}, namespace); err != nil { if err := r.Create(ctx, namespace); err != nil { - return fmt.Errorf("There was an error creating the namespace. Error was: %v", err) + return fmt.Errorf("there was an error creating the namespace. Error was: %v", err) } } @@ -191,10 +191,10 @@ func (r *LagoonBuildReconciler) getOrCreateNamespace(ctx context.Context, namesp }, }) if err := r.Patch(ctx, namespace, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { - return fmt.Errorf("There was an error patching the namespace. Error was: %v", err) + return fmt.Errorf("there was an error patching the namespace. Error was: %v", err) } if err := r.Get(ctx, types.NamespacedName{Name: ns}, namespace); err != nil { - return fmt.Errorf("There was an error getting the namespace. Error was: %v", err) + return fmt.Errorf("there was an error getting the namespace. Error was: %v", err) } // if local/regional harbor is enabled @@ -202,18 +202,18 @@ func (r *LagoonBuildReconciler) getOrCreateNamespace(ctx context.Context, namesp // create the harbor client lagoonHarbor, err := harbor.New(r.Harbor) if err != nil { - return fmt.Errorf("Error creating harbor client, check your harbor configuration. Error was: %v", err) + return fmt.Errorf("error creating harbor client, check your harbor configuration. Error was: %v", err) } // create the project in harbor robotCreds := &helpers.RegistryCredentials{} curVer, err := lagoonHarbor.GetHarborVersion(ctx) if err != nil { - return fmt.Errorf("Error getting harbor version, check your harbor configuration. Error was: %v", err) + return fmt.Errorf("error getting harbor version, check your harbor configuration. Error was: %v", err) } if lagoonHarbor.UseV2Functions(curVer) { hProject, err := lagoonHarbor.CreateProjectV2(ctx, lagoonBuild.Spec.Project.Name) if err != nil { - return fmt.Errorf("Error creating harbor project: %v", err) + return fmt.Errorf("error creating harbor project: %v", err) } // create or refresh the robot credentials robotCreds, err = lagoonHarbor.CreateOrRefreshRobotV2(ctx, @@ -224,12 +224,12 @@ func (r *LagoonBuildReconciler) getOrCreateNamespace(ctx context.Context, namesp lagoonHarbor.RobotAccountExpiry, false) if err != nil { - return fmt.Errorf("Error creating harbor robot account: %v", err) + return fmt.Errorf("error creating harbor robot account: %v", err) } } else { hProject, err := lagoonHarbor.CreateProject(ctx, lagoonBuild.Spec.Project.Name) if err != nil { - return fmt.Errorf("Error creating harbor project: %v", err) + return fmt.Errorf("error creating harbor project: %v", err) } // create or refresh the robot credentials robotCreds, err = lagoonHarbor.CreateOrRefreshRobot(ctx, @@ -240,7 +240,7 @@ func (r *LagoonBuildReconciler) getOrCreateNamespace(ctx context.Context, namesp time.Now().Add(lagoonHarbor.RobotAccountExpiry).Unix(), false) if err != nil { - return fmt.Errorf("Error creating harbor robot account: %v", err) + return fmt.Errorf("error creating harbor robot account: %v", err) } } // if we have robotcredentials to create, do that here @@ -250,7 +250,7 @@ func (r *LagoonBuildReconciler) getOrCreateNamespace(ctx context.Context, namesp "lagoon-internal-registry-secret", robotCreds) if err != nil { - return fmt.Errorf("Error upserting harbor robot account secret: %v", err) + return fmt.Errorf("error upserting harbor robot account secret: %v", err) } } return nil @@ -275,16 +275,16 @@ func (r *LagoonBuildReconciler) getCreateOrUpdateSSHKeySecret(ctx context.Contex }, sshKey) if err != nil { if err := r.Create(ctx, sshKey); err != nil { - return fmt.Errorf("There was an error creating the lagoon-sshkey. Error was: %v", err) + return fmt.Errorf("there was an error creating the lagoon-sshkey. Error was: %v", err) } } // if the keys are different, then load in the new key from the spec - if bytes.Compare(sshKey.Data["ssh-privatekey"], spec.Project.Key) != 0 { + if !bytes.Equal(sshKey.Data["ssh-privatekey"], spec.Project.Key) { sshKey.Data = map[string][]byte{ "ssh-privatekey": spec.Project.Key, } if err := r.Update(ctx, sshKey); err != nil { - return fmt.Errorf("There was an error updating the lagoon-sshkey. Error was: %v", err) + return fmt.Errorf("there was an error updating the lagoon-sshkey. Error was: %v", err) } } return nil @@ -300,7 +300,7 @@ func (r *LagoonBuildReconciler) getOrCreateConfigMap(ctx context.Context, cmName configMap.SetName(cmName) //we create it if err = r.Create(ctx, configMap); err != nil { - return fmt.Errorf("There was an error creating the configmap '%v'. Error was: %v", cmName, err) + return fmt.Errorf("there was an error creating the configmap '%v'. Error was: %v", cmName, err) } } return nil @@ -605,12 +605,12 @@ func (r *LagoonBuildReconciler) processBuild(ctx context.Context, opLog logr.Log Namespace: lagoonBuild.ObjectMeta.Namespace, Name: "lagoon-internal-registry-secret", }, robotCredential); err != nil { - return fmt.Errorf("Could not find Harbor RobotAccount credential") + return fmt.Errorf("could not find Harbor RobotAccount credential") } auths := helpers.Auths{} if secretData, ok := robotCredential.Data[".dockerconfigjson"]; ok { if err := json.Unmarshal(secretData, &auths); err != nil { - return fmt.Errorf("Could not unmarshal Harbor RobotAccount credential") + return fmt.Errorf("could not unmarshal Harbor RobotAccount credential") } // if the defined regional harbor key exists using the hostname if creds, ok := auths.Registries[r.Harbor.URL]; ok { @@ -865,7 +865,7 @@ func (r *LagoonBuildReconciler) processBuild(ctx context.Context, opLog logr.Log // if it doesn't exist, then create the build pod opLog.Info(fmt.Sprintf("Creating build pod for: %s", lagoonBuild.ObjectMeta.Name)) if err := r.Create(ctx, newPod); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to create build pod")) + opLog.Error(err, "Unable to create build pod") // log the error and just exit, don't continue to try and do anything // @TODO: should update the build to failed return nil @@ -886,10 +886,9 @@ func (r *LagoonBuildReconciler) updateQueuedBuild( if r.EnableDebug { opLog.Info(fmt.Sprintf("Updating build %s to queued: %s", lagoonBuild.ObjectMeta.Name, fmt.Sprintf("This build is currently queued in position %v/%v", queuePosition, queueLength))) } - var allContainerLogs []byte // if we get this handler, then it is likely that the build was in a pending or running state with no actual running pod // so just set the logs to be cancellation message - allContainerLogs = []byte(fmt.Sprintf(` + allContainerLogs := []byte(fmt.Sprintf(` ======================================== %s ======================================== @@ -941,8 +940,7 @@ func (r *LagoonBuildReconciler) cleanUpUndeployableBuild( Build cancelled ======================================== %s`, message)) - var buildCondition lagoonv1beta1.BuildStatusType - buildCondition = lagoonv1beta1.BuildStatusCancelled + buildCondition := lagoonv1beta1.BuildStatusCancelled lagoonBuild.Labels["lagoon.sh/buildStatus"] = buildCondition.String() mergePatch, _ := json.Marshal(map[string]interface{}{ "metadata": map[string]interface{}{ @@ -952,7 +950,7 @@ Build cancelled }, }) if err := r.Patch(ctx, &lagoonBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to update build status")) + opLog.Error(err, "Unable to update build status") } } // get the configmap for lagoon-env so we can use it for updating the deployment in lagoon @@ -980,50 +978,7 @@ Build cancelled // delete the build from the lagoon namespace in kubernetes entirely err = r.Delete(ctx, &lagoonBuild) if err != nil { - return fmt.Errorf("There was an error deleting the lagoon build. Error was: %v", err) - } - return nil -} - -func (r *LagoonBuildReconciler) cancelExtraBuilds(ctx context.Context, opLog logr.Logger, pendingBuilds *lagoonv1beta1.LagoonBuildList, ns string, status string) error { - listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ - client.InNamespace(ns), - client.MatchingLabels(map[string]string{"lagoon.sh/buildStatus": lagoonv1beta1.BuildStatusPending.String()}), - }) - if err := r.List(ctx, pendingBuilds, listOption); err != nil { - return fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err) - } - if len(pendingBuilds.Items) > 0 { - if r.EnableDebug { - opLog.Info(fmt.Sprintf("There are %v pending builds", len(pendingBuilds.Items))) - } - // if we have any pending builds, then grab the latest one and make it running - // if there are any other pending builds, cancel them so only the latest one runs - sort.Slice(pendingBuilds.Items, func(i, j int) bool { - return pendingBuilds.Items[i].ObjectMeta.CreationTimestamp.After(pendingBuilds.Items[j].ObjectMeta.CreationTimestamp.Time) - }) - for idx, pBuild := range pendingBuilds.Items { - pendingBuild := pBuild.DeepCopy() - if idx == 0 { - pendingBuild.Labels["lagoon.sh/buildStatus"] = status - } else { - // cancel any other pending builds - opLog.Info(fmt.Sprintf("Attempting to cancel build %s", pendingBuild.ObjectMeta.Name)) - pendingBuild.Labels["lagoon.sh/buildStatus"] = lagoonv1beta1.BuildStatusCancelled.String() - } - if err := r.Update(ctx, pendingBuild); err != nil { - return fmt.Errorf("There was an error updating the pending build. Error was: %v", err) - } - var lagoonBuild lagoonv1beta1.LagoonBuild - if err := r.Get(ctx, types.NamespacedName{ - Namespace: pendingBuild.ObjectMeta.Namespace, - Name: pendingBuild.ObjectMeta.Name, - }, &lagoonBuild); err != nil { - return helpers.IgnoreNotFound(err) - } - opLog.Info(fmt.Sprintf("Cleaning up build %s as cancelled extra build", lagoonBuild.ObjectMeta.Name)) - r.cleanUpUndeployableBuild(ctx, lagoonBuild, "This build was cancelled as a newer build was triggered.", opLog, true) - } + return fmt.Errorf("there was an error deleting the lagoon build. Error was: %v", err) } return nil } diff --git a/controllers/v1beta1/build_qoshandler.go b/controllers/v1beta1/build_qoshandler.go index f28cda4a..7a7016ab 100644 --- a/controllers/v1beta1/build_qoshandler.go +++ b/controllers/v1beta1/build_qoshandler.go @@ -21,8 +21,7 @@ type BuildQoS struct { func (r *LagoonBuildReconciler) qosBuildProcessor(ctx context.Context, opLog logr.Logger, - lagoonBuild lagoonv1beta1.LagoonBuild, - req ctrl.Request) (ctrl.Result, error) { + lagoonBuild lagoonv1beta1.LagoonBuild) (ctrl.Result, error) { // check if we get a lagoonbuild that hasn't got any buildstatus // this means it was created by the message queue handler // so we should do the steps required for a lagoon build and then copy the build @@ -34,7 +33,7 @@ func (r *LagoonBuildReconciler) qosBuildProcessor(ctx context.Context, return r.createNamespaceBuild(ctx, opLog, lagoonBuild) } if r.EnableDebug { - opLog.Info(fmt.Sprintf("Checking which build next")) + opLog.Info("Checking which build next") } // handle the QoS build process here return ctrl.Result{}, r.whichBuildNext(ctx, opLog) @@ -49,7 +48,7 @@ func (r *LagoonBuildReconciler) whichBuildNext(ctx context.Context, opLog logr.L }) runningBuilds := &lagoonv1beta1.LagoonBuildList{} if err := r.List(ctx, runningBuilds, listOption); err != nil { - return fmt.Errorf("Unable to list builds in the cluster, there may be none or something went wrong: %v", err) + return fmt.Errorf("unable to list builds in the cluster, there may be none or something went wrong: %v", err) } buildsToStart := r.BuildQoS.MaxBuilds - len(runningBuilds.Items) if len(runningBuilds.Items) >= r.BuildQoS.MaxBuilds { @@ -86,7 +85,7 @@ func (r *LagoonBuildReconciler) processQueue(ctx context.Context, opLog logr.Log if !runningProcessQueue { runningProcessQueue = true if r.EnableDebug { - opLog.Info(fmt.Sprintf("Processing queue")) + opLog.Info("Processing queue") } listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ client.MatchingLabels(map[string]string{ @@ -97,7 +96,7 @@ func (r *LagoonBuildReconciler) processQueue(ctx context.Context, opLog logr.Log pendingBuilds := &lagoonv1beta1.LagoonBuildList{} if err := r.List(ctx, pendingBuilds, listOption); err != nil { runningProcessQueue = false - return fmt.Errorf("Unable to list builds in the cluster, there may be none or something went wrong: %v", err) + return fmt.Errorf("unable to list builds in the cluster, there may be none or something went wrong: %v", err) } if len(pendingBuilds.Items) > 0 { if r.EnableDebug { @@ -124,7 +123,7 @@ func (r *LagoonBuildReconciler) processQueue(ctx context.Context, opLog logr.Log // list any builds that are running if err := r.List(ctx, runningNSBuilds, listOption); err != nil { runningProcessQueue = false - return fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err) + return fmt.Errorf("unable to list builds in the namespace, there may be none or something went wrong: %v", err) } // if there are no running builds, check if there are any pending builds that can be started if len(runningNSBuilds.Items) == 0 { diff --git a/controllers/v1beta1/build_standardhandler.go b/controllers/v1beta1/build_standardhandler.go index 6ce5fb41..9bcc8ca1 100644 --- a/controllers/v1beta1/build_standardhandler.go +++ b/controllers/v1beta1/build_standardhandler.go @@ -36,7 +36,7 @@ func (r *LagoonBuildReconciler) standardBuildProcessor(ctx context.Context, }) // list any builds that are running if err := r.List(ctx, runningBuilds, listOption); err != nil { - return ctrl.Result{}, fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err) + return ctrl.Result{}, fmt.Errorf("unable to list builds in the namespace, there may be none or something went wrong: %v", err) } for _, runningBuild := range runningBuilds.Items { // if the running build is the one from this request then process it diff --git a/controllers/v1beta1/podmonitor_buildhandlers.go b/controllers/v1beta1/podmonitor_buildhandlers.go index 496b18d9..9222c2f0 100644 --- a/controllers/v1beta1/podmonitor_buildhandlers.go +++ b/controllers/v1beta1/podmonitor_buildhandlers.go @@ -136,7 +136,7 @@ func (r *LagoonMonitorReconciler) handleBuildMonitor(ctx context.Context, // buildLogsToLagoonLogs sends the build logs to the lagoon-logs message queue // it contains the actual pod log output that is sent to elasticsearch, it is what eventually is displayed in the UI -func (r *LagoonMonitorReconciler) buildLogsToLagoonLogs(ctx context.Context, +func (r *LagoonMonitorReconciler) buildLogsToLagoonLogs( opLog logr.Logger, lagoonBuild *lagoonv1beta1.LagoonBuild, jobPod *corev1.Pod, @@ -227,7 +227,7 @@ Logs on pod %s, assigned to cluster %s // updateDeploymentAndEnvironmentTask sends the status of the build and deployment to the controllerhandler message queue in lagoon, // this is for the handler in lagoon to process. -func (r *LagoonMonitorReconciler) updateDeploymentAndEnvironmentTask(ctx context.Context, +func (r *LagoonMonitorReconciler) updateDeploymentAndEnvironmentTask( opLog logr.Logger, lagoonBuild *lagoonv1beta1.LagoonBuild, jobPod *corev1.Pod, @@ -357,7 +357,7 @@ func (r *LagoonMonitorReconciler) updateDeploymentAndEnvironmentTask(ctx context } // buildStatusLogsToLagoonLogs sends the logs to lagoon-logs message queue, used for general messaging -func (r *LagoonMonitorReconciler) buildStatusLogsToLagoonLogs(ctx context.Context, +func (r *LagoonMonitorReconciler) buildStatusLogsToLagoonLogs( opLog logr.Logger, lagoonBuild *lagoonv1beta1.LagoonBuild, jobPod *corev1.Pod, @@ -568,14 +568,14 @@ Build %s } // do any message publishing here, and update any pending messages if needed - pendingStatus, pendingStatusMessage := r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &jobPod, &lagoonEnv, namespace, buildCondition.ToLower()) - pendingEnvironment, pendingEnvironmentMessage := r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &jobPod, &lagoonEnv, namespace, buildCondition.ToLower()) + pendingStatus, pendingStatusMessage := r.buildStatusLogsToLagoonLogs(opLog, &lagoonBuild, &jobPod, &lagoonEnv, namespace, buildCondition.ToLower()) + pendingEnvironment, pendingEnvironmentMessage := r.updateDeploymentAndEnvironmentTask(opLog, &lagoonBuild, &jobPod, &lagoonEnv, namespace, buildCondition.ToLower()) var pendingBuildLog bool var pendingBuildLogMessage schema.LagoonLog // if the container logs can't be retrieved, we don't want to send any build logs back, as this will nuke // any previously received logs if !strings.Contains(string(allContainerLogs), "unable to retrieve container logs for containerd") { - pendingBuildLog, pendingBuildLogMessage = r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &jobPod, namespace, buildCondition.ToLower(), allContainerLogs) + pendingBuildLog, pendingBuildLogMessage = r.buildLogsToLagoonLogs(opLog, &lagoonBuild, &jobPod, namespace, buildCondition.ToLower(), allContainerLogs) } if pendingStatus || pendingEnvironment || pendingBuildLog { mergeMap["metadata"].(map[string]interface{})["labels"].(map[string]interface{})["lagoon.sh/pendingMessages"] = "true" @@ -599,7 +599,7 @@ Build %s if err := r.Get(ctx, req.NamespacedName, &lagoonBuild); err == nil { // if it does, try to patch it if err := r.Patch(ctx, &lagoonBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to update resource")) + opLog.Error(err, "Unable to update resource") } } // just delete the pod diff --git a/controllers/v1beta1/podmonitor_controller.go b/controllers/v1beta1/podmonitor_controller.go index da169187..f9f63a92 100644 --- a/controllers/v1beta1/podmonitor_controller.go +++ b/controllers/v1beta1/podmonitor_controller.go @@ -110,22 +110,22 @@ func (r *LagoonMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Reques Name: jobPod.ObjectMeta.Labels["lagoon.sh/buildName"], }, &lagoonBuild) if err != nil { - opLog.Info(fmt.Sprintf("The build that started this pod may have been deleted or not started yet, continuing with cancellation if required.")) + opLog.Info("The build that started this pod may have been deleted or not started yet, continuing with cancellation if required.") err = r.updateDeploymentWithLogs(ctx, req, lagoonBuild, jobPod, nil, true) if err != nil { - opLog.Error(err, fmt.Sprintf("Unable to update the LagoonBuild.")) + opLog.Error(err, "Unable to update the LagoonBuild.") } } else { if helpers.ContainsString( lagoonv1beta1.BuildRunningPendingStatus, lagoonBuild.Labels["lagoon.sh/buildStatus"], ) { - opLog.Info(fmt.Sprintf("Attempting to update the LagoonBuild with cancellation if required.")) + opLog.Info("Attempting to update the LagoonBuild with cancellation if required.") // this will update the deployment back to lagoon if it can do so // and should only update if the LagoonBuild is Pending or Running err = r.updateDeploymentWithLogs(ctx, req, lagoonBuild, jobPod, nil, true) if err != nil { - opLog.Error(err, fmt.Sprintf("Unable to update the LagoonBuild.")) + opLog.Error(err, "Unable to update the LagoonBuild.") } } } @@ -142,7 +142,7 @@ func (r *LagoonMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Reques // sorted by their creation timestamp and set the first to running if !r.LFFQoSEnabled { // if qos is not enabled, then handle the check for pending builds here - opLog.Info(fmt.Sprintf("Checking for any pending builds.")) + opLog.Info("Checking for any pending builds.") runningBuilds := &lagoonv1beta1.LagoonBuildList{} listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ client.InNamespace(req.Namespace), @@ -150,7 +150,7 @@ func (r *LagoonMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Reques }) // list all builds in the namespace that have the running buildstatus if err := r.List(ctx, runningBuilds, listOption); err != nil { - return ctrl.Result{}, fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err) + return ctrl.Result{}, fmt.Errorf("unable to list builds in the namespace, there may be none or something went wrong: %v", err) } // if we have no running builds, then check for any pending builds if len(runningBuilds.Items) == 0 { @@ -159,7 +159,7 @@ func (r *LagoonMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Reques } else { // since qos handles pending build checks as part of its own operations, we can skip the running pod check step with no-op if r.EnableDebug { - opLog.Info(fmt.Sprintf("No pending build check in namespaces when QoS is enabled")) + opLog.Info("No pending build check in namespaces when QoS is enabled") } } } @@ -209,7 +209,7 @@ func (r *LagoonMonitorReconciler) collectLogs(ctx context.Context, req reconcile for _, container := range jobPod.Spec.Containers { cLogs, err := getContainerLogs(ctx, container.Name, req) if err != nil { - return nil, fmt.Errorf("Unable to retrieve logs from pod: %v", err) + return nil, fmt.Errorf("unable to retrieve logs from pod: %v", err) } allContainerLogs = append(allContainerLogs, cLogs...) } diff --git a/controllers/v1beta1/podmonitor_taskhandlers.go b/controllers/v1beta1/podmonitor_taskhandlers.go index 0e25f851..d6ec6258 100644 --- a/controllers/v1beta1/podmonitor_taskhandlers.go +++ b/controllers/v1beta1/podmonitor_taskhandlers.go @@ -157,7 +157,7 @@ Logs on pod %s, assigned to cluster %s // updateLagoonTask sends the status of the task and deployment to the controllerhandler message queue in lagoon, // this is for the handler in lagoon to process. -func (r *LagoonMonitorReconciler) updateLagoonTask(ctx context.Context, opLog logr.Logger, +func (r *LagoonMonitorReconciler) updateLagoonTask(opLog logr.Logger, lagoonTask *lagoonv1beta1.LagoonTask, jobPod *corev1.Pod, condition string, @@ -345,7 +345,7 @@ Task %s // send any messages to lagoon message queues // update the deployment with the status pendingStatus, pendingStatusMessage := r.taskStatusLogsToLagoonLogs(opLog, &lagoonTask, &jobPod, taskCondition.ToLower()) - pendingEnvironment, pendingEnvironmentMessage := r.updateLagoonTask(ctx, opLog, &lagoonTask, &jobPod, taskCondition.ToLower()) + pendingEnvironment, pendingEnvironmentMessage := r.updateLagoonTask(opLog, &lagoonTask, &jobPod, taskCondition.ToLower()) var pendingTaskLog bool var pendingTaskLogMessage schema.LagoonLog // if the container logs can't be retrieved, we don't want to send any task logs back, as this will nuke @@ -375,7 +375,7 @@ Task %s if err := r.Get(ctx, req.NamespacedName, &lagoonTask); err == nil { // if it does, try to patch it if err := r.Patch(ctx, &lagoonTask, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to update resource")) + opLog.Error(err, "Unable to update resource") } } // just delete the pod diff --git a/controllers/v1beta1/task_controller.go b/controllers/v1beta1/task_controller.go index bcb0eb41..0b442464 100644 --- a/controllers/v1beta1/task_controller.go +++ b/controllers/v1beta1/task_controller.go @@ -143,7 +143,7 @@ func (r *LagoonTaskReconciler) getTaskPodDeployment(ctx context.Context, lagoonT err := r.List(ctx, deployments, listOption) if err != nil { return nil, fmt.Errorf( - "Unable to get deployments for project %s, environment %s: %v", + "unable to get deployments for project %s, environment %s: %v", lagoonTask.Spec.Project.Name, lagoonTask.Spec.Environment.Name, err, @@ -297,7 +297,7 @@ func (r *LagoonTaskReconciler) getTaskPodDeployment(ctx context.Context, lagoonT } if !hasService { return nil, fmt.Errorf( - "No matching service %s for project %s, environment %s: %v", + "no matching service %s for project %s, environment %s: %v", lagoonTask.Spec.Task.Service, lagoonTask.Spec.Project.Name, lagoonTask.Spec.Environment.Name, @@ -307,7 +307,7 @@ func (r *LagoonTaskReconciler) getTaskPodDeployment(ctx context.Context, lagoonT } // no deployments found return error return nil, fmt.Errorf( - "No deployments %s for project %s, environment %s: %v", + "no deployments %s for project %s, environment %s: %v", lagoonTask.ObjectMeta.Namespace, lagoonTask.Spec.Project.Name, lagoonTask.Spec.Environment.Name, diff --git a/controllers/v1beta1/task_helpers.go b/controllers/v1beta1/task_helpers.go index a423d4e9..899c85cb 100644 --- a/controllers/v1beta1/task_helpers.go +++ b/controllers/v1beta1/task_helpers.go @@ -35,7 +35,7 @@ func (r *LagoonTaskReconciler) createActiveStandbyRole(ctx context.Context, sour }, activeStandbyRoleBinding) if err != nil { if err := r.Create(ctx, activeStandbyRoleBinding); err != nil { - return fmt.Errorf("There was an error creating the lagoon-deployer-activestandby role binding. Error was: %v", err) + return fmt.Errorf("there was an error creating the lagoon-deployer-activestandby role binding. Error was: %v", err) } } return nil @@ -53,7 +53,7 @@ func (r *LagoonMonitorReconciler) deleteActiveStandbyRole(ctx context.Context, d } err = r.Delete(ctx, activeStandbyRoleBinding) if err != nil { - return fmt.Errorf("Unable to delete lagoon-deployer-activestandby role binding") + return fmt.Errorf("unable to delete lagoon-deployer-activestandby role binding") } return nil } diff --git a/controllers/v1beta2/build_controller.go b/controllers/v1beta2/build_controller.go index de1ecab6..67550833 100644 --- a/controllers/v1beta2/build_controller.go +++ b/controllers/v1beta2/build_controller.go @@ -148,7 +148,7 @@ func (r *LagoonBuildReconciler) Reconcile(ctx context.Context, req ctrl.Request) }) // list any builds that are running if err := r.List(ctx, runningNSBuilds, listOption); err != nil { - return ctrl.Result{}, fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err) + return ctrl.Result{}, fmt.Errorf("unable to list builds in the namespace, there may be none or something went wrong: %v", err) } for _, runningBuild := range runningNSBuilds.Items { // if the running build is the one from this request then process it @@ -162,7 +162,7 @@ func (r *LagoonBuildReconciler) Reconcile(ctx context.Context, req ctrl.Request) } // end check if running build is current LagoonBuild } // end loop for running builds // once running builds are processed, run the qos handler - return r.qosBuildProcessor(ctx, opLog, lagoonBuild, req) + return r.qosBuildProcessor(ctx, opLog, lagoonBuild) } // if qos is not enabled, just process it as a standard build return r.standardBuildProcessor(ctx, opLog, lagoonBuild, req) @@ -179,7 +179,7 @@ func (r *LagoonBuildReconciler) Reconcile(ctx context.Context, req ctrl.Request) ); err != nil { // if fail to delete the external dependency here, return with error // so that it can be retried - opLog.Error(err, fmt.Sprintf("Unable to delete external resources")) + opLog.Error(err, "Unable to delete external resources") return ctrl.Result{}, err } // remove our finalizer from the list and update it. @@ -274,7 +274,7 @@ func (r *LagoonBuildReconciler) createNamespaceBuild(ctx context.Context, }) // list all builds in the namespace that have the running buildstatus if err := r.List(ctx, runningBuilds, listOption); err != nil { - return ctrl.Result{}, fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err) + return ctrl.Result{}, fmt.Errorf("unable to list builds in the namespace, there may be none or something went wrong: %v", err) } // if there are running builds still, check if the pod exists or if the pod is complete/failed and attempt to get the status for _, rBuild := range runningBuilds.Items { diff --git a/controllers/v1beta2/build_deletionhandlers.go b/controllers/v1beta2/build_deletionhandlers.go index bfe90ccb..881383a3 100644 --- a/controllers/v1beta2/build_deletionhandlers.go +++ b/controllers/v1beta2/build_deletionhandlers.go @@ -42,7 +42,7 @@ func (r *LagoonBuildReconciler) deleteExternalResources( // only do it if the build status is Pending or Running though err = r.updateCancelledDeploymentWithLogs(ctx, req, *lagoonBuild) if err != nil { - opLog.Error(err, fmt.Sprintf("Unable to update the lagoon with LagoonBuild result")) + opLog.Error(err, "Unable to update the lagoon with LagoonBuild result") } } else { opLog.Info(fmt.Sprintf("Found build pod for %s, deleting it", lagoonBuild.ObjectMeta.Name)) @@ -94,7 +94,7 @@ func (r *LagoonBuildReconciler) deleteExternalResources( }) // list any builds that are running if err := r.List(ctx, runningBuilds, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list builds in the namespace, there may be none or something went wrong")) + opLog.Error(err, "Unable to list builds in the namespace, there may be none or something went wrong") // just return nil so the deletion of the resource isn't held up return nil } @@ -117,7 +117,7 @@ func (r *LagoonBuildReconciler) deleteExternalResources( }), }) if err := r.List(ctx, pendingBuilds, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list builds in the namespace, there may be none or something went wrong")) + opLog.Error(err, "Unable to list builds in the namespace, there may be none or something went wrong") // just return nil so the deletion of the resource isn't held up return nil } @@ -144,11 +144,11 @@ func (r *LagoonBuildReconciler) deleteExternalResources( }, }) if err := r.Patch(ctx, pendingBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to update pending build to running status")) + opLog.Error(err, "Unable to update pending build to running status") return nil } } else { - opLog.Info(fmt.Sprintf("No pending builds")) + opLog.Info("No pending builds") } } return nil @@ -177,15 +177,13 @@ func (r *LagoonBuildReconciler) updateCancelledDeploymentWithLogs( ), ) - var allContainerLogs []byte // if we get this handler, then it is likely that the build was in a pending or running state with no actual running pod // so just set the logs to be cancellation message - allContainerLogs = []byte(fmt.Sprintf(` + allContainerLogs := []byte(` ======================================== Build cancelled -========================================`)) - var buildCondition lagooncrd.BuildStatusType - buildCondition = lagooncrd.BuildStatusCancelled +========================================`) + buildCondition := lagooncrd.BuildStatusCancelled lagoonBuild.Labels["lagoon.sh/buildStatus"] = buildCondition.String() mergePatch, _ := json.Marshal(map[string]interface{}{ "metadata": map[string]interface{}{ @@ -195,7 +193,7 @@ Build cancelled }, }) if err := r.Patch(ctx, &lagoonBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to update build status")) + opLog.Error(err, "Unable to update build status") } // get the configmap for lagoon-env so we can use it for updating the deployment in lagoon var lagoonEnv corev1.ConfigMap @@ -251,7 +249,7 @@ func (r *LagoonBuildReconciler) buildLogsToLagoonLogs(ctx context.Context, }, } // add the actual build log message - msg.Message = fmt.Sprintf("%s", logs) + msg.Message = string(logs) msgBytes, err := json.Marshal(msg) if err != nil { opLog.Error(err, "Unable to encode message as JSON") @@ -440,7 +438,7 @@ func (r *LagoonBuildReconciler) updateEnvironmentMessage(ctx context.Context, }, }) if err := r.Patch(ctx, lagoonBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { - return fmt.Errorf("Unable to update status condition: %v", err) + return fmt.Errorf("unable to update status condition: %v", err) } return nil } @@ -462,7 +460,7 @@ func (r *LagoonBuildReconciler) updateBuildStatusMessage(ctx context.Context, }, }) if err := r.Patch(ctx, lagoonBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { - return fmt.Errorf("Unable to update status condition: %v", err) + return fmt.Errorf("unable to update status condition: %v", err) } return nil } @@ -484,7 +482,7 @@ func (r *LagoonBuildReconciler) removeBuildPendingMessageStatus(ctx context.Cont "statusMessages": nil, }) if err := r.Patch(ctx, lagoonBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { - return fmt.Errorf("Unable to update status condition: %v", err) + return fmt.Errorf("unable to update status condition: %v", err) } } } @@ -508,7 +506,7 @@ func (r *LagoonBuildReconciler) updateBuildLogMessage(ctx context.Context, }, }) if err := r.Patch(ctx, lagoonBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { - return fmt.Errorf("Unable to update status condition: %v", err) + return fmt.Errorf("unable to update status condition: %v", err) } return nil } diff --git a/controllers/v1beta2/build_helpers.go b/controllers/v1beta2/build_helpers.go index 983bc0f3..9b903cb8 100644 --- a/controllers/v1beta2/build_helpers.go +++ b/controllers/v1beta2/build_helpers.go @@ -22,6 +22,7 @@ import ( lagooncrd "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" "github.com/uselagoon/remote-controller/internal/harbor" "github.com/uselagoon/remote-controller/internal/helpers" + "github.com/uselagoon/remote-controller/internal/metrics" ) var ( @@ -53,7 +54,7 @@ func (r *LagoonBuildReconciler) getOrCreateServiceAccount(ctx context.Context, s }, serviceAccount) if err != nil { if err := r.Create(ctx, serviceAccount); err != nil { - return fmt.Errorf("There was an error creating the lagoon-deployer servicea account. Error was: %v", err) + return fmt.Errorf("there was an error creating the lagoon-deployer servicea account. Error was: %v", err) } } return nil @@ -83,7 +84,7 @@ func (r *LagoonBuildReconciler) getOrCreateSARoleBinding(ctx context.Context, sa }, saRoleBinding) if err != nil { if err := r.Create(ctx, saRoleBinding); err != nil { - return fmt.Errorf("There was an error creating the lagoon-deployer-admin role binding. Error was: %v", err) + return fmt.Errorf("there was an error creating the lagoon-deployer-admin role binding. Error was: %v", err) } } return nil @@ -150,7 +151,7 @@ func (r *LagoonBuildReconciler) getOrCreateNamespace(ctx context.Context, namesp if err := r.Get(ctx, types.NamespacedName{Name: ns}, namespace); err != nil { if helpers.IgnoreNotFound(err) != nil { - return fmt.Errorf("There was an error getting the namespace. Error was: %v", err) + return fmt.Errorf("there was an error getting the namespace. Error was: %v", err) } } if namespace.Status.Phase == corev1.NamespaceTerminating { @@ -179,7 +180,7 @@ func (r *LagoonBuildReconciler) getOrCreateNamespace(ctx context.Context, namesp // if kubernetes, just create it if it doesn't exist if err := r.Get(ctx, types.NamespacedName{Name: ns}, namespace); err != nil { if err := r.Create(ctx, namespace); err != nil { - return fmt.Errorf("There was an error creating the namespace. Error was: %v", err) + return fmt.Errorf("there was an error creating the namespace. Error was: %v", err) } } @@ -192,10 +193,10 @@ func (r *LagoonBuildReconciler) getOrCreateNamespace(ctx context.Context, namesp }, }) if err := r.Patch(ctx, namespace, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { - return fmt.Errorf("There was an error patching the namespace. Error was: %v", err) + return fmt.Errorf("there was an error patching the namespace. Error was: %v", err) } if err := r.Get(ctx, types.NamespacedName{Name: ns}, namespace); err != nil { - return fmt.Errorf("There was an error getting the namespace. Error was: %v", err) + return fmt.Errorf("there was an error getting the namespace. Error was: %v", err) } // if local/regional harbor is enabled @@ -203,18 +204,18 @@ func (r *LagoonBuildReconciler) getOrCreateNamespace(ctx context.Context, namesp // create the harbor client lagoonHarbor, err := harbor.New(r.Harbor) if err != nil { - return fmt.Errorf("Error creating harbor client, check your harbor configuration. Error was: %v", err) + return fmt.Errorf("error creating harbor client, check your harbor configuration. Error was: %v", err) } // create the project in harbor robotCreds := &helpers.RegistryCredentials{} curVer, err := lagoonHarbor.GetHarborVersion(ctx) if err != nil { - return fmt.Errorf("Error getting harbor version, check your harbor configuration. Error was: %v", err) + return fmt.Errorf("error getting harbor version, check your harbor configuration. Error was: %v", err) } if lagoonHarbor.UseV2Functions(curVer) { hProject, err := lagoonHarbor.CreateProjectV2(ctx, lagoonBuild.Spec.Project.Name) if err != nil { - return fmt.Errorf("Error creating harbor project: %v", err) + return fmt.Errorf("error creating harbor project: %v", err) } // create or refresh the robot credentials robotCreds, err = lagoonHarbor.CreateOrRefreshRobotV2(ctx, @@ -225,12 +226,12 @@ func (r *LagoonBuildReconciler) getOrCreateNamespace(ctx context.Context, namesp lagoonHarbor.RobotAccountExpiry, false) if err != nil { - return fmt.Errorf("Error creating harbor robot account: %v", err) + return fmt.Errorf("error creating harbor robot account: %v", err) } } else { hProject, err := lagoonHarbor.CreateProject(ctx, lagoonBuild.Spec.Project.Name) if err != nil { - return fmt.Errorf("Error creating harbor project: %v", err) + return fmt.Errorf("error creating harbor project: %v", err) } // create or refresh the robot credentials robotCreds, err = lagoonHarbor.CreateOrRefreshRobot(ctx, @@ -241,7 +242,7 @@ func (r *LagoonBuildReconciler) getOrCreateNamespace(ctx context.Context, namesp time.Now().Add(lagoonHarbor.RobotAccountExpiry).Unix(), false) if err != nil { - return fmt.Errorf("Error creating harbor robot account: %v", err) + return fmt.Errorf("error creating harbor robot account: %v", err) } } // if we have robotcredentials to create, do that here @@ -251,7 +252,7 @@ func (r *LagoonBuildReconciler) getOrCreateNamespace(ctx context.Context, namesp "lagoon-internal-registry-secret", robotCreds) if err != nil { - return fmt.Errorf("Error upserting harbor robot account secret: %v", err) + return fmt.Errorf("error upserting harbor robot account secret: %v", err) } } return nil @@ -276,16 +277,16 @@ func (r *LagoonBuildReconciler) getCreateOrUpdateSSHKeySecret(ctx context.Contex }, sshKey) if err != nil { if err := r.Create(ctx, sshKey); err != nil { - return fmt.Errorf("There was an error creating the lagoon-sshkey. Error was: %v", err) + return fmt.Errorf("there was an error creating the lagoon-sshkey. Error was: %v", err) } } // if the keys are different, then load in the new key from the spec - if bytes.Compare(sshKey.Data["ssh-privatekey"], spec.Project.Key) != 0 { + if !bytes.Equal(sshKey.Data["ssh-privatekey"], spec.Project.Key) { sshKey.Data = map[string][]byte{ "ssh-privatekey": spec.Project.Key, } if err := r.Update(ctx, sshKey); err != nil { - return fmt.Errorf("There was an error updating the lagoon-sshkey. Error was: %v", err) + return fmt.Errorf("there was an error updating the lagoon-sshkey. Error was: %v", err) } } return nil @@ -301,7 +302,7 @@ func (r *LagoonBuildReconciler) getOrCreateConfigMap(ctx context.Context, cmName configMap.SetName(cmName) //we create it if err = r.Create(ctx, configMap); err != nil { - return fmt.Errorf("There was an error creating the configmap '%v'. Error was: %v", cmName, err) + return fmt.Errorf("there was an error creating the configmap '%v'. Error was: %v", cmName, err) } } return nil @@ -606,12 +607,12 @@ func (r *LagoonBuildReconciler) processBuild(ctx context.Context, opLog logr.Log Namespace: lagoonBuild.ObjectMeta.Namespace, Name: "lagoon-internal-registry-secret", }, robotCredential); err != nil { - return fmt.Errorf("Could not find Harbor RobotAccount credential") + return fmt.Errorf("could not find Harbor RobotAccount credential") } auths := helpers.Auths{} if secretData, ok := robotCredential.Data[".dockerconfigjson"]; ok { if err := json.Unmarshal(secretData, &auths); err != nil { - return fmt.Errorf("Could not unmarshal Harbor RobotAccount credential") + return fmt.Errorf("could not unmarshal Harbor RobotAccount credential") } // if the defined regional harbor key exists using the hostname if creds, ok := auths.Registries[r.Harbor.URL]; ok { @@ -866,21 +867,21 @@ func (r *LagoonBuildReconciler) processBuild(ctx context.Context, opLog logr.Log // if it doesn't exist, then create the build pod opLog.Info(fmt.Sprintf("Creating build pod for: %s", lagoonBuild.ObjectMeta.Name)) if err := r.Create(ctx, newPod); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to create build pod")) + opLog.Error(err, "Unable to create build pod") // log the error and just exit, don't continue to try and do anything // @TODO: should update the build to failed return nil } - buildRunningStatus.With(prometheus.Labels{ + metrics.BuildRunningStatus.With(prometheus.Labels{ "build_namespace": lagoonBuild.ObjectMeta.Namespace, "build_name": lagoonBuild.ObjectMeta.Name, }).Set(1) - buildStatus.With(prometheus.Labels{ + metrics.BuildStatus.With(prometheus.Labels{ "build_namespace": lagoonBuild.ObjectMeta.Namespace, "build_name": lagoonBuild.ObjectMeta.Name, "build_step": "running", }).Set(1) - buildsStartedCounter.Inc() + metrics.BuildsStartedCounter.Inc() // then break out of the build } opLog.Info(fmt.Sprintf("Build pod already running for: %s", lagoonBuild.ObjectMeta.Name)) @@ -897,10 +898,9 @@ func (r *LagoonBuildReconciler) updateQueuedBuild( if r.EnableDebug { opLog.Info(fmt.Sprintf("Updating build %s to queued: %s", lagoonBuild.ObjectMeta.Name, fmt.Sprintf("This build is currently queued in position %v/%v", queuePosition, queueLength))) } - var allContainerLogs []byte // if we get this handler, then it is likely that the build was in a pending or running state with no actual running pod // so just set the logs to be cancellation message - allContainerLogs = []byte(fmt.Sprintf(` + allContainerLogs := []byte(fmt.Sprintf(` ======================================== %s ======================================== @@ -952,8 +952,7 @@ func (r *LagoonBuildReconciler) cleanUpUndeployableBuild( Build cancelled ======================================== %s`, message)) - var buildCondition lagooncrd.BuildStatusType - buildCondition = lagooncrd.BuildStatusCancelled + buildCondition := lagooncrd.BuildStatusCancelled lagoonBuild.Labels["lagoon.sh/buildStatus"] = buildCondition.String() mergePatch, _ := json.Marshal(map[string]interface{}{ "metadata": map[string]interface{}{ @@ -963,7 +962,7 @@ Build cancelled }, }) if err := r.Patch(ctx, &lagoonBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to update build status")) + opLog.Error(err, "Unable to update build status") } } // get the configmap for lagoon-env so we can use it for updating the deployment in lagoon @@ -991,50 +990,7 @@ Build cancelled // delete the build from the lagoon namespace in kubernetes entirely err = r.Delete(ctx, &lagoonBuild) if err != nil { - return fmt.Errorf("There was an error deleting the lagoon build. Error was: %v", err) - } - return nil -} - -func (r *LagoonBuildReconciler) cancelExtraBuilds(ctx context.Context, opLog logr.Logger, pendingBuilds *lagooncrd.LagoonBuildList, ns string, status string) error { - listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ - client.InNamespace(ns), - client.MatchingLabels(map[string]string{"lagoon.sh/buildStatus": lagooncrd.BuildStatusPending.String()}), - }) - if err := r.List(ctx, pendingBuilds, listOption); err != nil { - return fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err) - } - if len(pendingBuilds.Items) > 0 { - if r.EnableDebug { - opLog.Info(fmt.Sprintf("There are %v pending builds", len(pendingBuilds.Items))) - } - // if we have any pending builds, then grab the latest one and make it running - // if there are any other pending builds, cancel them so only the latest one runs - sort.Slice(pendingBuilds.Items, func(i, j int) bool { - return pendingBuilds.Items[i].ObjectMeta.CreationTimestamp.After(pendingBuilds.Items[j].ObjectMeta.CreationTimestamp.Time) - }) - for idx, pBuild := range pendingBuilds.Items { - pendingBuild := pBuild.DeepCopy() - if idx == 0 { - pendingBuild.Labels["lagoon.sh/buildStatus"] = status - } else { - // cancel any other pending builds - opLog.Info(fmt.Sprintf("Attempting to cancel build %s", pendingBuild.ObjectMeta.Name)) - pendingBuild.Labels["lagoon.sh/buildStatus"] = lagooncrd.BuildStatusCancelled.String() - } - if err := r.Update(ctx, pendingBuild); err != nil { - return fmt.Errorf("There was an error updating the pending build. Error was: %v", err) - } - var lagoonBuild lagooncrd.LagoonBuild - if err := r.Get(ctx, types.NamespacedName{ - Namespace: pendingBuild.ObjectMeta.Namespace, - Name: pendingBuild.ObjectMeta.Name, - }, &lagoonBuild); err != nil { - return helpers.IgnoreNotFound(err) - } - opLog.Info(fmt.Sprintf("Cleaning up build %s as cancelled extra build", lagoonBuild.ObjectMeta.Name)) - r.cleanUpUndeployableBuild(ctx, lagoonBuild, "This build was cancelled as a newer build was triggered.", opLog, true) - } + return fmt.Errorf("there was an error deleting the lagoon build. Error was: %v", err) } return nil } diff --git a/controllers/v1beta2/build_qoshandler.go b/controllers/v1beta2/build_qoshandler.go index 6cddf05c..555a4644 100644 --- a/controllers/v1beta2/build_qoshandler.go +++ b/controllers/v1beta2/build_qoshandler.go @@ -8,6 +8,7 @@ import ( "github.com/go-logr/logr" lagooncrd "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" "github.com/uselagoon/remote-controller/internal/helpers" + "github.com/uselagoon/remote-controller/internal/metrics" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -21,8 +22,7 @@ type BuildQoS struct { func (r *LagoonBuildReconciler) qosBuildProcessor(ctx context.Context, opLog logr.Logger, - lagoonBuild lagooncrd.LagoonBuild, - req ctrl.Request) (ctrl.Result, error) { + lagoonBuild lagooncrd.LagoonBuild) (ctrl.Result, error) { // check if we get a lagoonbuild that hasn't got any buildstatus // this means it was created by the message queue handler // so we should do the steps required for a lagoon build and then copy the build @@ -34,7 +34,7 @@ func (r *LagoonBuildReconciler) qosBuildProcessor(ctx context.Context, return r.createNamespaceBuild(ctx, opLog, lagoonBuild) } if r.EnableDebug { - opLog.Info(fmt.Sprintf("Checking which build next")) + opLog.Info("Checking which build next") } // handle the QoS build process here return ctrl.Result{}, r.whichBuildNext(ctx, opLog) @@ -49,7 +49,7 @@ func (r *LagoonBuildReconciler) whichBuildNext(ctx context.Context, opLog logr.L }) runningBuilds := &lagooncrd.LagoonBuildList{} if err := r.List(ctx, runningBuilds, listOption); err != nil { - return fmt.Errorf("Unable to list builds in the cluster, there may be none or something went wrong: %v", err) + return fmt.Errorf("unable to list builds in the cluster, there may be none or something went wrong: %v", err) } buildsToStart := r.BuildQoS.MaxBuilds - len(runningBuilds.Items) if len(runningBuilds.Items) >= r.BuildQoS.MaxBuilds { @@ -86,7 +86,7 @@ func (r *LagoonBuildReconciler) processQueue(ctx context.Context, opLog logr.Log if !runningProcessQueue { runningProcessQueue = true if r.EnableDebug { - opLog.Info(fmt.Sprintf("Processing queue")) + opLog.Info("Processing queue") } listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ client.MatchingLabels(map[string]string{ @@ -97,8 +97,9 @@ func (r *LagoonBuildReconciler) processQueue(ctx context.Context, opLog logr.Log pendingBuilds := &lagooncrd.LagoonBuildList{} if err := r.List(ctx, pendingBuilds, listOption); err != nil { runningProcessQueue = false - return fmt.Errorf("Unable to list builds in the cluster, there may be none or something went wrong: %v", err) + return fmt.Errorf("unable to list builds in the cluster, there may be none or something went wrong: %v", err) } + metrics.BuildsPendingGauge.Set(float64(len(pendingBuilds.Items))) if len(pendingBuilds.Items) > 0 { if r.EnableDebug { opLog.Info(fmt.Sprintf("There are %v pending builds", len(pendingBuilds.Items))) @@ -124,7 +125,7 @@ func (r *LagoonBuildReconciler) processQueue(ctx context.Context, opLog logr.Log // list any builds that are running if err := r.List(ctx, runningNSBuilds, listOption); err != nil { runningProcessQueue = false - return fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err) + return fmt.Errorf("unable to list builds in the namespace, there may be none or something went wrong: %v", err) } // if there are no running builds, check if there are any pending builds that can be started if len(runningNSBuilds.Items) == 0 { diff --git a/controllers/v1beta2/build_standardhandler.go b/controllers/v1beta2/build_standardhandler.go index 5315fd89..dd893c28 100644 --- a/controllers/v1beta2/build_standardhandler.go +++ b/controllers/v1beta2/build_standardhandler.go @@ -36,7 +36,7 @@ func (r *LagoonBuildReconciler) standardBuildProcessor(ctx context.Context, }) // list any builds that are running if err := r.List(ctx, runningBuilds, listOption); err != nil { - return ctrl.Result{}, fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err) + return ctrl.Result{}, fmt.Errorf("unable to list builds in the namespace, there may be none or something went wrong: %v", err) } for _, runningBuild := range runningBuilds.Items { // if the running build is the one from this request then process it diff --git a/controllers/v1beta2/metrics.go b/controllers/v1beta2/metrics.go deleted file mode 100644 index 900ca39b..00000000 --- a/controllers/v1beta2/metrics.go +++ /dev/null @@ -1,92 +0,0 @@ -package v1beta2 - -import ( - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" -) - -var ( - // general counters for builds - buildsRunningGauge = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "lagoon_builds_running_current", - Help: "The total number of Lagoon builds running", - }) - buildsPendingGauge = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "lagoon_builds_pending_current", - Help: "The total number of Lagoon builds pending", - }) - buildsStartedCounter = promauto.NewCounter(prometheus.CounterOpts{ - Name: "lagoon_builds_started_total", - Help: "The total number of Lagoon builds started", - }) - buildsCompletedCounter = promauto.NewCounter(prometheus.CounterOpts{ - Name: "lagoon_builds_completed_total", - Help: "The total number of Lagoon builds completed", - }) - buildsFailedCounter = promauto.NewCounter(prometheus.CounterOpts{ - Name: "lagoon_builds_failed_total", - Help: "The total number of Lagoon builds failed", - }) - buildsCancelledCounter = promauto.NewCounter(prometheus.CounterOpts{ - Name: "lagoon_builds_cancelled_total", - Help: "The total number of Lagoon builds cancelled", - }) - - // general counters for tasks - tasksRunningGauge = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "lagoon_tasks_running_current", - Help: "The total number of Lagoon tasks running", - }) - tasksStartedCounter = promauto.NewCounter(prometheus.CounterOpts{ - Name: "lagoon_tasks_started_total", - Help: "The total number of Lagoon tasks started", - }) - tasksCompletedCounter = promauto.NewCounter(prometheus.CounterOpts{ - Name: "lagoon_tasks_completed_total", - Help: "The total number of Lagoon tasks completed", - }) - tasksFailedCounter = promauto.NewCounter(prometheus.CounterOpts{ - Name: "lagoon_tasks_failed_total", - Help: "The total number of Lagoon tasks failed", - }) - tasksCancelledCounter = promauto.NewCounter(prometheus.CounterOpts{ - Name: "lagoon_tasks_cancelled_total", - Help: "The total number of Lagoon tasks cancelled", - }) - - // buildStatus will count the build transisiton steps - // when the build step changes, the count is removed and the new step metric is created - // this is useful to gauge how long particular steps take in a build - buildStatus = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "lagoon_build_status", - Help: "The status of running Lagoon builds", - }, - []string{ - "build_name", - "build_namespace", - "build_step", - }, - ) - - // RunningStatus will count when a build or task is running - // when the build or task is complete, the count is removed - // this is useful to gauge how long a build or task runs for - buildRunningStatus = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "lagoon_build_running_status", - Help: "The duration of running Lagoon builds", - }, - []string{ - "build_name", - "build_namespace", - }, - ) - taskRunningStatus = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "lagoon_task_running_status", - Help: "The duration of running Lagoon tasks", - }, - []string{ - "task_name", - "task_namespace", - }, - ) -) diff --git a/controllers/v1beta2/podmonitor_buildhandlers.go b/controllers/v1beta2/podmonitor_buildhandlers.go index 0bacdc78..f46b4340 100644 --- a/controllers/v1beta2/podmonitor_buildhandlers.go +++ b/controllers/v1beta2/podmonitor_buildhandlers.go @@ -15,6 +15,7 @@ import ( "github.com/uselagoon/machinery/api/schema" lagooncrd "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" "github.com/uselagoon/remote-controller/internal/helpers" + "github.com/uselagoon/remote-controller/internal/metrics" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -137,7 +138,7 @@ func (r *LagoonMonitorReconciler) handleBuildMonitor(ctx context.Context, // buildLogsToLagoonLogs sends the build logs to the lagoon-logs message queue // it contains the actual pod log output that is sent to elasticsearch, it is what eventually is displayed in the UI -func (r *LagoonMonitorReconciler) buildLogsToLagoonLogs(ctx context.Context, +func (r *LagoonMonitorReconciler) buildLogsToLagoonLogs( opLog logr.Logger, lagoonBuild *lagooncrd.LagoonBuild, jobPod *corev1.Pod, @@ -227,7 +228,7 @@ Logs on pod %s, assigned to cluster %s // updateDeploymentAndEnvironmentTask sends the status of the build and deployment to the controllerhandler message queue in lagoon, // this is for the handler in lagoon to process. -func (r *LagoonMonitorReconciler) updateDeploymentAndEnvironmentTask(ctx context.Context, +func (r *LagoonMonitorReconciler) updateDeploymentAndEnvironmentTask( opLog logr.Logger, lagoonBuild *lagooncrd.LagoonBuild, jobPod *corev1.Pod, @@ -247,7 +248,7 @@ func (r *LagoonMonitorReconciler) updateDeploymentAndEnvironmentTask(ctx context } if condition == "failed" || condition == "complete" || condition == "cancelled" { time.AfterFunc(31*time.Second, func() { - buildRunningStatus.Delete(prometheus.Labels{ + metrics.BuildRunningStatus.Delete(prometheus.Labels{ "build_namespace": lagoonBuild.ObjectMeta.Namespace, "build_name": lagoonBuild.ObjectMeta.Name, }) @@ -362,7 +363,7 @@ func (r *LagoonMonitorReconciler) updateDeploymentAndEnvironmentTask(ctx context } // buildStatusLogsToLagoonLogs sends the logs to lagoon-logs message queue, used for general messaging -func (r *LagoonMonitorReconciler) buildStatusLogsToLagoonLogs(ctx context.Context, +func (r *LagoonMonitorReconciler) buildStatusLogsToLagoonLogs( opLog logr.Logger, lagoonBuild *lagooncrd.LagoonBuild, jobPod *corev1.Pod, @@ -572,14 +573,14 @@ Build %s } // do any message publishing here, and update any pending messages if needed - pendingStatus, pendingStatusMessage := r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &jobPod, &lagoonEnv, namespace, buildCondition.ToLower()) - pendingEnvironment, pendingEnvironmentMessage := r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &jobPod, &lagoonEnv, namespace, buildCondition.ToLower()) + pendingStatus, pendingStatusMessage := r.buildStatusLogsToLagoonLogs(opLog, &lagoonBuild, &jobPod, &lagoonEnv, namespace, buildCondition.ToLower()) + pendingEnvironment, pendingEnvironmentMessage := r.updateDeploymentAndEnvironmentTask(opLog, &lagoonBuild, &jobPod, &lagoonEnv, namespace, buildCondition.ToLower()) var pendingBuildLog bool var pendingBuildLogMessage schema.LagoonLog // if the container logs can't be retrieved, we don't want to send any build logs back, as this will nuke // any previously received logs if !strings.Contains(string(allContainerLogs), "unable to retrieve container logs for containerd") { - pendingBuildLog, pendingBuildLogMessage = r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &jobPod, namespace, buildCondition.ToLower(), allContainerLogs) + pendingBuildLog, pendingBuildLogMessage = r.buildLogsToLagoonLogs(opLog, &lagoonBuild, &jobPod, namespace, buildCondition.ToLower(), allContainerLogs) } if pendingStatus || pendingEnvironment || pendingBuildLog { mergeMap["metadata"].(map[string]interface{})["labels"].(map[string]interface{})["lagoon.sh/pendingMessages"] = "true" @@ -603,7 +604,7 @@ Build %s if err := r.Get(ctx, req.NamespacedName, &lagoonBuild); err == nil { // if it does, try to patch it if err := r.Patch(ctx, &lagoonBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to update resource")) + opLog.Error(err, "Unable to update resource") } } // just delete the pod diff --git a/controllers/v1beta2/podmonitor_controller.go b/controllers/v1beta2/podmonitor_controller.go index 9ae9f340..08ce2516 100644 --- a/controllers/v1beta2/podmonitor_controller.go +++ b/controllers/v1beta2/podmonitor_controller.go @@ -77,7 +77,7 @@ func (r *LagoonMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Reques if jobPod.ObjectMeta.Labels["lagoon.sh/jobType"] == "task" { err := r.calculateTaskMetrics(ctx) if err != nil { - opLog.Error(err, fmt.Sprintf("Unable to generate metrics.")) + opLog.Error(err, "Unable to generate metrics.") } if jobPod.ObjectMeta.DeletionTimestamp.IsZero() { // pod is not being deleted @@ -102,7 +102,7 @@ func (r *LagoonMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Reques if jobPod.ObjectMeta.Labels["lagoon.sh/jobType"] == "build" { err := r.calculateBuildMetrics(ctx) if err != nil { - opLog.Error(err, fmt.Sprintf("Unable to generate metrics.")) + opLog.Error(err, "Unable to generate metrics.") } if jobPod.ObjectMeta.DeletionTimestamp.IsZero() { // pod is not being deleted @@ -118,22 +118,22 @@ func (r *LagoonMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Reques Name: jobPod.ObjectMeta.Labels["lagoon.sh/buildName"], }, &lagoonBuild) if err != nil { - opLog.Info(fmt.Sprintf("The build that started this pod may have been deleted or not started yet, continuing with cancellation if required.")) + opLog.Info("The build that started this pod may have been deleted or not started yet, continuing with cancellation if required.") err = r.updateDeploymentWithLogs(ctx, req, lagoonBuild, jobPod, nil, true) if err != nil { - opLog.Error(err, fmt.Sprintf("Unable to update the LagoonBuild.")) + opLog.Error(err, "Unable to update the LagoonBuild.") } } else { if helpers.ContainsString( lagooncrd.BuildRunningPendingStatus, lagoonBuild.Labels["lagoon.sh/buildStatus"], ) { - opLog.Info(fmt.Sprintf("Attempting to update the LagoonBuild with cancellation if required.")) + opLog.Info("Attempting to update the LagoonBuild with cancellation if required.") // this will update the deployment back to lagoon if it can do so // and should only update if the LagoonBuild is Pending or Running err = r.updateDeploymentWithLogs(ctx, req, lagoonBuild, jobPod, nil, true) if err != nil { - opLog.Error(err, fmt.Sprintf("Unable to update the LagoonBuild.")) + opLog.Error(err, "Unable to update the LagoonBuild.") } } } @@ -150,7 +150,7 @@ func (r *LagoonMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Reques // sorted by their creation timestamp and set the first to running if !r.LFFQoSEnabled { // if qos is not enabled, then handle the check for pending builds here - opLog.Info(fmt.Sprintf("Checking for any pending builds.")) + opLog.Info("Checking for any pending builds.") runningBuilds := &lagooncrd.LagoonBuildList{} listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ client.InNamespace(req.Namespace), @@ -158,7 +158,7 @@ func (r *LagoonMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Reques }) // list all builds in the namespace that have the running buildstatus if err := r.List(ctx, runningBuilds, listOption); err != nil { - return ctrl.Result{}, fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err) + return ctrl.Result{}, fmt.Errorf("unable to list builds in the namespace, there may be none or something went wrong: %v", err) } // if we have no running builds, then check for any pending builds if len(runningBuilds.Items) == 0 { @@ -167,7 +167,7 @@ func (r *LagoonMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Reques } else { // since qos handles pending build checks as part of its own operations, we can skip the running pod check step with no-op if r.EnableDebug { - opLog.Info(fmt.Sprintf("No pending build check in namespaces when QoS is enabled")) + opLog.Info("No pending build check in namespaces when QoS is enabled") } } } @@ -217,7 +217,7 @@ func (r *LagoonMonitorReconciler) collectLogs(ctx context.Context, req reconcile for _, container := range jobPod.Spec.Containers { cLogs, err := getContainerLogs(ctx, container.Name, req) if err != nil { - return nil, fmt.Errorf("Unable to retrieve logs from pod: %v", err) + return nil, fmt.Errorf("unable to retrieve logs from pod: %v", err) } allContainerLogs = append(allContainerLogs, cLogs...) } diff --git a/controllers/v1beta2/podmonitor_metrics.go b/controllers/v1beta2/podmonitor_metrics.go index 24520f51..f3599047 100644 --- a/controllers/v1beta2/podmonitor_metrics.go +++ b/controllers/v1beta2/podmonitor_metrics.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "github.com/uselagoon/remote-controller/internal/metrics" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -17,7 +18,7 @@ func (r *LagoonMonitorReconciler) calculateBuildMetrics(ctx context.Context) err }) buildPods := &corev1.PodList{} if err := r.List(ctx, buildPods, listOption); err != nil { - return fmt.Errorf("Unable to list builds in the cluster, there may be none or something went wrong: %v", err) + return fmt.Errorf("unable to list builds in the cluster, there may be none or something went wrong: %v", err) } runningBuilds := float64(0) for _, buildPod := range buildPods.Items { @@ -25,7 +26,7 @@ func (r *LagoonMonitorReconciler) calculateBuildMetrics(ctx context.Context) err runningBuilds = runningBuilds + 1 } } - buildsRunningGauge.Set(runningBuilds) + metrics.BuildsRunningGauge.Set(runningBuilds) return nil } @@ -38,7 +39,7 @@ func (r *LagoonMonitorReconciler) calculateTaskMetrics(ctx context.Context) erro }) taskPods := &corev1.PodList{} if err := r.List(ctx, taskPods, listOption); err != nil { - return fmt.Errorf("Unable to list tasks in the cluster, there may be none or something went wrong: %v", err) + return fmt.Errorf("unable to list tasks in the cluster, there may be none or something went wrong: %v", err) } runningTasks := float64(0) for _, taskPod := range taskPods.Items { @@ -46,6 +47,6 @@ func (r *LagoonMonitorReconciler) calculateTaskMetrics(ctx context.Context) erro runningTasks = runningTasks + 1 } } - tasksRunningGauge.Set(runningTasks) + metrics.TasksRunningGauge.Set(runningTasks) return nil } diff --git a/controllers/v1beta2/podmonitor_taskhandlers.go b/controllers/v1beta2/podmonitor_taskhandlers.go index 5868e543..2aee39f8 100644 --- a/controllers/v1beta2/podmonitor_taskhandlers.go +++ b/controllers/v1beta2/podmonitor_taskhandlers.go @@ -15,6 +15,7 @@ import ( "github.com/uselagoon/machinery/api/schema" lagooncrd "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" "github.com/uselagoon/remote-controller/internal/helpers" + "github.com/uselagoon/remote-controller/internal/metrics" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -158,7 +159,7 @@ Logs on pod %s, assigned to cluster %s // updateLagoonTask sends the status of the task and deployment to the controllerhandler message queue in lagoon, // this is for the handler in lagoon to process. -func (r *LagoonMonitorReconciler) updateLagoonTask(ctx context.Context, opLog logr.Logger, +func (r *LagoonMonitorReconciler) updateLagoonTask(opLog logr.Logger, lagoonTask *lagooncrd.LagoonTask, jobPod *corev1.Pod, condition string, @@ -166,7 +167,7 @@ func (r *LagoonMonitorReconciler) updateLagoonTask(ctx context.Context, opLog lo if r.EnableMQ && lagoonTask != nil { if condition == "failed" || condition == "complete" || condition == "cancelled" { time.AfterFunc(31*time.Second, func() { - taskRunningStatus.Delete(prometheus.Labels{ + metrics.TaskRunningStatus.Delete(prometheus.Labels{ "task_namespace": lagoonTask.ObjectMeta.Namespace, "task_name": lagoonTask.ObjectMeta.Name, }) @@ -352,7 +353,7 @@ Task %s // send any messages to lagoon message queues // update the deployment with the status pendingStatus, pendingStatusMessage := r.taskStatusLogsToLagoonLogs(opLog, &lagoonTask, &jobPod, taskCondition.ToLower()) - pendingEnvironment, pendingEnvironmentMessage := r.updateLagoonTask(ctx, opLog, &lagoonTask, &jobPod, taskCondition.ToLower()) + pendingEnvironment, pendingEnvironmentMessage := r.updateLagoonTask(opLog, &lagoonTask, &jobPod, taskCondition.ToLower()) var pendingTaskLog bool var pendingTaskLogMessage schema.LagoonLog // if the container logs can't be retrieved, we don't want to send any task logs back, as this will nuke @@ -382,7 +383,7 @@ Task %s if err := r.Get(ctx, req.NamespacedName, &lagoonTask); err == nil { // if it does, try to patch it if err := r.Patch(ctx, &lagoonTask, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to update resource")) + opLog.Error(err, "Unable to update resource") } } // just delete the pod diff --git a/controllers/v1beta2/predicates.go b/controllers/v1beta2/predicates.go index 0e4185aa..19b4faa3 100644 --- a/controllers/v1beta2/predicates.go +++ b/controllers/v1beta2/predicates.go @@ -7,6 +7,7 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" + "github.com/uselagoon/remote-controller/internal/metrics" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/predicate" ) @@ -79,14 +80,14 @@ func (p PodPredicates) Update(e event.UpdateEvent) bool { oldBuildStep = value } if newBuildStep != oldBuildStep { - buildStatus.With(prometheus.Labels{ + metrics.BuildStatus.With(prometheus.Labels{ "build_namespace": e.ObjectOld.GetNamespace(), "build_name": e.ObjectOld.GetName(), "build_step": newBuildStep, }).Set(1) } time.AfterFunc(31*time.Second, func() { - buildStatus.Delete(prometheus.Labels{ + metrics.BuildStatus.Delete(prometheus.Labels{ "build_namespace": e.ObjectOld.GetNamespace(), "build_name": e.ObjectOld.GetName(), "build_step": oldBuildStep, diff --git a/controllers/v1beta2/task_controller.go b/controllers/v1beta2/task_controller.go index e5796e1e..429a63b7 100644 --- a/controllers/v1beta2/task_controller.go +++ b/controllers/v1beta2/task_controller.go @@ -34,6 +34,7 @@ import ( lagooncrd "github.com/uselagoon/remote-controller/apis/lagoon/v1beta2" "github.com/uselagoon/remote-controller/internal/helpers" + "github.com/uselagoon/remote-controller/internal/metrics" ) // LagoonTaskReconciler reconciles a LagoonTask object @@ -128,7 +129,7 @@ func (r *LagoonTaskReconciler) getTaskPodDeployment(ctx context.Context, lagoonT err := r.List(ctx, deployments, listOption) if err != nil { return nil, fmt.Errorf( - "Unable to get deployments for project %s, environment %s: %v", + "unable to get deployments for project %s, environment %s: %v", lagoonTask.Spec.Project.Name, lagoonTask.Spec.Environment.Name, err, @@ -282,7 +283,7 @@ func (r *LagoonTaskReconciler) getTaskPodDeployment(ctx context.Context, lagoonT } if !hasService { return nil, fmt.Errorf( - "No matching service %s for project %s, environment %s: %v", + "no matching service %s for project %s, environment %s: %v", lagoonTask.Spec.Task.Service, lagoonTask.Spec.Project.Name, lagoonTask.Spec.Environment.Name, @@ -292,7 +293,7 @@ func (r *LagoonTaskReconciler) getTaskPodDeployment(ctx context.Context, lagoonT } // no deployments found return error return nil, fmt.Errorf( - "No deployments %s for project %s, environment %s: %v", + "no deployments %s for project %s, environment %s: %v", lagoonTask.ObjectMeta.Namespace, lagoonTask.Spec.Project.Name, lagoonTask.Spec.Environment.Name, @@ -332,11 +333,11 @@ func (r *LagoonTaskReconciler) createStandardTask(ctx context.Context, lagoonTas //@TODO: send msg back and update task to failed? return nil } - taskRunningStatus.With(prometheus.Labels{ + metrics.TaskRunningStatus.With(prometheus.Labels{ "task_namespace": lagoonTask.ObjectMeta.Namespace, "task_name": lagoonTask.ObjectMeta.Name, }).Set(1) - tasksStartedCounter.Inc() + metrics.TasksStartedCounter.Inc() } else { opLog.Info(fmt.Sprintf("Task pod already running for: %s", lagoonTask.ObjectMeta.Name)) } @@ -623,11 +624,11 @@ func (r *LagoonTaskReconciler) createAdvancedTask(ctx context.Context, lagoonTas ) return err } - taskRunningStatus.With(prometheus.Labels{ + metrics.TaskRunningStatus.With(prometheus.Labels{ "task_namespace": lagoonTask.ObjectMeta.Namespace, "task_name": lagoonTask.ObjectMeta.Name, }).Set(1) - tasksStartedCounter.Inc() + metrics.TasksStartedCounter.Inc() } else { opLog.Info(fmt.Sprintf("Advanced task pod already running for: %s", lagoonTask.ObjectMeta.Name)) } diff --git a/controllers/v1beta2/task_helpers.go b/controllers/v1beta2/task_helpers.go index 017857dc..a77a3c13 100644 --- a/controllers/v1beta2/task_helpers.go +++ b/controllers/v1beta2/task_helpers.go @@ -35,7 +35,7 @@ func (r *LagoonTaskReconciler) createActiveStandbyRole(ctx context.Context, sour }, activeStandbyRoleBinding) if err != nil { if err := r.Create(ctx, activeStandbyRoleBinding); err != nil { - return fmt.Errorf("There was an error creating the lagoon-deployer-activestandby role binding. Error was: %v", err) + return fmt.Errorf("there was an error creating the lagoon-deployer-activestandby role binding. Error was: %v", err) } } return nil @@ -53,7 +53,7 @@ func (r *LagoonMonitorReconciler) deleteActiveStandbyRole(ctx context.Context, d } err = r.Delete(ctx, activeStandbyRoleBinding) if err != nil { - return fmt.Errorf("Unable to delete lagoon-deployer-activestandby role binding") + return fmt.Errorf("unable to delete lagoon-deployer-activestandby role binding") } return nil } diff --git a/internal/harbor/harbor21x.go b/internal/harbor/harbor21x.go index dcb1f26c..a1025655 100644 --- a/internal/harbor/harbor21x.go +++ b/internal/harbor/harbor21x.go @@ -156,7 +156,7 @@ func (h *Harbor) CreateOrRefreshRobot(ctx context.Context, auths := helpers.Auths{} // unmarshal it if err := json.Unmarshal(secretData, &auths); err != nil { - return nil, fmt.Errorf("Could not unmarshal Harbor RobotAccount credential") + return nil, fmt.Errorf("could not unmarshal Harbor RobotAccount credential") } // set the force recreate robot account flag here forceRecreate = true @@ -168,7 +168,7 @@ func (h *Harbor) CreateOrRefreshRobot(ctx context.Context, } } for _, robot := range robots { - if h.matchRobotAccount(robot.Name, project.Name, environmentName) { + if h.matchRobotAccount(robot.Name, environmentName) { exists = true if forceRecreate || force { // if the secret doesn't exist in kubernetes, then force re-creation of the robot diff --git a/internal/harbor/harbor22x.go b/internal/harbor/harbor22x.go index bea48adb..f6416b30 100644 --- a/internal/harbor/harbor22x.go +++ b/internal/harbor/harbor22x.go @@ -154,7 +154,7 @@ func (h *Harbor) CreateOrRefreshRobotV2(ctx context.Context, auths := helpers.Auths{} // unmarshal it if err := json.Unmarshal(secretData, &auths); err != nil { - return nil, fmt.Errorf("Could not unmarshal Harbor RobotAccount credential") + return nil, fmt.Errorf("could not unmarshal Harbor RobotAccount credential") } // set the force recreate robot account flag here forceRecreate = true @@ -167,7 +167,7 @@ func (h *Harbor) CreateOrRefreshRobotV2(ctx context.Context, } tempRobots := robots[:0] for _, robot := range robots { - if h.matchRobotAccount(robot.Name, project.Name, environmentName) && !robot.Editable { + if h.matchRobotAccount(robot.Name, environmentName) && !robot.Editable { // this is an old (legacy) robot account, get rid of it // if accounts are disabled, and deletion of disabled accounts is enabled // then this will delete the account to get re-created diff --git a/internal/harbor/harbor_credentialrotation.go b/internal/harbor/harbor_credentialrotation.go index 43c97d90..01d1f82f 100644 --- a/internal/harbor/harbor_credentialrotation.go +++ b/internal/harbor/harbor_credentialrotation.go @@ -32,7 +32,7 @@ func (h *Harbor) RotateRobotCredentials(ctx context.Context, cl client.Client) { }, }) if err := cl.List(ctx, namespaces, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list namespaces created by Lagoon, there may be none or something went wrong")) + opLog.Error(err, "Unable to list namespaces created by Lagoon, there may be none or something went wrong") return } // go over every namespace that has a lagoon.sh label diff --git a/internal/harbor/harbor_helpers.go b/internal/harbor/harbor_helpers.go index 0acd1287..3bda0d5e 100644 --- a/internal/harbor/harbor_helpers.go +++ b/internal/harbor/harbor_helpers.go @@ -53,14 +53,10 @@ func (h *Harbor) generateRobotWithPrefix(str string) string { // matchRobotAccount will check if the robotaccount exists or not func (h *Harbor) matchRobotAccount(robotName string, - projectName string, environmentName string, ) bool { // pre global-robot-accounts (2.2.0+) - if robotName == h.generateRobotWithPrefix(fmt.Sprintf("%s-%s", environmentName, helpers.HashString(h.LagoonTargetName)[0:8])) { - return true - } - return false + return robotName == h.generateRobotWithPrefix(fmt.Sprintf("%s-%s", environmentName, helpers.HashString(h.LagoonTargetName)[0:8])) } // https://github.com/goharbor/harbor/pull/13685 @@ -90,10 +86,7 @@ func (h *Harbor) matchRobotAccountV2(robotName string, projectName string, environmentName string, ) bool { - if robotName == h.generateRobotWithPrefixV2(projectName, environmentName) { - return true - } - return false + return robotName == h.generateRobotWithPrefixV2(projectName, environmentName) } // already expired? diff --git a/internal/harbor/harbor_helpers_test.go b/internal/harbor/harbor_helpers_test.go index 6a1b426f..68c1718f 100644 --- a/internal/harbor/harbor_helpers_test.go +++ b/internal/harbor/harbor_helpers_test.go @@ -33,7 +33,7 @@ func TestHarbor_matchRobotAccount(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { h := &tt.args.harbor - if got := h.matchRobotAccount(tt.args.robotName, tt.args.projectName, tt.args.environmentName); got != tt.want { + if got := h.matchRobotAccount(tt.args.robotName, tt.args.environmentName); got != tt.want { t.Errorf("Harbor.matchRobotAccount() = %v, want %v", got, tt.want) } }) diff --git a/internal/helpers/helpers.go b/internal/helpers/helpers.go index 680141d4..cba6bc38 100644 --- a/internal/helpers/helpers.go +++ b/internal/helpers/helpers.go @@ -51,32 +51,28 @@ func RemoveString(slice []string, s string) (result []string) { // IntPtr . func IntPtr(i int) *int { - var iPtr *int - iPtr = new(int) + iPtr := new(int) *iPtr = i return iPtr } // Int32Ptr . func Int32Ptr(i int32) *int32 { - var iPtr *int32 - iPtr = new(int32) + iPtr := new(int32) *iPtr = i return iPtr } // Int64Ptr . func Int64Ptr(i int64) *int64 { - var iPtr *int64 - iPtr = new(int64) + iPtr := new(int64) *iPtr = i return iPtr } // UintPtr . func UintPtr(i uint) *uint { - var iPtr *uint - iPtr = new(uint) + iPtr := new(uint) *iPtr = i return iPtr } @@ -210,7 +206,7 @@ func GenerateNamespaceName(pattern, environmentName, projectname, prefix, contro ) // If there is a namespaceprefix defined, and random prefix is disabled // then add the prefix to the namespace - if prefix != "" && randomPrefix == false { + if prefix != "" && !randomPrefix { ns = fmt.Sprintf("%s-%s", prefix, ns) } // If the randomprefix is enabled, then add a prefix based on the hash of the controller namespace diff --git a/internal/messenger/consumer.go b/internal/messenger/consumer.go index 8ad7c8d7..28f3acac 100644 --- a/internal/messenger/consumer.go +++ b/internal/messenger/consumer.go @@ -208,7 +208,7 @@ func (m *Messenger) Consumer(targetName string) { //error { ns, project, branch, - fmt.Errorf("The controller label value %s does not match %s for this namespace", value, m.ControllerNamespace), + fmt.Errorf("the controller label value %s does not match %s for this namespace", value, m.ControllerNamespace), ), ) message.Ack(false) // ack to remove from queue @@ -221,7 +221,7 @@ func (m *Messenger) Consumer(targetName string) { //error { ns, project, branch, - fmt.Errorf("The controller ownership label does not exist on this namespace, nothing will be done for this removal request"), + fmt.Errorf("the controller ownership label does not exist on this namespace, nothing will be done for this removal request"), ), ) } diff --git a/internal/messenger/messenger.go b/internal/messenger/messenger.go index 2ce0eee6..76fdfcd2 100644 --- a/internal/messenger/messenger.go +++ b/internal/messenger/messenger.go @@ -17,12 +17,6 @@ type removeTask struct { NamespacePattern string `json:"namespacePattern,omitempty"` } -type messenger interface { - Consumer(string) - Publish(string, []byte) - GetPendingMessages() -} - // Messaging is used for the config and client information for the messaging queue. type Messenger struct { Config mq.Config diff --git a/internal/messenger/publish.go b/internal/messenger/publish.go index 9311390f..79c928c6 100644 --- a/internal/messenger/publish.go +++ b/internal/messenger/publish.go @@ -23,6 +23,6 @@ func (m *Messenger) Publish(queue string, message []byte) error { opLog.Info(fmt.Sprintf("Failed to get async producer: %v", err)) return err } - producer.Produce([]byte(fmt.Sprintf("%s", message))) + producer.Produce(message) return nil } diff --git a/internal/messenger/tasks_handler.go b/internal/messenger/tasks_handler.go index 3a88b464..35653d3e 100644 --- a/internal/messenger/tasks_handler.go +++ b/internal/messenger/tasks_handler.go @@ -33,11 +33,11 @@ func (m *Messenger) ActiveStandbySwitch(namespace string, jobSpec *lagoonv1beta2 asPayload := &ActiveStandbyPayload{} asPayloadDecoded, err := base64.StdEncoding.DecodeString(jobSpec.AdvancedTask.JSONPayload) if err != nil { - return fmt.Errorf("Unable to base64 decode payload: %v", err) + return fmt.Errorf("unable to base64 decode payload: %v", err) } err = json.Unmarshal([]byte(asPayloadDecoded), asPayload) if err != nil { - return fmt.Errorf("Unable to unmarshal json payload: %v", err) + return fmt.Errorf("unable to unmarshal json payload: %v", err) } return m.createAdvancedTask(namespace, jobSpec, map[string]string{ "lagoon.sh/activeStandby": "true", diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index c98b6e74..f87e87e0 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -6,6 +6,8 @@ import ( "time" "github.com/go-logr/logr" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promhttp" ) @@ -28,3 +30,89 @@ func NewServer(log logr.Logger, addr string) *http.Server { }() return &s } + +var ( + // general counters for builds + BuildsRunningGauge = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "lagoon_builds_running_current", + Help: "The total number of Lagoon builds running", + }) + BuildsPendingGauge = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "lagoon_builds_pending_current", + Help: "The total number of Lagoon builds pending or queued", + }) + BuildsStartedCounter = promauto.NewCounter(prometheus.CounterOpts{ + Name: "lagoon_builds_started_total", + Help: "The total number of Lagoon builds started", + }) + BuildsCompletedCounter = promauto.NewCounter(prometheus.CounterOpts{ + Name: "lagoon_builds_completed_total", + Help: "The total number of Lagoon builds completed", + }) + BuildsFailedCounter = promauto.NewCounter(prometheus.CounterOpts{ + Name: "lagoon_builds_failed_total", + Help: "The total number of Lagoon builds failed", + }) + BuildsCancelledCounter = promauto.NewCounter(prometheus.CounterOpts{ + Name: "lagoon_builds_cancelled_total", + Help: "The total number of Lagoon builds cancelled", + }) + + // general counters for tasks + TasksRunningGauge = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "lagoon_tasks_running_current", + Help: "The total number of Lagoon tasks running", + }) + TasksStartedCounter = promauto.NewCounter(prometheus.CounterOpts{ + Name: "lagoon_tasks_started_total", + Help: "The total number of Lagoon tasks started", + }) + TasksCompletedCounter = promauto.NewCounter(prometheus.CounterOpts{ + Name: "lagoon_tasks_completed_total", + Help: "The total number of Lagoon tasks completed", + }) + TasksFailedCounter = promauto.NewCounter(prometheus.CounterOpts{ + Name: "lagoon_tasks_failed_total", + Help: "The total number of Lagoon tasks failed", + }) + TasksCancelledCounter = promauto.NewCounter(prometheus.CounterOpts{ + Name: "lagoon_tasks_cancelled_total", + Help: "The total number of Lagoon tasks cancelled", + }) + + // buildStatus will count the build transisiton steps + // when the build step changes, the count is removed and the new step metric is created + // this is useful to gauge how long particular steps take in a build + BuildStatus = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "lagoon_build_status", + Help: "The status of running Lagoon builds", + }, + []string{ + "build_name", + "build_namespace", + "build_step", + }, + ) + + // RunningStatus will count when a build or task is running + // when the build or task is complete, the count is removed + // this is useful to gauge how long a build or task runs for + BuildRunningStatus = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "lagoon_build_running_status", + Help: "The duration of running Lagoon builds", + }, + []string{ + "build_name", + "build_namespace", + }, + ) + TaskRunningStatus = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "lagoon_task_running_status", + Help: "The duration of running Lagoon tasks", + }, + []string{ + "task_name", + "task_namespace", + }, + ) +) diff --git a/internal/utilities/deletions/process.go b/internal/utilities/deletions/process.go index 997868d1..3738e18e 100644 --- a/internal/utilities/deletions/process.go +++ b/internal/utilities/deletions/process.go @@ -25,7 +25,7 @@ func (d *Deletions) ProcessDeletion(ctx context.Context, opLog logr.Logger, name environment = val } if project == "" && environment == "" { - return fmt.Errorf("Namespace %s is not a lagoon environment", namespace.Name) + return fmt.Errorf("namespace %s is not a lagoon environment", namespace.Name) } /* @@ -99,43 +99,43 @@ func (d *Deletions) ProcessDeletion(ctx context.Context, opLog logr.Logger, name get any deployments/statefulsets/daemonsets then delete them */ - if del := lagoonv1beta1.DeleteLagoonTasks(ctx, opLog.WithName("DeleteLagoonTasks"), d.Client, namespace.ObjectMeta.Name, project, environment); del == false { + if del := lagoonv1beta1.DeleteLagoonTasks(ctx, opLog.WithName("DeleteLagoonTasks"), d.Client, namespace.ObjectMeta.Name, project, environment); !del { return fmt.Errorf("error deleting tasks") } - if del := lagoonv1beta2.DeleteLagoonTasks(ctx, opLog.WithName("DeleteLagoonTasks"), d.Client, namespace.ObjectMeta.Name, project, environment); del == false { + if del := lagoonv1beta2.DeleteLagoonTasks(ctx, opLog.WithName("DeleteLagoonTasks"), d.Client, namespace.ObjectMeta.Name, project, environment); !del { return fmt.Errorf("error deleting tasks") } - if del := lagoonv1beta1.DeleteLagoonBuilds(ctx, opLog.WithName("DeleteLagoonBuilds"), d.Client, namespace.ObjectMeta.Name, project, environment); del == false { + if del := lagoonv1beta1.DeleteLagoonBuilds(ctx, opLog.WithName("DeleteLagoonBuilds"), d.Client, namespace.ObjectMeta.Name, project, environment); !del { return fmt.Errorf("error deleting builds") } - if del := lagoonv1beta2.DeleteLagoonBuilds(ctx, opLog.WithName("DeleteLagoonBuilds"), d.Client, namespace.ObjectMeta.Name, project, environment); del == false { + if del := lagoonv1beta2.DeleteLagoonBuilds(ctx, opLog.WithName("DeleteLagoonBuilds"), d.Client, namespace.ObjectMeta.Name, project, environment); !del { return fmt.Errorf("error deleting builds") } - if del := d.DeleteDeployments(ctx, opLog.WithName("DeleteDeployments"), namespace.ObjectMeta.Name, project, environment); del == false { + if del := d.DeleteDeployments(ctx, opLog.WithName("DeleteDeployments"), namespace.ObjectMeta.Name, project, environment); !del { return fmt.Errorf("error deleting deployments") } - if del := d.DeleteStatefulSets(ctx, opLog.WithName("DeleteStatefulSets"), namespace.ObjectMeta.Name, project, environment); del == false { + if del := d.DeleteStatefulSets(ctx, opLog.WithName("DeleteStatefulSets"), namespace.ObjectMeta.Name, project, environment); !del { return fmt.Errorf("error deleting statefulsets") } - if del := d.DeleteDaemonSets(ctx, opLog.WithName("DeleteDaemonSets"), namespace.ObjectMeta.Name, project, environment); del == false { + if del := d.DeleteDaemonSets(ctx, opLog.WithName("DeleteDaemonSets"), namespace.ObjectMeta.Name, project, environment); !del { return fmt.Errorf("error deleting daemonsets") } - if del := d.DeleteIngress(ctx, opLog.WithName("DeleteIngress"), namespace.ObjectMeta.Name, project, environment); del == false { + if del := d.DeleteIngress(ctx, opLog.WithName("DeleteIngress"), namespace.ObjectMeta.Name, project, environment); !del { return fmt.Errorf("error deleting ingress") } - if del := d.DeleteJobs(ctx, opLog.WithName("DeleteJobs"), namespace.ObjectMeta.Name, project, environment); del == false { + if del := d.DeleteJobs(ctx, opLog.WithName("DeleteJobs"), namespace.ObjectMeta.Name, project, environment); !del { return fmt.Errorf("error deleting jobs") } - if del := d.DeletePods(ctx, opLog.WithName("DeletePods"), namespace.ObjectMeta.Name, project, environment); del == false { + if del := d.DeletePods(ctx, opLog.WithName("DeletePods"), namespace.ObjectMeta.Name, project, environment); !del { return fmt.Errorf("error deleting pods") } - if del := d.DeletePVCs(ctx, opLog.WithName("DeletePVCs"), namespace.ObjectMeta.Name, project, environment); del == false { + if del := d.DeletePVCs(ctx, opLog.WithName("DeletePVCs"), namespace.ObjectMeta.Name, project, environment); !del { return fmt.Errorf("error deleting pvcs") } /* then delete the namespace */ - if del := d.DeleteNamespace(ctx, opLog.WithName("DeleteNamespace"), namespace, project, environment); del == false { + if del := d.DeleteNamespace(ctx, opLog.WithName("DeleteNamespace"), namespace, project, environment); !del { return fmt.Errorf("error deleting namespace") } opLog.WithName("DeleteNamespace").Info( diff --git a/internal/utilities/pruner/namespace_pruner.go b/internal/utilities/pruner/namespace_pruner.go index ed3ce38d..653241ba 100644 --- a/internal/utilities/pruner/namespace_pruner.go +++ b/internal/utilities/pruner/namespace_pruner.go @@ -26,8 +26,11 @@ func (p *Pruner) NamespacePruner() { nsList := &corev1.NamespaceList{} markedForDeletionRequirement, err := labels.NewRequirement(ExpirationLabel, selection.Exists, []string{}) + if err != nil { + opLog.Info(fmt.Sprintf("bad requirement: %v", err)) + return + } markedForDeletionPausedRequirement, err := labels.NewRequirement(ExpirationPausedLabel, selection.DoesNotExist, []string{}) - if err != nil { opLog.Info(fmt.Sprintf("bad requirement: %v", err)) return @@ -38,7 +41,10 @@ func (p *Pruner) NamespacePruner() { } err = p.Client.List(ctx, nsList, &migrationCompleteSecretOptionSearch) - + if err != nil { + opLog.Info(fmt.Sprintf("bad requirement: %v", err)) + return + } for _, x := range nsList.Items { ns := &corev1.Namespace{} //client diff --git a/internal/utilities/pruner/old_process_pruner.go b/internal/utilities/pruner/old_process_pruner.go index 8e43db04..544a6f87 100644 --- a/internal/utilities/pruner/old_process_pruner.go +++ b/internal/utilities/pruner/old_process_pruner.go @@ -26,7 +26,7 @@ func (p *Pruner) LagoonOldProcPruner(pruneBuilds, pruneTasks bool) { }, }) if err := p.Client.List(context.Background(), namespaces, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list namespaces created by Lagoon, there may be none or something went wrong")) + opLog.Error(err, "Unable to list namespaces created by Lagoon, there may be none or something went wrong") return } @@ -61,7 +61,7 @@ func (p *Pruner) LagoonOldProcPruner(pruneBuilds, pruneTasks bool) { }, }) if err := p.Client.List(context.Background(), &podList, listOption); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to list pod resources, there may be none or something went wrong")) + opLog.Error(err, "Unable to list pod resources, there may be none or something went wrong") continue } @@ -75,13 +75,13 @@ func (p *Pruner) LagoonOldProcPruner(pruneBuilds, pruneTasks bool) { if pod.CreationTimestamp.Time.Before(removeTaskIfCreatedBefore) && pruneTasks { updatePod = true pod.ObjectMeta.Labels["lagoon.sh/cancelTask"] = "true" - pod.ObjectMeta.Annotations["lagoon.sh/cancelReason"] = fmt.Sprintf("Cancelled task due to timeout") + pod.ObjectMeta.Annotations["lagoon.sh/cancelReason"] = "Cancelled task due to timeout" } case "build": if pod.CreationTimestamp.Time.Before(removeBuildIfCreatedBefore) && pruneBuilds { updatePod = true pod.ObjectMeta.Labels["lagoon.sh/cancelBuild"] = "true" - pod.ObjectMeta.Annotations["lagoon.sh/cancelReason"] = fmt.Sprintf("Cancelled build due to timeout") + pod.ObjectMeta.Annotations["lagoon.sh/cancelReason"] = "Cancelled build due to timeout" } default: return diff --git a/internal/utilities/pruner/pruner.go b/internal/utilities/pruner/pruner.go index 058e4c35..419427d2 100644 --- a/internal/utilities/pruner/pruner.go +++ b/internal/utilities/pruner/pruner.go @@ -5,12 +5,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -type pruner interface { - LagoonBuildCleanup() - BuildPodCleanup() - TaskPodCleanup() -} - // Pruner is used for cleaning up old pods or resources. type Pruner struct { Client client.Client From 9dfae271cd0a0cad03122cc6da0f985d7fb0995e Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Fri, 5 Apr 2024 13:23:03 +1100 Subject: [PATCH 08/17] chore: port #249 into v1beta2 controller --- controllers/v1beta2/build_controller.go | 1 + controllers/v1beta2/build_helpers.go | 2 +- main.go | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/controllers/v1beta2/build_controller.go b/controllers/v1beta2/build_controller.go index 67550833..c33d0ded 100644 --- a/controllers/v1beta2/build_controller.go +++ b/controllers/v1beta2/build_controller.go @@ -75,6 +75,7 @@ type LagoonBuildReconciler struct { LagoonFeatureFlags map[string]string LagoonAPIConfiguration helpers.LagoonAPIConfiguration ProxyConfig ProxyConfig + UnauthenticatedRegistry string } // BackupConfig holds all the backup configuration settings diff --git a/controllers/v1beta2/build_helpers.go b/controllers/v1beta2/build_helpers.go index 9b903cb8..1d07dc85 100644 --- a/controllers/v1beta2/build_helpers.go +++ b/controllers/v1beta2/build_helpers.go @@ -508,7 +508,7 @@ func (r *LagoonBuildReconciler) processBuild(ctx context.Context, opLog logr.Log }) podEnvs = append(podEnvs, corev1.EnvVar{ Name: "REGISTRY", - Value: lagoonBuild.Spec.Project.Registry, + Value: r.UnauthenticatedRegistry, }) // this is enabled by default for now // eventually will be disabled by default because support for the generation/modification of this will diff --git a/main.go b/main.go index 65624be8..c722dc09 100644 --- a/main.go +++ b/main.go @@ -924,6 +924,7 @@ func main() { HTTPSProxy: httpsProxy, NoProxy: noProxy, }, + UnauthenticatedRegistry: unauthenticatedRegistry, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "LagoonBuild") os.Exit(1) From acf3e740c6d06d6afa004262c9bb8d3af4423e9c Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Mon, 8 Apr 2024 10:18:02 +1000 Subject: [PATCH 09/17] chore: bump dependencies --- go.mod | 59 +++++++++++++------------ go.sum | 137 ++++++++++++++++++++++++++++++++------------------------- 2 files changed, 107 insertions(+), 89 deletions(-) diff --git a/go.mod b/go.mod index c38f7648..b394be17 100644 --- a/go.mod +++ b/go.mod @@ -1,29 +1,29 @@ module github.com/uselagoon/remote-controller -go 1.20 +go 1.21 require ( github.com/cheshir/go-mq/v2 v2.0.1 github.com/coreos/go-semver v0.3.1 - github.com/go-logr/logr v1.2.4 - github.com/google/go-cmp v0.5.9 + github.com/go-logr/logr v1.3.0 + github.com/google/go-cmp v0.6.0 github.com/hashicorp/go-version v1.6.0 github.com/hashicorp/golang-lru/v2 v2.0.6 github.com/k8up-io/k8up/v2 v2.7.1 github.com/mittwald/goharbor-client/v3 v3.3.0 github.com/mittwald/goharbor-client/v5 v5.3.1 github.com/onsi/ginkgo v1.16.5 - github.com/onsi/gomega v1.27.7 - github.com/prometheus/client_golang v1.15.1 + github.com/onsi/gomega v1.29.0 + github.com/prometheus/client_golang v1.16.0 github.com/uselagoon/machinery v0.0.17-0.20240109062854-3b567fb41003 github.com/vshn/k8up v1.99.99 - github.com/xhit/go-str2duration/v2 v2.0.0 + github.com/xhit/go-str2duration/v2 v2.1.0 gopkg.in/matryer/try.v1 v1.0.0-20150601225556-312d2599e12e gopkg.in/robfig/cron.v2 v2.0.0-20150107220207-be2e0b0deed5 - k8s.io/api v0.27.3 - k8s.io/apiextensions-apiserver v0.27.3 - k8s.io/apimachinery v0.27.3 - k8s.io/client-go v0.27.3 + k8s.io/api v0.29.3 + k8s.io/apiextensions-apiserver v0.29.3 + k8s.io/apimachinery v0.29.3 + k8s.io/client-go v0.29.3 sigs.k8s.io/controller-runtime v0.15.0 ) @@ -34,9 +34,9 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/emicklei/go-restful/v3 v3.9.0 // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect - github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.2.4 // indirect github.com/go-openapi/analysis v0.21.4 // indirect @@ -52,8 +52,8 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/goharbor/harbor/src v0.0.0-20230220075213-6015b3efa7d0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect - github.com/google/gnostic v0.5.7-v3refs // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/gnostic-models v0.6.8 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/uuid v1.3.0 // indirect github.com/guregu/null v4.0.0+incompatible // indirect @@ -73,35 +73,36 @@ require ( github.com/pborman/uuid v1.2.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_model v0.4.0 // indirect - github.com/prometheus/common v0.42.0 // indirect - github.com/prometheus/procfs v0.9.0 // indirect + github.com/prometheus/common v0.44.0 // indirect + github.com/prometheus/procfs v0.10.1 // indirect github.com/rabbitmq/amqp091-go v1.7.0 // indirect github.com/spf13/pflag v1.0.5 // indirect go.mongodb.org/mongo-driver v1.11.2 // indirect - go.opentelemetry.io/otel v1.14.0 // indirect - go.opentelemetry.io/otel/trace v1.14.0 // indirect - go.uber.org/atomic v1.7.0 // indirect - go.uber.org/multierr v1.6.0 // indirect + go.opentelemetry.io/otel v1.19.0 // indirect + go.opentelemetry.io/otel/metric v1.19.0 // indirect + go.opentelemetry.io/otel/trace v1.19.0 // indirect + go.uber.org/atomic v1.10.0 // indirect + go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.24.0 // indirect - golang.org/x/net v0.10.0 // indirect - golang.org/x/oauth2 v0.5.0 // indirect + golang.org/x/net v0.19.0 // indirect + golang.org/x/oauth2 v0.10.0 // indirect golang.org/x/sys v0.15.0 // indirect golang.org/x/term v0.15.0 // indirect - golang.org/x/text v0.9.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/component-base v0.27.3 // indirect - k8s.io/klog/v2 v2.90.1 // indirect - k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect - k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect + k8s.io/component-base v0.29.3 // indirect + k8s.io/klog/v2 v2.110.1 // indirect + k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect + k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/go.sum b/go.sum index d7d22f0b..f689c8cf 100644 --- a/go.sum +++ b/go.sum @@ -379,8 +379,8 @@ github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkg github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= -github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -397,6 +397,7 @@ github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= +github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.2.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= @@ -410,8 +411,8 @@ github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoD github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= -github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/fsouza/go-dockerclient v1.9.4 h1:V7KGGCIoaQTRCQqhIMH1lvL1scnOWeZZq3jvQ0rLhUw= github.com/fsouza/go-dockerclient v1.9.4/go.mod h1:w8kPIZLpXeohzR5os7o4VWllLL5oitqYE8T3zX8RCJg= github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= @@ -443,8 +444,9 @@ github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTg github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.1/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.0/go.mod h1:YkVgnZu1ZjjL7xTxrfm/LLZBfkhTqSR1ydtm6jTKKwI= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= @@ -557,6 +559,7 @@ github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4= github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ= @@ -652,8 +655,8 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4= github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk= @@ -673,8 +676,8 @@ github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= -github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= -github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -687,8 +690,9 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-containerregistry v0.5.1/go.mod h1:Ct15B4yir3PLOP5jsy0GNeYVaIZs/MK/Jz5any1wFW0= github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -711,6 +715,7 @@ github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -845,6 +850,7 @@ github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw= +github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4= github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/knadh/koanf v1.2.1/go.mod h1:xpPTwMhsA/aaQLAilyCCqfpEiY1gpa160AiCuWHJUjY= @@ -857,6 +863,7 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= @@ -951,6 +958,7 @@ github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXy github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= github.com/moby/term v0.0.0-20221205130635-1aeaba878587 h1:HfkjXDfhgVaN5rmueG8cL8KKeFNecRCXFhaJ2qZ5SKA= +github.com/moby/term v0.0.0-20221205130635-1aeaba878587/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -1001,7 +1009,8 @@ github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vv github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= +github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4= +github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -1015,8 +1024,8 @@ github.com/onsi/gomega v1.11.0/go.mod h1:azGKhqFUon9Vuj0YmTfLSmx0FUwqXYSTl5re8lQ github.com/onsi/gomega v1.14.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.27.7 h1:fVih9JD6ogIiHUN6ePK7HJidyEDpWGVB5mzM7cWNXoU= -github.com/onsi/gomega v1.27.7/go.mod h1:1p8OOlwo2iUUDsHnOrjE5UKYJ+e3W8eQ3qSlRahPmr4= +github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg= +github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= @@ -1086,8 +1095,8 @@ github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQ github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= -github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= +github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= +github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -1104,8 +1113,8 @@ github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+ github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= -github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= +github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -1119,8 +1128,8 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= -github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= +github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/qri-io/starlib v0.4.2-0.20200213133954-ff2e8cd5ef8d/go.mod h1:7DPO4domFU579Ga6E61sB9VFNaniPVwJP5C4bBCu3wA= github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI= @@ -1135,6 +1144,7 @@ github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= @@ -1168,6 +1178,7 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= @@ -1217,7 +1228,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= @@ -1242,8 +1254,6 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= -github.com/uselagoon/machinery v0.0.17-0.20240109062106-ccc88289f3ca h1:Opwha2XIEUFIUbAmYy6xdxtbnk0DOjjYY4MkUSArzGI= -github.com/uselagoon/machinery v0.0.17-0.20240109062106-ccc88289f3ca/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU= github.com/uselagoon/machinery v0.0.17-0.20240109062854-3b567fb41003 h1:fLhncKjshd8f0DBFB45u7OWsF1H6OgNmpkswJYqwCA8= github.com/uselagoon/machinery v0.0.17-0.20240109062854-3b567fb41003/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU= github.com/uudashr/gocognit v0.0.0-20190926065955-1655d0de0517/go.mod h1:j44Ayx2KW4+oB6SWMv8KsmHzZrOInQav7D3cQMJ5JUM= @@ -1274,8 +1284,8 @@ github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHM github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= -github.com/xhit/go-str2duration/v2 v2.0.0 h1:uFtk6FWB375bP7ewQl+/1wBcn840GPhnySOdcz/okPE= -github.com/xhit/go-str2duration/v2 v2.0.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= +github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc= +github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= @@ -1327,24 +1337,27 @@ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.2 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= go.opentelemetry.io/otel v1.3.0/go.mod h1:PWIKzi6JCp7sM0k9yZ43VX+T345uNbAkDKwHVjb2PTs= -go.opentelemetry.io/otel v1.14.0 h1:/79Huy8wbf5DnIPhemGB+zEPVwnN6fuQybr/SRXa6hM= -go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU= +go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= +go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.3.0/go.mod h1:VpP4/RMn8bv8gNo9uK7/IMY4mtWLELsS+JIP0inH0h4= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.3.0/go.mod h1:hO1KLR7jcKaDDKDkvI9dP/FIhpmna5lkqPUQdEjFAM8= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.3.0/go.mod h1:keUU7UfnwWTWpJ+FWnyqmogPa82nuU5VUANFq49hlMY= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.3.0/go.mod h1:QNX1aly8ehqqX1LEa6YniTU7VY9I6R3X/oPxhGdTceE= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= +go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE= +go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= go.opentelemetry.io/otel/sdk v1.3.0/go.mod h1:rIo4suHNhQwBIPg9axF8V9CA72Wz2mKF1teNrup8yzs= -go.opentelemetry.io/otel/sdk v1.11.1 h1:F7KmQgoHljhUuJyA+9BiU+EkJfyX5nVVF4wyzWZpKxs= +go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o= +go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= go.opentelemetry.io/otel/trace v1.3.0/go.mod h1:c/VDhno8888bvQYmbYLqe41/Ldmr/KKunbvWM4/fEjk= -go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M= -go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8= +go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg= +go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.11.0/go.mod h1:QpEjXPrNQzrFDZgoTo49dgHR9RYRSrg3NAKnUGl9YpQ= go.starlark.net v0.0.0-20190528202925-30ae18b8564f/go.mod h1:c1/X6cHgvdXj6pUlmWKMkuqRnW4K8x2vwt6JAaaircg= @@ -1354,20 +1367,23 @@ go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= +go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/multierr v0.0.0-20180122172545-ddea229ff1df/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v0.0.0-20180814183419-67bc79d13d15/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= @@ -1445,7 +1461,8 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= +golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1514,8 +1531,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1529,8 +1546,8 @@ golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s= -golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= +golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= +golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1659,7 +1676,6 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220405210540-1e041c57c461/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -1681,8 +1697,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1779,7 +1795,8 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo= +golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= +golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1918,8 +1935,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= @@ -1991,13 +2008,13 @@ k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= k8s.io/api v0.21.3/go.mod h1:hUgeYHUbBp23Ue4qdX9tR8/ANi/g3ehylAqDn9NWVOg= k8s.io/api v0.22.5/go.mod h1:mEhXyLaSD1qTOf40rRiKXkc+2iCem09rWLlFwhCEiAs= -k8s.io/api v0.27.3 h1:yR6oQXXnUEBWEWcvPWS0jQL575KoAboQPfJAuKNrw5Y= -k8s.io/api v0.27.3/go.mod h1:C4BNvZnQOF7JA/0Xed2S+aUyJSfTGkGFxLXz9MnpIpg= +k8s.io/api v0.29.3 h1:2ORfZ7+bGC3YJqGpV0KSDDEVf8hdGQ6A03/50vj8pmw= +k8s.io/api v0.29.3/go.mod h1:y2yg2NTyHUUkIoTC+phinTnEa3KFM6RZ3szxt014a80= k8s.io/apiextensions-apiserver v0.0.0-20190918161926-8f644eb6e783/go.mod h1:xvae1SZB3E17UpV59AWc271W/Ph25N+bjPyR63X6tPY= k8s.io/apiextensions-apiserver v0.20.2/go.mod h1:F6TXp389Xntt+LUq3vw6HFOLttPa0V8821ogLGwb6Zs= k8s.io/apiextensions-apiserver v0.21.3/go.mod h1:kl6dap3Gd45+21Jnh6utCx8Z2xxLm8LGDkprcd+KbsE= -k8s.io/apiextensions-apiserver v0.27.3 h1:xAwC1iYabi+TDfpRhxh4Eapl14Hs2OftM2DN5MpgKX4= -k8s.io/apiextensions-apiserver v0.27.3/go.mod h1:BH3wJ5NsB9XE1w+R6SSVpKmYNyIiyIz9xAmBl8Mb+84= +k8s.io/apiextensions-apiserver v0.29.3 h1:9HF+EtZaVpFjStakF4yVufnXGPRppWFEQ87qnO91YeI= +k8s.io/apiextensions-apiserver v0.29.3/go.mod h1:po0XiY5scnpJfFizNGo6puNU6Fq6D70UJY2Cb2KwAVc= k8s.io/apimachinery v0.0.0-20190913080033-27d36303b655/go.mod h1:nL6pwRT8NgfF8TT68DBI8uEePRt89cSvoXUVqbkWHq4= k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= k8s.io/apimachinery v0.18.10/go.mod h1:PF5taHbXgTEJLU+xMypMmYTXTWPJ5LaW8bfsisxnEXk= @@ -2008,8 +2025,8 @@ k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MA k8s.io/apimachinery v0.21.3/go.mod h1:H/IM+5vH9kZRNJ4l3x/fXP/5bOPJaVP/guptnZPeCFI= k8s.io/apimachinery v0.22.1/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= k8s.io/apimachinery v0.22.5/go.mod h1:xziclGKwuuJ2RM5/rSFQSYAj0zdbci3DH8kj+WvyN0U= -k8s.io/apimachinery v0.27.3 h1:Ubye8oBufD04l9QnNtW05idcOe9Z3GQN8+7PqmuVcUM= -k8s.io/apimachinery v0.27.3/go.mod h1:XNfZ6xklnMCOGGFNqXG7bUrQCoR04dh/E7FprV6pb+E= +k8s.io/apimachinery v0.29.3 h1:2tbx+5L7RNvqJjn7RIuIKu9XTsIZ9Z5wX2G22XAa5EU= +k8s.io/apimachinery v0.29.3/go.mod h1:hx/S4V2PNW4OMg3WizRrHutyB5la0iCUbZym+W0EQIU= k8s.io/apiserver v0.0.0-20190918160949-bfa5e2e684ad/go.mod h1:XPCXEwhjaFN29a8NldXA901ElnKeKLrLtREO9ZhFyhg= k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= k8s.io/apiserver v0.20.2/go.mod h1:2nKd93WyMhZx4Hp3RfgH2K5PhwyTrprrkWYnI7id7jA= @@ -2026,8 +2043,8 @@ k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k= k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0= k8s.io/client-go v0.21.3/go.mod h1:+VPhCgTsaFmGILxR/7E1N0S+ryO010QBeNCv5JwRGYU= k8s.io/client-go v0.22.5/go.mod h1:cs6yf/61q2T1SdQL5Rdcjg9J1ElXSwbjSrW2vFImM4Y= -k8s.io/client-go v0.27.3 h1:7dnEGHZEJld3lYwxvLl7WoehK6lAq7GvgjxpA3nv1E8= -k8s.io/client-go v0.27.3/go.mod h1:2MBEKuTo6V1lbKy3z1euEGnhPfGZLKTS9tiJ2xodM48= +k8s.io/client-go v0.29.3 h1:R/zaZbEAxqComZ9FHeQwOh3Y1ZUs7FaHKZdQtIc2WZg= +k8s.io/client-go v0.29.3/go.mod h1:tkDisCvgPfiRpxGnOORfkljmS+UrW+WtXAy2fTvXJB0= k8s.io/code-generator v0.0.0-20190912054826-cd179ad6a269/go.mod h1:V5BD6M4CyaN5m+VthcclXWsVcT1Hu+glwa1bi3MIsyE= k8s.io/code-generator v0.19.7/go.mod h1:lwEq3YnLYb/7uVXLorOJfxg+cUu2oihFhHZ0n9NIla0= k8s.io/code-generator v0.20.2/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= @@ -2039,8 +2056,8 @@ k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGw k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM= k8s.io/component-base v0.21.3/go.mod h1:kkuhtfEHeZM6LkX0saqSK8PbdO7A0HigUngmhhrwfGQ= k8s.io/component-base v0.22.5/go.mod h1:VK3I+TjuF9eaa+Ln67dKxhGar5ynVbwnGrUiNF4MqCI= -k8s.io/component-base v0.27.3 h1:g078YmdcdTfrCE4fFobt7qmVXwS8J/3cI1XxRi/2+6k= -k8s.io/component-base v0.27.3/go.mod h1:JNiKYcGImpQ44iwSYs6dysxzR9SxIIgQalk4HaCNVUY= +k8s.io/component-base v0.29.3 h1:Oq9/nddUxlnrCuuR2K/jp6aflVvc0uDvxMzAWxnGzAo= +k8s.io/component-base v0.29.3/go.mod h1:Yuj33XXjuOk2BAaHsIGHhCKZQAgYKhqIxIjIr2UXYio= k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM= k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= @@ -2063,8 +2080,8 @@ k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= -k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= +k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= @@ -2073,8 +2090,8 @@ k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAG k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= -k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f h1:2kWPakN3i/k81b0gvD5C5FJ2kxm1WrQFanWchyKuqGg= -k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f/go.mod h1:byini6yhqGC14c3ebc/QwanvYwhuMWF6yz2F8uwW8eg= +k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= +k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= k8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= @@ -2083,8 +2100,8 @@ k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/ k8s.io/utils v0.0.0-20210722164352-7f3ee0f31471/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20230209194617-a36077c30491 h1:r0BAOLElQnnFhE/ApUsg3iHdVYYPBjNSSOMowRZxxsY= -k8s.io/utils v0.0.0-20230209194617-a36077c30491/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= @@ -2121,8 +2138,8 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= From 75efd6a4f02aa3c2bc3109130af166aa84a645ef Mon Sep 17 00:00:00 2001 From: Toby Bellwood Date: Wed, 24 Apr 2024 15:46:56 +1000 Subject: [PATCH 10/17] build: publish multiarch images --- .github/workflows/build_and_publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_and_publish.yml b/.github/workflows/build_and_publish.yml index 0a7ace83..c3bde2cf 100644 --- a/.github/workflows/build_and_publish.yml +++ b/.github/workflows/build_and_publish.yml @@ -52,6 +52,7 @@ jobs: uses: docker/build-push-action@v2 with: context: . + platforms: linux/amd64,linux/arm64 push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file From 90679787fae23db8081929cdcb601ec44d12a516 Mon Sep 17 00:00:00 2001 From: Toby Bellwood Date: Wed, 24 Apr 2024 15:50:50 +1000 Subject: [PATCH 11/17] build: use ${ARCH} for GOARCH in Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b9b6fd75..c651cc76 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ COPY controllers/ controllers/ COPY internal/ internal/ # Build -RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o manager main.go +RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} GO111MODULE=on go build -a -o manager main.go # Use distroless as minimal base image to package the manager binary # Refer to https://github.com/GoogleContainerTools/distroless for more details From 104fec62d9fa31ec198ab7ab34944f8109259d7f Mon Sep 17 00:00:00 2001 From: Toby Bellwood Date: Wed, 24 Apr 2024 18:08:18 +1000 Subject: [PATCH 12/17] build: enable PR image building --- .github/workflows/build_and_publish.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/build_and_publish.yml b/.github/workflows/build_and_publish.yml index c3bde2cf..55f1fe20 100644 --- a/.github/workflows/build_and_publish.yml +++ b/.github/workflows/build_and_publish.yml @@ -34,14 +34,12 @@ jobs: uses: docker/setup-buildx-action@v1 - name: Login to DockerHub - if: github.event_name != 'pull_request' uses: docker/login-action@v1 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to GHCR - if: github.event_name != 'pull_request' uses: docker/login-action@v1 with: registry: ghcr.io @@ -53,6 +51,6 @@ jobs: with: context: . platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} + push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file From a84b30bd48891ccee0675d0d52d593024c67805a Mon Sep 17 00:00:00 2001 From: Toby Bellwood Date: Wed, 24 Apr 2024 18:12:55 +1000 Subject: [PATCH 13/17] build: update github action versions --- .github/workflows/build_and_publish.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build_and_publish.yml b/.github/workflows/build_and_publish.yml index 55f1fe20..b376c404 100644 --- a/.github/workflows/build_and_publish.yml +++ b/.github/workflows/build_and_publish.yml @@ -16,11 +16,11 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Docker meta id: meta - uses: docker/metadata-action@v3 + uses: docker/metadata-action@v5 with: # list of Docker images to use as base name for tags images: | @@ -28,26 +28,26 @@ jobs: ghcr.io/uselagoon/remote-controller - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to GHCR - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v5 with: context: . platforms: linux/amd64,linux/arm64 From 5de8e7b256483da49b5b26bffccce43902ede011 Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Thu, 2 May 2024 11:14:52 +1000 Subject: [PATCH 14/17] chore: minor formatting fixes --- controllers/v1beta1/podmonitor_buildhandlers.go | 14 +++++++------- controllers/v1beta1/podmonitor_taskhandlers.go | 14 +++++++------- controllers/v1beta2/podmonitor_buildhandlers.go | 14 +++++++------- controllers/v1beta2/podmonitor_taskhandlers.go | 14 +++++++------- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/controllers/v1beta1/podmonitor_buildhandlers.go b/controllers/v1beta1/podmonitor_buildhandlers.go index 44686516..a75c227c 100644 --- a/controllers/v1beta1/podmonitor_buildhandlers.go +++ b/controllers/v1beta1/podmonitor_buildhandlers.go @@ -201,7 +201,7 @@ Logs on pod %s, assigned to cluster %s } msgBytes, err := json.Marshal(msg) if err != nil { - opLog.Error(err, "Unable to encode message as JSON") + opLog.Error(err, "unable to encode message as JSON") } if err := r.Messaging.Publish("lagoon-logs", msgBytes); err != nil { // if we can't publish the message, set it as a pending message @@ -334,7 +334,7 @@ func (r *LagoonMonitorReconciler) updateDeploymentAndEnvironmentTask( } msgBytes, err := json.Marshal(msg) if err != nil { - opLog.Error(err, "Unable to encode message as JSON") + opLog.Error(err, "unable to encode message as JSON") } if err := r.Messaging.Publish("lagoon-tasks:controller", msgBytes); err != nil { // if we can't publish the message, set it as a pending message @@ -429,7 +429,7 @@ func (r *LagoonMonitorReconciler) buildStatusLogsToLagoonLogs( ) msgBytes, err := json.Marshal(msg) if err != nil { - opLog.Error(err, "Unable to encode message as JSON") + opLog.Error(err, "unable to encode message as JSON") } if err := r.Messaging.Publish("lagoon-logs", msgBytes); err != nil { // if we can't publish the message, set it as a pending message @@ -568,16 +568,16 @@ Build %s // do any message publishing here, and update any pending messages if needed if err = r.buildStatusLogsToLagoonLogs(opLog, &lagoonBuild, &jobPod, &lagoonEnv, namespace, buildCondition.ToLower()); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to publish build status logs")) + opLog.Error(err, "unable to publish build status logs") } if err = r.updateDeploymentAndEnvironmentTask(opLog, &lagoonBuild, &jobPod, &lagoonEnv, namespace, buildCondition.ToLower()); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to publish build update")) + opLog.Error(err, "unable to publish build update") } // if the container logs can't be retrieved, we don't want to send any build logs back, as this will nuke // any previously received logs if !strings.Contains(string(allContainerLogs), "unable to retrieve container logs for containerd") { if err = r.buildLogsToLagoonLogs(opLog, &lagoonBuild, &jobPod, namespace, buildCondition.ToLower(), allContainerLogs); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to publish build logs")) + opLog.Error(err, "unable to publish build logs") } } mergePatch, _ := json.Marshal(mergeMap) @@ -585,7 +585,7 @@ Build %s if err := r.Get(ctx, req.NamespacedName, &lagoonBuild); err == nil { // if it does, try to patch it if err := r.Patch(ctx, &lagoonBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { - opLog.Error(err, "Unable to update resource") + opLog.Error(err, "unable to update resource") } } // just delete the pod diff --git a/controllers/v1beta1/podmonitor_taskhandlers.go b/controllers/v1beta1/podmonitor_taskhandlers.go index 7eadd880..af03a7c3 100644 --- a/controllers/v1beta1/podmonitor_taskhandlers.go +++ b/controllers/v1beta1/podmonitor_taskhandlers.go @@ -141,7 +141,7 @@ Logs on pod %s, assigned to cluster %s } msgBytes, err := json.Marshal(msg) if err != nil { - opLog.Error(err, "Unable to encode message as JSON") + opLog.Error(err, "unable to encode message as JSON") } if err := r.Messaging.Publish("lagoon-logs", msgBytes); err != nil { // if we can't publish the message, set it as a pending message @@ -200,7 +200,7 @@ func (r *LagoonMonitorReconciler) updateLagoonTask(opLog logr.Logger, } msgBytes, err := json.Marshal(msg) if err != nil { - opLog.Error(err, "Unable to encode message as JSON") + opLog.Error(err, "unable to encode message as JSON") } if err := r.Messaging.Publish("lagoon-tasks:controller", msgBytes); err != nil { // if we can't publish the message, set it as a pending message @@ -246,7 +246,7 @@ func (r *LagoonMonitorReconciler) taskStatusLogsToLagoonLogs(opLog logr.Logger, } msgBytes, err := json.Marshal(msg) if err != nil { - opLog.Error(err, "Unable to encode message as JSON") + opLog.Error(err, "unable to encode message as JSON") } if err := r.Messaging.Publish("lagoon-logs", msgBytes); err != nil { // if we can't publish the message, set it as a pending message @@ -344,16 +344,16 @@ Task %s // send any messages to lagoon message queues // update the deployment with the status if err = r.taskStatusLogsToLagoonLogs(opLog, &lagoonTask, &jobPod, taskCondition.ToLower()); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to publish task status logs")) + opLog.Error(err, "unable to publish task status logs") } if err = r.updateLagoonTask(opLog, &lagoonTask, &jobPod, taskCondition.ToLower()); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to publish task update")) + opLog.Error(err, "unable to publish task update") } // if the container logs can't be retrieved, we don't want to send any task logs back, as this will nuke // any previously received logs if !strings.Contains(string(allContainerLogs), "unable to retrieve container logs for containerd") { if err = r.taskLogsToLagoonLogs(opLog, &lagoonTask, &jobPod, taskCondition.ToLower(), allContainerLogs); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to publish task logs")) + opLog.Error(err, "unable to publish task logs") } } mergePatch, _ := json.Marshal(mergeMap) @@ -361,7 +361,7 @@ Task %s if err := r.Get(ctx, req.NamespacedName, &lagoonTask); err == nil { // if it does, try to patch it if err := r.Patch(ctx, &lagoonTask, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { - opLog.Error(err, "Unable to update resource") + opLog.Error(err, "unable to update resource") } } // just delete the pod diff --git a/controllers/v1beta2/podmonitor_buildhandlers.go b/controllers/v1beta2/podmonitor_buildhandlers.go index b0b3144f..b79a0bd6 100644 --- a/controllers/v1beta2/podmonitor_buildhandlers.go +++ b/controllers/v1beta2/podmonitor_buildhandlers.go @@ -202,7 +202,7 @@ Logs on pod %s, assigned to cluster %s } msgBytes, err := json.Marshal(msg) if err != nil { - opLog.Error(err, "Unable to encode message as JSON") + opLog.Error(err, "unable to encode message as JSON") } if err := r.Messaging.Publish("lagoon-logs", msgBytes); err != nil { // if we can't publish the message, set it as a pending message @@ -340,7 +340,7 @@ func (r *LagoonMonitorReconciler) updateDeploymentAndEnvironmentTask( } msgBytes, err := json.Marshal(msg) if err != nil { - opLog.Error(err, "Unable to encode message as JSON") + opLog.Error(err, "unable to encode message as JSON") } if err := r.Messaging.Publish("lagoon-tasks:controller", msgBytes); err != nil { // if we can't publish the message, set it as a pending message @@ -434,7 +434,7 @@ func (r *LagoonMonitorReconciler) buildStatusLogsToLagoonLogs( ) msgBytes, err := json.Marshal(msg) if err != nil { - opLog.Error(err, "Unable to encode message as JSON") + opLog.Error(err, "unable to encode message as JSON") } if err := r.Messaging.Publish("lagoon-logs", msgBytes); err != nil { // if we can't publish the message, set it as a pending message @@ -574,16 +574,16 @@ Build %s // do any message publishing here, and update any pending messages if needed if err = r.buildStatusLogsToLagoonLogs(opLog, &lagoonBuild, &jobPod, &lagoonEnv, namespace, buildCondition.ToLower()); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to publish build status logs")) + opLog.Error(err, "unable to publish build status logs") } if err = r.updateDeploymentAndEnvironmentTask(opLog, &lagoonBuild, &jobPod, &lagoonEnv, namespace, buildCondition.ToLower()); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to publish build update")) + opLog.Error(err, "unable to publish build update") } // if the container logs can't be retrieved, we don't want to send any build logs back, as this will nuke // any previously received logs if !strings.Contains(string(allContainerLogs), "unable to retrieve container logs for containerd") { if err = r.buildLogsToLagoonLogs(opLog, &lagoonBuild, &jobPod, namespace, buildCondition.ToLower(), allContainerLogs); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to publish build logs")) + opLog.Error(err, "unable to publish build logs") } } mergePatch, _ := json.Marshal(mergeMap) @@ -591,7 +591,7 @@ Build %s if err := r.Get(ctx, req.NamespacedName, &lagoonBuild); err == nil { // if it does, try to patch it if err := r.Patch(ctx, &lagoonBuild, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { - opLog.Error(err, "Unable to update resource") + opLog.Error(err, "unable to update resource") } } // just delete the pod diff --git a/controllers/v1beta2/podmonitor_taskhandlers.go b/controllers/v1beta2/podmonitor_taskhandlers.go index ebada680..53dd39e4 100644 --- a/controllers/v1beta2/podmonitor_taskhandlers.go +++ b/controllers/v1beta2/podmonitor_taskhandlers.go @@ -143,7 +143,7 @@ Logs on pod %s, assigned to cluster %s } msgBytes, err := json.Marshal(msg) if err != nil { - opLog.Error(err, "Unable to encode message as JSON") + opLog.Error(err, "unable to encode message as JSON") } if err := r.Messaging.Publish("lagoon-logs", msgBytes); err != nil { // if we can't publish the message, set it as a pending message @@ -208,7 +208,7 @@ func (r *LagoonMonitorReconciler) updateLagoonTask(opLog logr.Logger, } msgBytes, err := json.Marshal(msg) if err != nil { - opLog.Error(err, "Unable to encode message as JSON") + opLog.Error(err, "unable to encode message as JSON") } if err := r.Messaging.Publish("lagoon-tasks:controller", msgBytes); err != nil { // if we can't publish the message, set it as a pending message @@ -254,7 +254,7 @@ func (r *LagoonMonitorReconciler) taskStatusLogsToLagoonLogs(opLog logr.Logger, } msgBytes, err := json.Marshal(msg) if err != nil { - opLog.Error(err, "Unable to encode message as JSON") + opLog.Error(err, "unable to encode message as JSON") } if err := r.Messaging.Publish("lagoon-logs", msgBytes); err != nil { // if we can't publish the message, set it as a pending message @@ -353,16 +353,16 @@ Task %s // send any messages to lagoon message queues // update the deployment with the status if err = r.taskStatusLogsToLagoonLogs(opLog, &lagoonTask, &jobPod, taskCondition.ToLower()); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to publish task status logs")) + opLog.Error(err, "unable to publish task status logs") } if err = r.updateLagoonTask(opLog, &lagoonTask, &jobPod, taskCondition.ToLower()); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to publish task update")) + opLog.Error(err, "unable to publish task update") } // if the container logs can't be retrieved, we don't want to send any task logs back, as this will nuke // any previously received logs if !strings.Contains(string(allContainerLogs), "unable to retrieve container logs for containerd") { if err = r.taskLogsToLagoonLogs(opLog, &lagoonTask, &jobPod, taskCondition.ToLower(), allContainerLogs); err != nil { - opLog.Error(err, fmt.Sprintf("Unable to publish task logs")) + opLog.Error(err, "unable to publish task logs") } } mergePatch, _ := json.Marshal(mergeMap) @@ -370,7 +370,7 @@ Task %s if err := r.Get(ctx, req.NamespacedName, &lagoonTask); err == nil { // if it does, try to patch it if err := r.Patch(ctx, &lagoonTask, client.RawPatch(types.MergePatchType, mergePatch)); err != nil { - opLog.Error(err, "Unable to update resource") + opLog.Error(err, "unable to update resource") } } // just delete the pod From d1b45f2e367c41eb36da5f77db8474f4845be4cb Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Thu, 2 May 2024 11:18:02 +1000 Subject: [PATCH 15/17] fix: v1beta1 crd changes --- .../crd/bases/crd.lagoon.sh_lagoonbuilds.yaml | 371 ++++++++++++++++++ .../crd/bases/crd.lagoon.sh_lagoontasks.yaml | 371 ++++++++++++++++++ 2 files changed, 742 insertions(+) diff --git a/config/crd/bases/crd.lagoon.sh_lagoonbuilds.yaml b/config/crd/bases/crd.lagoon.sh_lagoonbuilds.yaml index 4d682ae5..4d8db47c 100644 --- a/config/crd/bases/crd.lagoon.sh_lagoonbuilds.yaml +++ b/config/crd/bases/crd.lagoon.sh_lagoonbuilds.yaml @@ -203,6 +203,377 @@ spec: format: byte type: string type: object + statusMessages: + description: LagoonStatusMessages is where unsent messages are stored + for re-sending. + properties: + buildLogMessage: + description: LagoonLog is used to sendToLagoonLogs messaging queue + this is general logging information + properties: + event: + type: string + message: + type: string + meta: + description: LagoonLogMeta is the metadata that is used by logging + in Lagoon. + properties: + advancedData: + type: string + branchName: + type: string + buildName: + type: string + buildPhase: + type: string + buildStatus: + type: string + buildStep: + type: string + clusterName: + type: string + endTime: + type: string + environment: + type: string + environmentId: + type: integer + jobName: + type: string + jobStatus: + type: string + jobStep: + type: string + key: + type: string + logLink: + type: string + project: + type: string + projectId: + type: integer + projectName: + type: string + remoteId: + type: string + route: + type: string + routes: + items: + type: string + type: array + services: + items: + type: string + type: array + startTime: + type: string + task: + description: LagoonTaskInfo defines what a task can use to + communicate with Lagoon via SSH/API. + properties: + apiHost: + type: string + command: + type: string + id: + type: string + name: + type: string + service: + type: string + sshHost: + type: string + sshPort: + type: string + taskName: + type: string + required: + - id + type: object + type: object + project: + type: string + severity: + type: string + uuid: + type: string + type: object + environmentMessage: + description: LagoonMessage is used for sending build info back to + Lagoon messaging queue to update the environment or deployment + properties: + meta: + description: LagoonLogMeta is the metadata that is used by logging + in Lagoon. + properties: + advancedData: + type: string + branchName: + type: string + buildName: + type: string + buildPhase: + type: string + buildStatus: + type: string + buildStep: + type: string + clusterName: + type: string + endTime: + type: string + environment: + type: string + environmentId: + type: integer + jobName: + type: string + jobStatus: + type: string + jobStep: + type: string + key: + type: string + logLink: + type: string + project: + type: string + projectId: + type: integer + projectName: + type: string + remoteId: + type: string + route: + type: string + routes: + items: + type: string + type: array + services: + items: + type: string + type: array + startTime: + type: string + task: + description: LagoonTaskInfo defines what a task can use to + communicate with Lagoon via SSH/API. + properties: + apiHost: + type: string + command: + type: string + id: + type: string + name: + type: string + service: + type: string + sshHost: + type: string + sshPort: + type: string + taskName: + type: string + required: + - id + type: object + type: object + namespace: + type: string + type: + type: string + type: object + statusMessage: + description: LagoonLog is used to sendToLagoonLogs messaging queue + this is general logging information + properties: + event: + type: string + message: + type: string + meta: + description: LagoonLogMeta is the metadata that is used by logging + in Lagoon. + properties: + advancedData: + type: string + branchName: + type: string + buildName: + type: string + buildPhase: + type: string + buildStatus: + type: string + buildStep: + type: string + clusterName: + type: string + endTime: + type: string + environment: + type: string + environmentId: + type: integer + jobName: + type: string + jobStatus: + type: string + jobStep: + type: string + key: + type: string + logLink: + type: string + project: + type: string + projectId: + type: integer + projectName: + type: string + remoteId: + type: string + route: + type: string + routes: + items: + type: string + type: array + services: + items: + type: string + type: array + startTime: + type: string + task: + description: LagoonTaskInfo defines what a task can use to + communicate with Lagoon via SSH/API. + properties: + apiHost: + type: string + command: + type: string + id: + type: string + name: + type: string + service: + type: string + sshHost: + type: string + sshPort: + type: string + taskName: + type: string + required: + - id + type: object + type: object + project: + type: string + severity: + type: string + uuid: + type: string + type: object + taskLogMessage: + description: LagoonLog is used to sendToLagoonLogs messaging queue + this is general logging information + properties: + event: + type: string + message: + type: string + meta: + description: LagoonLogMeta is the metadata that is used by logging + in Lagoon. + properties: + advancedData: + type: string + branchName: + type: string + buildName: + type: string + buildPhase: + type: string + buildStatus: + type: string + buildStep: + type: string + clusterName: + type: string + endTime: + type: string + environment: + type: string + environmentId: + type: integer + jobName: + type: string + jobStatus: + type: string + jobStep: + type: string + key: + type: string + logLink: + type: string + project: + type: string + projectId: + type: integer + projectName: + type: string + remoteId: + type: string + route: + type: string + routes: + items: + type: string + type: array + services: + items: + type: string + type: array + startTime: + type: string + task: + description: LagoonTaskInfo defines what a task can use to + communicate with Lagoon via SSH/API. + properties: + apiHost: + type: string + command: + type: string + id: + type: string + name: + type: string + service: + type: string + sshHost: + type: string + sshPort: + type: string + taskName: + type: string + required: + - id + type: object + type: object + project: + type: string + severity: + type: string + uuid: + type: string + type: object + type: object type: object served: true storage: false diff --git a/config/crd/bases/crd.lagoon.sh_lagoontasks.yaml b/config/crd/bases/crd.lagoon.sh_lagoontasks.yaml index 54b434ab..0f88b0bd 100644 --- a/config/crd/bases/crd.lagoon.sh_lagoontasks.yaml +++ b/config/crd/bases/crd.lagoon.sh_lagoontasks.yaml @@ -185,6 +185,377 @@ spec: format: byte type: string type: object + statusMessages: + description: LagoonStatusMessages is where unsent messages are stored + for re-sending. + properties: + buildLogMessage: + description: LagoonLog is used to sendToLagoonLogs messaging queue + this is general logging information + properties: + event: + type: string + message: + type: string + meta: + description: LagoonLogMeta is the metadata that is used by logging + in Lagoon. + properties: + advancedData: + type: string + branchName: + type: string + buildName: + type: string + buildPhase: + type: string + buildStatus: + type: string + buildStep: + type: string + clusterName: + type: string + endTime: + type: string + environment: + type: string + environmentId: + type: integer + jobName: + type: string + jobStatus: + type: string + jobStep: + type: string + key: + type: string + logLink: + type: string + project: + type: string + projectId: + type: integer + projectName: + type: string + remoteId: + type: string + route: + type: string + routes: + items: + type: string + type: array + services: + items: + type: string + type: array + startTime: + type: string + task: + description: LagoonTaskInfo defines what a task can use to + communicate with Lagoon via SSH/API. + properties: + apiHost: + type: string + command: + type: string + id: + type: string + name: + type: string + service: + type: string + sshHost: + type: string + sshPort: + type: string + taskName: + type: string + required: + - id + type: object + type: object + project: + type: string + severity: + type: string + uuid: + type: string + type: object + environmentMessage: + description: LagoonMessage is used for sending build info back to + Lagoon messaging queue to update the environment or deployment + properties: + meta: + description: LagoonLogMeta is the metadata that is used by logging + in Lagoon. + properties: + advancedData: + type: string + branchName: + type: string + buildName: + type: string + buildPhase: + type: string + buildStatus: + type: string + buildStep: + type: string + clusterName: + type: string + endTime: + type: string + environment: + type: string + environmentId: + type: integer + jobName: + type: string + jobStatus: + type: string + jobStep: + type: string + key: + type: string + logLink: + type: string + project: + type: string + projectId: + type: integer + projectName: + type: string + remoteId: + type: string + route: + type: string + routes: + items: + type: string + type: array + services: + items: + type: string + type: array + startTime: + type: string + task: + description: LagoonTaskInfo defines what a task can use to + communicate with Lagoon via SSH/API. + properties: + apiHost: + type: string + command: + type: string + id: + type: string + name: + type: string + service: + type: string + sshHost: + type: string + sshPort: + type: string + taskName: + type: string + required: + - id + type: object + type: object + namespace: + type: string + type: + type: string + type: object + statusMessage: + description: LagoonLog is used to sendToLagoonLogs messaging queue + this is general logging information + properties: + event: + type: string + message: + type: string + meta: + description: LagoonLogMeta is the metadata that is used by logging + in Lagoon. + properties: + advancedData: + type: string + branchName: + type: string + buildName: + type: string + buildPhase: + type: string + buildStatus: + type: string + buildStep: + type: string + clusterName: + type: string + endTime: + type: string + environment: + type: string + environmentId: + type: integer + jobName: + type: string + jobStatus: + type: string + jobStep: + type: string + key: + type: string + logLink: + type: string + project: + type: string + projectId: + type: integer + projectName: + type: string + remoteId: + type: string + route: + type: string + routes: + items: + type: string + type: array + services: + items: + type: string + type: array + startTime: + type: string + task: + description: LagoonTaskInfo defines what a task can use to + communicate with Lagoon via SSH/API. + properties: + apiHost: + type: string + command: + type: string + id: + type: string + name: + type: string + service: + type: string + sshHost: + type: string + sshPort: + type: string + taskName: + type: string + required: + - id + type: object + type: object + project: + type: string + severity: + type: string + uuid: + type: string + type: object + taskLogMessage: + description: LagoonLog is used to sendToLagoonLogs messaging queue + this is general logging information + properties: + event: + type: string + message: + type: string + meta: + description: LagoonLogMeta is the metadata that is used by logging + in Lagoon. + properties: + advancedData: + type: string + branchName: + type: string + buildName: + type: string + buildPhase: + type: string + buildStatus: + type: string + buildStep: + type: string + clusterName: + type: string + endTime: + type: string + environment: + type: string + environmentId: + type: integer + jobName: + type: string + jobStatus: + type: string + jobStep: + type: string + key: + type: string + logLink: + type: string + project: + type: string + projectId: + type: integer + projectName: + type: string + remoteId: + type: string + route: + type: string + routes: + items: + type: string + type: array + services: + items: + type: string + type: array + startTime: + type: string + task: + description: LagoonTaskInfo defines what a task can use to + communicate with Lagoon via SSH/API. + properties: + apiHost: + type: string + command: + type: string + id: + type: string + name: + type: string + service: + type: string + sshHost: + type: string + sshPort: + type: string + taskName: + type: string + required: + - id + type: object + type: object + project: + type: string + severity: + type: string + uuid: + type: string + type: object + type: object type: object served: true storage: false From 48f4ee477d7c69cb610db6b43dc872a76cbe622e Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Thu, 2 May 2024 11:19:42 +1000 Subject: [PATCH 16/17] test: add main-v1beta2 --- .github/workflows/remote-controller.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/remote-controller.yaml b/.github/workflows/remote-controller.yaml index 1288a60d..1a1c8be9 100644 --- a/.github/workflows/remote-controller.yaml +++ b/.github/workflows/remote-controller.yaml @@ -8,6 +8,7 @@ on: pull_request: branches: - main + - main-v1beta2 jobs: test-suite: From 7d5190e68a7ca934ce814758e4118ae15290dea1 Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Thu, 2 May 2024 13:08:09 +1000 Subject: [PATCH 17/17] fix: add environment services changes to v1beta2 controller and other fixes --- apis/lagoon/v1beta1/zz_generated.deepcopy.go | 103 ------------------ .../crd/bases/crd.lagoon.sh_lagoonbuilds.yaml | 44 ++++++++ .../crd/bases/crd.lagoon.sh_lagoontasks.yaml | 44 ++++++++ controllers/v1beta1/build_deletionhandlers.go | 8 +- .../v1beta1/podmonitor_buildhandlers.go | 8 +- controllers/v1beta2/build_deletionhandlers.go | 22 +++- .../v1beta2/podmonitor_buildhandlers.go | 24 ++-- go.mod | 10 +- go.sum | 20 ++-- 9 files changed, 144 insertions(+), 139 deletions(-) diff --git a/apis/lagoon/v1beta1/zz_generated.deepcopy.go b/apis/lagoon/v1beta1/zz_generated.deepcopy.go index d57f59fa..c11c785c 100644 --- a/apis/lagoon/v1beta1/zz_generated.deepcopy.go +++ b/apis/lagoon/v1beta1/zz_generated.deepcopy.go @@ -199,94 +199,6 @@ func (in *LagoonBuildStatus) DeepCopy() *LagoonBuildStatus { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -<<<<<<< HEAD -func (in *LagoonLog) DeepCopyInto(out *LagoonLog) { - *out = *in - if in.Meta != nil { - in, out := &in.Meta, &out.Meta - *out = new(LagoonLogMeta) - (*in).DeepCopyInto(*out) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonLog. -func (in *LagoonLog) DeepCopy() *LagoonLog { - if in == nil { - return nil - } - out := new(LagoonLog) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *LagoonLogMeta) DeepCopyInto(out *LagoonLogMeta) { - *out = *in - if in.EnvironmentID != nil { - in, out := &in.EnvironmentID, &out.EnvironmentID - *out = new(uint) - **out = **in - } - if in.ProjectID != nil { - in, out := &in.ProjectID, &out.ProjectID - *out = new(uint) - **out = **in - } - if in.Routes != nil { - in, out := &in.Routes, &out.Routes - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.Services != nil { - in, out := &in.Services, &out.Services - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.EnvironmentServices != nil { - in, out := &in.EnvironmentServices, &out.EnvironmentServices - *out = make([]LagoonService, len(*in)) - copy(*out, *in) - } - if in.Task != nil { - in, out := &in.Task, &out.Task - *out = new(LagoonTaskInfo) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonLogMeta. -func (in *LagoonLogMeta) DeepCopy() *LagoonLogMeta { - if in == nil { - return nil - } - out := new(LagoonLogMeta) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *LagoonMessage) DeepCopyInto(out *LagoonMessage) { - *out = *in - if in.Meta != nil { - in, out := &in.Meta, &out.Meta - *out = new(LagoonLogMeta) - (*in).DeepCopyInto(*out) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonMessage. -func (in *LagoonMessage) DeepCopy() *LagoonMessage { - if in == nil { - return nil - } - out := new(LagoonMessage) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -======= ->>>>>>> main-v1beta2 func (in *LagoonMiscBackupInfo) DeepCopyInto(out *LagoonMiscBackupInfo) { *out = *in } @@ -326,21 +238,6 @@ func (in *LagoonMiscInfo) DeepCopy() *LagoonMiscInfo { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *LagoonService) DeepCopyInto(out *LagoonService) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LagoonService. -func (in *LagoonService) DeepCopy() *LagoonService { - if in == nil { - return nil - } - out := new(LagoonService) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LagoonStatusMessages) DeepCopyInto(out *LagoonStatusMessages) { *out = *in diff --git a/config/crd/bases/crd.lagoon.sh_lagoonbuilds.yaml b/config/crd/bases/crd.lagoon.sh_lagoonbuilds.yaml index d9d377e2..ed182224 100644 --- a/config/crd/bases/crd.lagoon.sh_lagoonbuilds.yaml +++ b/config/crd/bases/crd.lagoon.sh_lagoonbuilds.yaml @@ -241,7 +241,18 @@ spec: type: integer environmentServices: items: + description: EnvironmentService is based on the Lagoon + API type. properties: + containers: + items: + description: ServiceContainer is based on the Lagoon + API type. + properties: + name: + type: string + type: object + type: array created: type: string id: @@ -345,7 +356,18 @@ spec: type: integer environmentServices: items: + description: EnvironmentService is based on the Lagoon + API type. properties: + containers: + items: + description: ServiceContainer is based on the Lagoon + API type. + properties: + name: + type: string + type: object + type: array created: type: string id: @@ -451,7 +473,18 @@ spec: type: integer environmentServices: items: + description: EnvironmentService is based on the Lagoon + API type. properties: + containers: + items: + description: ServiceContainer is based on the Lagoon + API type. + properties: + name: + type: string + type: object + type: array created: type: string id: @@ -559,7 +592,18 @@ spec: type: integer environmentServices: items: + description: EnvironmentService is based on the Lagoon + API type. properties: + containers: + items: + description: ServiceContainer is based on the Lagoon + API type. + properties: + name: + type: string + type: object + type: array created: type: string id: diff --git a/config/crd/bases/crd.lagoon.sh_lagoontasks.yaml b/config/crd/bases/crd.lagoon.sh_lagoontasks.yaml index e4343f9e..5be4d4aa 100644 --- a/config/crd/bases/crd.lagoon.sh_lagoontasks.yaml +++ b/config/crd/bases/crd.lagoon.sh_lagoontasks.yaml @@ -223,7 +223,18 @@ spec: type: integer environmentServices: items: + description: EnvironmentService is based on the Lagoon + API type. properties: + containers: + items: + description: ServiceContainer is based on the Lagoon + API type. + properties: + name: + type: string + type: object + type: array created: type: string id: @@ -327,7 +338,18 @@ spec: type: integer environmentServices: items: + description: EnvironmentService is based on the Lagoon + API type. properties: + containers: + items: + description: ServiceContainer is based on the Lagoon + API type. + properties: + name: + type: string + type: object + type: array created: type: string id: @@ -433,7 +455,18 @@ spec: type: integer environmentServices: items: + description: EnvironmentService is based on the Lagoon + API type. properties: + containers: + items: + description: ServiceContainer is based on the Lagoon + API type. + properties: + name: + type: string + type: object + type: array created: type: string id: @@ -541,7 +574,18 @@ spec: type: integer environmentServices: items: + description: EnvironmentService is based on the Lagoon + API type. properties: + containers: + items: + description: ServiceContainer is based on the Lagoon + API type. + properties: + name: + type: string + type: object + type: array created: type: string id: diff --git a/controllers/v1beta1/build_deletionhandlers.go b/controllers/v1beta1/build_deletionhandlers.go index 6c24e670..c7f7292b 100644 --- a/controllers/v1beta1/build_deletionhandlers.go +++ b/controllers/v1beta1/build_deletionhandlers.go @@ -306,24 +306,24 @@ func (r *LagoonBuildReconciler) updateDeploymentAndEnvironmentTask(ctx context.C }) podList := &corev1.PodList{} serviceNames := []string{} - services := []lagoonv1beta1.LagoonService{} + services := []schema.EnvironmentService{} if err := r.List(context.TODO(), podList, listOption); err == nil { // generate the list of services to add to the environment for _, pod := range podList.Items { var serviceName, serviceType string - containers := []lagoonv1beta1.EnvironmentContainer{} + containers := []schema.ServiceContainer{} if name, ok := pod.ObjectMeta.Labels["lagoon.sh/service"]; ok { serviceName = name serviceNames = append(serviceNames, serviceName) for _, container := range pod.Spec.Containers { - containers = append(containers, lagoonv1beta1.EnvironmentContainer{Name: container.Name}) + containers = append(containers, schema.ServiceContainer{Name: container.Name}) } } if sType, ok := pod.ObjectMeta.Labels["lagoon.sh/service-type"]; ok { serviceType = sType } // probably need to collect dbaas consumers too at some stage - services = append(services, lagoonv1beta1.LagoonService{ + services = append(services, schema.EnvironmentService{ Name: serviceName, Type: serviceType, Containers: containers, diff --git a/controllers/v1beta1/podmonitor_buildhandlers.go b/controllers/v1beta1/podmonitor_buildhandlers.go index 220e0692..96a69d34 100644 --- a/controllers/v1beta1/podmonitor_buildhandlers.go +++ b/controllers/v1beta1/podmonitor_buildhandlers.go @@ -290,24 +290,24 @@ func (r *LagoonMonitorReconciler) updateDeploymentAndEnvironmentTask( }) depList := &appsv1.DeploymentList{} serviceNames := []string{} - services := []lagoonv1beta1.LagoonService{} + services := []schema.EnvironmentService{} if err := r.List(context.TODO(), depList, listOption); err == nil { // generate the list of services to add or update to the environment for _, deployment := range depList.Items { var serviceName, serviceType string - containers := []lagoonv1beta1.EnvironmentContainer{} + containers := []schema.ServiceContainer{} if name, ok := deployment.ObjectMeta.Labels["lagoon.sh/service"]; ok { serviceName = name serviceNames = append(serviceNames, serviceName) for _, container := range deployment.Spec.Template.Spec.Containers { - containers = append(containers, lagoonv1beta1.EnvironmentContainer{Name: container.Name}) + containers = append(containers, schema.ServiceContainer{Name: container.Name}) } } if sType, ok := deployment.ObjectMeta.Labels["lagoon.sh/service-type"]; ok { serviceType = sType } // probably need to collect dbaas consumers too at some stage - services = append(services, lagoonv1beta1.LagoonService{ + services = append(services, schema.EnvironmentService{ Name: serviceName, Type: serviceType, Containers: containers, diff --git a/controllers/v1beta2/build_deletionhandlers.go b/controllers/v1beta2/build_deletionhandlers.go index 51096a47..e9fb7f26 100644 --- a/controllers/v1beta2/build_deletionhandlers.go +++ b/controllers/v1beta2/build_deletionhandlers.go @@ -305,21 +305,31 @@ func (r *LagoonBuildReconciler) updateDeploymentAndEnvironmentTask(ctx context.C }) podList := &corev1.PodList{} serviceNames := []string{} + services := []schema.EnvironmentService{} if err := r.List(context.TODO(), podList, listOption); err == nil { // generate the list of services to add to the environment for _, pod := range podList.Items { - if _, ok := pod.ObjectMeta.Labels["lagoon.sh/service"]; ok { + var serviceName, serviceType string + containers := []schema.ServiceContainer{} + if name, ok := pod.ObjectMeta.Labels["lagoon.sh/service"]; ok { + serviceName = name + serviceNames = append(serviceNames, serviceName) for _, container := range pod.Spec.Containers { - serviceNames = append(serviceNames, container.Name) + containers = append(containers, schema.ServiceContainer{Name: container.Name}) } } - if _, ok := pod.ObjectMeta.Labels["service"]; ok { - for _, container := range pod.Spec.Containers { - serviceNames = append(serviceNames, container.Name) - } + if sType, ok := pod.ObjectMeta.Labels["lagoon.sh/service-type"]; ok { + serviceType = sType } + // probably need to collect dbaas consumers too at some stage + services = append(services, schema.EnvironmentService{ + Name: serviceName, + Type: serviceType, + Containers: containers, + }) } msg.Meta.Services = serviceNames + msg.Meta.EnvironmentServices = services } // if we aren't being provided the lagoon config, we can skip adding the routes etc if lagoonEnv != nil { diff --git a/controllers/v1beta2/podmonitor_buildhandlers.go b/controllers/v1beta2/podmonitor_buildhandlers.go index b79a0bd6..9d2ae42a 100644 --- a/controllers/v1beta2/podmonitor_buildhandlers.go +++ b/controllers/v1beta2/podmonitor_buildhandlers.go @@ -296,21 +296,31 @@ func (r *LagoonMonitorReconciler) updateDeploymentAndEnvironmentTask( }) depList := &appsv1.DeploymentList{} serviceNames := []string{} + services := []schema.EnvironmentService{} if err := r.List(context.TODO(), depList, listOption); err == nil { - // generate the list of services to add to the environment + // generate the list of services to add or update to the environment for _, deployment := range depList.Items { - if _, ok := deployment.ObjectMeta.Labels["lagoon.sh/service"]; ok { + var serviceName, serviceType string + containers := []schema.ServiceContainer{} + if name, ok := deployment.ObjectMeta.Labels["lagoon.sh/service"]; ok { + serviceName = name + serviceNames = append(serviceNames, serviceName) for _, container := range deployment.Spec.Template.Spec.Containers { - serviceNames = append(serviceNames, container.Name) + containers = append(containers, schema.ServiceContainer{Name: container.Name}) } } - if _, ok := deployment.ObjectMeta.Labels["service"]; ok { - for _, container := range deployment.Spec.Template.Spec.Containers { - serviceNames = append(serviceNames, container.Name) - } + if sType, ok := deployment.ObjectMeta.Labels["lagoon.sh/service-type"]; ok { + serviceType = sType } + // probably need to collect dbaas consumers too at some stage + services = append(services, schema.EnvironmentService{ + Name: serviceName, + Type: serviceType, + Containers: containers, + }) } msg.Meta.Services = serviceNames + msg.Meta.EnvironmentServices = services } // if we aren't being provided the lagoon config, we can skip adding the routes etc if lagoonEnv != nil { diff --git a/go.mod b/go.mod index b394be17..3f0b7efb 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.29.0 github.com/prometheus/client_golang v1.16.0 - github.com/uselagoon/machinery v0.0.17-0.20240109062854-3b567fb41003 + github.com/uselagoon/machinery v0.0.20 github.com/vshn/k8up v1.99.99 github.com/xhit/go-str2duration/v2 v2.1.0 gopkg.in/matryer/try.v1 v1.0.0-20150601225556-312d2599e12e @@ -84,10 +84,10 @@ require ( go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.24.0 // indirect - golang.org/x/net v0.19.0 // indirect - golang.org/x/oauth2 v0.10.0 // indirect - golang.org/x/sys v0.15.0 // indirect - golang.org/x/term v0.15.0 // indirect + golang.org/x/net v0.21.0 // indirect + golang.org/x/oauth2 v0.17.0 // indirect + golang.org/x/sys v0.17.0 // indirect + golang.org/x/term v0.17.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect diff --git a/go.sum b/go.sum index f689c8cf..2fecfb90 100644 --- a/go.sum +++ b/go.sum @@ -1254,8 +1254,8 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= -github.com/uselagoon/machinery v0.0.17-0.20240109062854-3b567fb41003 h1:fLhncKjshd8f0DBFB45u7OWsF1H6OgNmpkswJYqwCA8= -github.com/uselagoon/machinery v0.0.17-0.20240109062854-3b567fb41003/go.mod h1:Duljjz/3d/7m0jbmF1nVRDTNaMxMr6m+5LkgjiRrQaU= +github.com/uselagoon/machinery v0.0.20 h1:4pGGX/Y5UwbT86TU9vaP7QyjGk8wvikUVDvkTpesvbs= +github.com/uselagoon/machinery v0.0.20/go.mod h1:NbgtEofjK2XY0iUpk9aMYazIo+W/NI56+UF72jv8zVY= github.com/uudashr/gocognit v0.0.0-20190926065955-1655d0de0517/go.mod h1:j44Ayx2KW4+oB6SWMv8KsmHzZrOInQav7D3cQMJ5JUM= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s= @@ -1531,8 +1531,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1546,8 +1546,8 @@ golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= -golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= +golang.org/x/oauth2 v0.17.0 h1:6m3ZPmLEFdVxKKWnKq4VqZ60gutO35zm+zrAHVmHyDQ= +golang.org/x/oauth2 v0.17.0/go.mod h1:OzPDGQiuQMguemayvdylqddI7qcD9lnSDb+1FiwQ5HA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1676,16 +1676,16 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220405210540-1e041c57c461/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=