Skip to content

Commit

Permalink
feat: support for harbor retention policies
Browse files Browse the repository at this point in the history
  • Loading branch information
@shreddedbacon
shreddedbacon committed Dec 12, 2024
1 parent b59050b commit 30e0986
Show file tree
Hide file tree
Showing 17 changed files with 507 additions and 262 deletions.
2 changes: 1 addition & 1 deletion controllers/v1beta1/build_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -326,7 +326,7 @@ func (r *LagoonBuildReconciler) createNamespaceBuild(ctx context.Context,
continue
}
// send the status change to lagoon
r.updateDeploymentAndEnvironmentTask(ctx, opLog, runningBuild, nil, buildCondition, "cancelled")
r.updateDeploymentAndEnvironmentTask(opLog, runningBuild, nil, buildCondition, "cancelled")
continue
}
// handle processing running but no pod/failed pod builds
Expand Down
12 changes: 6 additions & 6 deletions controllers/v1beta1/build_deletionhandlers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -212,16 +212,16 @@ Build cancelled
}
// send any messages to lagoon message queues
// update the deployment with the status of cancelled in lagoon
r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusCancelled, "cancelled")
r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusCancelled, "cancelled")
r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, allContainerLogs, lagoonv1beta1.BuildStatusCancelled)
r.buildStatusLogsToLagoonLogs(opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusCancelled, "cancelled")
r.updateDeploymentAndEnvironmentTask(opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusCancelled, "cancelled")
r.buildLogsToLagoonLogs(opLog, &lagoonBuild, allContainerLogs, lagoonv1beta1.BuildStatusCancelled)
}
return nil
}

// buildLogsToLagoonLogs sends the build logs to the lagoon-logs message queue
// it contains the actual pod log output that is sent to elasticsearch, it is what eventually is displayed in the UI
func (r *LagoonBuildReconciler) buildLogsToLagoonLogs(ctx context.Context,
func (r *LagoonBuildReconciler) buildLogsToLagoonLogs(
opLog logr.Logger,
lagoonBuild *lagoonv1beta1.LagoonBuild,
logs []byte,
Expand Down Expand Up @@ -267,7 +267,7 @@ func (r *LagoonBuildReconciler) buildLogsToLagoonLogs(ctx context.Context,

// updateDeploymentAndEnvironmentTask sends the status of the build and deployment to the controllerhandler message queue in lagoon,
// this is for the handler in lagoon to process.
func (r *LagoonBuildReconciler) updateDeploymentAndEnvironmentTask(ctx context.Context,
func (r *LagoonBuildReconciler) updateDeploymentAndEnvironmentTask(
opLog logr.Logger,
lagoonBuild *lagoonv1beta1.LagoonBuild,
lagoonEnv *corev1.ConfigMap,
Expand Down Expand Up @@ -361,7 +361,7 @@ func (r *LagoonBuildReconciler) updateDeploymentAndEnvironmentTask(ctx context.C
}

// buildStatusLogsToLagoonLogs sends the logs to lagoon-logs message queue, used for general messaging
func (r *LagoonBuildReconciler) buildStatusLogsToLagoonLogs(ctx context.Context,
func (r *LagoonBuildReconciler) buildStatusLogsToLagoonLogs(
opLog logr.Logger,
lagoonBuild *lagoonv1beta1.LagoonBuild,
lagoonEnv *corev1.ConfigMap,
Expand Down
23 changes: 12 additions & 11 deletions controllers/v1beta1/build_helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@ func (r *LagoonBuildReconciler) getOrCreateNamespace(ctx context.Context, namesp
return fmt.Errorf("error getting harbor version, check your harbor configuration. Error was: %v", err)
}
if lagoonHarbor.UseV2Functions(curVer) {
hProject, err := lagoonHarbor.CreateProjectV2(ctx, lagoonBuild.Spec.Project.Name)
hProject, err := lagoonHarbor.CreateProjectV2(ctx, *namespace)
if err != nil {
return fmt.Errorf("error creating harbor project: %v", err)
}
Expand Down Expand Up @@ -330,8 +330,9 @@ func (r *LagoonBuildReconciler) processBuild(ctx context.Context, opLog logr.Log
}

var serviceaccountTokenSecret string
reg, _ := regexp.Compile("^lagoon-deployer-token")
for _, secret := range serviceAccount.Secrets {
match, _ := regexp.MatchString("^lagoon-deployer-token", secret.Name)
match := reg.MatchString(secret.Name)
if match {
serviceaccountTokenSecret = secret.Name
break
Expand Down Expand Up @@ -896,13 +897,13 @@ func (r *LagoonBuildReconciler) updateQueuedBuild(
// send any messages to lagoon message queues
// update the deployment with the status, lagoon v2.12.0 supports queued status, otherwise use pending
if lagoonv1beta1.CheckLagoonVersion(&lagoonBuild, "2.12.0") {
r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusQueued, fmt.Sprintf("queued %v/%v", queuePosition, queueLength))
r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusQueued, fmt.Sprintf("queued %v/%v", queuePosition, queueLength))
r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, allContainerLogs, lagoonv1beta1.BuildStatusQueued)
r.buildStatusLogsToLagoonLogs(opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusQueued, fmt.Sprintf("queued %v/%v", queuePosition, queueLength))
r.updateDeploymentAndEnvironmentTask(opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusQueued, fmt.Sprintf("queued %v/%v", queuePosition, queueLength))
r.buildLogsToLagoonLogs(opLog, &lagoonBuild, allContainerLogs, lagoonv1beta1.BuildStatusQueued)
} else {
r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusPending, fmt.Sprintf("queued %v/%v", queuePosition, queueLength))
r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusPending, fmt.Sprintf("queued %v/%v", queuePosition, queueLength))
r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, allContainerLogs, lagoonv1beta1.BuildStatusPending)
r.buildStatusLogsToLagoonLogs(opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusPending, fmt.Sprintf("queued %v/%v", queuePosition, queueLength))
r.updateDeploymentAndEnvironmentTask(opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusPending, fmt.Sprintf("queued %v/%v", queuePosition, queueLength))
r.buildLogsToLagoonLogs(opLog, &lagoonBuild, allContainerLogs, lagoonv1beta1.BuildStatusPending)

}
return nil
Expand Down Expand Up @@ -955,10 +956,10 @@ Build cancelled
}
// send any messages to lagoon message queues
// update the deployment with the status of cancelled in lagoon
r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusCancelled, "cancelled")
r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusCancelled, "cancelled")
r.buildStatusLogsToLagoonLogs(opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusCancelled, "cancelled")
r.updateDeploymentAndEnvironmentTask(opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusCancelled, "cancelled")
if cancelled {
r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, allContainerLogs, lagoonv1beta1.BuildStatusCancelled)
r.buildLogsToLagoonLogs(opLog, &lagoonBuild, allContainerLogs, lagoonv1beta1.BuildStatusCancelled)
}
// delete the build from the lagoon namespace in kubernetes entirely
err = r.Delete(ctx, &lagoonBuild)
Expand Down
57 changes: 29 additions & 28 deletions controllers/v1beta1/task_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ func (r *LagoonTaskReconciler) Reconcile(ctx context.Context, req ctrl.Request)
// The object is being deleted
if helpers.ContainsString(lagoonTask.ObjectMeta.Finalizers, taskFinalizer) {
// our finalizer is present, so lets handle any external dependency
if err := r.deleteExternalResources(ctx, &lagoonTask, req.NamespacedName.Namespace); err != nil {
if err := r.deleteExternalResources(); err != nil {
// if fail to delete the external dependency here, return with error
// so that it can be retried
return ctrl.Result{}, err
Expand Down Expand Up @@ -130,7 +130,7 @@ func (r *LagoonTaskReconciler) SetupWithManager(mgr ctrl.Manager) error {
Complete(r)
}

func (r *LagoonTaskReconciler) deleteExternalResources(ctx context.Context, lagoonTask *lagoonv1beta1.LagoonTask, namespace string) error {
func (r *LagoonTaskReconciler) deleteExternalResources() error {
// delete any external resources if required
return nil
}
Expand Down Expand Up @@ -266,34 +266,34 @@ func (r *LagoonTaskReconciler) getTaskPodDeployment(ctx context.Context, lagoonT
lagoonTask.Spec.Task.Command,
}
dep.Spec.Template.Spec.RestartPolicy = "Never"
taskPod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: lagoonTask.ObjectMeta.Name,
Namespace: lagoonTask.ObjectMeta.Namespace,
Labels: map[string]string{
"lagoon.sh/jobType": "task",
"lagoon.sh/taskName": lagoonTask.ObjectMeta.Name,
"lagoon.sh/crdVersion": crdVersion,
"lagoon.sh/controller": r.ControllerNamespace,
},
OwnerReferences: []metav1.OwnerReference{
{
APIVersion: fmt.Sprintf("%v", lagoonv1beta1.GroupVersion),
Kind: "LagoonTask",
Name: lagoonTask.ObjectMeta.Name,
UID: lagoonTask.UID,
},
}
taskPod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: lagoonTask.ObjectMeta.Name,
Namespace: lagoonTask.ObjectMeta.Namespace,
Labels: map[string]string{
"lagoon.sh/jobType": "task",
"lagoon.sh/taskName": lagoonTask.ObjectMeta.Name,
"lagoon.sh/crdVersion": crdVersion,
"lagoon.sh/controller": r.ControllerNamespace,
},
OwnerReferences: []metav1.OwnerReference{
{
APIVersion: fmt.Sprintf("%v", lagoonv1beta1.GroupVersion),
Kind: "LagoonTask",
Name: lagoonTask.ObjectMeta.Name,
UID: lagoonTask.UID,
},
},
Spec: dep.Spec.Template.Spec,
}
// set the organization labels on task pods
if lagoonTask.Spec.Project.Organization != nil {
taskPod.ObjectMeta.Labels["organization.lagoon.sh/id"] = fmt.Sprintf("%d", *lagoonTask.Spec.Project.Organization.ID)
taskPod.ObjectMeta.Labels["organization.lagoon.sh/name"] = lagoonTask.Spec.Project.Organization.Name
}
return taskPod, nil
},
Spec: dep.Spec.Template.Spec,
}
// set the organization labels on task pods
if lagoonTask.Spec.Project.Organization != nil {
taskPod.ObjectMeta.Labels["organization.lagoon.sh/id"] = fmt.Sprintf("%d", *lagoonTask.Spec.Project.Organization.ID)
taskPod.ObjectMeta.Labels["organization.lagoon.sh/name"] = lagoonTask.Spec.Project.Organization.Name
}
return taskPod, nil
}
}
if !hasService {
Expand Down Expand Up @@ -425,8 +425,9 @@ func (r *LagoonTaskReconciler) createAdvancedTask(ctx context.Context, lagoonTas
return err
}
var serviceaccountTokenSecret string
reg, _ := regexp.Compile("^lagoon-deployer-token")
for _, secret := range serviceAccount.Secrets {
match, _ := regexp.MatchString("^lagoon-deployer-token", secret.Name)
match := reg.MatchString(secret.Name)
if match {
serviceaccountTokenSecret = secret.Name
break
Expand Down
2 changes: 1 addition & 1 deletion controllers/v1beta2/build_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -307,7 +307,7 @@ func (r *LagoonBuildReconciler) createNamespaceBuild(ctx context.Context,
continue
}
// send the status change to lagoon
r.updateDeploymentAndEnvironmentTask(ctx, opLog, runningBuild, nil, buildCondition, "cancelled")
r.updateDeploymentAndEnvironmentTask(opLog, runningBuild, nil, buildCondition, "cancelled")
continue
}
// handle processing running but no pod/failed pod builds
Expand Down
12 changes: 6 additions & 6 deletions controllers/v1beta2/build_deletionhandlers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -212,16 +212,16 @@ Build cancelled
}
// send any messages to lagoon message queues
// update the deployment with the status of cancelled in lagoon
r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &lagoonEnv, lagooncrd.BuildStatusCancelled, "cancelled")
r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagooncrd.BuildStatusCancelled, "cancelled")
r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, allContainerLogs, lagooncrd.BuildStatusCancelled)
r.buildStatusLogsToLagoonLogs(opLog, &lagoonBuild, &lagoonEnv, lagooncrd.BuildStatusCancelled, "cancelled")
r.updateDeploymentAndEnvironmentTask(opLog, &lagoonBuild, &lagoonEnv, lagooncrd.BuildStatusCancelled, "cancelled")
r.buildLogsToLagoonLogs(opLog, &lagoonBuild, allContainerLogs, lagooncrd.BuildStatusCancelled)
}
return nil
}

// buildLogsToLagoonLogs sends the build logs to the lagoon-logs message queue
// it contains the actual pod log output that is sent to elasticsearch, it is what eventually is displayed in the UI
func (r *LagoonBuildReconciler) buildLogsToLagoonLogs(ctx context.Context,
func (r *LagoonBuildReconciler) buildLogsToLagoonLogs(
opLog logr.Logger,
lagoonBuild *lagooncrd.LagoonBuild,
logs []byte,
Expand Down Expand Up @@ -266,7 +266,7 @@ func (r *LagoonBuildReconciler) buildLogsToLagoonLogs(ctx context.Context,

// updateDeploymentAndEnvironmentTask sends the status of the build and deployment to the controllerhandler message queue in lagoon,
// this is for the handler in lagoon to process.
func (r *LagoonBuildReconciler) updateDeploymentAndEnvironmentTask(ctx context.Context,
func (r *LagoonBuildReconciler) updateDeploymentAndEnvironmentTask(
opLog logr.Logger,
lagoonBuild *lagooncrd.LagoonBuild,
lagoonEnv *corev1.ConfigMap,
Expand Down Expand Up @@ -360,7 +360,7 @@ func (r *LagoonBuildReconciler) updateDeploymentAndEnvironmentTask(ctx context.C
}

// buildStatusLogsToLagoonLogs sends the logs to lagoon-logs message queue, used for general messaging
func (r *LagoonBuildReconciler) buildStatusLogsToLagoonLogs(ctx context.Context,
func (r *LagoonBuildReconciler) buildStatusLogsToLagoonLogs(
opLog logr.Logger,
lagoonBuild *lagooncrd.LagoonBuild,
lagoonEnv *corev1.ConfigMap,
Expand Down
23 changes: 12 additions & 11 deletions controllers/v1beta2/build_helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -212,7 +212,7 @@ func (r *LagoonBuildReconciler) getOrCreateNamespace(ctx context.Context, namesp
return fmt.Errorf("error getting harbor version, check your harbor configuration. Error was: %v", err)
}
if lagoonHarbor.UseV2Functions(curVer) {
hProject, err := lagoonHarbor.CreateProjectV2(ctx, lagoonBuild.Spec.Project.Name)
hProject, err := lagoonHarbor.CreateProjectV2(ctx, *namespace)
if err != nil {
return fmt.Errorf("error creating harbor project: %v", err)
}
Expand Down Expand Up @@ -332,8 +332,9 @@ func (r *LagoonBuildReconciler) processBuild(ctx context.Context, opLog logr.Log
}

var serviceaccountTokenSecret string
reg, _ := regexp.Compile("^lagoon-deployer-token")
for _, secret := range serviceAccount.Secrets {
match, _ := regexp.MatchString("^lagoon-deployer-token", secret.Name)
match := reg.MatchString(secret.Name)
if match {
serviceaccountTokenSecret = secret.Name
break
Expand Down Expand Up @@ -920,13 +921,13 @@ func (r *LagoonBuildReconciler) updateQueuedBuild(
// send any messages to lagoon message queues
// update the deployment with the status, lagoon v2.12.0 supports queued status, otherwise use pending
if lagooncrd.CheckLagoonVersion(&lagoonBuild, "2.12.0") {
r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &lagoonEnv, lagooncrd.BuildStatusQueued, fmt.Sprintf("queued %v/%v", queuePosition, queueLength))
r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagooncrd.BuildStatusQueued, fmt.Sprintf("queued %v/%v", queuePosition, queueLength))
r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, allContainerLogs, lagooncrd.BuildStatusQueued)
r.buildStatusLogsToLagoonLogs(opLog, &lagoonBuild, &lagoonEnv, lagooncrd.BuildStatusQueued, fmt.Sprintf("queued %v/%v", queuePosition, queueLength))
r.updateDeploymentAndEnvironmentTask(opLog, &lagoonBuild, &lagoonEnv, lagooncrd.BuildStatusQueued, fmt.Sprintf("queued %v/%v", queuePosition, queueLength))
r.buildLogsToLagoonLogs(opLog, &lagoonBuild, allContainerLogs, lagooncrd.BuildStatusQueued)
} else {
r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &lagoonEnv, lagooncrd.BuildStatusPending, fmt.Sprintf("queued %v/%v", queuePosition, queueLength))
r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagooncrd.BuildStatusPending, fmt.Sprintf("queued %v/%v", queuePosition, queueLength))
r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, allContainerLogs, lagooncrd.BuildStatusPending)
r.buildStatusLogsToLagoonLogs(opLog, &lagoonBuild, &lagoonEnv, lagooncrd.BuildStatusPending, fmt.Sprintf("queued %v/%v", queuePosition, queueLength))
r.updateDeploymentAndEnvironmentTask(opLog, &lagoonBuild, &lagoonEnv, lagooncrd.BuildStatusPending, fmt.Sprintf("queued %v/%v", queuePosition, queueLength))
r.buildLogsToLagoonLogs(opLog, &lagoonBuild, allContainerLogs, lagooncrd.BuildStatusPending)

}
return nil
Expand Down Expand Up @@ -979,10 +980,10 @@ Build cancelled
}
// send any messages to lagoon message queues
// update the deployment with the status of cancelled in lagoon
r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &lagoonEnv, lagooncrd.BuildStatusCancelled, "cancelled")
r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagooncrd.BuildStatusCancelled, "cancelled")
r.buildStatusLogsToLagoonLogs(opLog, &lagoonBuild, &lagoonEnv, lagooncrd.BuildStatusCancelled, "cancelled")
r.updateDeploymentAndEnvironmentTask(opLog, &lagoonBuild, &lagoonEnv, lagooncrd.BuildStatusCancelled, "cancelled")
if cancelled {
r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, allContainerLogs, lagooncrd.BuildStatusCancelled)
r.buildLogsToLagoonLogs(opLog, &lagoonBuild, allContainerLogs, lagooncrd.BuildStatusCancelled)
}
// delete the build from the lagoon namespace in kubernetes entirely
err = r.Delete(ctx, &lagoonBuild)
Expand Down
Loading

0 comments on commit 30e0986

Please sign in to comment.