From 9f071d5e7b19fbfb3aafcadb2b66ba263740255f Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Mon, 25 Nov 2024 07:55:13 +1100 Subject: [PATCH] chore: backport podconfig to pr-379 --- cmd/template_backups.go | 9 ++ cmd/template_backups_test.go | 25 +++- .../templating/backups/template_podconfig.go | 131 ++++++++++++++++++ .../backups/template_podconfig_test.go | 103 ++++++++++++++ .../backups/template_prebackuppod.go | 20 ++- .../templating/backups/template_schedule.go | 5 + .../test-resources/result-podconfig1.yaml | 28 ++++ .../k8up-lagoon-backup-schedule.yaml | 44 ++++++ .../k8up-rootless-workload-podconfig.yaml | 28 ++++ 9 files changed, 380 insertions(+), 13 deletions(-) create mode 100644 internal/templating/backups/template_podconfig.go create mode 100644 internal/templating/backups/template_podconfig_test.go create mode 100644 internal/templating/backups/test-resources/result-podconfig1.yaml create mode 100644 internal/testdata/node/backup-templates/test-generic-backup-rootless-workloads/k8up-lagoon-backup-schedule.yaml create mode 100644 internal/testdata/node/backup-templates/test-generic-backup-rootless-workloads/k8up-rootless-workload-podconfig.yaml diff --git a/cmd/template_backups.go b/cmd/template_backups.go index 33ff8ef9..dbd3cf17 100644 --- a/cmd/template_backups.go +++ b/cmd/template_backups.go @@ -77,6 +77,15 @@ func BackupTemplateGeneration(g generator.GeneratorInput, helpers.WriteTemplateFile(fmt.Sprintf("%s/%s.yaml", savedTemplates, "k8up-lagoon-backup-schedule"), templateYAML) } + // generate the backup schedule templates + templateYAML, err = backuptemplate.GenerateBackupPodConfig(*lagoonBuild.BuildValues) + if err != nil { + return fmt.Errorf("couldn't generate template: %v", err) + } + if len(templateYAML) > 0 { + helpers.WriteTemplateFile(fmt.Sprintf("%s/%s.yaml", savedTemplates, "k8up-rootless-workload-podconfig"), templateYAML) + } + // generate any prebackuppod templates templateYAML, err = backuptemplate.GeneratePreBackupPod(*lagoonBuild.BuildValues) if err != nil { diff --git a/cmd/template_backups_test.go b/cmd/template_backups_test.go index 7fb56aef..5693f498 100644 --- a/cmd/template_backups_test.go +++ b/cmd/template_backups_test.go @@ -6,6 +6,7 @@ import ( "reflect" "testing" + "github.com/andreyvit/diff" "github.com/uselagoon/build-deploy-tool/internal/dbaasclient" "github.com/uselagoon/build-deploy-tool/internal/helpers" "github.com/uselagoon/build-deploy-tool/internal/lagoon" @@ -240,6 +241,27 @@ func TestBackupTemplateGeneration(t *testing.T) { templatePath: "testdata/output", want: "internal/testdata/node/backup-templates/backup-8", }, + { + name: "test-generic-backup-rootless-workloads", + args: testdata.GetSeedData( + testdata.TestData{ + ProjectName: "example-project", + EnvironmentName: "main", + Branch: "main", + EnvironmentType: "production", + LagoonYAML: "internal/testdata/node/lagoon.yml", + K8UPVersion: "v2", + ProjectVariables: []lagoon.EnvironmentVariable{ + { + Name: "LAGOON_FEATURE_FLAG_ROOTLESS_WORKLOAD", + Value: "enabled", + Scope: "build", + }, + }, + }, true), + templatePath: "testoutput", + want: "internal/testdata/node/backup-templates/test-generic-backup-rootless-workloads", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -307,8 +329,7 @@ func TestBackupTemplateGeneration(t *testing.T) { t.Errorf("couldn't read file %v: %v", tt.want, err) } if !reflect.DeepEqual(f1, r1) { - fmt.Println(string(f1)) - t.Errorf("resulting templates do not match") + t.Errorf("TemplateLagoonServices() = \n%v", diff.LineDiff(string(r1), string(f1))) } } } diff --git a/internal/templating/backups/template_podconfig.go b/internal/templating/backups/template_podconfig.go new file mode 100644 index 00000000..9edd95f8 --- /dev/null +++ b/internal/templating/backups/template_podconfig.go @@ -0,0 +1,131 @@ +package backups + +import ( + "fmt" + + "github.com/uselagoon/build-deploy-tool/internal/generator" + "github.com/uselagoon/build-deploy-tool/internal/helpers" + + k8upv1 "github.com/k8up-io/k8up/v2/api/v1" + corev1 "k8s.io/api/core/v1" + apivalidation "k8s.io/apimachinery/pkg/api/validation" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + metavalidation "k8s.io/apimachinery/pkg/apis/meta/v1/validation" + + "sigs.k8s.io/yaml" +) + +func GenerateBackupPodConfig( + lValues generator.BuildValues, +) ([]byte, error) { + // generate the template spec + + var result []byte + separator := []byte("---\n") + + // create the podconfig + if lValues.BackupsEnabled { + switch lValues.Backup.K8upVersion { + case "v2": + if lValues.PodSecurityContext.RunAsUser != 0 { + podConfig := &k8upv1.PodConfig{ + TypeMeta: metav1.TypeMeta{ + Kind: "PodConfig", + APIVersion: k8upv1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "k8up-rootless-workload-podconfig", + }, + Spec: k8upv1.PodConfigSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + SecurityContext: &corev1.PodSecurityContext{ + RunAsUser: helpers.Int64Ptr(lValues.PodSecurityContext.RunAsUser), + RunAsGroup: helpers.Int64Ptr(lValues.PodSecurityContext.RunAsGroup), + FSGroup: helpers.Int64Ptr(lValues.PodSecurityContext.FsGroup), + }, + }, + }, + }, + } + // add the default labels + podConfig.ObjectMeta.Labels = map[string]string{ + "app.kubernetes.io/name": "k8up-podconfig", + "app.kubernetes.io/instance": "k8up-rootless-workload-podconfig", + "app.kubernetes.io/managed-by": "build-deploy-tool", + "lagoon.sh/template": fmt.Sprintf("%s-%s", "k8up-podconfig", "0.1.0"), + "lagoon.sh/service": "k8up-rootless-workload-podconfig", + "lagoon.sh/service-type": "k8up-podconfig", + "lagoon.sh/project": lValues.Project, + "lagoon.sh/environment": lValues.Environment, + "lagoon.sh/environmentType": lValues.EnvironmentType, + "lagoon.sh/buildType": lValues.BuildType, + } + + // add the default annotations + podConfig.ObjectMeta.Annotations = map[string]string{ + "lagoon.sh/version": lValues.LagoonVersion, + } + + // add any additional labels + additionalLabels := map[string]string{} + additionalAnnotations := map[string]string{} + if lValues.BuildType == "branch" { + additionalAnnotations["lagoon.sh/branch"] = lValues.Branch + } else if lValues.BuildType == "pullrequest" { + additionalAnnotations["lagoon.sh/prNumber"] = lValues.PRNumber + additionalAnnotations["lagoon.sh/prHeadBranch"] = lValues.PRHeadBranch + additionalAnnotations["lagoon.sh/prBaseBranch"] = lValues.PRBaseBranch + + } + for key, value := range additionalLabels { + podConfig.ObjectMeta.Labels[key] = value + } + // add any additional annotations + for key, value := range additionalAnnotations { + podConfig.ObjectMeta.Annotations[key] = value + } + // validate any annotations + if err := apivalidation.ValidateAnnotations(podConfig.ObjectMeta.Annotations, nil); err != nil { + if len(err) != 0 { + return nil, fmt.Errorf("the annotations for %s are not valid: %v", "k8up-rootless-workload-podconfig", err) + } + } + // validate any labels + if err := metavalidation.ValidateLabels(podConfig.ObjectMeta.Labels, nil); err != nil { + if len(err) != 0 { + return nil, fmt.Errorf("the labels for %s are not valid: %v", "k8up-rootless-workload-podconfig", err) + } + } + + // check length of labels + err := helpers.CheckLabelLength(podConfig.ObjectMeta.Labels) + if err != nil { + return nil, err + } + // @TODO: we should review this in the future when we stop doing `kubectl apply` in the builds :) + // marshal the resulting ingress + podconfigBytes, err := yaml.Marshal(podConfig) + if err != nil { + return nil, err + } + podconfigBytes, _ = CleanupPodConfigYAML(podconfigBytes) + // add the seperator to the template so that it can be `kubectl apply` in bulk as part + // of the current build process + result = append(separator[:], podconfigBytes[:]...) + } + } + } + return result, nil +} + +// helper function to remove data from the yaml spec so that kubectl will apply without validation errors +// this is only needed because we use kubectl in builds for now +func CleanupPodConfigYAML(a []byte) ([]byte, error) { + tmpMap := map[string]interface{}{} + yaml.Unmarshal(a, &tmpMap) + delete(tmpMap["spec"].(map[string]interface{})["template"].(map[string]interface{}), "metadata") + delete(tmpMap["spec"].(map[string]interface{})["template"].(map[string]interface{})["spec"].(map[string]interface{}), "containers") + b, _ := yaml.Marshal(tmpMap) + return b, nil +} diff --git a/internal/templating/backups/template_podconfig_test.go b/internal/templating/backups/template_podconfig_test.go new file mode 100644 index 00000000..c0009071 --- /dev/null +++ b/internal/templating/backups/template_podconfig_test.go @@ -0,0 +1,103 @@ +package backups + +import ( + "os" + "reflect" + "testing" + "time" + + "github.com/andreyvit/diff" + "github.com/uselagoon/build-deploy-tool/internal/dbaasclient" + "github.com/uselagoon/build-deploy-tool/internal/generator" +) + +func TestGenerateBackupPodConfig(t *testing.T) { + type args struct { + lValues generator.BuildValues + } + tests := []struct { + name string + description string + args args + want string + wantErr bool + wantEmpty bool + }{ + { + name: "test-k8up-v1-rootless", + description: "this will generate a podconfig if the environment is configured for rootless workloads", + args: args{ + lValues: generator.BuildValues{ + Project: "example-project", + Environment: "environment", + EnvironmentType: "production", + Namespace: "myexample-project-environment", + BuildType: "branch", + LagoonVersion: "v2.x.x", + Kubernetes: "generator.local", + Branch: "environment", + BackupsEnabled: true, + Backup: generator.BackupConfiguration{ + K8upVersion: "v2", + }, + FeatureFlags: map[string]bool{ + "rootlessworkloads": true, + }, + PodSecurityContext: generator.PodSecurityContext{ + RunAsGroup: 0, + RunAsUser: 10000, + FsGroup: 10001, + }, + }, + }, + want: "test-resources/result-podconfig1.yaml", + }, + { + name: "test-k8up-v1-root", + description: "this will not generate a podconfig if the environment is not configured for rootless workloads", + args: args{ + lValues: generator.BuildValues{ + Project: "example-project", + Environment: "environment", + EnvironmentType: "production", + Namespace: "myexample-project-environment", + BuildType: "branch", + LagoonVersion: "v2.x.x", + Kubernetes: "generator.local", + Branch: "environment", + BackupsEnabled: true, + Backup: generator.BackupConfiguration{ + K8upVersion: "v2", + }, + }, + }, + wantEmpty: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // add dbaasclient overrides for tests + tt.args.lValues.DBaaSClient = dbaasclient.NewClient(dbaasclient.Client{ + RetryMax: 5, + RetryWaitMin: time.Duration(10) * time.Millisecond, + RetryWaitMax: time.Duration(50) * time.Millisecond, + }) + got, err := GenerateBackupPodConfig(tt.args.lValues) + if err != nil { + t.Errorf("couldn't generate template %v: %v", tt.want, err) + } + if tt.wantEmpty && len(got) > 0 { + t.Errorf("wanted empty, but got data:\n%v", string(got)) + } + if !tt.wantEmpty { + r1, err := os.ReadFile(tt.want) + if err != nil { + t.Errorf("couldn't read file %v: %v", tt.want, err) + } + if !reflect.DeepEqual(string(got), string(r1)) { + t.Errorf("GenerateBackupPodConfig() = \n%v", diff.LineDiff(string(r1), string(got))) + } + } + }) + } +} diff --git a/internal/templating/backups/template_prebackuppod.go b/internal/templating/backups/template_prebackuppod.go index 43ce0c9c..302c99e9 100644 --- a/internal/templating/backups/template_prebackuppod.go +++ b/internal/templating/backups/template_prebackuppod.go @@ -149,7 +149,7 @@ func GeneratePreBackupPod( return nil, err } - pbpBytes, _ := RemoveYAML(prebackuppodBytes) + pbpBytes, _ := CleanupPreBackupPodYAML(prebackuppodBytes) // add the seperator to the template so that it can be `kubectl apply` in bulk as part // of the current build process restoreResult := append(separator[:], pbpBytes[:]...) @@ -238,7 +238,7 @@ func GeneratePreBackupPod( if err != nil { return nil, err } - pbpBytes, _ := RemoveYAML(prebackuppodBytes) + pbpBytes, _ := CleanupPreBackupPodYAML(prebackuppodBytes) // add the seperator to the template so that it can be `kubectl apply` in bulk as part // of the current build process restoreResult := append(separator[:], pbpBytes[:]...) @@ -249,16 +249,14 @@ func GeneratePreBackupPod( return result, nil } -// helper function to remove the creationtimestamp from the prebackuppod pod spec so that kubectl will apply without validation errors -func RemoveYAML(a []byte) ([]byte, error) { +// helper function to remove data from the yaml spec so that kubectl will apply without validation errors +// this is only needed because we use kubectl in builds for now +func CleanupPreBackupPodYAML(a []byte) ([]byte, error) { tmpMap := map[string]interface{}{} yaml.Unmarshal(a, &tmpMap) - if _, ok := tmpMap["spec"].(map[string]interface{})["pod"].(map[string]interface{})["metadata"].(map[string]interface{})["creationTimestamp"]; ok { - delete(tmpMap["spec"].(map[string]interface{})["pod"].(map[string]interface{})["metadata"].(map[string]interface{}), "creationTimestamp") - b, _ := yaml.Marshal(tmpMap) - return b, nil - } - return a, nil + delete(tmpMap["spec"].(map[string]interface{})["pod"].(map[string]interface{})["metadata"].(map[string]interface{}), "creationTimestamp") + b, _ := yaml.Marshal(tmpMap) + return b, nil } var funcMap = template.FuncMap{ @@ -267,7 +265,7 @@ var funcMap = template.FuncMap{ // varfix just uppercases and replaces - with _ for variable names func varFix(s string) string { - return fmt.Sprintf("%s", strings.ToUpper(strings.Replace(s, "-", "_", -1))) + return strings.ToUpper(strings.Replace(s, "-", "_", -1)) } // this is just the first run at doing this, once the service template generator is introduced, this will need to be re-evaluated diff --git a/internal/templating/backups/template_schedule.go b/internal/templating/backups/template_schedule.go index d37bfd75..df2ea7e6 100644 --- a/internal/templating/backups/template_schedule.go +++ b/internal/templating/backups/template_schedule.go @@ -221,6 +221,11 @@ func GenerateBackupSchedule( }, }, } + if lValues.PodSecurityContext.RunAsUser != 0 { + schedule.Spec.Backup.PodConfigRef = &corev1.LocalObjectReference{ + Name: "k8up-rootless-workload-podconfig", + } + } // add the default labels schedule.ObjectMeta.Labels = map[string]string{ "app.kubernetes.io/name": "k8up-schedule", diff --git a/internal/templating/backups/test-resources/result-podconfig1.yaml b/internal/templating/backups/test-resources/result-podconfig1.yaml new file mode 100644 index 00000000..6d5fa242 --- /dev/null +++ b/internal/templating/backups/test-resources/result-podconfig1.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: k8up.io/v1 +kind: PodConfig +metadata: + annotations: + lagoon.sh/branch: environment + lagoon.sh/version: v2.x.x + creationTimestamp: null + labels: + app.kubernetes.io/instance: k8up-rootless-workload-podconfig + app.kubernetes.io/managed-by: build-deploy-tool + app.kubernetes.io/name: k8up-podconfig + lagoon.sh/buildType: branch + lagoon.sh/environment: environment + lagoon.sh/environmentType: production + lagoon.sh/project: example-project + lagoon.sh/service: k8up-rootless-workload-podconfig + lagoon.sh/service-type: k8up-podconfig + lagoon.sh/template: k8up-podconfig-0.1.0 + name: k8up-rootless-workload-podconfig +spec: + template: + spec: + securityContext: + fsGroup: 10001 + runAsGroup: 0 + runAsUser: 10000 +status: {} diff --git a/internal/testdata/node/backup-templates/test-generic-backup-rootless-workloads/k8up-lagoon-backup-schedule.yaml b/internal/testdata/node/backup-templates/test-generic-backup-rootless-workloads/k8up-lagoon-backup-schedule.yaml new file mode 100644 index 00000000..1f7e83bb --- /dev/null +++ b/internal/testdata/node/backup-templates/test-generic-backup-rootless-workloads/k8up-lagoon-backup-schedule.yaml @@ -0,0 +1,44 @@ +--- +apiVersion: k8up.io/v1 +kind: Schedule +metadata: + annotations: + lagoon.sh/branch: main + lagoon.sh/version: v2.7.x + creationTimestamp: null + labels: + app.kubernetes.io/instance: k8up-lagoon-backup-schedule + app.kubernetes.io/managed-by: build-deploy-tool + app.kubernetes.io/name: k8up-schedule + lagoon.sh/buildType: branch + lagoon.sh/environment: main + lagoon.sh/environmentType: production + lagoon.sh/project: example-project + lagoon.sh/service: k8up-lagoon-backup-schedule + lagoon.sh/service-type: k8up-schedule + lagoon.sh/template: k8up-schedule-0.1.0 + name: k8up-lagoon-backup-schedule +spec: + backend: + repoPasswordSecretRef: + key: repo-pw + name: baas-repo-pw + s3: + bucket: baas-example-project + backup: + podConfigRef: + name: k8up-rootless-workload-podconfig + resources: {} + schedule: 48 22 * * * + check: + resources: {} + schedule: 48 5 * * 1 + prune: + resources: {} + retention: + keepDaily: 7 + keepMonthly: 1 + keepWeekly: 6 + schedule: 48 3 * * 0 + resourceRequirementsTemplate: {} +status: {} diff --git a/internal/testdata/node/backup-templates/test-generic-backup-rootless-workloads/k8up-rootless-workload-podconfig.yaml b/internal/testdata/node/backup-templates/test-generic-backup-rootless-workloads/k8up-rootless-workload-podconfig.yaml new file mode 100644 index 00000000..87286c6b --- /dev/null +++ b/internal/testdata/node/backup-templates/test-generic-backup-rootless-workloads/k8up-rootless-workload-podconfig.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: k8up.io/v1 +kind: PodConfig +metadata: + annotations: + lagoon.sh/branch: main + lagoon.sh/version: v2.7.x + creationTimestamp: null + labels: + app.kubernetes.io/instance: k8up-rootless-workload-podconfig + app.kubernetes.io/managed-by: build-deploy-tool + app.kubernetes.io/name: k8up-podconfig + lagoon.sh/buildType: branch + lagoon.sh/environment: main + lagoon.sh/environmentType: production + lagoon.sh/project: example-project + lagoon.sh/service: k8up-rootless-workload-podconfig + lagoon.sh/service-type: k8up-podconfig + lagoon.sh/template: k8up-podconfig-0.1.0 + name: k8up-rootless-workload-podconfig +spec: + template: + spec: + securityContext: + fsGroup: 10001 + runAsGroup: 0 + runAsUser: 10000 +status: {}