From 9b7078cebced4ec75aa4305ecfb11d4cdfb1ce6a Mon Sep 17 00:00:00 2001 From: Ben Jackson Date: Sun, 17 Dec 2023 09:37:35 +1100 Subject: [PATCH] feat: enable retention policies --- controllers/v1beta1/build_helpers.go | 2 +- go.mod | 1 + go.sum | 2 + internal/harbor/harbor.go | 8 + internal/harbor/harbor22x.go | 66 +++- internal/harbor/harbor_credentialrotation.go | 2 +- internal/harbor/harbor_helpers.go | 93 ++++++ internal/harbor/harbor_helpers_test.go | 56 ++++ internal/helpers/helpers_cron.go | 331 +++++++++++++++++++ internal/helpers/helpers_cron_test.go | 236 +++++++++++++ main.go | 33 ++ 11 files changed, 827 insertions(+), 3 deletions(-) create mode 100644 internal/helpers/helpers_cron.go create mode 100644 internal/helpers/helpers_cron_test.go diff --git a/controllers/v1beta1/build_helpers.go b/controllers/v1beta1/build_helpers.go index 81831ab1..5ddd140a 100644 --- a/controllers/v1beta1/build_helpers.go +++ b/controllers/v1beta1/build_helpers.go @@ -212,7 +212,7 @@ func (r *LagoonBuildReconciler) getOrCreateNamespace(ctx context.Context, namesp return fmt.Errorf("Error getting harbor version, check your harbor configuration. Error was: %v", err) } if lagoonHarbor.UseV2Functions(curVer) { - hProject, err := lagoonHarbor.CreateProjectV2(ctx, lagoonBuild.Spec.Project.Name) + hProject, err := lagoonHarbor.CreateProjectV2(ctx, *namespace) if err != nil { return fmt.Errorf("Error creating harbor project: %v", err) } diff --git a/go.mod b/go.mod index 88770eda..0ddf3766 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.20 require ( github.com/cheshir/go-mq/v2 v2.0.1 github.com/coreos/go-semver v0.3.1 + github.com/cxmcc/unixsums v0.0.0-20131125091133-89564297d82f github.com/go-logr/logr v1.2.4 github.com/google/go-cmp v0.5.9 github.com/hashicorp/go-version v1.6.0 diff --git a/go.sum b/go.sum index 53473ad0..9e9ffa35 100644 --- a/go.sum +++ b/go.sum @@ -334,6 +334,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cxmcc/unixsums v0.0.0-20131125091133-89564297d82f h1:PkAFGgVtJnasAxOaiEY1RYPx8W+7X7l66vi8T2apKCM= +github.com/cxmcc/unixsums v0.0.0-20131125091133-89564297d82f/go.mod h1:XJq7OckzkOtlgeEKFwkH2gFbc1+1WRFUBf7QnvfyrzQ= github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ= diff --git a/internal/harbor/harbor.go b/internal/harbor/harbor.go index d2e698d3..834b526d 100644 --- a/internal/harbor/harbor.go +++ b/internal/harbor/harbor.go @@ -35,6 +35,14 @@ type Harbor struct { WebhookEventTypes []string LagoonTargetName string Config *config.Options + TagRetention TagRetention +} + +type TagRetention struct { + Enabled bool `json:"enabled"` + BranchRetention int `json:"branchRetention"` + PullRequestRetention int `json:"pullrequestRetention"` + Schedule string `json:"schedule"` } // New create a new harbor connection. diff --git a/internal/harbor/harbor22x.go b/internal/harbor/harbor22x.go index bea48adb..b527d4d3 100644 --- a/internal/harbor/harbor22x.go +++ b/internal/harbor/harbor22x.go @@ -16,7 +16,8 @@ import ( ) // CreateProjectV2 will create a project if one doesn't exist, but will update as required. -func (h *Harbor) CreateProjectV2(ctx context.Context, projectName string) (*harborclientv5model.Project, error) { +func (h *Harbor) CreateProjectV2(ctx context.Context, namespace corev1.Namespace) (*harborclientv5model.Project, error) { + projectName := namespace.Labels["lagoon.sh/project"] exists, err := h.ClientV5.ProjectExists(ctx, projectName) if err != nil { h.Log.Info(fmt.Sprintf("Error checking project %s exists, err: %v", projectName, err)) @@ -64,6 +65,45 @@ func (h *Harbor) CreateProjectV2(ctx context.Context, projectName string) (*harb // fmt.Println(x) // } + // get the retention policy from the namespace annotations if the annotation exists + projectRetention := h.retentionOverrides(namespace) + // handle the creation and updating of retention policies as required + // generate a somewhat random schedule from the retention schedule template, using the harbor projectname as the seed + schedule, err := helpers.ConvertCrontab(projectName, projectRetention.Schedule) + if err != nil { + h.Log.Info(fmt.Sprintf("Error generating retention schedule %s: %v", project.Name, err)) + } + schedule = fmt.Sprintf("0 %s", schedule) // harbor needs seconds :\ + // create the retention policy as required + retentionPolicy := h.generateEmptyRetentionPolicy(int64(project.ProjectID)) + if projectRetention.Enabled { + // if a retention policy is enabled, configure it here + retentionPolicy = h.generateRetentionPolicy(int64(project.ProjectID), projectRetention) + } + // get the existing one if one exists + existingPolicy, err := h.ClientV5.GetRetentionPolicyByProject(ctx, projectName) + if err != nil { + h.Log.Info(fmt.Sprintf("Error getting retention policy %s: %v", project.Name, err)) + } + if existingPolicy != nil { + retentionPolicy.ID = existingPolicy.ID + r1, _ := json.Marshal(existingPolicy) + r2, _ := json.Marshal(retentionPolicy) + // if the policy differs, then we need to update it with our new policy + if string(r1) != string(r2) { + err := h.ClientV5.UpdateRetentionPolicy(ctx, retentionPolicy) + if err != nil { + f, _ := json.Marshal(err) + h.Log.Info(fmt.Sprintf("Error updating retention policy %s: %v", project.Name, string(f))) + } + } + } else { + // create it if it doesn't + if err := h.ClientV5.NewRetentionPolicy(ctx, retentionPolicy); err != nil { + h.Log.Info(fmt.Sprintf("Error creating retention policy %s: %v", project.Name, err)) + } + } + if h.WebhookAddition { wps, err := h.ClientV5.ListProjectWebhookPolicies(ctx, int(project.ProjectID)) if err != nil { @@ -278,6 +318,14 @@ func (h *Harbor) DeleteRepository(ctx context.Context, projectName, branch strin if err != nil { h.Log.Info(fmt.Sprintf("Error deleting harbor repository %s", repo.Name)) } + h.Log.Info( + fmt.Sprintf( + "Deleted harbor repository %s in project %s, environment %s", + repo.Name, + projectName, + environmentName, + ), + ) } } if len(listRepositories) > 100 { @@ -293,6 +341,14 @@ func (h *Harbor) DeleteRepository(ctx context.Context, projectName, branch strin if err != nil { h.Log.Info(fmt.Sprintf("Error deleting harbor repository %s", repo.Name)) } + h.Log.Info( + fmt.Sprintf( + "Deleted harbor repository %s in project %s, environment %s", + repo.Name, + projectName, + environmentName, + ), + ) } } } @@ -321,6 +377,14 @@ func (h *Harbor) DeleteRobotAccount(ctx context.Context, projectName, branch str h.Log.Info(fmt.Sprintf("Error deleting project %s robot account %s", projectName, robot.Name)) return } + h.Log.Info( + fmt.Sprintf( + "Deleted harbor robot account %s in project %s, environment %s", + robot.Name, + projectName, + environmentName, + ), + ) } } } diff --git a/internal/harbor/harbor_credentialrotation.go b/internal/harbor/harbor_credentialrotation.go index 5256e316..1acb2f05 100644 --- a/internal/harbor/harbor_credentialrotation.go +++ b/internal/harbor/harbor_credentialrotation.go @@ -94,7 +94,7 @@ func (h *Harbor) RotateRobotCredential(ctx context.Context, cl client.Client, ns return false, fmt.Errorf("error checking harbor version: %v", err) } if h.UseV2Functions(curVer) { - hProject, err := h.CreateProjectV2(ctx, ns.Labels["lagoon.sh/project"]) + hProject, err := h.CreateProjectV2(ctx, ns) if err != nil { return false, fmt.Errorf("error getting or creating project: %v", err) } diff --git a/internal/harbor/harbor_helpers.go b/internal/harbor/harbor_helpers.go index 0acd1287..8cfade53 100644 --- a/internal/harbor/harbor_helpers.go +++ b/internal/harbor/harbor_helpers.go @@ -10,6 +10,7 @@ import ( "encoding/json" "time" + harborclientv5model "github.com/mittwald/goharbor-client/v5/apiv2/model" "github.com/uselagoon/remote-controller/internal/helpers" corev1 "k8s.io/api/core/v1" @@ -204,3 +205,95 @@ func (h *Harbor) UpsertHarborSecret(ctx context.Context, cl client.Client, ns, n } return false, nil } + +func (h *Harbor) generateRetentionPolicy(projectID int64, policy TagRetention) *harborclientv5model.RetentionPolicy { + return &harborclientv5model.RetentionPolicy{ + Algorithm: "or", + Rules: []*harborclientv5model.RetentionRule{ + { // create a retention policy for all images + Action: "retain", + Params: map[string]interface{}{ + "latestPulledN": policy.BranchRetention, + }, + ScopeSelectors: map[string][]harborclientv5model.RetentionSelector{ + "repository": { + { + Decoration: "repoMatches", + Kind: "doublestar", + Pattern: "[^pr\\-]*/*", // exclude pullrequest repository images https://github.com/bmatcuk/doublestar#patterns + }, + }, + }, + TagSelectors: []*harborclientv5model.RetentionSelector{ + { + Decoration: "matches", + Extras: "{\"untagged\":true}", + Kind: "doublestar", + Pattern: "**", + }, + }, + Template: "latestPulledN", + }, + { // create a retention policy specifically for pullrequests + Action: "retain", + Params: map[string]interface{}{ + "latestPulledN": policy.BranchRetention, + }, + ScopeSelectors: map[string][]harborclientv5model.RetentionSelector{ + "repository": { + { + Decoration: "repoMatches", + Kind: "doublestar", + Pattern: "pr-*", + }, + }, + }, + TagSelectors: []*harborclientv5model.RetentionSelector{ + { + Decoration: "matches", + Extras: "{\"untagged\":true}", + Kind: "doublestar", + Pattern: "**", + }, + }, + Template: "latestPulledN", + }, + }, + Scope: &harborclientv5model.RetentionPolicyScope{ + Level: "project", + Ref: projectID, + }, + Trigger: &harborclientv5model.RetentionRuleTrigger{ + Kind: "Schedule", + Settings: map[string]string{ + "cron": policy.Schedule, + }, + }, + } +} + +func (h *Harbor) generateEmptyRetentionPolicy(projectID int64) *harborclientv5model.RetentionPolicy { + return &harborclientv5model.RetentionPolicy{ + Algorithm: "or", + Rules: []*harborclientv5model.RetentionRule{}, + Scope: &harborclientv5model.RetentionPolicyScope{ + Level: "project", + Ref: projectID, + }, + Trigger: &harborclientv5model.RetentionRuleTrigger{ + Kind: "Schedule", + Settings: map[string]string{ + "cron": "", + }, + }} +} + +func (h *Harbor) retentionOverrides(namespace corev1.Namespace) TagRetention { + // set the default retention policy + retention := h.TagRetention + // check if the annotation is set on the namespace + if ret, ok := namespace.ObjectMeta.Annotations["harbor.lagoon.sh/retention-policy"]; ok { + json.Unmarshal([]byte(ret), &retention) + } + return retention +} diff --git a/internal/harbor/harbor_helpers_test.go b/internal/harbor/harbor_helpers_test.go index 6a1b426f..6c3af52f 100644 --- a/internal/harbor/harbor_helpers_test.go +++ b/internal/harbor/harbor_helpers_test.go @@ -1,7 +1,11 @@ package harbor import ( + "reflect" "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestHarbor_matchRobotAccount(t *testing.T) { @@ -198,3 +202,55 @@ func TestHarbor_generateRobotName(t *testing.T) { }) } } + +func TestHarbor_retentionOverrides(t *testing.T) { + type fields struct { + TagRetention TagRetention + } + type args struct { + namespace corev1.Namespace + } + tests := []struct { + name string + fields fields + args args + want TagRetention + }{ + { + name: "test1", + fields: fields{ + TagRetention: TagRetention{ + Enabled: false, + Schedule: "M H(22-2) D(5-25) * *", + PullRequestRetention: 2, + BranchRetention: 5, + }, + }, + args: args{ + namespace: corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "harbor.lagoon.sh/retention-policy": "{\"enabled\":true,\"schedule\":\"M H(2-15) D(5-15) * *\", \"pullrequestRetention\": 3, \"branchRetention\": 6}", + }, + }, + }, + }, + want: TagRetention{ + Enabled: true, + Schedule: "M H(2-15) D(5-15) * *", + PullRequestRetention: 3, + BranchRetention: 6, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + h := &Harbor{ + TagRetention: tt.fields.TagRetention, + } + if got := h.retentionOverrides(tt.args.namespace); !reflect.DeepEqual(got, tt.want) { + t.Errorf("Harbor.retentionOverrides() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/helpers/helpers_cron.go b/internal/helpers/helpers_cron.go new file mode 100644 index 00000000..a7fbb7ba --- /dev/null +++ b/internal/helpers/helpers_cron.go @@ -0,0 +1,331 @@ +package helpers + +import ( + "fmt" + "math" + "regexp" + "strconv" + "strings" + + "github.com/cxmcc/unixsums/cksum" +) + +func ConvertCrontab(seedstring, cron string) (string, error) { + // Seed is used to generate pseudo random numbers. + // The seed is based on the seedstring, so will not change + seed := cksum.Cksum([]byte(fmt.Sprintf("%s\n", seedstring))) + var minutes, hours, days, months, dayweek string + splitCron := strings.Split(cron, " ") + // check the provided cron splits into 5 + if len(splitCron) == 5 { + for idx, val := range splitCron { + if idx == 0 { + match1, _ := regexp.MatchString("^(M|H)$", val) + if match1 { + // If just an `M` or `H` (for backwards compatibility) is defined, we + // generate a pseudo random minute. + minutes = strconv.Itoa(int(math.Mod(float64(seed), 60))) + continue + } + match2, _ := regexp.MatchString("^(M|H|\\*)/([0-5]?[0-9])$", val) + if match2 { + // A Minute like M/15 (or H/15 or */15 for backwards compatibility) is defined, create a list of minutes with a random start + // like 4,19,34,49 or 6,21,36,51 + params := getCaptureBlocks("^(?PM|H|\\*)/(?P[0-5]?[0-9])$", val) + step, err := strconv.Atoi(params["P2"]) + if err != nil { + return "", fmt.Errorf("cron definition '%s' is invalid, unable to determine minutes value", cron) + } + counter := int(math.Mod(float64(seed), float64(step))) + var minutesArr []string + for counter < 60 { + minutesArr = append(minutesArr, fmt.Sprintf("%d", counter)) + counter += step + } + minutes = strings.Join(minutesArr, ",") + continue + } + if isInCSVRange(val, 0, 59) { + // A minute like 0,10,15,30,59 + minutes = val + continue + } + if isInRange(val, 0, 59) { + // A minute like 0-59 + minutes = val + continue + } + if val == "*" { + // otherwise pass the * through + minutes = val + continue + } + // if the value is not valid, return an error with where the issue is + if minutes == "" { + return "", fmt.Errorf("cron definition '%s' is invalid, unable to determine minutes value", cron) + } + } + if idx == 1 { + match1, _ := regexp.MatchString("^H$", val) + if match1 { + // If just an `H` is defined, we generate a pseudo random hour. + hours = strconv.Itoa(int(math.Mod(float64(seed), 24))) + continue + } + match2, _ := regexp.MatchString("^H\\(([01]?[0-9]|2[0-3])-([01]?[0-9]|2[0-3])\\)$", val) + if match2 { + // If H is defined with a given range, example: H(2-4), we generate a random hour between 2-4 + params := getCaptureBlocks("^H\\((?P[01]?[0-9]|2[0-3])-(?P[01]?[0-9]|2[0-3])\\)$", val) + hFrom, err := strconv.Atoi(params["P1"]) + if err != nil { + return "", fmt.Errorf("cron definition '%s' is invalid, unable to determine hours value", cron) + } + hTo, err := strconv.Atoi(params["P2"]) + if err != nil { + return "", fmt.Errorf("cron definition '%s' is invalid, unable to determine hours value", cron) + } + if hFrom < hTo { + // Example: HOUR_FROM: 2, HOUR_TO: 4 + // Calculate the difference between the two hours (in example will be 2) + maxDiff := float64(hTo - hFrom) + // Generate a difference based on the SEED (in example will be 0, 1 or 2) + diff := int(math.Mod(float64(seed), maxDiff)) + // Add the generated difference to the FROM hour (in example will be 2, 3 or 4) + hours = strconv.Itoa(hFrom + diff) + continue + } + if hFrom > hTo { + // If the FROM is larger than the TO, we have a range like 22-2 + // Calculate the difference between the two hours with a 24 hour jump (in example will be 4) + maxDiff := float64(24 - hFrom + hTo) + // Generate a difference based on the SEED (in example will be 0, 1, 2, 3 or 4) + diff := int(math.Mod(float64(seed), maxDiff)) + // Add the generated difference to the FROM hour (in example will be 22, 23, 24, 25 or 26) + if hFrom+diff >= 24 { + // If the hour is higher than 24, we subtract 24 to handle the midnight change + hours = strconv.Itoa(hFrom + diff - 24) + continue + } + hours = strconv.Itoa(hFrom + diff) + continue + } + if hFrom == hTo { + hours = strconv.Itoa(hFrom) + continue + } + } + if isInCSVRange(val, 0, 23) { + hours = val + continue + } + if isInRange(val, 0, 23) { + hours = val + continue + } + if val == "*" { + hours = val + continue + } + // if the value is not valid, return an error with where the issue is + if hours == "" { + return "", fmt.Errorf("cron definition '%s' is invalid, unable to determine hours value", cron) + } + } + if idx == 2 { + match1, _ := regexp.MatchString("^D$", val) + if match1 { + // If just an `D` is defined, we generate a pseudo random day of the month, but only generated up to 31 + // so february is never skipped + days = strconv.Itoa(int(math.Mod(float64(seed), 32))) + // days can't be 0, support 1-31 only + if days == "0" { + days = "1" + } + continue + } + match2, _ := regexp.MatchString("^D\\(([01]?[0-9]|2[0-9]|3[0-1])-([01]?[0-9]|2[0-9]|3[0-1])\\)$", val) + if match2 { + // If D is defined with a given range, example: D(2-4), we generate a random day of the month between 2-4 + params := getCaptureBlocks("^D\\((?P[01]?[0-9]|2[0-9]|3[0-1])-(?P[01]?[0-9]|2[0-9]|3[0-1])\\)$", val) + hFrom, err := strconv.Atoi(params["P1"]) + if err != nil { + return "", fmt.Errorf("cron definition '%s' is invalid, unable to determine day of month value", cron) + } + if hFrom == 0 { + return "", fmt.Errorf("cron definition '%s' is invalid, unable to determine day of month value, starting day can't be 0", cron) + } + hTo, err := strconv.Atoi(params["P2"]) + if err != nil { + return "", fmt.Errorf("cron definition '%s' is invalid, unable to determine day of month value", cron) + } + if hFrom < hTo { + maxDiff := float64(hTo - hFrom) + diff := int(math.Mod(float64(seed), maxDiff)) + days = strconv.Itoa(hFrom + diff) + continue + } + if hFrom > hTo { + maxDiff := float64(29 - hFrom + hTo) + diff := int(math.Mod(float64(seed), maxDiff)) + if hFrom+diff >= 29 { + days = strconv.Itoa(hFrom + diff - 29) + continue + } + days = strconv.Itoa(hFrom + diff) + continue + } + if hFrom == hTo { + days = strconv.Itoa(hFrom) + continue + } + } + if isInCSVRange(val, 1, 31) { + days = val + continue + } + if isInRange(val, 1, 31) { + days = val + continue + } + if val == "*" { + days = val + continue + } + // if the value is not valid, return an error with where the issue is + if days == "" { + return "", fmt.Errorf("cron definition '%s' is invalid, unable to determine days value", cron) + } + } + if idx == 3 { + if isInCSVRange(val, 1, 12) { + months = val + continue + } + if isInRange(val, 1, 12) { + months = val + continue + } + if val == "*" { + months = val + continue + } + // if the value is not valid, return an error with where the issue is + if months == "" { + return "", fmt.Errorf("cron definition '%s' is invalid, unable to determine months value", cron) + } + } + if idx == 4 { + match1, _ := regexp.MatchString("^D$", val) + if match1 { + // If just an `D` is defined, we generate a pseudo random day of the week. + dayweek = strconv.Itoa(int(math.Mod(float64(seed), 6))) + continue + } + match2, _ := regexp.MatchString("^D\\(([0-6])-([0-6])\\)$", val) + if match2 { + // If D is defined with a given range, example: D(2-4), we generate a random day of the week between 2-4 + params := getCaptureBlocks("^D\\((?P[0-6])-(?P[0-6])\\)$", val) + hFrom, err := strconv.Atoi(params["P1"]) + if err != nil { + return "", fmt.Errorf("cron definition '%s' is invalid, unable to determine day value", cron) + } + hTo, err := strconv.Atoi(params["P2"]) + if err != nil { + return "", fmt.Errorf("cron definition '%s' is invalid, unable to determine day value", cron) + } + if hFrom < hTo { + maxDiff := float64(hTo - hFrom) + diff := int(math.Mod(float64(seed), maxDiff)) + dayweek = strconv.Itoa(hFrom + diff) + continue + } + if hFrom > hTo { + maxDiff := float64(6 - hFrom + hTo) + diff := int(math.Mod(float64(seed), maxDiff)) + if hFrom+diff >= 6 { + dayweek = strconv.Itoa(hFrom + diff - 6) + continue + } + dayweek = strconv.Itoa(hFrom + diff) + continue + } + if hFrom == hTo { + dayweek = strconv.Itoa(hFrom) + continue + } + } + if isInCSVRange(val, 0, 6) { + dayweek = val + continue + } + if isInRange(val, 0, 6) { + dayweek = val + continue + } + if val == "*" { + dayweek = val + continue + } + // if the value is not valid, return an error with where the issue is + if dayweek == "" { + return "", fmt.Errorf("cron definition '%s' is invalid, unable to determine day(week) value", cron) + } + } + } + return fmt.Sprintf("%v %v %v %v %v", minutes, hours, days, months, dayweek), nil + } + return "", fmt.Errorf("cron definition '%s' is invalid", cron) +} + +func getCaptureBlocks(regex, val string) (captureMap map[string]string) { + var regexComp = regexp.MustCompile(regex) + match := regexComp.FindStringSubmatch(val) + captureMap = make(map[string]string) + for i, name := range regexComp.SubexpNames() { + if i > 0 && i <= len(match) { + captureMap[name] = match[i] + } + } + return captureMap +} + +// check if the provided cron time definition is a valid `1,2,4,8` type range +func isInCSVRange(s string, min, max int) bool { + items := strings.Split(s, ",") + for _, val := range items { + num, err := strconv.Atoi(val) + if err != nil { + // not a number, return false + return false + } + if num < min || num > max { + // outside range, return false + return false + } + } + return true +} + +// check if the provided cron time definition is a valid `1-2` type range +func isInRange(s string, min, max int) bool { + items := strings.Split(s, "-") + if len(items) > 2 || len(items) < 1 { + // too many or not enough items split by - + return false + } + hFrom, err := strconv.Atoi(items[0]) + if err != nil { + // not a number or error checking if it is, return false + return false + } + hTo, err := strconv.Atoi(items[1]) + if err != nil { + // not a number or error checking if it is, return false + return false + } + if hFrom > hTo || hFrom < min || hFrom > max || hTo < min || hTo > max { + // numbers in range are not in valid format of LOW-HIGH + return false + } + return true +} diff --git a/internal/helpers/helpers_cron_test.go b/internal/helpers/helpers_cron_test.go new file mode 100644 index 00000000..187189a3 --- /dev/null +++ b/internal/helpers/helpers_cron_test.go @@ -0,0 +1,236 @@ +package helpers + +import ( + "testing" +) + +func TestConvertCrontab(t *testing.T) { + type args struct { + namespace string + cron string + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "test1", + args: args{ + namespace: "example-com-main", + cron: "M * * * *", + }, + want: "31 * * * *", + }, + { + name: "test2", + args: args{ + namespace: "example-com-main", + cron: "M/5 * * * *", + }, + want: "1,6,11,16,21,26,31,36,41,46,51,56 * * * *", + }, + { + name: "test3", + args: args{ + namespace: "example-com-main", + cron: "M H(2-4) * * *", + }, + want: "31 3 * * *", + }, + { + name: "test4", + args: args{ + namespace: "example-com-main", + cron: "M H(22-2) * * *", + }, + want: "31 1 * * *", + }, + { + name: "test5", + args: args{ + namespace: "example-com-main", + cron: "M/15 H(22-2) * * *", + }, + want: "1,16,31,46 1 * * *", + }, + { + name: "test6 - invalid minutes definition", + args: args{ + namespace: "example-com-main", + cron: "M/H5 H(22-2) * * *", + }, + wantErr: true, + }, + { + name: "test7 - invalid hour definiton", + args: args{ + namespace: "example-com-main", + cron: "M/15 H(H2-2) * * *", + }, + wantErr: true, + }, + { + name: "test8", + args: args{ + namespace: "example-com-main", + cron: "M/15 H(22-2) 3,5 * *", + }, + want: "1,16,31,46 1 3,5 * *", + }, + { + name: "test9", + args: args{ + namespace: "example-com-main", + cron: "M/15 H(22-2) * 10-12 *", + }, + want: "1,16,31,46 1 * 10-12 *", + }, + { + name: "test10 - invalid dayofweek range", + args: args{ + namespace: "example-com-main", + cron: "M/15 H(22-2) * * 1-8", + }, + wantErr: true, + }, + { + name: "test11", + args: args{ + namespace: "example-com-main", + cron: "15 * * * 1,2,3,6", + }, + want: "15 * * * 1,2,3,6", + }, + { + name: "test12", + args: args{ + namespace: "example-com-main", + cron: "15 * 1-31 * *", + }, + want: "15 * 1-31 * *", + }, + { + name: "test13 - invalid day range", + args: args{ + namespace: "example-com-main", + cron: "15 * 1-32 * *", + }, + wantErr: true, + }, + { + name: "test14 - set hours", + args: args{ + namespace: "example-com-main", + cron: "M/15 23 * * 0-5", + }, + want: "1,16,31,46 23 * * 0-5", + }, + { + name: "test15 - set day", + args: args{ + namespace: "example-com-main", + cron: "M/15 * 31 * 0-5", + }, + want: "1,16,31,46 * 31 * 0-5", + }, + { + name: "test16 - set month", + args: args{ + namespace: "example-com-main", + cron: "M/15 * * 11 0-5", + }, + want: "1,16,31,46 * * 11 0-5", + }, + { + name: "test17 - pick day of week between range", + args: args{ + namespace: "example-com-main", + cron: "M/15 * * * D(0-4)", + }, + want: "1,16,31,46 * * * 3", + }, + { + name: "test18 - pick day of week between range", + args: args{ + namespace: "example-com-main", + cron: "M/15 * * * D(2-6)", + }, + want: "1,16,31,46 * * * 5", + }, + { + name: "test19 - pick day of week random", + args: args{ + namespace: "example-com-main", + cron: "M/15 * * * D", + }, + want: "1,16,31,46 * * * 1", + }, + { + name: "test20 - pick day of month between range", + args: args{ + namespace: "example-com-main", + cron: "M/15 * D(1-31) * *", + }, + want: "1,16,31,46 * 2 * *", + }, + { + name: "test21 - pick day of month between range", + args: args{ + namespace: "example-com-main", + cron: "M/15 * D(15-25) * *", + }, + want: "1,16,31,46 * 16 * *", + }, + { + name: "test22 - pick day of month random", + args: args{ + namespace: "example-com-main", + cron: "M/15 * D * *", + }, + want: "1,16,31,46 * 27 * *", + }, + { + name: "test23 - pick day of month between range with invalid end day (only support 1-31)", + args: args{ + namespace: "example-com-main", + cron: "M/15 * D(1-32) * *", + }, + wantErr: true, + }, + { + name: "test24 - pick day of week between range with invalid end day (only support 0-6)", + args: args{ + namespace: "example-com-main", + cron: "M/15 * * * D(0-7)", + }, + wantErr: true, + }, + { + name: "test26 - pick day of month between range with invalid start day (only support 1-31)", + args: args{ + namespace: "example-com-main", + cron: "M/15 * D(0-28) * *", + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ConvertCrontab(tt.args.namespace, tt.args.cron) + if err != nil { + if !tt.wantErr { + t.Errorf("ConvertCrontab() error = %v, wantErr %v", err, tt.wantErr) + } + } + if got != tt.want { + if !tt.wantErr { + t.Errorf("ConvertCrontab() = %v, want %v", got, tt.want) + } else { + t.Errorf("ConvertCrontab() = %v, wantErr %v", got, tt.wantErr) + } + } + }) + } +} diff --git a/main.go b/main.go index 7238b1ab..104fb31a 100644 --- a/main.go +++ b/main.go @@ -175,6 +175,11 @@ func main() { var enablePodProxy bool var podsUseDifferentProxy bool + // @TODO: policy control in remote-controller disabled for now + // var enableHarborRetentionPolicy bool + // var harborRetentionBranch, harborRetentionPullrequest int + // var harborRetentionSchedule string + flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.") flag.StringVar(&lagoonTargetName, "lagoon-target-name", "ci-local-control-k8s", @@ -368,6 +373,15 @@ func main() { flag.IntVar(&pvcRetryAttempts, "delete-pvc-retry-attempts", 30, "How many attempts to check that PVCs have been removed (default 30).") flag.IntVar(&pvcRetryInterval, "delete-pvc-retry-interval", 10, "The number of seconds between each retry attempt (default 10).") + // retention policy configuration + // @TODO: policy control in remote-controller disabled for now + // flag.BoolVar(&enableHarborRetentionPolicy, "enable-harbor-retention-policy", false, + // "Flag to have this controller create and manage retention policies.\n The Lagoon API can also configure per project retention policies which will override the controller defined policy.") + // flag.IntVar(&harborRetentionBranch, "harbor-retention-branch", 5, "The number of latest pulled image tags for branch environments to retain.") + // flag.IntVar(&harborRetentionPullrequest, "harbor-retention-pullrequest", 1, "The number of latest pulled image tags for pullrequest environments to retain.") + // flag.StringVar(&harborRetentionSchedule, "harbor-retention-schedule", "M H(22-2) D(5-25) * *", + // "The schedule to use for harbor tag retentions, the lagoon project name will influence any replacement values that impact the time this could run per harbor project.") + flag.Parse() // get overrides from environment variables @@ -459,6 +473,18 @@ func main() { } } + // @TODO: policy control in remote-controller disabled for now + // enableHarborRetentionPolicy = helpers.GetEnvBool("HARBOR_RETENTION_POLICIES_ENABLED", enableHarborRetentionPolicy) + // harborRetentionBranch = helpers.GetEnvInt("HARBOR_RETENTION_POLICY_BRANCH", harborRetentionBranch) + // harborRetentionPullrequest = helpers.GetEnvInt("HARBOR_RETENTION_POLICY_PULLREQUEST", harborRetentionPullrequest) + // harborRetentionSchedule = helpers.GetEnv("HARBOR_RETENTION_POLICY_SCHEDULE", harborRetentionSchedule) + // validate provided retention policy and crash the controller if it is invalid + // _, err := helpers.ConvertCrontab("seed-data", harborRetentionSchedule) + // if err != nil { + // setupLog.Error(err, "unable to start manager - harbor retention schedule is formatted incorrectly") + // os.Exit(1) + // } + ctrl.SetLogger(zap.New(func(o *zap.Options) { o.Development = true })) @@ -624,6 +650,13 @@ func main() { WebhookURL: harborLagoonWebhook, LagoonTargetName: lagoonTargetName, WebhookEventTypes: strings.Split(harborWebhookEventTypes, ","), + // @TODO: policy control in remote-controller disabled for now + // TagRetention: harbor.TagRetention{ + // Enabled: enableHarborRetentionPolicy, + // BranchRetention: harborRetentionBranch, + // PullRequestRetention: harborRetentionPullrequest, + // Schedule: harborRetentionSchedule, + // }, } deletion := deletions.New(mgr.GetClient(),