feat: support for pod spread constraints and additional spot controls (…

…#364)
uselagoon · Sep 13, 2024 · 5b92b0e · 5b92b0e
1 parent 25bb7b8
commit 5b92b0e
Show file tree

Hide file tree

Showing 20 changed files with 514 additions and 3 deletions.
diff --git a/cmd/identify_dbaas_test.go b/cmd/identify_dbaas_test.go
@@ -134,6 +134,7 @@ func TestIdentifyDBaaSConsumers(t *testing.T) {
 			}
 			t.Cleanup(func() {
 				helpers.UnsetEnvVars(tt.vars)
+				helpers.UnsetEnvVars(tt.args.BuildPodVariables)
 			})
 		})
 	}

diff --git a/cmd/identify_feature_test.go b/cmd/identify_feature_test.go
@@ -184,6 +184,7 @@ func TestIdentifyFeatureFlag(t *testing.T) {
 			}
 			t.Cleanup(func() {
 				helpers.UnsetEnvVars(tt.vars)
+				helpers.UnsetEnvVars(tt.args.BuildPodVariables)
 			})
 		})
 	}

diff --git a/cmd/identify_imagebuild_test.go b/cmd/identify_imagebuild_test.go
@@ -795,6 +795,7 @@ func TestImageBuildConfigurationIdentification(t *testing.T) {
 			}
 			t.Cleanup(func() {
 				helpers.UnsetEnvVars(tt.vars)
+				helpers.UnsetEnvVars(tt.args.BuildPodVariables)
 			})
 		})
 	}

diff --git a/cmd/identify_ingress_test.go b/cmd/identify_ingress_test.go
@@ -529,6 +529,10 @@ func TestCreatedIngressIdentification(t *testing.T) {
 			if string(retJSON) != tt.wantJSON {
 				t.Errorf("returned autogen %v doesn't match want %v", string(retJSON), tt.wantJSON)
 			}
+			t.Cleanup(func() {
+				helpers.UnsetEnvVars(nil)
+				helpers.UnsetEnvVars(tt.args.BuildPodVariables)
+			})
 		})
 	}
 }
diff --git a/cmd/identify_lagoonservices_test.go b/cmd/identify_lagoonservices_test.go
@@ -341,6 +341,7 @@ func TestIdentifyLagoonServices(t *testing.T) {
 			}
 			t.Cleanup(func() {
 				helpers.UnsetEnvVars(nil)
+				helpers.UnsetEnvVars(tt.args.BuildPodVariables)
 			})
 		})
 	}

diff --git a/cmd/identify_native_cronjobs_test.go b/cmd/identify_native_cronjobs_test.go
@@ -99,6 +99,7 @@ func TestIdentifyNativeCronjobs(t *testing.T) {
 
 			t.Cleanup(func() {
 				helpers.UnsetEnvVars(nil)
+				helpers.UnsetEnvVars(tt.args.BuildPodVariables)
 			})
 		})
 	}

diff --git a/cmd/template_autogen_ingress_test.go b/cmd/template_autogen_ingress_test.go
@@ -625,6 +625,7 @@ func TestAutogeneratedIngressGeneration(t *testing.T) {
 			}
 			t.Cleanup(func() {
 				helpers.UnsetEnvVars(nil)
+				helpers.UnsetEnvVars(tt.args.BuildPodVariables)
 			})
 		})
 	}

diff --git a/cmd/template_backups_test.go b/cmd/template_backups_test.go
@@ -325,6 +325,7 @@ func TestBackupTemplateGeneration(t *testing.T) {
 			}
 			t.Cleanup(func() {
 				helpers.UnsetEnvVars(nil)
+				helpers.UnsetEnvVars(tt.args.BuildPodVariables)
 			})
 		})
 	}

diff --git a/cmd/template_dbaas_test.go b/cmd/template_dbaas_test.go
@@ -202,6 +202,7 @@ func TestDBaaSTemplateGeneration(t *testing.T) {
 			}
 			t.Cleanup(func() {
 				helpers.UnsetEnvVars(nil)
+				helpers.UnsetEnvVars(tt.args.BuildPodVariables)
 			})
 		})
 	}

diff --git a/cmd/template_ingress_test.go b/cmd/template_ingress_test.go
@@ -511,6 +511,7 @@ func TestTemplateRoutes(t *testing.T) {
 			}
 			t.Cleanup(func() {
 				helpers.UnsetEnvVars(nil)
+				helpers.UnsetEnvVars(tt.args.BuildPodVariables)
 			})
 		})
 	}

diff --git a/cmd/template_lagoonservices_test.go b/cmd/template_lagoonservices_test.go
@@ -441,6 +441,45 @@ func TestTemplateLagoonServices(t *testing.T) {
 			templatePath: "testoutput",
 			want:         "internal/testdata/basic/service-templates/test15-basic-custom-volume-no-backup",
 		},
+		{
+			name: "test-basic-spot-affinity",
+			args: testdata.GetSeedData(
+				testdata.TestData{
+					ProjectName:     "example-project",
+					EnvironmentName: "main",
+					Branch:          "main",
+					EnvironmentType: "development",
+					LagoonYAML:      "internal/testdata/basic/lagoon.yml",
+					ImageReferences: map[string]string{
+						"node": "harbor.example/example-project/main/node@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8",
+					},
+					BuildPodVariables: []helpers.EnvironmentVariable{
+						{
+							Name:  "LAGOON_FEATURE_FLAG_DEFAULT_POD_SPREADCONSTRAINTS",
+							Value: "enabled",
+						},
+						{
+							Name:  "LAGOON_FEATURE_FLAG_DEFAULT_ROOTLESS_WORKLOAD",
+							Value: "enabled",
+						},
+						{
+							Name:  "LAGOON_FEATURE_FLAG_DEFAULT_SPOT_INSTANCE_DEVELOPMENT",
+							Value: "enabled",
+						},
+						{
+							Name:  "LAGOON_FEATURE_FLAG_DEFAULT_SPOT_INSTANCE_DEVELOPMENT_TYPES",
+							Value: "basic,basic-persistent",
+						},
+						{
+							// `ADMIN_` are only configurable by the remote-controller
+							Name:  "ADMIN_LAGOON_FEATURE_FLAG_SPOT_TYPE_REPLICAS_DEVELOPMENT",
+							Value: "basic,basic-persistent",
+						},
+					},
+				}, true),
+			templatePath: "testoutput",
+			want:         "internal/testdata/basic/service-templates/test-basic-spot-affinity",
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -533,6 +572,7 @@ func TestTemplateLagoonServices(t *testing.T) {
 			}
 			t.Cleanup(func() {
 				helpers.UnsetEnvVars(tt.vars)
+				helpers.UnsetEnvVars(tt.args.BuildPodVariables)
 			})
 		})
 	}

diff --git a/internal/generator/buildvalues.go b/internal/generator/buildvalues.go
@@ -82,6 +82,7 @@ type BuildValues struct {
 	SSHPrivateKey                 string                       `json:"sshPrivateKey"`
 	ForcePullImages               []string                     `json:"forcePullImages"`
 	Volumes                       []ComposeVolume              `json:"volumes,omitempty" description:"stores any additional persistent volume definitions"`
+	PodAntiAffinity               bool                         `json:"podAntiAffinity"`
 }
 
 type Resources struct {

diff --git a/internal/generator/generator.go b/internal/generator/generator.go
@@ -270,6 +270,12 @@ func NewGenerator(
 		return nil, err
 	}
 
+	// feature to enable pod antiaffinity on deployments
+	podAntiAffinity := CheckFeatureFlag("POD_SPREADCONSTRAINTS", buildValues.EnvironmentVariables, false)
+	if podAntiAffinity == "enabled" {
+		buildValues.PodAntiAffinity = true
+	}
+
 	// check for readwritemany to readwriteonce flag, disabled by default
 	rwx2rwo := CheckFeatureFlag("RWX_TO_RWO", buildValues.EnvironmentVariables, generator.Debug)
 	if rwx2rwo == "enabled" {

diff --git a/internal/generator/services.go b/internal/generator/services.go
@@ -399,8 +399,14 @@ func composeToServiceValues(
 
 		// these services can support multiple replicas in production
 		// @TODO this should probably be an admin only feature flag though
-		spotReplicaTypes := "nginx,nginx-persistent,nginx-php,nginx-php-persistent"
-		// spotReplicaTypes := CheckAdminFeatureFlag("SPOT_REPLICAS_PRODUCTION", buildValues.EnvironmentVariables, debug) // doesn't exist yet
+		prodSpotReplicaTypes := CheckAdminFeatureFlag("SPOT_TYPE_REPLICAS_PRODUCTION", debug)
+		if prodSpotReplicaTypes == "" {
+			prodSpotReplicaTypes = "nginx,nginx-persistent,nginx-php,nginx-php-persistent"
+		}
+		devSpotReplicaTypes := CheckAdminFeatureFlag("SPOT_TYPE_REPLICAS_DEVELOPMENT", debug)
+		if devSpotReplicaTypes == "" {
+			devSpotReplicaTypes = ""
+		}
 
 		productionSpot := CheckFeatureFlag("SPOT_INSTANCE_PRODUCTION", buildValues.EnvironmentVariables, debug)
 		developmentSpot := CheckFeatureFlag("SPOT_INSTANCE_DEVELOPMENT", buildValues.EnvironmentVariables, debug)
@@ -439,13 +445,20 @@ func composeToServiceValues(
 			}
 		}
 		// check if the this service is production and can support 2 replicas on spot
-		for _, t := range strings.Split(spotReplicaTypes, ",") {
+		for _, t := range strings.Split(prodSpotReplicaTypes, ",") {
 			if t != "" {
 				if t == lagoonType && buildValues.EnvironmentType == "production" && useSpot {
 					spotReplicas = 2
 				}
 			}
 		}
+		for _, t := range strings.Split(devSpotReplicaTypes, ",") {
+			if t != "" {
+				if t == lagoonType && buildValues.EnvironmentType == "development" && useSpot {
+					spotReplicas = 2
+				}
+			}
+		}
 		// end spot instance handling
 
 		// work out cronjobs for this service

diff --git a/internal/templating/services/templates_deployment.go b/internal/templating/services/templates_deployment.go
@@ -138,6 +138,92 @@ func GenerateDeploymentTemplate(
 					}
 				}
 			}
+			if buildValues.PodAntiAffinity {
+				// if deployment.Spec.Template.Spec.Affinity == nil {
+				// 	deployment.Spec.Template.Spec.Affinity = &corev1.Affinity{}
+				// }
+				deployment.Spec.Template.Spec.TopologySpreadConstraints = []corev1.TopologySpreadConstraint{
+					{
+						MaxSkew:           1,
+						WhenUnsatisfiable: corev1.ScheduleAnyway,
+						TopologyKey:       "kubernetes.io/hostname",
+						LabelSelector: &metav1.LabelSelector{
+							MatchExpressions: []metav1.LabelSelectorRequirement{
+								{
+									Key:      "app.kubernetes.io/name",
+									Operator: metav1.LabelSelectorOpIn,
+									Values: []string{
+										serviceTypeValues.Name,
+									},
+								},
+								{
+									Key:      "app.kubernetes.io/instance",
+									Operator: metav1.LabelSelectorOpIn,
+									Values: []string{
+										serviceValues.OverrideName,
+									},
+								},
+								{
+									Key:      "lagoon.sh/project",
+									Operator: metav1.LabelSelectorOpIn,
+									Values: []string{
+										buildValues.Project,
+									},
+								},
+								{
+									Key:      "lagoon.sh/environment",
+									Operator: metav1.LabelSelectorOpIn,
+									Values: []string{
+										buildValues.Environment,
+									},
+								},
+							},
+						},
+					},
+				}
+				// deployment.Spec.Template.Spec.Affinity.PodAntiAffinity = &corev1.PodAntiAffinity{
+				// 	PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{
+				// 		{
+				// 			Weight: 100,
+				// 			PodAffinityTerm: corev1.PodAffinityTerm{
+				// 				TopologyKey: "kubernetes.io/hostname",
+				// 				LabelSelector: &metav1.LabelSelector{
+				// 					MatchExpressions: []metav1.LabelSelectorRequirement{
+				// 						{
+				// 							Key:      "app.kubernetes.io/name",
+				// 							Operator: metav1.LabelSelectorOpIn,
+				// 							Values: []string{
+				// 								serviceTypeValues.Name,
+				// 							},
+				// 						},
+				// 						{
+				// 							Key:      "app.kubernetes.io/instance",
+				// 							Operator: metav1.LabelSelectorOpIn,
+				// 							Values: []string{
+				// 								serviceValues.OverrideName,
+				// 							},
+				// 						},
+				// 						{
+				// 							Key:      "lagoon.sh/project",
+				// 							Operator: metav1.LabelSelectorOpIn,
+				// 							Values: []string{
+				// 								buildValues.Project,
+				// 							},
+				// 						},
+				// 						{
+				// 							Key:      "lagoon.sh/environment",
+				// 							Operator: metav1.LabelSelectorOpIn,
+				// 							Values: []string{
+				// 								buildValues.Environment,
+				// 							},
+				// 						},
+				// 					},
+				// 				},
+				// 			},
+				// 		},
+				// 	},
+				// }
+			}
 
 			for key, value := range additionalLabels {
 				deployment.ObjectMeta.Labels[key] = value

diff --git a/internal/templating/services/templates_deployment_test.go b/internal/templating/services/templates_deployment_test.go
@@ -856,6 +856,62 @@ func TestGenerateDeploymentTemplate(t *testing.T) {
 			},
 			want: "test-resources/deployment/result-postgres-1.yaml",
 		},
+		{
+			name: "test-basic-antiaffinity",
+			args: args{
+				buildValues: generator.BuildValues{
+					Project:         "example-project",
+					Environment:     "environment-name",
+					EnvironmentType: "production",
+					Namespace:       "myexample-project-environment-name",
+					BuildType:       "branch",
+					LagoonVersion:   "v2.x.x",
+					Kubernetes:      "generator.local",
+					Branch:          "environment-name",
+					Resources: generator.Resources{
+						Limits: generator.ResourceLimits{
+							Memory:           "16Gi",
+							EphemeralStorage: "160Gi",
+						},
+						Requests: generator.ResourceRequests{
+							EphemeralStorage: "1Gi",
+						},
+					},
+					GitSHA:       "0",
+					ConfigMapSha: "32bf1359ac92178c8909f0ef938257b477708aa0d78a5a15ad7c2d7919adf273",
+					ImageReferences: map[string]string{
+						"myservice": "harbor.example.com/example-project/environment-name/myservice@latest",
+					},
+					PodAntiAffinity: true,
+					Services: []generator.ServiceValues{
+						{
+							Name:             "myservice",
+							OverrideName:     "myservice",
+							Type:             "basic",
+							DBaaSEnvironment: "production",
+							AdditionalServicePorts: []generator.AdditionalServicePort{
+								{
+									ServiceName: "myservice-8191",
+									ServicePort: types.ServicePortConfig{
+										Target:   8191,
+										Protocol: "tcp",
+									},
+								},
+								{
+									ServiceName: "myservice-8211",
+									ServicePort: types.ServicePortConfig{
+										Target:   8211,
+										Protocol: "tcp",
+									},
+								},
+							},
+							Replicas: 2,
+						},
+					},
+				},
+			},
+			want: "test-resources/deployment/result-basic-4.yaml",
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-Original file line number
+Diff line change
@@ Expand Up / @@ -134,6 +134,7 @@ func TestIdentifyDBaaSConsumers(t *testing.T) { @@
     			}
     			t.Cleanup(func() {
     				helpers.UnsetEnvVars(tt.vars)
+    				helpers.UnsetEnvVars(tt.args.BuildPodVariables)
     			})
     		})
     	}
@@ Expand Down @@